From 4d13d704dcd876bc6f4d182bdf3673fe1efd5a47 Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Sat, 5 Oct 2024 19:48:25 +0200 Subject: [PATCH] Clean up Redis logic Simplify functions and move internal database interfaces into a more appropriate data structure. Configure all database numbers using a redis plugin group, instead of using a custom setting just for the whois database to make the setup more logical. Signed-off-by: Georg Pfuetzenreuter --- config.py | 51 +++++++++++++++--------------- plugin.py | 93 +++++++++++++++++-------------------------------------- 2 files changed, 54 insertions(+), 90 deletions(-) diff --git a/config.py b/config.py index 14654c5..f58e176 100644 --- a/config.py +++ b/config.py @@ -104,20 +104,10 @@ conf.registerGlobalValue( # REDIS related settings below: ### conf.registerGroup(SnoParser, "redis") +conf.registerGroup(SnoParser.redis, "db") conf.registerGlobalValue( - SnoParser.redis, - "db1", - registry.Integer( - 1, - """ - Redis: Database number for counting of NICKNAMES. - """, - private=True, - ), -) -conf.registerGlobalValue( - SnoParser.redis, - "db2", + SnoParser.redis.db, + "ips", registry.Integer( 2, """ @@ -126,11 +116,33 @@ conf.registerGlobalValue( private=True, ), ) +conf.registerGlobalValue( + SnoParser.redis.db, + "nicks", + registry.Integer( + 1, + """ + Redis: Database number for counting of NICKNAMES. + """, + private=True, + ), +) +conf.registerGlobalValue( + SnoParser.redis.db, + "whois", + registry.Integer( + 0, + """ + Redis: Database number for WHOIS query caching. + """, + ), +) + conf.registerGlobalValue( SnoParser.redis, "host", registry.String( - "127.0.0.1", + "localhost", """ Redis: IP address or hostname. """, @@ -218,16 +230,5 @@ conf.registerGlobalValue( private=True, ), ) -conf.registerGroup(SnoParser.whois, "redis") -conf.registerGlobalValue( - SnoParser.whois.redis, - "db", - registry.Integer( - 0, - """ - Redis: Database number for WHOIS query caching. - """, - ), -) # vim:set shiftwidth=4 tabstop=4 expandtab textwidth=79: diff --git a/plugin.py b/plugin.py index ce6e9fd..501ee3c 100644 --- a/plugin.py +++ b/plugin.py @@ -57,62 +57,38 @@ class SnoParser(callbacks.Plugin): threaded = True - def redis_connect_whois(self) -> redis.client.Redis: + def _redis_connect(self, db) -> redis.client.Redis: try: - redis_client_whois = redis.Redis( + redis_client = redis.Redis( host=self.registryValue("redis.host"), port=self.registryValue("redis.port"), password=self.registryValue("redis.password"), username=self.registryValue("redis.username"), - db=self.registryValue("whois.redis.db"), + db=self.registryValue(f"redis.db.{db}"), socket_timeout=int(self.registryValue("redis.timeout")), ) - ping = redis_client_whois.ping() + ping = redis_client.ping() if ping is True: - return redis_client_whois - except redis.AuthenticationError: - print("Could not authenticate to Redis backend.") - - def redis_connect_nicks(self) -> redis.client.Redis: - try: - redis_client_nicks = redis.Redis( - host=self.registryValue("redis.host"), - port=self.registryValue("redis.port"), - password=self.registryValue("redis.password"), - username=self.registryValue("redis.username"), - db=self.registryValue("redis.db1"), - socket_timeout=int(self.registryValue("redis.timeout")), - ) - ping = redis_client_nicks.ping() - if ping is True: - return redis_client_nicks - except redis.AuthenticationError: - print("Could not authenticate to Redis backend.") - - def redis_connect_ips(self) -> redis.client.Redis: - try: - redis_client_ips = redis.Redis( - host=self.registryValue("redis.host"), - port=self.registryValue("redis.port"), - password=self.registryValue("redis.password"), - username=self.registryValue("redis.username"), - db=self.registryValue("redis.db2"), - socket_timeout=int(self.registryValue("redis.timeout")), - ) - ping = redis_client_ips.ping() - if ping is True: - return redis_client_ips + return redis_client except redis.AuthenticationError: print("Could not authenticate to Redis backend.") def __init__(self, irc): super().__init__(irc) - self.redis_client_whois = self.redis_connect_whois() - self.redis_client_nicks = self.redis_connect_nicks() - self.redis_client_ips = self.redis_connect_ips() + self.redis_clients = { + 'ips': self._redis_connect("ips"), + 'nicks': self._redis_connect("nicks"), + 'whois': self._redis_connect("whois"), + } + + def _get_from_cache(self, db, key): + """ Get value from Redis cache """ + + return self.redis_clients[db].get(key) def whois_fresh(self, sourceip: str) -> dict: """Data from WHOIS backend (IANA or respective RIR).""" + asn = 0 subnet = "" try: @@ -131,31 +107,19 @@ class SnoParser(callbacks.Plugin): response = whoisout return response - def whois_get_cache(self, key: str) -> str: - """Data from Redis.""" - - k = self.redis_client_whois.get(key) - - # self = SnoParser() - # val = self.redis_client_whois.get(key) - val = k - return val - def whois_set_cache(self, key: str, value: str) -> bool: """Data to Redis.""" - duration = int(self.registryValue("whois.ttl")) - state = self.redis_client_whois.setex( + return self.redis_clients['whois'].setex( key, - timedelta(seconds=duration), + timedelta(seconds=int(self.registryValue("whois.ttl"))), value=value, ) - return state def whois_run(self, sourceip: str) -> dict: """Whois query router.""" - data = self.whois_get_cache(key=sourceip) + data = self._get_from_cache("whois", sourceip) if data is not None: data = json.loads(data) if self.registryValue("whois.debug"): @@ -191,14 +155,14 @@ class SnoParser(callbacks.Plugin): def nick_run(self, nickname: str) -> dict: """Tracks nicknames""" - data = self.redis_client_nicks.get(nickname) + data = self._get_from_cache("nicks", nickname) if data is not None: if self.registryValue("debug"): print("SNOPARSER DEBUG - NICK_RUN, SEEN: TRUE") print(nickname) print(data) - self.redis_client_nicks.incrby(nickname, amount=1) + self.redis_clients['nicks'].incrby(nickname, amount=1) if data: decoded = data.decode("utf-8") return decoded @@ -209,7 +173,7 @@ class SnoParser(callbacks.Plugin): print("SNOPARSER DEBUG _ NICK_RUN, SEEN: FALSE") print(nickname) print(data) - self.redis_client_nicks.set(nickname, value="1") + self.redis_clients['nicks'].set(nickname, value="1") if data: decoded = data.decode("utf-8") return decoded @@ -219,14 +183,13 @@ class SnoParser(callbacks.Plugin): def ip_run(self, ipaddress: str) -> dict: """Tracks IP addresses""" - data = self.redis_client_ips.get(ipaddress) - + data = self._get_from_cache("ips", ipaddresses) if data is not None: if self.registryValue("debug"): print("SNOPARSER DEBUG - IP_RUN, SEEN: TRUE") print(ipaddress) print(data) - self.redis_client_ips.incrby(ipaddress, amount=1) + self.redis_clients['ips'].incrby(ipaddress, amount=1) if data: decoded = data.decode("utf-8") return decoded @@ -237,7 +200,7 @@ class SnoParser(callbacks.Plugin): print("SNOPARSER DEBUG _ IP_RUN, SEEN: FALSE") print(ipaddress) print(data) - self.redis_client_ips.set(ipaddress, value="1") + self.redis_clients['ips'].set(ipaddress, value="1") if data: decoded = data.decode("utf-8") return decoded @@ -249,10 +212,10 @@ class SnoParser(callbacks.Plugin): Queries the cache for an address. """ - data = self.whois_get_cache(key=ipaddress) + data = self.redis_clients['whois'].get(sourceip) decoded_data = data.decode("utf-8") - ttl = self.redis_client_whois.ttl(ipaddress) - count = self.redis_client_ips.get(ipaddress) + ttl = self.redis_clients['whois'].ttl(ipaddress) + count = self.redis_clients['ips'].get(ipaddress) decoded_count = count.decode("utf-8") print("SnoParser manual query: ", data, " ", ttl) irc.reply(f"{decoded_data} - Count: {decoded_count} - Remaining: {ttl}s")