From 20797b44cd58e8a9eafac05d3534debfe938308f Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Sat, 5 Oct 2024 19:48:09 +0200 Subject: [PATCH 1/5] Clean up imports Signed-off-by: Georg Pfuetzenreuter --- plugin.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/plugin.py b/plugin.py index b0d13bd..ce6e9fd 100644 --- a/plugin.py +++ b/plugin.py @@ -29,14 +29,8 @@ ### from supybot import ( - utils, - plugins, ircutils, callbacks, - ircdb, - conf, - log, - world, ircmsgs, ) from supybot.commands import * @@ -51,15 +45,11 @@ except ImportError: _ = lambda x: x import re -import os -import sys -import time -import sqlite3 import redis import json from datetime import timedelta from ipwhois import IPWhois -import ipwhois +from ipwhois.exceptions import IPDefinedError class SnoParser(callbacks.Plugin): @@ -135,7 +125,7 @@ class SnoParser(callbacks.Plugin): country = whoisres["asn_country_code"] description = whoisres["asn_description"] whoisout = asn + " " + country + " " + description - except ipwhois.exceptions.IPDefinedError: + except IPDefinedError: whoisout = "RFC 4291 (Local)" response = whoisout -- 2.35.3 From 4d13d704dcd876bc6f4d182bdf3673fe1efd5a47 Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Sat, 5 Oct 2024 19:48:25 +0200 Subject: [PATCH 2/5] Clean up Redis logic Simplify functions and move internal database interfaces into a more appropriate data structure. Configure all database numbers using a redis plugin group, instead of using a custom setting just for the whois database to make the setup more logical. Signed-off-by: Georg Pfuetzenreuter --- config.py | 51 +++++++++++++++--------------- plugin.py | 93 +++++++++++++++++-------------------------------------- 2 files changed, 54 insertions(+), 90 deletions(-) diff --git a/config.py b/config.py index 14654c5..f58e176 100644 --- a/config.py +++ b/config.py @@ -104,20 +104,10 @@ conf.registerGlobalValue( # REDIS related settings below: ### conf.registerGroup(SnoParser, "redis") +conf.registerGroup(SnoParser.redis, "db") conf.registerGlobalValue( - SnoParser.redis, - "db1", - registry.Integer( - 1, - """ - Redis: Database number for counting of NICKNAMES. - """, - private=True, - ), -) -conf.registerGlobalValue( - SnoParser.redis, - "db2", + SnoParser.redis.db, + "ips", registry.Integer( 2, """ @@ -126,11 +116,33 @@ conf.registerGlobalValue( private=True, ), ) +conf.registerGlobalValue( + SnoParser.redis.db, + "nicks", + registry.Integer( + 1, + """ + Redis: Database number for counting of NICKNAMES. + """, + private=True, + ), +) +conf.registerGlobalValue( + SnoParser.redis.db, + "whois", + registry.Integer( + 0, + """ + Redis: Database number for WHOIS query caching. + """, + ), +) + conf.registerGlobalValue( SnoParser.redis, "host", registry.String( - "127.0.0.1", + "localhost", """ Redis: IP address or hostname. """, @@ -218,16 +230,5 @@ conf.registerGlobalValue( private=True, ), ) -conf.registerGroup(SnoParser.whois, "redis") -conf.registerGlobalValue( - SnoParser.whois.redis, - "db", - registry.Integer( - 0, - """ - Redis: Database number for WHOIS query caching. - """, - ), -) # vim:set shiftwidth=4 tabstop=4 expandtab textwidth=79: diff --git a/plugin.py b/plugin.py index ce6e9fd..501ee3c 100644 --- a/plugin.py +++ b/plugin.py @@ -57,62 +57,38 @@ class SnoParser(callbacks.Plugin): threaded = True - def redis_connect_whois(self) -> redis.client.Redis: + def _redis_connect(self, db) -> redis.client.Redis: try: - redis_client_whois = redis.Redis( + redis_client = redis.Redis( host=self.registryValue("redis.host"), port=self.registryValue("redis.port"), password=self.registryValue("redis.password"), username=self.registryValue("redis.username"), - db=self.registryValue("whois.redis.db"), + db=self.registryValue(f"redis.db.{db}"), socket_timeout=int(self.registryValue("redis.timeout")), ) - ping = redis_client_whois.ping() + ping = redis_client.ping() if ping is True: - return redis_client_whois - except redis.AuthenticationError: - print("Could not authenticate to Redis backend.") - - def redis_connect_nicks(self) -> redis.client.Redis: - try: - redis_client_nicks = redis.Redis( - host=self.registryValue("redis.host"), - port=self.registryValue("redis.port"), - password=self.registryValue("redis.password"), - username=self.registryValue("redis.username"), - db=self.registryValue("redis.db1"), - socket_timeout=int(self.registryValue("redis.timeout")), - ) - ping = redis_client_nicks.ping() - if ping is True: - return redis_client_nicks - except redis.AuthenticationError: - print("Could not authenticate to Redis backend.") - - def redis_connect_ips(self) -> redis.client.Redis: - try: - redis_client_ips = redis.Redis( - host=self.registryValue("redis.host"), - port=self.registryValue("redis.port"), - password=self.registryValue("redis.password"), - username=self.registryValue("redis.username"), - db=self.registryValue("redis.db2"), - socket_timeout=int(self.registryValue("redis.timeout")), - ) - ping = redis_client_ips.ping() - if ping is True: - return redis_client_ips + return redis_client except redis.AuthenticationError: print("Could not authenticate to Redis backend.") def __init__(self, irc): super().__init__(irc) - self.redis_client_whois = self.redis_connect_whois() - self.redis_client_nicks = self.redis_connect_nicks() - self.redis_client_ips = self.redis_connect_ips() + self.redis_clients = { + 'ips': self._redis_connect("ips"), + 'nicks': self._redis_connect("nicks"), + 'whois': self._redis_connect("whois"), + } + + def _get_from_cache(self, db, key): + """ Get value from Redis cache """ + + return self.redis_clients[db].get(key) def whois_fresh(self, sourceip: str) -> dict: """Data from WHOIS backend (IANA or respective RIR).""" + asn = 0 subnet = "" try: @@ -131,31 +107,19 @@ class SnoParser(callbacks.Plugin): response = whoisout return response - def whois_get_cache(self, key: str) -> str: - """Data from Redis.""" - - k = self.redis_client_whois.get(key) - - # self = SnoParser() - # val = self.redis_client_whois.get(key) - val = k - return val - def whois_set_cache(self, key: str, value: str) -> bool: """Data to Redis.""" - duration = int(self.registryValue("whois.ttl")) - state = self.redis_client_whois.setex( + return self.redis_clients['whois'].setex( key, - timedelta(seconds=duration), + timedelta(seconds=int(self.registryValue("whois.ttl"))), value=value, ) - return state def whois_run(self, sourceip: str) -> dict: """Whois query router.""" - data = self.whois_get_cache(key=sourceip) + data = self._get_from_cache("whois", sourceip) if data is not None: data = json.loads(data) if self.registryValue("whois.debug"): @@ -191,14 +155,14 @@ class SnoParser(callbacks.Plugin): def nick_run(self, nickname: str) -> dict: """Tracks nicknames""" - data = self.redis_client_nicks.get(nickname) + data = self._get_from_cache("nicks", nickname) if data is not None: if self.registryValue("debug"): print("SNOPARSER DEBUG - NICK_RUN, SEEN: TRUE") print(nickname) print(data) - self.redis_client_nicks.incrby(nickname, amount=1) + self.redis_clients['nicks'].incrby(nickname, amount=1) if data: decoded = data.decode("utf-8") return decoded @@ -209,7 +173,7 @@ class SnoParser(callbacks.Plugin): print("SNOPARSER DEBUG _ NICK_RUN, SEEN: FALSE") print(nickname) print(data) - self.redis_client_nicks.set(nickname, value="1") + self.redis_clients['nicks'].set(nickname, value="1") if data: decoded = data.decode("utf-8") return decoded @@ -219,14 +183,13 @@ class SnoParser(callbacks.Plugin): def ip_run(self, ipaddress: str) -> dict: """Tracks IP addresses""" - data = self.redis_client_ips.get(ipaddress) - + data = self._get_from_cache("ips", ipaddresses) if data is not None: if self.registryValue("debug"): print("SNOPARSER DEBUG - IP_RUN, SEEN: TRUE") print(ipaddress) print(data) - self.redis_client_ips.incrby(ipaddress, amount=1) + self.redis_clients['ips'].incrby(ipaddress, amount=1) if data: decoded = data.decode("utf-8") return decoded @@ -237,7 +200,7 @@ class SnoParser(callbacks.Plugin): print("SNOPARSER DEBUG _ IP_RUN, SEEN: FALSE") print(ipaddress) print(data) - self.redis_client_ips.set(ipaddress, value="1") + self.redis_clients['ips'].set(ipaddress, value="1") if data: decoded = data.decode("utf-8") return decoded @@ -249,10 +212,10 @@ class SnoParser(callbacks.Plugin): Queries the cache for an address. """ - data = self.whois_get_cache(key=ipaddress) + data = self.redis_clients['whois'].get(sourceip) decoded_data = data.decode("utf-8") - ttl = self.redis_client_whois.ttl(ipaddress) - count = self.redis_client_ips.get(ipaddress) + ttl = self.redis_clients['whois'].ttl(ipaddress) + count = self.redis_clients['ips'].get(ipaddress) decoded_count = count.decode("utf-8") print("SnoParser manual query: ", data, " ", ttl) irc.reply(f"{decoded_data} - Count: {decoded_count} - Remaining: {ttl}s") -- 2.35.3 From 3ba723b111aef387499b8219ef114713a53c3ca8 Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Sun, 6 Oct 2024 16:45:27 +0200 Subject: [PATCH 3/5] Correct OPER regex Signed-off-by: Georg Pfuetzenreuter --- plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin.py b/plugin.py index 501ee3c..0398c5e 100644 --- a/plugin.py +++ b/plugin.py @@ -386,7 +386,7 @@ class SnoParser(callbacks.Plugin): self._sendSnotice(irc, msg, repl) if "OPER" in text and "Client opered up" in text: - operregex = "^-OPER- Client opered up \[(.*)\, \ (.*)\]$" + operregex = "^-OPER- Client opered up \[(.*), (.*)\]$" couple = re.match(operregex, text) hostmask = couple.group(1) oper = couple.group(2) -- 2.35.3 From 58f1e22dde319e92f32611fc3fd5c22f7b428471 Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Sun, 6 Oct 2024 17:20:07 +0200 Subject: [PATCH 4/5] Correct ipaddress argument Signed-off-by: Georg Pfuetzenreuter --- plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin.py b/plugin.py index 0398c5e..e0f6771 100644 --- a/plugin.py +++ b/plugin.py @@ -183,7 +183,7 @@ class SnoParser(callbacks.Plugin): def ip_run(self, ipaddress: str) -> dict: """Tracks IP addresses""" - data = self._get_from_cache("ips", ipaddresses) + data = self._get_from_cache("ips", ipaddress) if data is not None: if self.registryValue("debug"): print("SNOPARSER DEBUG - IP_RUN, SEEN: TRUE") -- 2.35.3 From 2b0207d12bdd72d37069bcdbb65f474d0d52535f Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Sun, 6 Oct 2024 17:21:10 +0200 Subject: [PATCH 5/5] Reduce data variables Signed-off-by: Georg Pfuetzenreuter --- plugin.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/plugin.py b/plugin.py index e0f6771..de93e94 100644 --- a/plugin.py +++ b/plugin.py @@ -212,13 +212,15 @@ class SnoParser(callbacks.Plugin): Queries the cache for an address. """ - data = self.redis_clients['whois'].get(sourceip) - decoded_data = data.decode("utf-8") + data = self.redis_clients['whois'].get(ipaddress) + if data is not None: + data = data.decode("utf-8") ttl = self.redis_clients['whois'].ttl(ipaddress) count = self.redis_clients['ips'].get(ipaddress) - decoded_count = count.decode("utf-8") + if count is not None: + count = count.decode("utf-8") print("SnoParser manual query: ", data, " ", ttl) - irc.reply(f"{decoded_data} - Count: {decoded_count} - Remaining: {ttl}s") + irc.reply(f"{data} - Count: {count} - Remaining: {ttl}s") ipquery = wrap(ipquery, ["ip"]) -- 2.35.3