config.py

@@ -1,5 +1,5 @@
 ###
-# Copyright (c) 2021, mogad0n
+# Copyright (c) 2021, mogad0n and Georg Pfuetzenreuter <georg@lysergic.dev>
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -61,4 +61,98 @@ conf.registerGlobalValue(SnoParser, 'AutoVhost',
 conf.registerGlobalValue(SnoParser, 'preventHighlight',
     registry.Boolean(True, ("""Toggles in channel highlights with ZWSP""")))
 
+conf.registerGlobalValue(SnoParser, 'debug',
+    registry.Boolean('false',
+    """
+    SnoParser: Verbose output. Note: there is a seperate debug option for the `whois` client.
+    """
+    , private=True
+))
+
+###
+# REDIS related settings below:
+###
+conf.registerGroup(SnoParser, 'redis')
+conf.registerGlobalValue(SnoParser.redis, 'db1',
+    registry.Integer('1',
+    """
+    Redis: Database number for counting of NICKNAMES.
+    """
+    , private=True
+))
+conf.registerGlobalValue(SnoParser.redis, 'db2',
+    registry.Integer('2',
+    """
+    Redis: Database number for counting of IP ADDRESSES.
+    """
+    , private=True
+))
+conf.registerGlobalValue(SnoParser.redis, 'host',
+    registry.String('127.0.0.1',
+    """
+    Redis: IP address or hostname.
+    """
+    , private=True
+))
+conf.registerGlobalValue(SnoParser.redis, 'port',
+    registry.Integer('6379',
+    """
+    Redis: Port.
+    """
+    , private=True
+))
+conf.registerGlobalValue(SnoParser.redis, 'username',
+    registry.String('',
+    """
+    Redis: Username. This is optional and has not been tested. It is recommended to perform initial tests on a local instance with no username and password.
+    """
+    , private=True
+))
+conf.registerGlobalValue(SnoParser.redis, 'password',
+    registry.String('',
+    """
+    Redis: Password. This is optional and has not been tested. It is recommended to perform initial tests on a local instance with no username and password.
+    """
+))
+conf.registerGlobalValue(SnoParser.redis, 'timeout',
+    registry.Integer('5',
+    """
+    Redis: Socket Timeout. The developer does not know what to recommend here, but `5` seems to yield good results.
+    """
+))
+
+
+###
+##  WHOIS related settings below:
+###
+conf.registerGroup(SnoParser, 'whois')
+conf.registerGlobalValue(SnoParser.whois, 'debug',
+    registry.Boolean('false',
+    """
+    SnoParser: True: Very verbose console output. False: Mostly silent operation.
+    """
+    , private=True
+))
+conf.registerGlobalValue(SnoParser.whois, 'sample',
+    registry.String('',
+    """
+    SnoParser: This allows to set a testing IP address, if the plugin shall be evaluated on i.e. a local network. This will override all IP addresses from SNOTICES!
+    """
+    , private=True
+))
+conf.registerGlobalValue(SnoParser.whois, 'ttl',
+    registry.Integer('3600',
+    """
+    SnoParser: How long to cache WHOIS entries for.
+    """
+    , private=True
+))
+conf.registerGroup(SnoParser.whois, 'redis')
+conf.registerGlobalValue(SnoParser.whois.redis, 'db',
+    registry.Integer('0',
+    """
+    Redis: Database number for WHOIS query caching.
+    """
+))
+
 # vim:set shiftwidth=4 tabstop=4 expandtab textwidth=79:

plugin.py

@@ -1,5 +1,5 @@
 ###
-# Copyright (c) 2021, mogad0n
+# Copyright (c) 2021, mogad0n and Georg Pfuetzenreuter <georg@lysergic.dev>
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -43,17 +43,218 @@ import os
 import sys
 import time
 import sqlite3
+import redis
+import json
+from datetime import timedelta
+from ipwhois import IPWhois
+import ipwhois
 
 class SnoParser(callbacks.Plugin):
     """Parses the Server Notices from ErgoIRCd"""
     threaded = True
 
+    def redis_connect_whois(self) -> redis.client.Redis:
+        try:
+            redis_client_whois = redis.Redis(
+                    host = self.registryValue('redis.host'),
+                    port = self.registryValue('redis.port'),
+                    password = self.registryValue('redis.password'),
+                    username = self.registryValue('redis.username'),
+                    db = self.registryValue('whois.redis.db'),
+                    socket_timeout = int(self.registryValue('redis.timeout'))
+            )
+            ping = redis_client_whois.ping()
+            if ping is True:
+                return redis_client_whois
+        except redis.AuthenticationError:
+            print("Could not authenticate to Redis backend.")
+
+    def redis_connect_nicks(self) -> redis.client.Redis:
+        try:
+            redis_client_nicks = redis.Redis(
+                    host = self.registryValue('redis.host'),
+                    port = self.registryValue('redis.port'),
+                    password = self.registryValue('redis.password'),
+                    username = self.registryValue('redis.username'),
+                    db = self.registryValue('redis.db1'),
+                    socket_timeout = int(self.registryValue('redis.timeout'))
+            )
+            ping = redis_client_nicks.ping()
+            if ping is True:
+                return redis_client_nicks
+        except redis.AuthenticationError:
+            print("Could not authenticate to Redis backend.")
+
+    def redis_connect_ips(self) -> redis.client.Redis:
+        try:
+            redis_client_ips = redis.Redis(
+                    host = self.registryValue('redis.host'),
+                    port = self.registryValue('redis.port'),
+                    password = self.registryValue('redis.password'),
+                    username = self.registryValue('redis.username'),
+                    db = self.registryValue('redis.db2'),
+                    socket_timeout = int(self.registryValue('redis.timeout'))
+            )
+            ping = redis_client_ips.ping()
+            if ping is True:
+                return redis_client_ips
+        except redis.AuthenticationError:
+            print("Could not authenticate to Redis backend.")
+
+
+    def __init__(self, irc):
+        super().__init__(irc)
+        self.redis_client_whois = self.redis_connect_whois()
+        self.redis_client_nicks = self.redis_connect_nicks()
+        self.redis_client_ips = self.redis_connect_ips()
+
+    def whois_fresh(self, sourceip: str) -> dict:
+        """Data from cache."""
+        asn = 0
+        subnet = ''
+        try:
+            whois = IPWhois(sourceip)
+            whoisres = whois.lookup_rdap(depth=1,retry_count=0)
+            results = whoisres
+            if self.registryValue('whois.debug'):
+                print(results)
+            asn = whoisres['asn_registry']
+            country = whoisres['asn_country_code']
+            description = whoisres['asn_description']
+            whoisout = asn + ' ' + country + ' ' + description
+        except ipwhois.exceptions.IPDefinedError:
+            whoisout = 'RFC 4291 (Local)'
+
+        response = whoisout
+        return response
+
+    def whois_get_cache(self, key: str) -> str:
+        """Data from Redis."""
+
+        k = self.redis_client_whois.get(key)
+
+#        self = SnoParser()
+#        val = self.redis_client_whois.get(key)
+        val = k
+        return val
+
+    def whois_set_cache(self, key: str, value: str) -> bool:
+        """Data to Redis."""
+
+        duration = self.registryValue('whois.ttl')
+        state = self.redis_client_whois.setex(key, timedelta(seconds=duration), value=value,)
+        return state
+
+    def whois_run(self, sourceip: str) -> dict:
+        """Whois query router."""
+
+        data = self.whois_get_cache(key=sourceip)
+        if data is not None:
+            data = json.loads(data)
+            if self.registryValue('whois.debug'):
+                print("SNOPARSER DEBUG - WHOIS_RUN WITH CACHE: TRUE")
+                print(data)
+                print(sourceip)
+            return data
+        else:
+            data = self.whois_fresh(sourceip)
+            if self.registryValue('whois.debug'):
+                print("SNOPARSER DEBUG - WHOIS_RUN WITH CACHE: FALSE")
+                print(data)
+                print(sourceip)
+            if data.startswith:
+                if self.registryValue('whois.debug'):
+                    print("SNOPARSER DEBUG - WHOIS_RUN WITH CACHE: FALSE AND CORRECT STARTING CHARACTER")
+                    print(data)
+                data = json.dumps(data)
+                state = self.whois_set_cache(key=sourceip, value=data)
+                
+                if state is True:
+                    return json.loads(data)
+            else:
+                if self.registryValue('whois.debug'):
+                    print("SNOPARSER DEBUG _ WHOIS_RUN WITH CACHE: FALSE AND WRONG STARTING CHARACTER")
+                    print(data)
+            return data
+
+    def nick_run(self, nickname: str) -> dict:
+        """Tracks nicknames"""
+
+        data = self.redis_client_nicks.get(nickname)
+
+        if data is not None:
+            if self.registryValue('debug'):
+                print("SNOPARSER DEBUG - NICK_RUN, SEEN: TRUE")
+                print(nickname)
+                print(data)
+            self.redis_client_nicks.incrby(nickname,amount=1)
+            if data:
+                decoded = data.decode('utf-8')
+                return decoded
+            else:
+                return 0
+        else:
+            if self.registryValue('debug'):
+                print("SNOPARSER DEBUG _ NICK_RUN, SEEN: FALSE")
+                print(nickname)
+                print(data)
+            self.redis_client_nicks.set(nickname,value='1')
+            if data:
+                decoded = data.decode('utf-8')
+                return decoded
+            else:
+                return 0
+
+    def ip_run(self, ipaddress: str) -> dict:
+        """Tracks IP addresses"""
+
+        data = self.redis_client_ips.get(ipaddress)
+
+        if data is not None:
+            if self.registryValue('debug'):
+                print("SNOPARSER DEBUG - IP_RUN, SEEN: TRUE")
+                print(ipaddress)
+                print(data)
+            self.redis_client_ips.incrby(ipaddress,amount=1)
+            if data:
+                decoded = data.decode('utf-8')
+                return decoded
+            else:
+                return 0
+        else:
+            if self.registryValue('debug'):
+                print("SNOPARSER DEBUG _ IP_RUN, SEEN: FALSE")
+                print(ipaddress)
+                print(data)
+            self.redis_client_ips.set(ipaddress,value='1')
+            if data:
+                decoded = data.decode('utf-8')
+                return decoded
+            else:
+                return 0
+
+    def ipquery(self, irc, msg, args, ipaddress):
+        """<IP address>
+        Queries the cache for an address.
+        """
+
+        data = self.whois_get_cache(key=ipaddress)
+        decoded_data = data.decode('utf-8')
+        ttl = self.redis_client_whois.ttl(ipaddress)
+        count = self.redis_client_ips.get(ipaddress)
+        decoded_count = count.decode('utf-8')
+
+        print('SnoParser manual query: ', data, ' ', ttl)
+        irc.reply(f'{decoded_data} - Count: {decoded_count} - Remaining: {ttl}s')
+        
+    ipquery = wrap(ipquery, ['anything'])
+
+
     def doNotice(self, irc, msg):
         (target, text) = msg.args
+
         if target == irc.nick:
-    
             # server notices CONNECT, KILL, XLINE, NICK, ACCOUNT
-    
             text = ircutils.stripFormatting(text)
             if 'CONNECT' in text:
                 connregex = "^-CONNECT- Client connected \[(.+)\] \[u\:~(.+)\] \[h\:(.+)\] \[ip\:(.+)\] \[r\:(.+)\]$"
@@ -61,12 +262,20 @@ class SnoParser(callbacks.Plugin):
                 nickname = couple.group(1)
                 username = couple.group(2)
                 host = couple.group(3)
-                ip = couple.group(4)
+                if self.registryValue('whois.sample'):
+                    ip = self.registryValue('whois.sample')
+                else:
+                    ip = couple.group(4)
                 realname = couple.group(5)
-                ip_seen = 0
-                nick_seen = 0
+
+                ip_seen = self.ip_run(ipaddress=ip)
+                nick_seen = self.nick_run(nickname=nickname)
+                whois = self.whois_run(sourceip=ip)
+
                 DictFromSnotice = {'notice': 'connect', 'nickname': nickname, 'username': username, 'host': host, 'ip': ip, 'realname': realname, 'ipCount': ip_seen, 'nickCount': nick_seen}
-                repl = f"\x02\x1FNOTICE: {DictFromSnotice['notice']} \x0F\x11\x0303==>>\x0F \x02Nick:\x0F {DictFromSnotice['nickname']} \x02Username:\x0F {DictFromSnotice['username']} \x02Hostname:\x0F {DictFromSnotice['host']} \x02IP:\x0F {DictFromSnotice['ip']} \x02Realname:\x0F {DictFromSnotice['realname']} \x02IPcount:\x0F {DictFromSnotice['ipCount']} \x02NickCount:\x0F {DictFromSnotice['nickCount']}"
+                #repl = f"\x02\x1FNOTICE: {DictFromSnotice['notice']} \x0F\x11\x0303==>>\x0F \x02Nick:\x0F {DictFromSnotice['nickname']} \x02Username:\x0F {DictFromSnotice['username']} \x02Hostname:\x0F {DictFromSnotice['host']} \x02IP:\x0F {DictFromSnotice['ip']} \x02Realname:\x0F {DictFromSnotice['realname']} \x02IPcount:\x0F {DictFromSnotice['ipCount']} \x02NickCount:\x0F {DictFromSnotice['nickCount']}"
+                repl = f"\x02\x1F{DictFromSnotice['notice']} \x0F\x11\x0303==>>\x0F \x02Nick:\x0F {DictFromSnotice['nickname']} \x02Username:\x0F {DictFromSnotice['username']} \x02Hostname:\x0F {DictFromSnotice['host']} \x02IP:\x0F {DictFromSnotice['ip']} {whois} \x02Realname:\x0F {DictFromSnotice['realname']} \x02IPcount:\x0F {DictFromSnotice['ipCount']} \x02NickCount:\x0F {DictFromSnotice['nickCount']}"
+
                 self._sendSnotice(irc, msg, repl)
             if 'XLINE' in text and 'temporary' in text:
                 xlineregex = "^-XLINE- (.+) \[(.+)\] added temporary \((.+)\) (K-Line|D-Line) for (.+)$"