Georg/SnoParser
1
0
Fork 0
forked from LimnoriaPlugins/SnoParser
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request 'regex-cleanup' (#19) from regex-cleanup into master

Browse Source 
Reviewed-on: LimnoriaPlugins/SnoParser#19
This commit is contained in:
Pratyush Desai 2023-05-03 13:01:49 +02:00
commit f6c9705fc5
5 changed files with 387 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,2 +1,3 @@
env/
.vscode
__pycache__/

7
README.rst
View File

@ -1,8 +1,11 @@
SnoParser
=========
SnoParser + Redis
=================
Parses the Server Notices from ErgoIRCd.
This branch contains Redis functionality:
Performs and caches WHOIS lookups for connecting IP addresses. Counts nicknames and IP addresses.
Assumptions
-----------

111
config.py
View File

@ -1,5 +1,5 @@
###
# Copyright (c) 2021, mogad0n
# Copyright (c) 2021, mogad0n and Georg Pfuetzenreuter <georg@lysergic.dev>
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
@ -28,7 +28,7 @@
###
from supybot import conf, registry
from supybot import conf, registry, ircutils
try:
from supybot.i18n import PluginInternationalization
_ = PluginInternationalization('SnoParser')
@ -37,6 +37,12 @@ except:
# without the i18n module
_ = lambda x: x
#class ValidChannel(registry.string):
# """Value must be a valid channel"""
# def setValue(self, v):
# if not (ircutils.isChannel(v)):
# self.error()
# registry.String.setValue(self, v)
def configure(advanced):
# This will be called by supybot to configure this module. advanced is
@ -52,8 +58,11 @@ SnoParser = conf.registerPlugin('SnoParser')
# conf.registerGlobalValue(SnoParser, 'someConfigVariableName',
# registry.Boolean(False, _("""Help for someConfigVariableName.""")))
conf.registerGlobalValue(SnoParser, 'targetChannel',
registry.String(None, ("""Sends reformatted snolines to the <channel>""")))
#conf.registerNetworkValue(SnoParser, 'targetChannel', ValidChannel,
# ("", ("""Determines which channel the bot should send snolines""")))
conf.registerNetworkValue(SnoParser, 'targetChannel',
registry.String("", ("""Determines which channel the bot should snolines example: `#snotices`""")))
conf.registerGlobalValue(SnoParser, 'AutoVhost',
registry.String('libcasa/user/', ("""Configure the vhost eg. libcasa/user/$account""")))
@ -61,4 +70,98 @@ conf.registerGlobalValue(SnoParser, 'AutoVhost',
conf.registerGlobalValue(SnoParser, 'preventHighlight',
registry.Boolean(True, ("""Toggles in channel highlights with ZWSP""")))
conf.registerGlobalValue(SnoParser, 'debug',
registry.Boolean('false',
"""
SnoParser: Verbose output. Note: there is a seperate debug option for the `whois` client.
"""
, private=True
))
###
# REDIS related settings below:
###
conf.registerGroup(SnoParser, 'redis')
conf.registerGlobalValue(SnoParser.redis, 'db1',
registry.Integer(1,
"""
Redis: Database number for counting of NICKNAMES.
"""
, private=True
))
conf.registerGlobalValue(SnoParser.redis, 'db2',
registry.Integer(2,
"""
Redis: Database number for counting of IP ADDRESSES.
"""
, private=True
))
conf.registerGlobalValue(SnoParser.redis, 'host',
registry.String('127.0.0.1',
"""
Redis: IP address or hostname.
"""
, private=True
))
conf.registerGlobalValue(SnoParser.redis, 'port',
registry.Integer(6379,
"""
Redis: Port.
"""
, private=True
))
conf.registerGlobalValue(SnoParser.redis, 'username',
registry.String('',
"""
Redis: Username. This is optional and has not been tested. It is recommended to perform initial tests on a local instance with no username and password.
"""
, private=True
))
conf.registerGlobalValue(SnoParser.redis, 'password',
registry.String('',
"""
Redis: Password. This is optional and has not been tested. It is recommended to perform initial tests on a local instance with no username and password.
"""
))
conf.registerGlobalValue(SnoParser.redis, 'timeout',
registry.Integer(5,
"""
Redis: Socket Timeout. The developer does not know what to recommend here, but `5` seems to yield good results.
"""
))
###
## WHOIS related settings below:
###
conf.registerGroup(SnoParser, 'whois')
conf.registerGlobalValue(SnoParser.whois, 'debug',
registry.Boolean('false',
"""
SnoParser: True: Very verbose console output. False: Mostly silent operation.
"""
, private=True
))
conf.registerGlobalValue(SnoParser.whois, 'sample',
registry.String('',
"""
SnoParser: This allows to set a testing IP address, if the plugin shall be evaluated on i.e. a local network. This will override all IP addresses from SNOTICES!
"""
, private=True
))
conf.registerGlobalValue(SnoParser.whois, 'ttl',
registry.Integer(3600,
"""
SnoParser: How long to cache WHOIS entries for.
"""
, private=True
))
conf.registerGroup(SnoParser.whois, 'redis')
conf.registerGlobalValue(SnoParser.whois.redis, 'db',
registry.Integer(0,
"""
Redis: Database number for WHOIS query caching.
"""
))
# vim:set shiftwidth=4 tabstop=4 expandtab textwidth=79:

293
plugin.py
View File

@ -1,5 +1,5 @@
###
# Copyright (c) 2021, mogad0n
# Copyright (c) 2021, mogad0n and Georg Pfuetzenreuter <georg@lysergic.dev>
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
@ -43,41 +43,251 @@ import os
import sys
import time
import sqlite3
import redis
import json
from datetime import timedelta
from ipwhois import IPWhois
import ipwhois
class SnoParser(callbacks.Plugin):
"""Parses the Server Notices from ErgoIRCd"""
threaded = True
def redis_connect_whois(self) -> redis.client.Redis:
try:
redis_client_whois = redis.Redis(
host = self.registryValue('redis.host'),
port = self.registryValue('redis.port'),
password = self.registryValue('redis.password'),
username = self.registryValue('redis.username'),
db = self.registryValue('whois.redis.db'),
socket_timeout = int(self.registryValue('redis.timeout'))
)
ping = redis_client_whois.ping()
if ping is True:
return redis_client_whois
except redis.AuthenticationError:
print("Could not authenticate to Redis backend.")
def redis_connect_nicks(self) -> redis.client.Redis:
try:
redis_client_nicks = redis.Redis(
host = self.registryValue('redis.host'),
port = self.registryValue('redis.port'),
password = self.registryValue('redis.password'),
username = self.registryValue('redis.username'),
db = self.registryValue('redis.db1'),
socket_timeout = int(self.registryValue('redis.timeout'))
)
ping = redis_client_nicks.ping()
if ping is True:
return redis_client_nicks
except redis.AuthenticationError:
print("Could not authenticate to Redis backend.")
def redis_connect_ips(self) -> redis.client.Redis:
try:
redis_client_ips = redis.Redis(
host = self.registryValue('redis.host'),
port = self.registryValue('redis.port'),
password = self.registryValue('redis.password'),
username = self.registryValue('redis.username'),
db = self.registryValue('redis.db2'),
socket_timeout = int(self.registryValue('redis.timeout'))
)
ping = redis_client_ips.ping()
if ping is True:
return redis_client_ips
except redis.AuthenticationError:
print("Could not authenticate to Redis backend.")
def __init__(self, irc):
super().__init__(irc)
self.redis_client_whois = self.redis_connect_whois()
self.redis_client_nicks = self.redis_connect_nicks()
self.redis_client_ips = self.redis_connect_ips()
def whois_fresh(self, sourceip: str) -> dict:
"""Data from WHOIS backend (IANA or respective RIR)."""
asn = 0
subnet = ''
try:
whois = IPWhois(sourceip)
whoisres = whois.lookup_rdap(depth=1,retry_count=0)
results = whoisres
if self.registryValue('whois.debug'):
print(results)
asn = whoisres['asn_registry']
country = whoisres['asn_country_code']
description = whoisres['asn_description']
whoisout = asn + ' ' + country + ' ' + description
except ipwhois.exceptions.IPDefinedError:
whoisout = 'RFC 4291 (Local)'
response = whoisout
return response
def whois_get_cache(self, key: str) -> str:
"""Data from Redis."""
k = self.redis_client_whois.get(key)
# self = SnoParser()
# val = self.redis_client_whois.get(key)
val = k
return val
def whois_set_cache(self, key: str, value: str) -> bool:
"""Data to Redis."""
duration = int(self.registryValue('whois.ttl'))
state = self.redis_client_whois.setex(key, timedelta(seconds=duration), value=value,)
return state
def whois_run(self, sourceip: str) -> dict:
"""Whois query router."""
data = self.whois_get_cache(key=sourceip)
if data is not None:
data = json.loads(data)
if self.registryValue('whois.debug'):
print("SNOPARSER DEBUG - WHOIS_RUN WITH CACHE: TRUE")
print(data)
print(sourceip)
return data
else:
data = self.whois_fresh(sourceip)
if self.registryValue('whois.debug'):
print("SNOPARSER DEBUG - WHOIS_RUN WITH CACHE: FALSE")
print(data)
print(sourceip)
if data.startswith:
if self.registryValue('whois.debug'):
print("SNOPARSER DEBUG - WHOIS_RUN WITH CACHE: FALSE AND CORRECT STARTING CHARACTER")
print(data)
data = json.dumps(data)
state = self.whois_set_cache(key=sourceip, value=data)
if state is True:
return json.loads(data)
else:
if self.registryValue('whois.debug'):
print("SNOPARSER DEBUG _ WHOIS_RUN WITH CACHE: FALSE AND WRONG STARTING CHARACTER")
print(data)
return data
def nick_run(self, nickname: str) -> dict:
"""Tracks nicknames"""
data = self.redis_client_nicks.get(nickname)
if data is not None:
if self.registryValue('debug'):
print("SNOPARSER DEBUG - NICK_RUN, SEEN: TRUE")
print(nickname)
print(data)
self.redis_client_nicks.incrby(nickname,amount=1)
if data:
decoded = data.decode('utf-8')
return decoded
else:
return 0
else:
if self.registryValue('debug'):
print("SNOPARSER DEBUG _ NICK_RUN, SEEN: FALSE")
print(nickname)
print(data)
self.redis_client_nicks.set(nickname,value='1')
if data:
decoded = data.decode('utf-8')
return decoded
else:
return 0
def ip_run(self, ipaddress: str) -> dict:
"""Tracks IP addresses"""
data = self.redis_client_ips.get(ipaddress)
if data is not None:
if self.registryValue('debug'):
print("SNOPARSER DEBUG - IP_RUN, SEEN: TRUE")
print(ipaddress)
print(data)
self.redis_client_ips.incrby(ipaddress,amount=1)
if data:
decoded = data.decode('utf-8')
return decoded
else:
return 0
else:
if self.registryValue('debug'):
print("SNOPARSER DEBUG _ IP_RUN, SEEN: FALSE")
print(ipaddress)
print(data)
self.redis_client_ips.set(ipaddress,value='1')
if data:
decoded = data.decode('utf-8')
return decoded
else:
return 0
def ipquery(self, irc, msg, args, ipaddress):
"""<IP address>
Queries the cache for an address.
"""
data = self.whois_get_cache(key=ipaddress)
decoded_data = data.decode('utf-8')
ttl = self.redis_client_whois.ttl(ipaddress)
count = self.redis_client_ips.get(ipaddress)
decoded_count = count.decode('utf-8')
print('SnoParser manual query: ', data, ' ', ttl)
irc.reply(f'{decoded_data} - Count: {decoded_count} - Remaining: {ttl}s')
ipquery = wrap(ipquery, ['ip'])
def doNotice(self, irc, msg):
(target, text) = msg.args
if target == irc.nick:
# server notices CONNECT, KILL, XLINE, NICK, ACCOUNT
# server notices CONNECT, KILL, XLINE, NICK, ACCOUNT, OPER, QUIT,
text = ircutils.stripFormatting(text)
if 'CONNECT' in text:
connregex = "^-CONNECT- Client connected \[(.+)\] \[u\:~(.+)\] \[h\:(.+)\] \[ip\:(.+)\] \[r\:(.+)\]$"
couple = re.match(connregex, text)
# if 'CONNECT' in text:
RE_CLICONN = re.compile(r"^-CONNECT- Client connected \[(.+)\] \[u\:~(.+)\] \[h\:(.+)\] \[ip\:(.+)\] \[r\:(.+)\]$")
couple = RE_CLICONN.match(text)
# check `if couple:` ie was there a match even? if yes proceed
if couple:
nickname = couple.group(1)
username = couple.group(2)
host = couple.group(3)
ip = couple.group(4)
if self.registryValue('whois.sample'):
ip = self.registryValue('whois.sample')
else:
ip = couple.group(4)
realname = couple.group(5)
ip_seen = 0
nick_seen = 0
DictFromSnotice = {'notice': 'connect', 'nickname': nickname, 'username': username, 'host': host, 'ip': ip, 'realname': realname, 'ipCount': ip_seen, 'nickCount': nick_seen}
repl = f"\x02\x1FNOTICE: {DictFromSnotice['notice']} \x0F\x11\x0303==>>\x0F \x02Nick:\x0F {DictFromSnotice['nickname']} \x02Username:\x0F {DictFromSnotice['username']} \x02Hostname:\x0F {DictFromSnotice['host']} \x02IP:\x0F {DictFromSnotice['ip']} \x02Realname:\x0F {DictFromSnotice['realname']} \x02IPcount:\x0F {DictFromSnotice['ipCount']} \x02NickCount:\x0F {DictFromSnotice['nickCount']}"
ip_seen = self.ip_run(ipaddress=ip)
nick_seen = self.nick_run(nickname=nickname)
whois = self.whois_run(sourceip=ip)
snote_dict = {'notice': 'connect', 'nickname': nickname, 'username': username, 'host': host, 'ip': ip, 'realname': realname, 'ipCount': ip_seen, 'nickCount': nick_seen}
repl = f"\x02\x1F{snote_dict['notice']} \x0F\x11\x0303==>>\x0F \x02Nick:\x0F {snote_dict['nickname']} \x02Username:\x0F {snote_dict['username']} \x02Hostname:\x0F {snote_dict['host']} \x02IP:\x0F {snote_dict['ip']} {whois} \x02Realname:\x0F {snote_dict['realname']} \x02IPcount:\x0F {snote_dict['ipCount']} \x02NickCount:\x0F {snote_dict['nickCount']}"
self._sendSnotice(irc, msg, repl)
if 'XLINE' in text and 'temporary' in text:
xlineregex = "^-XLINE- (.+) \[(.+)\] added temporary \((.+)\) (K-Line|D-Line) for (.+)$"
couple = re.match(xlineregex, text)
# if 'XLINE' in text and 'temporary' in text:
RE_XLINE = re.compile(r"^-XLINE- (.+) \[(.+)\] added temporary \((.+)\) (K-Line|D-Line) for (.+)$")
couple = RE_XLINE.match(text)
if couple:
who = couple.group(1)
who_operator = couple.group(2)
duration = couple.group(3)
which_line = couple.group(4)
host_or_ip = couple.group(5)
DictFromSnotice = {'notice': 'tempban', 'who': who, 'operator': who_operator, 'duration': duration, 'type': which_line, 'target': host_or_ip}
repl = f"\x02\x1FNOTICE: {DictFromSnotice['notice']}\x0F \x11\x0303 X_X \x0F \x02BannedBy:\x0F {DictFromSnotice['who']} \x02BannedByOper:\x0F {DictFromSnotice['operator']} \x02Duration:\x0F {DictFromSnotice['duration']} \x02XLINE Type:\x0F {DictFromSnotice['type']} \x02Nick:\x0F {DictFromSnotice['target']}"
snote_dict = {'notice': 'tempban', 'who': who, 'operator': who_operator, 'duration': duration, 'type': which_line, 'target': host_or_ip}
repl = f"\x02\x1FNOTICE: {snote_dict['notice']}\x0F \x11\x0303 X_X \x0F \x02BannedBy:\x0F {snote_dict['who']} \x02BannedByOper:\x0F {snote_dict['operator']} \x02Duration:\x0F {snote_dict['duration']} \x02XLINE Type:\x0F {snote_dict['type']} \x02Nick:\x0F {snote_dict['target']}"
self._sendSnotice(irc, msg, repl)
# WHY THE FUCK IS IT elif ??
elif 'XLINE' in text and 'temporary' not in text and 'removed' not in text:
@ -87,8 +297,8 @@ class SnoParser(callbacks.Plugin):
who_operator = couple.group(2)
which_line = couple.group(3)
host_or_ip = couple.group(4)
DictFromSnotice = {'notice': 'Permaban', 'who': who, 'operator': who_operator, 'type': which_line, 'target': host_or_ip}
repl = f"\x02\x1FNOTICE: {DictFromSnotice['notice']} \x0F \x11\x0303 X_X \x0F \x02BannedBy:\x0F {DictFromSnotice['who']} \x02BannedByOper:\x0F {DictFromSnotice['operator']} \x02XLINE Type:\x0F {DictFromSnotice['type']} \x02Host/IP:\x0F {DictFromSnotice['target']}"
snote_dict = {'notice': 'Permaban', 'who': who, 'operator': who_operator, 'type': which_line, 'target': host_or_ip}
repl = f"\x02\x1FNOTICE: {snote_dict['notice']} \x0F \x11\x0303 X_X \x0F \x02BannedBy:\x0F {snote_dict['who']} \x02BannedByOper:\x0F {snote_dict['operator']} \x02XLINE Type:\x0F {snote_dict['type']} \x02Host/IP:\x0F {snote_dict['target']}"
self._sendSnotice(irc, msg, repl)
elif 'XLINE' in text and 'removed' in text:
unxlineregex = "^-XLINE- (.+) removed (D-Line|K-Line) for (.+)$"
@ -96,8 +306,8 @@ class SnoParser(callbacks.Plugin):
who = couple.group(1)
which_line = couple.group(2)
host_or_ip = couple.group(3)
DictFromSnotice = {'notice': 'unxline', 'who': who, 'type': which_line, 'target': host_or_ip}
repl = f"\x02\x1FNOTICE: {DictFromSnotice['notice']} \x0F\x11\x0303 :=D\x0F \x02UnbannedBy:\x0F {DictFromSnotice['who']} \x02XLINE type:\x0F {DictFromSnotice['type']} \x02Host/IP:\x0F {DictFromSnotice['target']}"
snote_dict = {'notice': 'unxline', 'who': who, 'type': which_line, 'target': host_or_ip}
repl = f"\x02\x1FNOTICE: {snote_dict['notice']} \x0F\x11\x0303 :=D\x0F \x02UnbannedBy:\x0F {snote_dict['who']} \x02XLINE type:\x0F {snote_dict['type']} \x02Host/IP:\x0F {snote_dict['target']}"
self._sendSnotice(irc, msg, repl)
if 'KILL' in text:
killregex = "^-KILL- (.+) \[(.+)\] killed (\d) clients with a (KLINE|DLINE) \[(.+)\]$"
@ -107,22 +317,22 @@ class SnoParser(callbacks.Plugin):
clients = couple.group(3)
which_line = couple.group(4)
nick = couple.group(5)
DictFromSnotice = {'notice': 'kill', 'who': who, 'operator': who_operator, "client": clients, 'type': which_line, 'nick': nick}
repl = f"\x02\x1FNOTICE: {DictFromSnotice['notice']} \x0F\x11\x0303☠\x0F \x02KilledBy:\x0F {DictFromSnotice['who']} \x02KilledByOper:\x0F {DictFromSnotice['operator']} \x02NumofClientsAffected:\x0F {DictFromSnotice['client']} \x02XLINE Type:\x0F {DictFromSnotice['type']} \x02Nick:\x0F {DictFromSnotice['nick']}"
snote_dict = {'notice': 'kill', 'who': who, 'operator': who_operator, "client": clients, 'type': which_line, 'nick': nick}
repl = f"\x02\x1FNOTICE: {snote_dict['notice']} \x0F\x11\x0303☠\x0F \x02KilledBy:\x0F {snote_dict['who']} \x02KilledByOper:\x0F {snote_dict['operator']} \x02NumofClientsAffected:\x0F {snote_dict['client']} \x02XLINE Type:\x0F {snote_dict['type']} \x02Nick:\x0F {snote_dict['nick']}"
self._sendSnotice(irc, msg, repl)
if 'NICK' in text and 'changed nickname to' in text:
nickregex = "^-NICK- (.+) changed nickname to (.+)$"
couple = re.match(nickregex, text)
old_nick = couple.group(1)
new_nick = couple.group(2)
DictFromSnotice = {'notice': 'nick change', 'old_nick': old_nick, 'new_nick': new_nick}
repl = f"\x02\x1FNOTICE: {DictFromSnotice['notice']} ==> {DictFromSnotice['old_nick']} changed their nick to {DictFromSnotice['new_nick']}"
snote_dict = {'notice': 'nick change', 'old_nick': old_nick, 'new_nick': new_nick}
repl = f"\x02\x1FNOTICE: {snote_dict['notice']} ==> {snote_dict['old_nick']} changed their nick to {snote_dict['new_nick']}"
self._sendSnotice(irc, msg, repl)
if 'QUIT' in text and 'exited' in text:
quitregex = "^-QUIT- (.+) exited the network$"
couple = re.match(quitregex, text)
nick = couple.group(1)
DictFromSnotice = {'notice': 'quit', 'nick': nick}
snote_dict = {'notice': 'quit', 'nick': nick}
repl = f"\x02\x1FNOTICE: quit nick: {nick} has exited the network"
self._sendSnotice(irc, msg, repl)
if 'ACCOUNT' in text and 'registered account' in text:
@ -131,24 +341,47 @@ class SnoParser(callbacks.Plugin):
hostmask = couple.group(1)
account = couple.group(2)
ip = couple.group(3)
DictFromSnotice = {'notice': 'accreg', 'hostmask': hostmask, 'account': account, 'ip': ip}
snote_dict = {'notice': 'accreg', 'hostmask': hostmask, 'account': account, 'ip': ip}
repl = f"\x02\x1FNOTICE: accreg -> [{account}] was registered by hostmask [{hostmask}] from IP {ip}"
# Trigger HS SET
self._setvhost(irc, msg, account)
self._sendSnotice(irc, msg, repl)
if 'ACCOUNT' in text and 'registered account' in text and 'SAREGISTER' in text:
accregex = "^-ACCOUNT- Operator \[(.*)\] registered account \[(.*)\] with SAREGISTER$"
couple = re.match(accregex, text)
oper = couple.group(1)
account = couple.group(2)
DictFromSnotice = {'notice': 'sareg', 'oper': oper, 'account': account}
snote_dict = {'notice': 'sareg', 'oper': oper, 'account': account}
repl = f"\x02\x1FNOTICE: sareg -> [{account}] was registered by operator [{oper}]"
self._setvhost(irc, msg, account)
self._sendSnotice(irc, msg, repl)
if 'OPER' in text and 'Client opered up' in text:
operregex = "^-OPER- Client opered up \[(.*)\, \ (.*)\]$"
couple = re.match(operregex, text)
hostmask = couple.group(1)
oper = couple.group(2)
print(couple)
snote_dict = {'notice': 'opered', 'hostmask': hostmask, 'oper': oper}
repl = f"\x02\x1FNOTICE:\x0F [{hostmask}] opered up as [{oper}]."
self._sendSnotice(irc, msg, repl)
if 'OPER' in text and 'Client deopered' in text:
operregex = "^-OPER- Client deopered \[(.*)\]"
couple = re.match(operregex, text)
account = couple.group(1)
snote_dict = {'notice': 'deopered', 'name': account}
repl = f"\x02\x1FNOTICE:\x0F [{account}] opered down."
self._sendSnotice(irc, msg, repl)
# Post Registration
@ -166,7 +399,7 @@ class SnoParser(callbacks.Plugin):
def _sendSnotice(self, irc, msg, repl):
try:
channel = self.registryValue('targetChannel')
if channel[0] == '#':
if irc.isChannel(channel):
irc.queueMsg(msg=ircmsgs.IrcMsg(command='NOTICE',
args=(channel, repl)))
# what sort of exception does one raise

3
requirements.txt Normal file
View File

@ -0,0 +1,3 @@
dnspython==2.0.0
ipwhois==1.2.0
redis==3.5.3