Add `Aliases` table to MessageHistory

Improve linking of known aliases for users by using an aliases table
to track the linkages.

Improve check-bans implementation to use new aliases table for
significant performance gains and reduced resource usage.
This commit is contained in:
Pragmatic Software 2015-05-11 21:27:22 -07:00
parent 3776bef88a
commit fa6dad12cd
4 changed files with 297 additions and 119 deletions

View File

@ -542,7 +542,6 @@ sub check_bans {
$self->{pbot}->{logger}->log("anti-flood: [check-bans] checking for bans on $mask in $channel\n") if $debug_checkban >= 3;
my @nickserv_accounts = $self->{pbot}->{messagehistory}->{database}->get_nickserv_accounts($message_account);
my $current_nickserv_account = $self->{pbot}->{messagehistory}->{database}->get_current_nickserv_account($message_account);
if($current_nickserv_account) {
@ -562,60 +561,36 @@ sub check_bans {
$self->{pbot}->{logger}->log("anti-flood: [check-bans] no account for $mask; marking for later validation\n") if $debug_checkban >= 2;
}
my ($nick, $host) = $mask =~ m/^([^!]+)![^@]+\@(.*)$/;
my $hostmasks = $self->{pbot}->{messagehistory}->{database}->get_hostmasks_for_channel($channel);
foreach my $nickserv_account (@nickserv_accounts) {
my $nickserv_hostmasks = $self->{pbot}->{messagehistory}->{database}->get_hostmasks_for_nickserv($nickserv_account);
push @$hostmasks, @$nickserv_hostmasks;
}
my ($nick) = $mask =~ m/^([^!]+)/;
my %aliases = $self->{pbot}->{messagehistory}->{database}->get_also_known_as($nick);
my ($do_not_validate, $bans);
foreach my $hostmask (@$hostmasks) {
my $check_ban = 0;
foreach my $alias (keys %aliases) {
next if $alias =~ /^Guest\d+(?:!.*)?$/;
# check if nickserv accounts match
if (exists $hostmask->{nickserv}) {
$self->{pbot}->{logger}->log("anti-flood: [check-bans] nickserv account for $hostmask->{hostmask} matches $hostmask->{nickserv}\n") if $debug_checkban;
$check_ban = 1;
goto CHECKBAN;
$self->{pbot}->{logger}->log("anti-flood: [check-bans] checking blacklist for $alias in channel $channel\n") if $debug_checkban >= 4;
if ($self->{pbot}->{blacklist}->check_blacklist($alias, $channel)) {
my $baninfo = {};
$baninfo->{banmask} = $alias;
$baninfo->{channel} = $channel;
$baninfo->{owner} = 'blacklist';
$baninfo->{when} = 0;
$baninfo->{type} = 'blacklist';
push @$bans, $baninfo;
next;
}
my @nickservs;
if (exists $aliases{$alias}->{nickserv}) {
@nickservs = split /,/, $aliases{$alias}->{nickserv};
} else {
$hostmask->{nickserv} = undef;
@nickservs = (undef);
}
# check if hosts match
my ($account_host) = $hostmask->{hostmask} =~ m/\@(.*)$/;
if($host eq $account_host) {
$self->{pbot}->{logger}->log("anti-flood: [check-bans] host for $hostmask->{hostmask} matches $mask\n") if $debug_checkban;
$check_ban = 1;
goto CHECKBAN;
}
# check if nicks match
my ($account_nick) = $hostmask->{hostmask} =~ m/^([^!]+)/;
if($nick eq $account_nick) {
$self->{pbot}->{logger}->log("anti-flood: [check-bans] nick for $hostmask->{hostmask} matches $mask\n") if $debug_checkban;
$check_ban = 1;
goto CHECKBAN;
}
CHECKBAN:
if($check_ban) {
$self->{pbot}->{logger}->log("anti-flood: [check-bans] checking blacklist for $hostmask->{hostmask} in channel $channel\n") if $debug_checkban >= 4;
if ($self->{pbot}->{blacklist}->check_blacklist($hostmask->{hostmask}, $channel)) {
my $baninfo = {};
$baninfo->{banmask} = $hostmask->{hostmask};
$baninfo->{channel} = $channel;
$baninfo->{owner} = 'blacklist';
$baninfo->{when} = 0;
$baninfo->{type} = 'blacklist';
push @$bans, $baninfo;
next;
}
$self->{pbot}->{logger}->log("anti-flood: [check-bans] checking for bans in $channel on $hostmask->{hostmask} using account " . (defined $hostmask->{nickserv} ? $hostmask->{nickserv} : "[undefined]") . "\n") if $debug_checkban >= 4;
my $baninfos = $self->{pbot}->{bantracker}->get_baninfo($hostmask->{hostmask}, $channel, $hostmask->{nickserv});
foreach my $nickserv (@nickservs) {
$self->{pbot}->{logger}->log("anti-flood: [check-bans] checking for bans in $channel on $alias using nickserv " . (defined $nickserv ? $nickserv : "[undefined]") . "\n") if $debug_checkban >= 4;
my $baninfos = $self->{pbot}->{bantracker}->get_baninfo($alias, $channel, $nickserv);
if(defined $baninfos) {
foreach my $baninfo (@$baninfos) {
@ -649,17 +624,10 @@ sub check_bans {
next;
}
my $skip_quiet_nickserv_mask = 0;
foreach my $nickserv_account (@nickserv_accounts) {
if($baninfo->{type} eq '+q' and $baninfo->{banmask} =~ /^\$a:(.*)/ and lc $1 eq $nickserv_account and $nickserv_account eq $current_nickserv_account) {
$self->{pbot}->{logger}->log("anti-flood: [check-bans] Hostmask ($mask) matches quiet on account ($nickserv_account), disregarding\n");
$skip_quiet_nickserv_mask = 1;
} elsif($baninfo->{type} eq '+b' and $baninfo->{banmask} =~ /^\$a:(.*)/ and lc $1 eq $nickserv_account) {
$skip_quiet_nickserv_mask = 0;
last;
}
if(defined $nickserv and $baninfo->{type} eq '+q' and $baninfo->{banmask} =~ /^\$a:(.*)/ and lc $1 eq $nickserv and $nickserv eq $current_nickserv_account) {
$self->{pbot}->{logger}->log("anti-flood: [check-bans] Hostmask ($mask) matches quiet on account ($nickserv), disregarding\n");
next;
}
next if $skip_quiet_nickserv_mask;
if(not defined $bans) {
$bans = [];
@ -758,6 +726,11 @@ sub on_endofwhois {
my ($self, $event_type, $event) = @_;
my $nick = $event->{event}->{args}[1];
delete $self->{whois_pending}->{$nick};
my ($id, $hostmask) = $self->{pbot}->{messagehistory}->{database}->find_message_account_by_nick($nick);
$self->{pbot}->{logger}->log("endofwhois: Found [$id][$hostmask] for [$nick]\n");
$self->{pbot}->{messagehistory}->{database}->link_aliases($id, $hostmask) if $id;
return 0;
}
@ -769,6 +742,11 @@ sub on_whoisaccount {
delete $self->{whois_pending}->{$nick};
$self->{pbot}->{logger}->log("$nick is using NickServ account [$account]\n");
$self->check_nickserv_accounts($nick, $account);
my ($id, $hostmask) = $self->{pbot}->{messagehistory}->{database}->find_message_account_by_nick($nick);
$self->{pbot}->{logger}->log("whoisaccount: Found [$id][$hostmask][$account] for [$nick]\n");
$self->{pbot}->{messagehistory}->{database}->link_aliases($id, undef, $account) if $id;
return 0;
}

View File

@ -46,8 +46,9 @@ sub initialize {
$self->{pbot}->{registry}->add_default('text', 'messagehistory', 'max_messages', $conf{max_messages} // 32);
$self->{pbot}->{commands}->register(sub { $self->recall_message(@_) }, "recall", 0);
$self->{pbot}->{commands}->register(sub { $self->list_also_known_as(@_) }, "aka", 0);
$self->{pbot}->{commands}->register(sub { $self->recall_message(@_) }, "recall", 0);
$self->{pbot}->{commands}->register(sub { $self->list_also_known_as(@_) }, "aka", 0);
$self->{pbot}->{commands}->register(sub { $self->rebuild_aliases(@_) }, "rebuildaliases", 90);
$self->{pbot}->{atexit}->register(sub { $self->{database}->end(); return; });
}
@ -62,10 +63,16 @@ sub add_message {
$self->{database}->add_message($account, $mask, $channel, { timestamp => scalar gettimeofday, msg => $text, mode => $mode });
}
sub rebuild_aliases {
my ($self, $from, $nick, $user, $host, $arguments) = @_;
$self->{database}->rebuild_aliases_table;
}
sub list_also_known_as {
my ($self, $from, $nick, $user, $host, $arguments) = @_;
my $usage = "Usage: aka [-h] <nick>";
my $usage = "Usage: aka [-h] [-i] [-n] <nick>; -h show hostmasks; -i show ids; -n show nickserv accounts";
if(not length $arguments) {
return $usage;
@ -77,33 +84,38 @@ sub list_also_known_as {
chomp $getopt_error;
};
my $show_hostmasks;
my ($show_hostmasks, $show_nickserv, $show_id, $dont_use_aliases_table);
my ($ret, $args) = GetOptionsFromString($arguments,
'h' => \$show_hostmasks);
'h' => \$show_hostmasks,
'n' => \$show_nickserv,
'nt' => \$dont_use_aliases_table,
'i' => \$show_id);
return "$getopt_error -- $usage" if defined $getopt_error;
return "Too many arguments -- $usage" if @$args > 1;
return "Missing argument -- $usage" if @$args != 1;
my @akas = $self->{database}->get_also_known_as(@$args[0]);
if(@akas) {
my %akas = $self->{database}->get_also_known_as(@$args[0], $dont_use_aliases_table);
if(%akas) {
my $result = "@$args[0] also known as:\n";
my %uniq;
foreach my $aka (@akas) {
if (not $show_hostmasks) {
my ($nick) = $aka =~ /^([^!]+)!/;
$uniq{$nick} = $nick;
} else {
$uniq{$aka} = $aka;
}
}
my %nicks;
my $sep = "";
foreach my $aka (sort keys %uniq) {
next if $aka =~ /^Guest\d+(!.*)?$/;
$result .= "$sep$aka";
if ($show_hostmasks) {
foreach my $aka (sort keys %akas) {
next if $aka =~ /^Guest\d+(?:!.*)?$/;
if (not $show_hostmasks) {
my ($nick) = $aka =~ m/([^!]+)/;
next if exists $nicks{$nick} and $nicks{$nick}->{id} == $akas{$aka}->{id};
$nicks{$nick}->{id} = $akas{$aka}->{id};
$result .= "$sep$nick";
} else {
$result .= "$sep$aka";
}
$result .= " ($akas{$aka}->{nickserv})" if $show_nickserv and exists $akas{$aka}->{nickserv};
$result .= " [$akas{$aka}->{id}]" if $show_id;
if ($show_hostmasks or $show_nickserv or $show_id) {
$sep = ",\n";
} else {
$sep = ", ";
@ -197,6 +209,8 @@ sub recall_message {
if(not defined $account) {
return "I don't know anybody named $recall_nick.";
}
$found_nick =~ s/!.*$//;
}
my $message;

View File

@ -117,6 +117,12 @@ CREATE TABLE IF NOT EXISTS Messages (
)
SQL
$self->{dbh}->do(<<SQL);
CREATE TABLE IF NOT EXISTS Aliases (
id INTEGER,
alias INTEGER
)
SQL
$self->{dbh}->do('CREATE INDEX IF NOT EXISTS MsgIdx1 ON Messages(id, channel, mode)');
@ -244,7 +250,7 @@ sub find_message_account_by_nick {
my ($self, $nick) = @_;
my ($id, $hostmask) = eval {
my $sth = $self->{dbh}->prepare('SELECT id,hostmask FROM Hostmasks WHERE hostmask LIKE ? ESCAPE "\" LIMIT 1');
my $sth = $self->{dbh}->prepare('SELECT id, hostmask FROM Hostmasks WHERE hostmask LIKE ? ESCAPE "\" ORDER BY last_seen DESC LIMIT 1');
my $qnick = quotemeta $nick;
$qnick =~ s/_/\\_/g;
$sth->bind_param(1, "$qnick!%");
@ -254,7 +260,6 @@ sub find_message_account_by_nick {
};
$self->{pbot}->{logger}->log($@) if $@;
$hostmask =~ s/!.*$// if defined $hostmask;
return ($id, $hostmask);
}
@ -768,42 +773,189 @@ sub devalidate_all_channels {
$self->{pbot}->{logger}->log($@) if $@;
}
sub link_aliases {
my ($self, $account, $hostmask, $nickserv) = @_;
# $self->{pbot}->{logger}->log("Linking [$account][" . ($hostmask?$hostmask:'undef') . "][" . ($nickserv?$nickserv:'undef') . "]\n");
eval {
my %ids;
if ($hostmask) {
my ($nick) = $hostmask =~ m/([^!]+)/;
unless ($nick =~ m/^Guest\d+$/) {
my $qnick = quotemeta $nick;
$qnick =~ s/_/\\_/g;
my $sth = $self->{dbh}->prepare('SELECT id FROM Hostmasks WHERE hostmask LIKE ? ESCAPE "\"');
$sth->bind_param(1, "$qnick!%");
$sth->execute();
my $rows = $sth->fetchall_arrayref({});
foreach my $row (@$rows) {
$ids{$row->{id}} = $row->{id};
# $self->{pbot}->{logger}->log("found matching id $row->{id} for nick [$qnick]\n");
}
}
my ($host) = $hostmask =~ /(\@.*)$/;
my $sth = $self->{dbh}->prepare('SELECT id FROM Hostmasks WHERE hostmask LIKE ?');
$sth->bind_param(1, "\%$host");
$sth->execute();
my $rows = $sth->fetchall_arrayref({});
foreach my $row (@$rows) {
$ids{$row->{id}} = $row->{id};
# $self->{pbot}->{logger}->log("found matching id $row->{id} for host [$host]\n");
}
}
if ($nickserv) {
my $sth = $self->{dbh}->prepare('SELECT id FROM Nickserv WHERE nickserv = ?');
$sth->bind_param(1, $nickserv);
$sth->execute();
my $rows = $sth->fetchall_arrayref({});
foreach my $row (@$rows) {
$ids{$row->{id}} = $row->{id};
# $self->{pbot}->{logger}->log("found matching id $row->{id} for nickserv [$nickserv]\n");
}
}
my $sth = $self->{dbh}->prepare('INSERT INTO Aliases SELECT ?, ? WHERE NOT EXISTS (SELECT 1 FROM Aliases WHERE id = ? AND alias = ?)');
$sth->bind_param(1, $account);
$sth->bind_param(3, $account);
foreach my $id (sort keys %ids) {
next if $account == $id;
$sth->bind_param(2, $id);
$sth->bind_param(4, $id);
$sth->execute();
if ($sth->rows) {
# $self->{pbot}->{logger}->log("Linked $account to $id\n");
$self->{new_entries}++;
}
}
};
$self->{pbot}->{logger}->log($@) if $@;
}
sub vacuum {
my $self = shift;
eval {
$self->{dbh}->commit();
};
$self->{pbot}->{logger}->log("SQLite error $@ when committing $self->{new_entries} entries.\n") if $@;
$self->{dbh}->do("VACUUM");
$self->{dbh}->begin_work();
$self->{new_entries} = 0;
}
sub rebuild_aliases_table {
my $self = shift;
eval {
$self->{dbh}->do('DELETE FROM Aliases');
$self->vacuum;
my $sth = $self->{dbh}->prepare('SELECT id, hostmask FROM Hostmasks ORDER BY id');
$sth->execute();
my $rows = $sth->fetchall_arrayref({});
$sth = $self->{dbh}->prepare('SELECT nickserv FROM Nickserv WHERE id = ?');
foreach my $row (@$rows) {
$self->{pbot}->{logger}->log("Link [$row->{id}][$row->{hostmask}]\n");
$self->link_aliases($row->{id}, $row->{hostmask});
$sth->bind_param(1, $row->{id});
$sth->execute();
my $nrows = $sth->fetchall_arrayref({});
foreach my $nrow (@$nrows) {
$self->link_aliases($row->{id}, undef, $nrow->{nickserv});
}
}
};
$self->{pbot}->{logger}->log($@) if $@;
}
sub get_also_known_as {
my ($self, $nick) = @_;
my ($self, $nick, $dont_use_aliases_table) = @_;
$self->{pbot}->{logger}->log("Looking for AKAs for nick [$nick]\n");
# $self->{pbot}->{logger}->log("Looking for AKAs for nick [$nick]\n");
my @list = eval {
my %aka_hostmasks;
my $sth = $self->{dbh}->prepare('SELECT hostmask FROM Hostmasks WHERE hostmask LIKE ? ESCAPE "\" ORDER BY last_seen DESC');
my %akas = eval {
my (%akas, %hostmasks, %ids);
if (not $dont_use_aliases_table) {
my ($id, $hostmask) = $self->find_message_account_by_nick($nick);
if (not defined $id) {
return %akas;
}
$akas{$hostmask} = { hostmask => $hostmask, id => $id };
$ids{$id} = $id;
my $sth = $self->{dbh}->prepare('SELECT alias FROM Aliases WHERE id = ?');
$sth->bind_param(1, $id);
$sth->execute();
my $rows = $sth->fetchall_arrayref({});
foreach my $row (@$rows) {
$ids{$row->{alias}} = $row->{alias};
}
foreach my $id (keys %ids) {
$sth = $self->{dbh}->prepare('SELECT hostmask FROM Hostmasks WHERE id = ?');
$sth->bind_param(1, $id);
$sth->execute();
$rows = $sth->fetchall_arrayref({});
foreach my $row (@$rows) {
$akas{$row->{hostmask}} = { hostmask => $row->{hostmask}, id => $id };
}
$sth = $self->{dbh}->prepare('SELECT nickserv FROM Nickserv WHERE id = ?');
$sth->bind_param(1, $id);
$sth->execute();
$rows = $sth->fetchall_arrayref({});
foreach my $row (@$rows) {
foreach my $aka (keys %akas) {
if ($akas{$aka}->{id} == $id) {
if (exists $akas{$aka}->{nickserv}) {
$akas{$aka}->{nickserv} .= ",$row->{nickserv}";
} else {
$akas{$aka}->{nickserv} = $row->{nickserv};
}
}
}
}
}
return %akas;
}
my $sth = $self->{dbh}->prepare('SELECT id, hostmask FROM Hostmasks WHERE hostmask LIKE ? ESCAPE "\" ORDER BY last_seen DESC');
my $qnick = quotemeta $nick;
$qnick =~ s/_/\\_/g;
$sth->bind_param(1, "$qnick!%");
$sth->execute();
my $rows = $sth->fetchall_arrayref({});
my %hostmasks;
foreach my $row (@$rows) {
$hostmasks{$row->{hostmask}} = $row->{hostmask};
$self->{pbot}->{logger}->log("Found matching nick for hostmask $row->{hostmask}\n");
}
my %ids;
foreach my $hostmask (keys %hostmasks) {
my $id = $self->get_message_account_id($hostmask);
next if exists $ids{$id};
$ids{$id} = $id;
$sth = $self->{dbh}->prepare('SELECT hostmask FROM Hostmasks WHERE id == ?');
$sth->bind_param(1, $id);
$sth->execute();
$rows = $sth->fetchall_arrayref({});
foreach my $row (@$rows) {
$aka_hostmasks{$row->{hostmask}} = $row->{hostmask};
$self->{pbot}->{logger}->log("Adding matching id AKA hostmask $row->{hostmask}\n");
$hostmasks{$row->{hostmask}} = $row->{hostmask};
}
$hostmasks{$row->{hostmask}} = $row->{id};
$ids{$row->{id}} = $row->{hostmask};
$akas{$row->{hostmask}} = { hostmask => $row->{hostmask}, id => $row->{id} };
$self->{pbot}->{logger}->log("Found matching nick [$nick] for hostmask $row->{hostmask} with id $row->{id}\n");
}
foreach my $hostmask (keys %hostmasks) {
@ -822,9 +974,10 @@ sub get_also_known_as {
$sth->execute();
my $rows = $sth->fetchall_arrayref({});
foreach my $row (@$rows) {
$aka_hostmasks{$row->{hostmask}} = $row->{hostmask};
$self->{pbot}->{logger}->log("Adding matching host AKA hostmask $row->{hostmask}\n");
foreach my $nrow (@$rows) {
next if exists $akas{$nrow->{hostmask}};
$akas{$nrow->{hostmask}} = { hostmask => $nrow->{hostmask}, id => $row->{id} };
$self->{pbot}->{logger}->log("Adding matching host [$hostmask] and id [$row->{id}] AKA hostmask $nrow->{hostmask}\n");
}
}
}
@ -837,11 +990,21 @@ sub get_also_known_as {
$rows = $sth->fetchall_arrayref({});
foreach my $row (@$rows) {
$nickservs{$row->{nickserv}} = $row->{nickserv};
$nickservs{$row->{nickserv}} = $id;
}
}
foreach my $nickserv (keys %nickservs) {
foreach my $nickserv (sort keys %nickservs) {
foreach my $aka (keys %akas) {
if ($akas{$aka}->{id} == $nickservs{$nickserv}) {
if (exists $akas{$aka}->{nickserv}) {
$akas{$aka}->{nickserv} .= ",$nickserv";
} else {
$akas{$aka}->{nickserv} = $nickserv;
}
}
}
$sth = $self->{dbh}->prepare('SELECT id FROM Nickserv WHERE nickserv == ?');
$sth->bind_param(1, $nickserv);
$sth->execute();
@ -856,18 +1019,39 @@ sub get_also_known_as {
$sth->execute();
my $rows = $sth->fetchall_arrayref({});
foreach my $row (@$rows) {
$aka_hostmasks{$row->{hostmask}} = $row->{hostmask};
$self->{pbot}->{logger}->log("Adding matching nickserv AKA hostmask $row->{hostmask}\n");
foreach my $nrow (@$rows) {
if (exists $akas{$nrow->{hostmask}}) {
if (exists $akas{$nrow->{hostmask}}->{nickserv}) {
$akas{$nrow->{hostmask}}->{nickserv} .= ",$nickserv";
} else {
$akas{$nrow->{hostmask}}->{nickserv} = $nickserv;
}
} else {
$akas{$nrow->{hostmask}} = { hostmask => $nrow->{hostmask}, id => $row->{id}, nickserv => $nickserv };
$self->{pbot}->{logger}->log("Adding matching nickserv [$nickserv] and id [$row->{id}] AKA hostmask $nrow->{hostmask}\n");
}
}
}
}
return sort keys %aka_hostmasks;
foreach my $id (keys %ids) {
$sth = $self->{dbh}->prepare('SELECT hostmask FROM Hostmasks WHERE id == ?');
$sth->bind_param(1, $id);
$sth->execute();
$rows = $sth->fetchall_arrayref({});
foreach my $row (@$rows) {
next if exists $akas{$row->{hostmask}};
$akas{$row->{hostmask}} = { hostmask => $row->{hostmask}, id => $id };
$self->{pbot}->{logger}->log("Adding matching id [$id] AKA hostmask $row->{hostmask}\n");
}
}
return %akas;
};
$self->{pbot}->{logger}->log($@) if $@;
return @list;
return %akas;
}
# End of public API, the remaining are internal support routines for this module
@ -906,7 +1090,7 @@ sub commit_message_history {
my $self = shift;
if($self->{new_entries} > 0) {
#$self->{pbot}->{logger}->log("Commiting $self->{new_entries} messages to SQLite\n");
# $self->{pbot}->{logger}->log("Commiting $self->{new_entries} messages to SQLite\n");
eval {
$self->{dbh}->commit();
};

View File

@ -186,6 +186,8 @@ sub grab_quotegrab {
return "I don't know anybody named $grab_nick";
}
$found_nick =~ s/!.*$//;
$grab_nick = $found_nick; # convert nick to proper casing
my $message;