From b154fe2b5ba7d2f64a29e0a233a7dba590609b64 Mon Sep 17 00:00:00 2001 From: Pragmatic Software Date: Sat, 17 Jul 2021 17:04:12 -0700 Subject: [PATCH] Significant speed-up when invoking Factoids using DualIndexSQLiteObject as backend --- lib/PBot/DualIndexSQLiteObject.pm | 36 ++++++++++++++++++++++--------- lib/PBot/Factoids.pm | 12 ++++++----- 2 files changed, 33 insertions(+), 15 deletions(-) diff --git a/lib/PBot/DualIndexSQLiteObject.pm b/lib/PBot/DualIndexSQLiteObject.pm index 329a0541..4c235302 100644 --- a/lib/PBot/DualIndexSQLiteObject.pm +++ b/lib/PBot/DualIndexSQLiteObject.pm @@ -53,9 +53,8 @@ sub begin { $self->{pbot}->{logger}->log("Opening $self->{name} database ($self->{filename})\n"); - $self->{dbh} = DBI->connect( - "dbi:SQLite:dbname=$self->{filename}", "", "", - {RaiseError => 1, PrintError => 0, AutoInactiveDestroy => 1, sqlite_unicode => 1} + $self->{dbh} = DBI->connect("dbi:SQLite:dbname=$self->{filename}", undef, undef, + { AutoCommit => 0, RaiseError => 1, PrintError => 0, AutoInactiveDestroy => 1, sqlite_unicode => 1 } ) or die $DBI::errstr; eval { @@ -88,6 +87,12 @@ sub load { $self->create_cache; } +sub save { + my ($self) = @_; + return if not $self->{dbh}; + $self->{dbh}->commit; +} + sub create_database { my ($self) = @_; @@ -158,6 +163,7 @@ sub trim_cache { sub create_metadata { my ($self, $columns) = @_; + return if not $self->{dbh}; $self->{columns} = $columns; @@ -166,8 +172,6 @@ sub create_metadata { my %existing = (); foreach my $col (@{$self->{dbh}->selectall_arrayref("PRAGMA TABLE_INFO(Stuff)")}) { $existing{$col->[1]} = $col->[2]; } - $self->{dbh}->begin_work; - foreach my $col (sort keys %$columns) { unless (exists $existing{$col}) { $self->{dbh}->do("ALTER TABLE Stuff ADD COLUMN \"$col\" $columns->{$col}"); } } @@ -183,6 +187,7 @@ sub create_metadata { sub levenshtein_matches { my ($self, $index1, $index2, $distance, $strictnamespace) = @_; + my $comma = ''; my $result = ''; @@ -588,7 +593,6 @@ sub add { eval { my $sth; - $self->{dbh}->begin_work; if (not $self->exists($index1, $index2)) { $sth = $self->{dbh}->prepare('INSERT INTO Stuff (index1, index2) VALUES (?, ?)'); @@ -620,8 +624,6 @@ sub add { $sth->bind_param($param++, $index2); $sth->execute(); - $self->{dbh}->commit; - # no errors updating SQL -- now we update cache my ($lc_index1, $lc_index2) = (lc $index1, lc $index2); $self->{cache}->{$lc_index1}->{_name} = $index1 if $index1 ne $lc_index1 and not exists $self->{cache}->{$lc_index1}->{_name}; @@ -645,6 +647,7 @@ sub add { $index1 = 'global' if $index1 eq '.*'; $index2 = "\"$index2\"" if $index2 =~ / /; $self->{pbot}->{logger}->log("$self->{name}: [$index1]: $index2 added.\n") unless $quiet; + $self->save unless $quiet; return "$index2 added to $name1."; } @@ -674,6 +677,8 @@ sub remove { return "Error removing $name1: $@"; } + $self->save unless $dont_save; + return "$name1 removed."; } @@ -700,6 +705,7 @@ sub remove { return "Error removing $name2 from $name1: $@"; } + $self->save unless $dont_save; return "$name2 removed from $name1."; } @@ -720,6 +726,7 @@ sub remove { return "Error unsetting $data_index from $name2: $@"; } + $self->save unless $dont_save; return "$name2.$data_index unset."; } @@ -764,18 +771,24 @@ sub set { return "$self->{name} have no such metadata $key."; } - if (not defined $value) { $value = $self->get_data($index1, $index2, $key); } + if (not defined $value) { + $value = $self->get_data($index1, $index2, $key); + } else { eval { my $sth = $self->{dbh}->prepare("UPDATE Stuff SET '$key' = ? WHERE index1 = ? AND index2 = ?"); + $sth->execute($value, $index1, $index2); my ($lc_index1, $lc_index2) = (lc $index1, lc $index2); + if (exists $self->{cache}->{$lc_index1} and exists $self->{cache}->{$lc_index1}->{$lc_index2} and exists $self->{cache}->{$lc_index1}->{$lc_index2}->{$key}) { $self->{cache}->{$lc_index1}->{$lc_index2}->{$key} = $value; } + + $self->save; }; if ($@) { @@ -814,14 +827,18 @@ sub unset { eval { my $sth = $self->{dbh}->prepare("UPDATE Stuff SET '$key' = ? WHERE index1 = ? AND index2 = ?"); + $sth->execute(undef, $index1, $index2); my ($lc_index1, $lc_index2) = (lc $index1, lc $index2); + if (exists $self->{cache}->{$lc_index1} and exists $self->{cache}->{$lc_index1}->{$lc_index2} and exists $self->{cache}->{$lc_index1}->{$lc_index2}->{$key}) { $self->{cache}->{$lc_index1}->{$lc_index2}->{$key} = undef; } + + $self->save; }; if ($@) { @@ -834,7 +851,6 @@ sub unset { # nothing to do here for SQLite # kept for compatibility with DualIndexHashObject -sub save { } sub clear { } 1; diff --git a/lib/PBot/Factoids.pm b/lib/PBot/Factoids.pm index f17df818..85f241bb 100644 --- a/lib/PBot/Factoids.pm +++ b/lib/PBot/Factoids.pm @@ -66,7 +66,6 @@ sub initialize { my $filename = $conf{filename}; $self->{pbot} = $self->{pbot}; - $self->{pbot}->{atexit}->register(sub { $self->save_factoids; return; }); $self->{storage} = PBot::DualIndexSQLiteObject->new(name => 'Factoids', filename => $filename, pbot => $self->{pbot}); @@ -78,18 +77,21 @@ sub initialize { $self->{pbot}->{registry}->add_default('text', 'factoids', 'max_channel_length', 20); $self->load_factoids; + + # save and export factoids at exit + $self->{pbot}->{atexit}->register(sub { $self->save_factoids(1) }); } sub load_factoids { - my $self = shift; + my ($self) = @_; $self->{storage}->load; $self->{storage}->create_metadata(\%factoid_metadata); } sub save_factoids { - my $self = shift; + my ($self, $export) = @_; $self->{storage}->save; - $self->export_factoids; + $self->export_factoids if $export; } sub get_meta { @@ -1239,7 +1241,7 @@ sub interpreter { last_referenced_on => scalar gettimeofday, last_referenced_in => $context->{from} || 'stdin', }; - $self->{storage}->add($channel, $keyword, $update_data, 1, 1); + $self->{storage}->add($channel, $keyword, $update_data, 1); my $action;