Fix Unicode encoding issues

PBot/DualIndexHashObject.pm

@@ -96,7 +96,7 @@ sub save {
 
   my $json = JSON->new;
   $json->space_before(0);
-  my $json_text = $json->pretty->encode($self->{hash});
+  my $json_text = $json->pretty->utf8->encode($self->{hash});
 
   open(FILE, "> $filename") or die "Couldn't open $filename: $!\n";
   print FILE "$json_text\n";

PBot/FactoidModuleLauncher.pm

@@ -113,6 +113,8 @@ sub execute_module {
     $stuff->{result} = `./$module $stuff->{arguments} 2>> $module-stderr`;
     chomp $stuff->{result};
 
+    utf8::decode($stuff->{result});
+
     my $json = encode_json $stuff;
     print $writer "$json\n";
     exit 0;

PBot/Factoids.pm

@@ -387,10 +387,6 @@ sub find_factoid {
 sub escape_json {
   my ($self, $text) = @_;
   my $thing = {thing => $text};
-  # not sure why we need this here, but it seems to stop strange
-  # text encoding issues in the following encode_json call
-  use Encode;
-  $thing->{thing} = decode('utf8', $thing->{thing});
   my $json = encode_json $thing;
   $json =~ s/^{".*":"//;
   $json =~ s/"}$//;
@@ -595,11 +591,6 @@ sub expand_action_arguments {
     %h = (args => $input);
   }
 
-  # not sure why we need this here, but it seems to stop strange
-  # text encoding issues in the following encode_json call
-  use Encode;
-  $h{args} = decode('utf8', $h{args});
-
   my $jsonargs = encode_json \%h;
   $jsonargs =~ s/^{".*":"//;
   $jsonargs =~ s/"}$//;
@@ -712,11 +703,6 @@ sub execute_code_factoid_using_vm {
     $h{'persist-key'} = $self->{factoids}->hash->{$stuff->{channel}}->{$stuff->{keyword}}->{'persist-key'};
   }
 
-  # not sure why we need this here, but it seems to stop strange
-  # text encoding issues in the following encode_json call
-  use Encode;
-  $h{arguments} = decode('utf8', $h{arguments});
-
   my $json = encode_json \%h;
 
   $stuff->{special} = 'code-factoid';

PBot/HashObject.pm

@@ -78,7 +78,6 @@ sub load {
 	};
 
   $self->{hash} = decode_json $contents;
-
   close FILE;
 }
 
@@ -97,7 +96,7 @@ sub save {
 
   my $json = JSON->new;
   $json->space_before(0);
-  my $json_text = $json->pretty->encode($self->{hash});
+  my $json_text = $json->pretty->utf8->encode($self->{hash});
 
   open(FILE, "> $filename") or die "Couldn't open $filename: $!\n";
   print FILE "$json_text\n";

PBot/IRC/Connection.pm

@@ -21,6 +21,7 @@ use IO::Socket;
 use IO::Socket::INET;
 use Symbol;
 use Carp;
+use Encode;
 
 # all this junk below just to conditionally load a module
 # sometimes even perl is braindead...
@@ -41,18 +42,20 @@ use vars (
 
 
 # The names of the methods to be handled by &AUTOLOAD.
-my %autoloaded = ( 'ircname'  => undef,
-		   'port'     => undef,
-		   'username' => undef,
-		   'socket'   => undef,
-		   'verbose'  => undef,
-		   'parent'   => undef,
-                   'hostname' => undef,
-		   'pacing'   => undef,
-                   'ssl'      => undef,
-                   'ssl_ca_path' => undef,
-                   'ssl_ca_file' => undef,
-		 );
+my %autoloaded = (
+  'ircname'  => undef,
+  'port'     => undef,
+  'username' => undef,
+  'socket'   => undef,
+  'verbose'  => undef,
+  'parent'   => undef,
+  'hostname' => undef,
+  'pacing'   => undef,
+  'utf8'     => undef,
+  'ssl'      => undef,
+  'ssl_ca_path' => undef,
+  'ssl_ca_file' => undef,
+);
 
 # This hash will contain any global default handlers that the user specifies.
 
@@ -81,6 +84,7 @@ sub new {
     _ssl	=>  0,       # no ssl by default
     _ssl_ca_path => undef,
     _ssl_ca_file => undef,
+    _utf8       => 0,
     _format     => { 'default' => "[%f:%t]  %m  <%d>", },
   };
 
@@ -231,6 +235,7 @@ sub connect {
     $self->ircname($arg{'Ircname'}) if exists $arg{'Ircname'};
     $self->username($arg{'Username'}) if exists $arg{'Username'};
     $self->pacing($arg{'Pacing'}) if exists $arg{'Pacing'};
+    $self->utf8($arg{'UTF8'}) if exists $arg{'UTF8'};
     $self->ssl($arg{'SSL'}) if exists $arg{'SSL'};
     $self->ssl_ca_path($arg{'SSL_ca_path'}) if exists $arg{'SSL_ca_path'};
     $self->ssl_ca_file($arg{'SSL_ca_file'}) if exists $arg{'SSL_ca_file'};
@@ -890,6 +895,9 @@ sub parse {
   }
 
  PARSELOOP: foreach $line (@lines) {
+   if ($self->{_utf8}) {
+     utf8::decode($line);
+   }
 
    # Clean the lint filter every 2 weeks...
    $line =~ s/[\012\015]+$//;
@@ -1439,13 +1447,24 @@ sub sl_real {
 
   return unless defined $self->socket;
 
-  # RFC compliance can be kinda nice...
-  my $rv = $self->ssl ?
-      $self->socket->print("$line\015\012") :
-      $self->socket->send("$line\015\012", 0);
-  unless ($rv) {
-    $self->handler("sockerror");
-    return;
+  if ($self->{_utf8}) {
+    $line = encode('UTF-8', $line);
+  }
+
+  my $rv = eval {
+    # RFC compliance can be kinda nice...
+    my $rv = $self->ssl ?
+    $self->socket->print("$line\015\012") :
+    $self->socket->send("$line\015\012", 0);
+    unless ($rv) {
+      $self->handler("sockerror");
+      return;
+    }
+    return $rv;
+  };
+
+  if ($@) {
+    print "Attempt to send bad line: [$line]\n";
   }
   return $rv;
 }

PBot/MessageHistory_SQLite.pm

@@ -72,7 +72,7 @@ sub begin {
 
   $self->{pbot}->{logger}->log("Opening message history SQLite database: $self->{filename}\n");
 
-  $self->{dbh} = DBI->connect("dbi:SQLite:dbname=$self->{filename}", "", "", { RaiseError => 1, PrintError => 0, AutoInactiveDestroy => 1 }) or die $DBI::errstr;
+  $self->{dbh} = DBI->connect("dbi:SQLite:dbname=$self->{filename}", "", "", { RaiseError => 1, PrintError => 0, AutoInactiveDestroy => 1, sqlite_unicode => 1 }) or die $DBI::errstr;
 
   $self->{dbh}->sqlite_enable_load_extension(my $_enabled = 1);
   $self->{dbh}->prepare("SELECT load_extension('/usr/lib/sqlite3/pcre.so')");

PBot/PBot.pm

@@ -7,8 +7,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-# $Id$
-
 package PBot::PBot;
 
 use strict;
@@ -182,6 +180,7 @@ sub connect {
       Ircname      => $self->{registry}->get_value('irc', 'ircname'),
       Server       => $server,
       Pacing       => 1,
+      UTF8         => 1,
       SSL          => $self->{registry}->get_value('irc', 'SSL'),
       SSL_ca_file  => $self->{registry}->get_value('irc', 'SSL_ca_file'),
       SSL_ca_path  => $self->{registry}->get_value('irc', 'SSL_ca_path'),

PBot/Plugins/ActionTrigger.pm

@@ -72,7 +72,7 @@ SQL
 sub dbi_begin {
   my ($self) = @_;
   eval {
-    $self->{dbh} = DBI->connect("dbi:SQLite:dbname=$self->{filename}", "", "", { RaiseError => 1, PrintError => 0, AutoInactiveDestroy => 1 }) or die $DBI::errstr;
+    $self->{dbh} = DBI->connect("dbi:SQLite:dbname=$self->{filename}", "", "", { RaiseError => 1, PrintError => 0, AutoInactiveDestroy => 1, sqlite_unicode => 1 }) or die $DBI::errstr;
   };
 
   if ($@) {

PBot/Plugins/Counter.pm

@@ -57,7 +57,7 @@ sub create_database {
   my $self = shift;
 
   eval {
-    $self->{dbh} = DBI->connect("dbi:SQLite:dbname=$self->{filename}", "", "", { RaiseError => 1, PrintError => 0, AutoInactiveDestroy => 1 }) or die $DBI::errstr;
+    $self->{dbh} = DBI->connect("dbi:SQLite:dbname=$self->{filename}", "", "", { RaiseError => 1, PrintError => 0, AutoInactiveDestroy => 1, sqlite_unicode => 1 }) or die $DBI::errstr;
 
     $self->{dbh}->do(<<SQL);
 CREATE TABLE IF NOT EXISTS Counters (

PBot/Plugins/Quotegrabs/Quotegrabs_SQLite.pm

@@ -39,7 +39,7 @@ sub begin {
 
   $self->{pbot}->{logger}->log("Opening quotegrabs SQLite database: $self->{filename}\n");
 
-  $self->{dbh} = DBI->connect("dbi:SQLite:dbname=$self->{filename}", "", "", { RaiseError => 1, PrintError => 0 }) or die $DBI::errstr;
+  $self->{dbh} = DBI->connect("dbi:SQLite:dbname=$self->{filename}", "", "", { RaiseError => 1, PrintError => 0, sqlite_unicode => 1 }) or die $DBI::errstr;
 
   eval {
     $self->{dbh}->do(<< 'SQL');

PBot/Plugins/RemindMe.pm

@@ -77,7 +77,7 @@ SQL
 sub dbi_begin {
   my ($self) = @_;
   eval {
-    $self->{dbh} = DBI->connect("dbi:SQLite:dbname=$self->{filename}", "", "", { RaiseError => 1, PrintError => 0, AutoInactiveDestroy => 1 }) or die $DBI::errstr;
+    $self->{dbh} = DBI->connect("dbi:SQLite:dbname=$self->{filename}", "", "", { RaiseError => 1, PrintError => 0, AutoInactiveDestroy => 1, sqlite_unicode => 1 }) or die $DBI::errstr;
   };
 
   if ($@) {

PBot/Plugins/Spinach.pm

@@ -121,8 +121,9 @@ sub load_questions {
       return "Failed to load $filename";
     };
     local $/;
-    <$fh>;
+    my $text = <$fh>;
     close $fh;
+    $text;
   };
 
   $self->{loaded_filename} = $filename;