Fix some more Unicode encoding issues

This commit is contained in:
Pragmatic Software 2019-07-10 18:40:53 -07:00
parent a1dcabf553
commit 3553692b5a
73 changed files with 161 additions and 11 deletions

View File

@ -16,6 +16,8 @@ package PBot::AntiFlood;
use warnings;
use strict;
use feature 'unicode_strings';
use feature 'switch';
no if $] >= 5.018, warnings => "experimental::smartmatch";
@ -405,7 +407,7 @@ sub check_flood {
}
if ($self->whitelisted($channel, "$nick!$user\@$host", 'antiflood')) {
$self->{pbot}->{logger}->log("$nick!$user\@$host anti-flood whitelisted, disgregarding ban\n");
$self->{pbot}->{logger}->log("$nick!$user\@$host anti-flood whitelisted, skipping anti-flood checking\n");
next;
}

View File

@ -12,6 +12,8 @@ package PBot::AntiSpam;
use warnings;
use strict;
use feature 'unicode_strings';
use feature 'switch';
no if $] >= 5.018, warnings => "experimental::smartmatch";

View File

@ -15,6 +15,8 @@ package PBot::BanTracker;
use warnings;
use strict;
use feature 'unicode_strings';
use Time::HiRes qw/gettimeofday/;
use Time::Duration;
use Data::Dumper;

View File

@ -12,6 +12,8 @@ package PBot::BlackList;
use warnings;
use strict;
use feature 'unicode_strings';
use feature 'switch';
no if $] >= 5.018, warnings => "experimental::smartmatch";

View File

@ -12,6 +12,8 @@ package PBot::BotAdminCommands;
use warnings;
use strict;
use feature 'unicode_strings';
use feature 'switch';
no if $] >= 5.018, warnings => "experimental::smartmatch";

View File

@ -12,6 +12,8 @@ package PBot::BotAdmins;
use warnings;
use strict;
use feature 'unicode_strings';
use PBot::DualIndexHashObject;
use PBot::BotAdminCommands;

View File

@ -12,6 +12,8 @@ package PBot::ChanOpCommands;
use warnings;
use strict;
use feature 'unicode_strings';
use Carp ();
use Time::Duration;

View File

@ -12,6 +12,8 @@ package PBot::ChanOps;
use warnings;
use strict;
use feature 'unicode_strings';
use PBot::ChanOpCommands;
use Time::HiRes qw(gettimeofday);

View File

@ -12,6 +12,8 @@ package PBot::Channels;
use warnings;
use strict;
use feature 'unicode_strings';
use Carp ();
use PBot::HashObject;

View File

@ -15,6 +15,8 @@ package PBot::Commands;
use warnings;
use strict;
use feature 'unicode_strings';
use base 'PBot::Registerable';
use Carp ();

View File

@ -13,6 +13,8 @@ package PBot::DualIndexHashObject;
use warnings;
use strict;
use feature 'unicode_strings';
use Text::Levenshtein qw(fastdistance);
use JSON;
use Carp ();

View File

@ -7,6 +7,8 @@ package PBot::EventDispatcher;
use warnings;
use strict;
use feature 'unicode_strings';
use IO::Select;
use Carp ();

View File

@ -12,6 +12,8 @@ package PBot::FactoidCommands;
use warnings;
use strict;
use feature 'unicode_strings';
use Carp ();
use Time::Duration;
use Time::HiRes qw(gettimeofday);

View File

@ -12,11 +12,14 @@ package PBot::FactoidModuleLauncher;
use warnings;
use strict;
use feature 'unicode_strings';
use POSIX qw(WNOHANG);
use Carp ();
use Text::Balanced qw(extract_delimited);
use JSON;
use IPC::Run qw/run timeout/;
use Encode;
# automatically reap children processes in background
$SIG{CHLD} = sub { while (waitpid(-1, WNOHANG) > 0) {} };
@ -110,11 +113,17 @@ sub execute_module {
}
my ($exitval, $stdout, $stderr) = eval {
my @cmdline = ("./$module", $self->{pbot}->{interpreter}->split_line($stuff->{arguments}));
my $args = $stuff->{arguments};
if (not $stuff->{args_utf8}) {
$args = encode('UTF-8', $args);
}
my @cmdline = ("./$module", $self->{pbot}->{interpreter}->split_line($args));
my $timeout = $self->{pbot}->{registry}->get_value('general', 'module_timeout') // 30;
my ($stdin, $stdout, $stderr);
run \@cmdline, \$stdin, \$stdout, \$stderr, timeout($timeout);
my $exitval = $? >> 8;
utf8::decode($stdout);
utf8::decode($stderr);
return ($exitval, $stdout, $stderr);
};
@ -139,8 +148,6 @@ sub execute_module {
$stuff->{result} = $stdout;
chomp $stuff->{result};
utf8::decode($stuff->{result});
my $json = encode_json $stuff;
print $writer "$json\n";
exit 0;

View File

@ -12,6 +12,8 @@ package PBot::Factoids;
use warnings;
use strict;
use feature 'unicode_strings';
use feature 'switch';
no if $] >= 5.018, warnings => "experimental::smartmatch";
@ -711,6 +713,7 @@ sub execute_code_factoid_using_vm {
$stuff->{root_channel} = $stuff->{channel};
$stuff->{keyword} = 'compiler';
$stuff->{arguments} = $json;
$stuff->{args_utf8} = 1;
$self->{pbot}->{factoids}->{factoidmodulelauncher}->execute_module($stuff);
return "";

View File

@ -13,6 +13,8 @@ package PBot::HashObject;
use warnings;
use strict;
use feature 'unicode_strings';
use Text::Levenshtein qw(fastdistance);
use Carp ();
use JSON;

View File

@ -23,6 +23,7 @@ use PBot::IRC::EventQueue; # pragma_ 2011/01/21
use IO::Select;
use Carp;
use feature 'unicode_strings';
# grab the drop-in replacement for time() from Time::HiRes, if it's available
BEGIN {

View File

@ -15,6 +15,8 @@
package PBot::IRC::Connection; # pragma_ 2011/21/01
use feature 'unicode_strings';
use PBot::IRC::Event; # pragma_ 2011/21/01
use PBot::IRC::DCC; # pragma_ 2011/21/01
use IO::Socket;

View File

@ -17,7 +17,7 @@ package PBot::IRC::DCC; # pragma_ 2011/21/01
use strict;
use feature 'unicode_strings';
# --- #perl was here! ---
#

View File

@ -24,6 +24,8 @@
package PBot::IRC::Event; # pragma_ 2011/21/01
use feature 'unicode_strings';
use strict;
our %_names;

View File

@ -1,5 +1,7 @@
package PBot::IRC::EventQueue; # pragma_ 2011/21/01
use feature 'unicode_strings';
use PBot::IRC::EventQueue::Entry; # pragma_ 2011/21/01
use strict;

View File

@ -2,6 +2,8 @@ package PBot::IRC::EventQueue::Entry; # pragma_ 2011/21/01
use strict;
use feature 'unicode_strings';
my $id = 0;
sub new {

View File

@ -12,6 +12,8 @@ package PBot::IRCHandlers;
use warnings;
use strict;
use feature 'unicode_strings';
use Carp();
use Time::HiRes qw(gettimeofday);
use Data::Dumper;

View File

@ -12,6 +12,8 @@ package PBot::IgnoreList;
use warnings;
use strict;
use feature 'unicode_strings';
use PBot::IgnoreListCommands;
use Time::HiRes qw(gettimeofday);

View File

@ -12,6 +12,8 @@ package PBot::IgnoreListCommands;
use warnings;
use strict;
use feature 'unicode_strings';
use Time::HiRes qw(gettimeofday);
use Time::Duration;
use Carp ();

View File

@ -12,6 +12,8 @@ package PBot::Interpreter;
use warnings;
use strict;
use feature 'unicode_strings';
use base 'PBot::Registerable';
use Time::HiRes qw/gettimeofday/;
@ -329,6 +331,7 @@ sub interpret {
# set arguments as a plain string
$stuff->{arguments} = $arguments;
$stuff->{original_arguments} = $arguments;
delete $stuff->{args_utf8};
# set arguments as an array
$stuff->{arglist} = $self->make_args($arguments);
@ -340,6 +343,7 @@ sub interpret {
last if defined $result;
# reset any manipulated arguments
$stuff->{arguments} = $stuff->{original_arguments};
delete $stuff->{args_utf8};
}
return $result;
}

View File

@ -13,6 +13,8 @@ package PBot::LagChecker;
use warnings;
use strict;
use feature 'unicode_strings';
use feature 'switch';
use Time::HiRes qw(gettimeofday tv_interval);

View File

@ -7,6 +7,8 @@ package PBot::Logger;
use warnings;
use strict;
use feature 'unicode_strings';
use Carp ();
sub new {

View File

@ -16,6 +16,8 @@ package PBot::MessageHistory;
use warnings;
use strict;
use feature 'unicode_strings';
use Getopt::Long qw(GetOptionsFromString);
use Time::HiRes qw(gettimeofday tv_interval);
use Time::Duration;

View File

@ -12,6 +12,8 @@ package PBot::MessageHistory_SQLite;
use warnings;
use strict;
use feature 'unicode_strings';
use DBI;
use Carp qw(shortmess);
use Time::HiRes qw(gettimeofday);

View File

@ -14,6 +14,8 @@ package PBot::NickList;
use warnings;
use strict;
use feature 'unicode_strings';
use Text::Levenshtein qw/fastdistance/;
use Data::Dumper;
$Data::Dumper::Sortkeys = 1;

View File

@ -12,6 +12,8 @@ package PBot::PBot;
use strict;
use warnings;
use feature 'unicode_strings';
# unbuffer stdout
STDOUT->autoflush(1);

View File

@ -12,6 +12,8 @@ package PBot::Plugins;
use warnings;
use strict;
use feature 'unicode_strings';
use File::Basename;
use Carp ();

View File

@ -7,6 +7,8 @@ package PBot::Plugins::ActionTrigger;
use warnings;
use strict;
use feature 'unicode_strings';
use feature 'switch';
no if $] >= 5.018, warnings => "experimental::smartmatch";

View File

@ -12,6 +12,8 @@ package PBot::Plugins::AntiAway;
use warnings;
use strict;
use feature 'unicode_strings';
use Carp ();
sub new {

View File

@ -12,6 +12,8 @@ package PBot::Plugins::AntiKickAutoRejoin;
use warnings;
use strict;
use feature 'unicode_strings';
use Carp ();
use Time::HiRes qw/gettimeofday/;
use Time::Duration;

View File

@ -13,6 +13,8 @@ package PBot::Plugins::AntiNickSpam;
use warnings;
use strict;
use feature 'unicode_strings';
use Carp ();
use Time::Duration qw/duration/;
use Time::HiRes qw/gettimeofday/;

View File

@ -10,6 +10,8 @@ use strict;
use feature 'switch';
no if $] >= 5.018, warnings => "experimental::smartmatch";
use feature 'unicode_strings';
use Carp ();
use String::LCSS qw/lcss/;

View File

@ -13,6 +13,8 @@ package PBot::Plugins::AntiTwitter;
use warnings;
use strict;
use feature 'unicode_strings';
use Carp ();
use Time::HiRes qw/gettimeofday/;
use Time::Duration qw/duration/;

View File

@ -12,6 +12,8 @@ package PBot::Plugins::AutoRejoin;
use warnings;
use strict;
use feature 'unicode_strings';
use Carp ();
use Time::HiRes qw/gettimeofday/;
use Time::Duration;

View File

@ -10,6 +10,8 @@ use strict;
use feature 'switch';
no if $] >= 5.018, warnings => "experimental::smartmatch";
use feature 'unicode_strings';
use Carp ();
use Time::Duration qw/concise duration/;
use Data::Dumper;

View File

@ -10,6 +10,8 @@ use strict;
use feature 'switch';
no if $] >= 5.018, warnings => "experimental::smartmatch";
use feature 'unicode_strings';
use Carp ();
use Time::Duration qw/concise duration/;
use Data::Dumper;

View File

@ -10,6 +10,8 @@ use strict;
use feature 'switch';
no if $] >= 5.018, warnings => "experimental::smartmatch";
use feature 'unicode_strings';
use Carp ();
use DBI;
use Time::Duration qw/duration/;

View File

@ -12,6 +12,8 @@ package PBot::Plugins::MagicCommand;
use warnings;
use strict;
use feature 'unicode_strings';
use Carp ();
sub new {

View File

@ -9,6 +9,8 @@ package PBot::Plugins::ParseDate;
use warnings;
use strict;
use feature 'unicode_strings';
use Carp ();
use Time::Duration qw/duration/;

View File

@ -12,6 +12,8 @@ package PBot::Plugins::Quotegrabs;
use warnings;
use strict;
use feature 'unicode_strings';
use HTML::Entities;
use Time::Duration;
use Time::HiRes qw(gettimeofday);

View File

@ -12,6 +12,8 @@ package PBot::Plugins::Quotegrabs::Quotegrabs_Hashtable;
use warnings;
use strict;
use feature 'unicode_strings';
use HTML::Entities;
use Time::Duration;
use Time::HiRes qw(gettimeofday);

View File

@ -12,6 +12,8 @@ package PBot::Plugins::Quotegrabs::Quotegrabs_SQLite;
use warnings;
use strict;
use feature 'unicode_strings';
use DBI;
use Carp qw(shortmess);

View File

@ -4,6 +4,8 @@ package PBot::Plugins::RelayUnreg;
use warnings;
use strict;
use feature 'unicode_strings';
use Carp ();
use Time::HiRes qw/gettimeofday/;

View File

@ -7,6 +7,8 @@ package PBot::Plugins::RemindMe;
use warnings;
use strict;
use feature 'unicode_strings';
use feature 'switch';
no if $] >= 5.018, warnings => "experimental::smartmatch";

View File

@ -13,6 +13,8 @@ use lib "$FindBin::RealBin/../..";
use feature 'switch';
no if $] >= 5.018, warnings => "experimental::smartmatch";
use feature 'unicode_strings';
use Carp ();
use JSON;
@ -1532,7 +1534,7 @@ sub normalize_question {
sub normalize_text {
my ($self, $text) = @_;
$text = unidecode decode('utf8', $text);
$text = unidecode $text;
$text =~ s/^\s+|\s+$//g;
$text =~ s/\s+/ /g;

View File

@ -9,6 +9,8 @@ package PBot::Plugins::Spinach::Rank;
use warnings;
use strict;
use feature 'unicode_strings';
use FindBin;
use lib "$FindBin::RealBin/../../..";

View File

@ -9,6 +9,8 @@ package PBot::Plugins::Spinach::Stats;
use warnings;
use strict;
use feature 'unicode_strings';
use DBI;
use Carp qw(shortmess);

View File

@ -12,6 +12,8 @@ package PBot::Plugins::UrlTitles;
use warnings;
use strict;
use feature 'unicode_strings';
use Carp ();
sub new {

View File

@ -7,6 +7,8 @@ package PBot::Plugins::_Example;
use warnings;
use strict;
use feature 'unicode_strings';
use Carp ();
sub new {

View File

@ -13,6 +13,8 @@ package PBot::Refresher;
use warnings;
use strict;
use feature 'unicode_strings';
use Module::Refresh;
use Carp ();

View File

@ -12,6 +12,8 @@ package PBot::Registerable;
use warnings;
use strict;
use feature 'unicode_strings';
use Carp ();
sub new {

View File

@ -13,6 +13,8 @@ package PBot::Registry;
use warnings;
use strict;
use feature 'unicode_strings';
use Time::HiRes qw(gettimeofday);
use Carp ();

View File

@ -12,6 +12,8 @@ package PBot::RegistryCommands;
use warnings;
use strict;
use feature 'unicode_strings';
use Carp ();
sub new {

View File

@ -13,6 +13,8 @@ package PBot::SQLiteLogger;
use strict;
use warnings;
use feature 'unicode_strings';
use Carp;
use Time::HiRes qw(gettimeofday);

View File

@ -12,6 +12,8 @@ package PBot::SQLiteLoggerLayer;
use strict;
use warnings;
use feature 'unicode_strings';
sub PUSHED
{
my ($class, $mode, $fh) = @_;

View File

@ -7,6 +7,8 @@ package PBot::SelectHandler;
use warnings;
use strict;
use feature 'unicode_strings';
use IO::Select;
use Carp ();

View File

@ -7,6 +7,8 @@ package PBot::StdinReader;
use warnings;
use strict;
use feature 'unicode_strings';
use POSIX qw(tcgetpgrp getpgrp); # to check whether process is in background or foreground
use Carp ();

View File

@ -14,6 +14,8 @@ package PBot::Timer;
use warnings;
use strict;
use feature 'unicode_strings';
use Carp ();
our $min_timeout = 1;

View File

@ -1,6 +1,8 @@
package PBot::Utils::Indefinite;
use 5.010; use warnings;
use feature 'unicode_strings';
require Exporter;
our @ISA = qw/Exporter/;
our @EXPORT = qw/prepend_indefinite_article select_indefinite_article/;

View File

@ -7,6 +7,8 @@
use warnings;
use strict;
use feature 'unicode_strings';
package PBot::Utils::ParseDate;
use DateTime;

View File

@ -1,5 +1,6 @@
package PBot::Utils::SafeFilename;
use 5.010; use warnings;
use feature 'unicode_strings';
require Exporter;
our @ISA = qw/Exporter/;

View File

@ -3,6 +3,8 @@ package PBot::Utils::ValidateString;
use warnings;
use strict;
use feature 'unicode_strings';
require Exporter;
our @ISA = qw/Exporter/;
our @EXPORT = qw/validate_string/;

View File

@ -14,6 +14,8 @@ package PBot::VERSION;
use strict;
use warnings;
use feature 'unicode_strings';
BEGIN {
use Exporter;
our @ISA = 'Exporter';

View File

@ -12,6 +12,8 @@ package PBot::WebPaste;
use warnings;
use strict;
use feature 'unicode_strings';
use Time::HiRes qw/gettimeofday/;
use Time::Duration;
use LWP::UserAgent;

View File

@ -4,7 +4,7 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
use warnings;
no warnings;
use strict;
use IPC::Open2;
@ -84,6 +84,7 @@ sub execute {
next if not length $line;
<$out> and next if $line =~ m/^\(gdb\) No line \d+ in/;
next if $line =~ m/No default breakpoint address/;
next if $line =~ m/^\(gdb\) No symbol table/;
next if $line =~ m/^\[Detaching after/;
next if $line =~ m/^\[New Thread/;

View File

@ -3,6 +3,7 @@
use warnings;
use strict;
use feature "switch";
use feature 'unicode_strings';
no if $] >= 5.018, warnings => "experimental::smartmatch";
@ -15,6 +16,7 @@ use Time::HiRes qw/gettimeofday/;
use Text::Balanced qw/extract_delimited/;
use JSON;
use Getopt::Long qw/GetOptionsFromArray :config pass_through no_ignore_case no_auto_abbrev/;
use Encode;
my $EXECUTE_PORT = '3333';
@ -278,7 +280,7 @@ sub paste_ixio {
return "error pasting: " . $response->status_line;
}
my $result = $response->content;
my $result = $response->decoded_content;
$result =~ s/^\s+//;
$result =~ s/\s+$//;
return $result;
@ -371,7 +373,7 @@ sub execute {
$compile_in->{'persist-key'} = $self->{'persist-key'} if length $self->{'persist-key'};
my $compile_json = encode_json($compile_in);
$compile_json .= "\n:end:\n";
$compile_json .= encode('UTF-8', "\n:end:\n");
my $length = length $compile_json;
my $sent = 0;
@ -388,7 +390,7 @@ sub execute {
#print FILE "Sending chunk [$chunk]\n";
$chunks_sent += length $chunk;
my $ret = syswrite($compiler, "$chunk\n");
my $ret = syswrite($compiler, $chunk);
if (not defined $ret) {
print FILE "Error sending: $!\n";
@ -412,7 +414,8 @@ sub execute {
my $got_result = 0;
while(my $line = <$compiler_output>) {
print STDERR "Read [$line]\n";
utf8::decode($line);
print STDERR "Read from vm [$line]\n";
$line =~ s/[\r\n]+$//;
last if $line =~ /^result:end$/;

View File

@ -12,6 +12,8 @@
my $VERSION = "1.0.0";
use feature 'unicode_strings';
use strict;
use warnings;