From e736051de514df66a1513e34015a32deafec7ad8 Mon Sep 17 00:00:00 2001
From: Pragmatic Software <pragma78@gmail.com>
Date: Sun, 12 Mar 2023 16:06:04 -0700
Subject: [PATCH] Fix UTF-8 encoding

---
 applets/pbot-vm/guest/lib/Languages/_default.pm |  5 ++++-
 applets/pbot-vm/host/bin/vm-server              | 11 +++++++----
 applets/pbot-vm/host/lib/Languages/_default.pm  |  4 ++--
 applets/urban                                   |  1 +
 lib/PBot/Core/Applets.pm                        | 13 +++++++------
 lib/PBot/VERSION.pm                             |  4 ++--
 6 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/applets/pbot-vm/guest/lib/Languages/_default.pm b/applets/pbot-vm/guest/lib/Languages/_default.pm
index 38d158bc..cdb2eaaf 100755
--- a/applets/pbot-vm/guest/lib/Languages/_default.pm
+++ b/applets/pbot-vm/guest/lib/Languages/_default.pm
@@ -77,13 +77,16 @@ sub execute {
 
     $stdin //= '';
 
+    $stdin = encode('UTF-8', $stdin);
+    @cmdline = map { encode('UTF-8', $_) } @cmdline;
+
     print STDERR "execute ($timeout) [$stdin] @cmdline\n";
 
     my ($exitval, $stdout, $stderr) = eval {
         my ($stdout, $stderr);
         run \@cmdline, \$stdin, \$stdout, \$stderr, timeout($timeout);
         my $exitval = $? >> 8;
-        return ($exitval, $stdout, $stderr);
+        return ($exitval, decode('UTF-8', $stdout), decode('UTF-8', $stderr));
     };
 
     if (my $exception = $@) {
diff --git a/applets/pbot-vm/host/bin/vm-server b/applets/pbot-vm/host/bin/vm-server
index 3f0b5cde..2b353b77 100755
--- a/applets/pbot-vm/host/bin/vm-server
+++ b/applets/pbot-vm/host/bin/vm-server
@@ -43,7 +43,7 @@ sub execute($command) {
     # to get $? from pipe
     local $SIG{CHLD} = 'DEFAULT';
 
-    my $pid = open(my $fh, '-|', split / /, $command);
+    my $pid = open(my $fh, '-|', split / /, encode('UTF-8', $command));
 
     if (not defined $pid) {
         print "Couldn't fork: $!\n";
@@ -55,7 +55,7 @@ sub execute($command) {
         local $SIG{ALRM} = sub { kill 9, $pid; die "Timed-out: $output\n"; };
         alarm(COMPILE_TIMEOUT);
 
-        while (my $line = <$fh>) {
+        while (my $line = decode('UTF-8', <$fh>)) {
             $output .= $line;
         }
 
@@ -182,7 +182,7 @@ sub handle_client($client, $heartbeat) {
         local $SIG{ALRM} = sub { die "Client I/O timed-out\n"; };
         alarm 5;
 
-        while (my $line = <$client>) {
+        while (my $line = decode('UTF-8', <$client>)) {
             $line =~ s/[\r\n]+$//;
             next if $line =~ m/^\s*$/;
 
@@ -214,7 +214,7 @@ sub handle_client($client, $heartbeat) {
                 $timed_out = 1;
             }
 
-            print $client $result . "\n";
+            print $client encode('UTF-8', $result . "\n");
             last;
         }
     };
@@ -238,6 +238,9 @@ sub handle_client($client, $heartbeat) {
 }
 
 sub main() {
+    binmode(STDOUT, ':utf8');
+    binmode(STDERR, ':utf8');
+
     # let OS clean-up child exits
     $SIG{CHLD} = 'IGNORE';
 
diff --git a/applets/pbot-vm/host/lib/Languages/_default.pm b/applets/pbot-vm/host/lib/Languages/_default.pm
index a8350f9e..27cc165c 100755
--- a/applets/pbot-vm/host/lib/Languages/_default.pm
+++ b/applets/pbot-vm/host/lib/Languages/_default.pm
@@ -210,7 +210,7 @@ sub execute {
     my $stdin = $self->{options}->{'-stdin'};
 
     if (not length $stdin) {
-        $stdin = `fortune -u -s`;
+        $stdin = decode('UTF-8', `fortune -u -s`);
         $stdin =~ s/[\n\r\t]/ /msg;
         $stdin =~ s/:/ - /g;
         $stdin =~ s/\s+/ /g;
@@ -270,7 +270,7 @@ sub execute {
     $compile_in->{'persist-key'} = $self->{'persist-key'} if length $self->{'persist-key'};
 
     my $compile_json = encode_json($compile_in);
-    $compile_json .= encode('UTF-8', "\n:end:\n");
+    $compile_json .= "\n:end:\n";
 
     my $length = length $compile_json;
     my $sent = 0;
diff --git a/applets/urban b/applets/urban
index bb89bc2b..3583a361 100755
--- a/applets/urban
+++ b/applets/urban
@@ -5,6 +5,7 @@
 
 use warnings;
 use strict;
+use utf8;
 
 use WebService::UrbanDictionary;
 use Getopt::Long qw(GetOptionsFromString);
diff --git a/lib/PBot/Core/Applets.pm b/lib/PBot/Core/Applets.pm
index 1a672bba..edeff1e5 100644
--- a/lib/PBot/Core/Applets.pm
+++ b/lib/PBot/Core/Applets.pm
@@ -78,10 +78,6 @@ sub launch_applet {
     my ($exitval, $stdout, $stderr) = eval {
         my $args = $context->{arguments};
 
-        if (not $context->{args_utf8}) {
-            $args = encode('UTF-8', $args);
-        }
-
         my $strip_quotes = 1;
 
         $strip_quotes = 0 if $self->{pbot}->{factoids}->{data}->{storage}->get_data($channel, $trigger, 'keep-quotes');
@@ -92,12 +88,17 @@ sub launch_applet {
 
         my ($stdin, $stdout, $stderr);
 
+        # encode as UTF-8 if not already encoded (e.g. by encode_json)
+        if (not $context->{args_utf8}) {
+            @cmdline = map { encode('UTF-8', $_) } @cmdline;
+        }
+
         run \@cmdline, \$stdin, \$stdout, \$stderr, timeout($timeout);
 
         my $exitval = $? >> 8;
 
-        utf8::decode $stdout;
-        utf8::decode $stderr;
+        $stdout = decode('UTF-8', $stdout);
+        $stderr = decode('UTF-8', $stderr);
 
         return ($exitval, $stdout, $stderr);
     };
diff --git a/lib/PBot/VERSION.pm b/lib/PBot/VERSION.pm
index f5c779ba..e12b5dae 100644
--- a/lib/PBot/VERSION.pm
+++ b/lib/PBot/VERSION.pm
@@ -25,8 +25,8 @@ use PBot::Imports;
 # These are set by the /misc/update_version script
 use constant {
     BUILD_NAME     => "PBot",
-    BUILD_REVISION => 4624,
-    BUILD_DATE     => "2023-02-24",
+    BUILD_REVISION => 4626,
+    BUILD_DATE     => "2023-03-12",
 };
 
 sub initialize {}