First stab at whole vm thing.

.github/workflows/candide.yaml

@@ -1,4 +1,4 @@
-name: candide
+name: candide on Libera
 
 on:
   push:
@@ -12,11 +12,12 @@ env:
   DEPLOYMENT_NAME: candide
   NAMESPACE: candide
   REPOSITORY: pbot
-  IMAGE: pbot
+  IMAGE_PBOT: pbot
+  IMAGE_VM: pbot-vm
 
 jobs:
-  build:
+  pbot:
-    name: Build & Deploy
+    name: PBot
     runs-on: ubuntu-latest
 
     permissions:
@@ -41,21 +42,17 @@ jobs:
         cluster_name: ${{ env.GKE_CLUSTER }}
         location: ${{ env.GKE_ZONE }}
 
-    - name: Testing
-      run: |
-        kubectl get pods
-
     - name: Authenticate Docker
       run: |
         gcloud auth configure-docker
 
     - name: Build image
       run: |-
-        docker build --tag "gcr.io/$PROJECT_ID/$REPOSITORY/$IMAGE:$GITHUB_SHA" .
+        docker build --tag "gcr.io/$PROJECT_ID/$REPOSITORY/$IMAGE_PBOT:$GITHUB_SHA" .
     
     - name: Push image
       run: |-
-        docker push "gcr.io/$PROJECT_ID/$REPOSITORY/$IMAGE:$GITHUB_SHA"
+        docker push "gcr.io/$PROJECT_ID/$REPOSITORY/$IMAGE_PBOT:$GITHUB_SHA"
 
     - name: Get all changed non-plugin files
       id: changed-non-plugin-files
@@ -83,5 +80,43 @@ jobs:
     - name: Deploy with restart
       if: steps.changed-non-plugin-files.outputs.any_changed == 'true'
       run: |-
-        sed -i -e "s/PROJECT_ID/$PROJECT_ID/g" -e "s/REPOSITORY/$REPOSITORY/g" -e "s/IMAGE/$IMAGE/g" -e "s/GITHUB_SHA/$GITHUB_SHA/g" k8s/deployment.yaml
+        sed -i -e "s/PROJECT_ID/$PROJECT_ID/g" -e "s/REPOSITORY/$REPOSITORY/g" -e "s/IMAGE/$IMAGE_PBOT/g" -e "s/GITHUB_SHA/$GITHUB_SHA/g" k8s/deployment.yaml
-        kubectl apply -f k8s/deployment.yaml
+        kubectl apply -f k8s/candide-pbot.yaml
+  
+  vms:
+    name: VMs
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with: 
+        submodules: recursive
+
+    - name: 'Authenticate to Google Cloud'
+      id: 'auth'
+      uses: 'google-github-actions/auth@v1'
+      with:
+        credentials_json: '${{ secrets.GCP_CREDENTIALS }}'
+
+    - name: Set up GKE credentials
+      uses: google-github-actions/get-gke-credentials@v2
+      with:
+        cluster_name: ${{ env.GKE_CLUSTER }}
+        location: ${{ env.GKE_ZONE }}
+
+    - name: Authenticate Docker
+      run: |
+        gcloud auth configure-docker
+
+    - name: Build image
+      run: |-
+        docker build --tag "gcr.io/$PROJECT_ID/$REPOSITORY/$IMAGE_VM:$GITHUB_SHA" -f applets/pbot-vm/Dockerfile .
+
+    - name: Push image
+      run: |-
+        docker push "gcr.io/$PROJECT_ID/$REPOSITORY/$IMAGE_VM:$GITHUB_SHA"
+
+    - name: Deploy
+      run: |-
+        sed -i -e "s/PROJECT_ID/$PROJECT_ID/g" -e "s/REPOSITORY/$REPOSITORY/g" -e "s/IMAGE/$IMAGE_VM/g" -e "s/GITHUB_SHA/$GITHUB_SHA/g" k8s/deployment.yaml
+        kubectl apply -f k8s/candide-vm.yaml

Dockerfile

@@ -11,14 +11,14 @@ RUN apt-get install -y libfribidi0 libfribidi-bin gawk libsigsegv2 translate-she
 RUN useradd -ms /bin/bash pbot
 
 # Location for perl libraries.
-RUN cpanm --local-lib=/home/pbot/perl5 local::lib && eval $(perl -I /home/pbot/perl5/lib/perl5/ -Mlocal::lib)
+RUN su -u pbot cpanm --local-lib=/home/pbot/perl5 local::lib && eval $(perl -I /home/pbot/perl5/lib/perl5/ -Mlocal::lib)
 
-# Install pbot from sources and get dependencies (replace nitrix with pragma- eventually).
+# Install pbot from sources and get dependencies.
-RUN cd /opt && git clone --recursive --depth=1 https://github.com/nitrix/pbot
+COPY . /opt/pbot
 RUN cd /opt/pbot && cpanm -n --installdeps . --with-all-features --without-feature=compiler_vm_win32
 
 # Wiktionary parser.
-RUN pip install git+https://github.com/pragma-/WiktionaryParser --break-system-packages
+RUN pip install git+https://github.com/pragma-/WiktionaryParser
 
 # Mount point to persist the bot's data.
 RUN mkdir /mnt/persistent

Dockerfile.guest

@@ -1,14 +0,0 @@
-FROM debian:bookworm
-
-# Install system dependencies.
-RUN apt-get update && apt-get install -y clang gcc clang g++ tcl ghc
-
-# Prefer a non-root user to run the guest.
-RUN useradd -ms /bin/bash dummy
-USER dummy
-
-# Just in case files are created in the working directory. 
-WORKDIR /home/dummy
-
-# Executable.
-ENTRYPOINT /usr/bin/bash

applets/pbot-vm/Dockerfile

@@ -0,0 +1,27 @@
+FROM perl:5.34
+
+# Install system dependencies.
+RUN apt-get update && apt-get install -y clang gcc clang g++ tcl ghc git cpanminus
+
+# Need the sources.
+COPY . /opt/pbot
+RUN cd /opt/pbot && cpanm -n --installdeps . --with-all-features --without-feature=compiler_vm_win32
+
+# Serial emulation.
+RUN apt-get install -y socat
+RUN sed -i -e "s/\/dev\/ttyS2/\/tmp\/ttyS2/g" /opt/pbot/applets/pbot-vm/guest/bin/guest-server
+RUN sed -i -e "s/\/dev\/ttyS3/\/tmp\/ttyS3/g" /opt/pbot/applets/pbot-vm/guest/bin/guest-server
+
+# Setup the guest server.
+RUN cd /opt/pbot/applets/pbot-vm/guest/bin && ln -al
+RUN cd /opt/pbot/applets/pbot-vm && ./guest/bin/setup-guest
+
+# Prefer a non-root user.
+RUN useradd -ms /bin/bash vm
+USER vm
+
+# Just in case files are created in the working directory. 
+WORKDIR /home/vm
+
+# Single entry point for both the host and the guest.
+ENTRYPOINT ["/opt/pbot/applets/pbot-vm/entrypoint.sh"]

applets/pbot-vm/entrypoint.sh

@@ -0,0 +1,8 @@
+#/bin/bash
+
+socat -d -d pty,raw,link=/tmp/ttyS2,echo=0 pty,raw,link=/tmp/ttyS2x,echo=0 &
+socat -d -d pty,raw,link=/tmp/ttyS3,echo=0 pty,raw,link=/tmp/ttyS3x,echo=0 &
+
+guest-server &
+/opt/pbot/applets/pbot-vm/docker-server &
+/opt/pbot/applets/pbot-vm/vm-server

applets/pbot-vm/host/bin/docker-server

@@ -0,0 +1,188 @@
+#!/usr/bin/env perl
+
+# File: docker-server
+#
+# Purpose: Unlike the `vm-server`, the `docker-server` does not manage virtual machines and leaves that task to
+# orchestration (e.g. Kubernetes) outside of its own control. Meaning the host and guest are one and the same.
+# The rest of the code remains similar, listening for incoming commands from vm-client and invoking `vm-exec` to
+# send commands to the `guest-server`.
+
+# SPDX-FileCopyrightText: 2021 Pragmatic Software <pragma78@gmail.com>
+# SPDX-FileContributor: 2024 Alex Belanger for Docker support
+# SPDX-License-Identifier: MIT
+
+use 5.020;
+
+use warnings;
+use strict;
+
+use feature qw(signatures);
+no warnings qw(experimental::signatures);
+
+use IO::Socket;
+use Net::hostent;
+use IPC::Shareable;
+use Time::HiRes qw/gettimeofday/;
+use Encode;
+
+use constant {
+    SERVER_PORT     => $ENV{SERVER_PORT}  // 9000,
+    COMPILE_TIMEOUT => $ENV{COMPILE_TIMEOUT} // 10,
+};
+
+sub execute($command) {
+    print "execute ($command)\n";
+
+    # to get $? from pipe
+    local $SIG{CHLD} = 'DEFAULT';
+
+    my $pid = open(my $fh, '-|', split / /, encode('UTF-8', $command));
+
+    if (not defined $pid) {
+        print "Couldn't fork: $!\n";
+        return (-13, "[Fatal error]");
+    }
+
+    my $result = eval {
+        my $output = '';
+        local $SIG{ALRM} = sub { kill 9, $pid; die "Timed-out: $output\n"; };
+        alarm(COMPILE_TIMEOUT);
+
+        while (my $line = decode('UTF-8', <$fh>)) {
+            $output .= $line;
+        }
+
+        return $output;
+    };
+
+    alarm 0;
+    close $fh;
+
+    my $ret = $? >> 8;
+
+    if (my $exception = $@) {
+        # handle time-out exception
+        if ($exception =~ /Timed-out: (.*)/) {
+            return (-13, "[Timed-out] $1");
+        }
+
+        # propagate unhandled exception
+        die $exception;
+    }
+
+    return ($ret, $result);
+}
+
+sub server_listen($port) {
+    my $server = IO::Socket::INET->new (
+        Proto     => 'tcp',
+        LocalPort => $port,
+        Listen    => SOMAXCONN,
+        ReuseAddr => 1,
+        Reuse     => 1,
+    );
+    die "Can't setup server: $!" unless $server;
+    print "Server $0 accepting clients at :$port\n";
+    return $server;
+}
+
+sub do_server() {
+    print "Starting PBot VM Server on port " . SERVER_PORT . "\n";
+    my $server = eval { server_listen(SERVER_PORT) };
+
+    if ($@) {
+        print STDERR $@;
+        return;
+    }
+
+    while ($running and my $client = $server->accept) {
+        print '-' x 20, "\n";
+        my $hostinfo = gethostbyaddr($client->peeraddr);
+        print "Connect from ", $client->peerhost, " at ", scalar localtime, "\n";
+        handle_client($client);
+    }
+
+    print "Shutting down server.\n";
+}
+
+sub handle_client($client) {
+    my ($timed_out, $killed) = (0, 0);
+
+    my $r = fork;
+
+    if (not defined $r) {
+        print "Could not fork to handle client: $!\n";
+        print $client "Fatal error.\n";
+        close $client;
+        return;
+    }
+
+    if ($r > 0) {
+        # nothing for parent to do with client
+        close $client;
+        return;
+    }
+
+    $client->autoflush(1);
+
+    eval {
+        # give client 5 seconds to send a line
+        local $SIG{ALRM} = sub { die "Client I/O timed-out\n"; };
+        alarm 5;
+
+        while (my $line = decode('UTF-8', <$client>)) {
+            $line =~ s/[\r\n]+$//;
+            next if $line =~ m/^\s*$/;
+
+            # give client 5 more seconds
+            alarm 5;
+
+            print "[$$] Read [$line]\n";
+
+            # disable client time-out
+            alarm 0;
+
+            my ($ret, $result) = execute("perl vm-exec $line");
+
+            $result =~ s/\s+$//;
+            print "Ret: $ret; result: [$result]\n";
+
+            if ($result =~ m/\[Killed\]$/) {
+                $killed = 1;
+                $ret = -14;
+            }
+
+            if ($ret == -13 && $result =~ m/\[Timed-out\]/) {
+                $timed_out = 1;
+            }
+
+            print $client encode('UTF-8', $result . "\n");
+            last;
+        }
+    };
+
+    # print client time-out exception
+    print "[$$] $@" if $@;
+
+    alarm 0;
+    close $client;
+
+    print "[$$] timed out: $timed_out; killed: $killed\n";
+
+    # child done
+    print "[$$] client exiting\n";
+    print "=" x 20, "\n";
+    exit;
+}
+
+sub main() {
+    binmode(STDOUT, ':utf8');
+    binmode(STDERR, ':utf8');
+
+    # let OS clean-up child exits
+    $SIG{CHLD} = 'IGNORE';
+
+    do_server();
+}
+
+main();

applets/pbot-vm/host/bin/vm-client

@@ -2,13 +2,14 @@
 
 # File: vm-client
 #
-# Purpose: Interfaces with the PBot VM Host server hosted by `vm-server`
+# Purpose: Interfaces with the PBot VM Host server hosted by `vm-server` or `docker-server`
-# at PeerAddr/PeerPort defined below. This allows us to host instances
+# at PeerAddr/PeerPort defined below. This allows us to host the virtual machines on remote servers
-# of virtual machines on remote servers.
+# and to abstract the choice of virtualization.
 #
 # This script is intended to be invoked by a PBot command such as `cc`.
 
 # SPDX-FileCopyrightText: 2021 Pragmatic Software <pragma78@gmail.com>
+# SPDX-FileContributor: 2024 Alex Belanger for Docker support
 # SPDX-License-Identifier: MIT
 
 use warnings;
@@ -17,12 +18,13 @@ use strict;
 use IO::Socket;
 
 use constant {
+    SERVER_HOST => $ENV{PBOTVM_HOST} // '127.0.0.1',
     SERVER_PORT => $ENV{PBOTVM_SERVER} // 9000,
 };
 
 # TODO: extend to take a list of server/ports to cycle for load-balancing
 my $sock = IO::Socket::INET->new(
-    PeerAddr => '127.0.0.1',
+    PeerAddr => SERVER_HOST,
     PeerPort => SERVER_PORT,
     Proto    => 'tcp'
 );

doc/VirtualMachine.md

@@ -20,16 +20,17 @@ the command should be executed.
 Many commands can be configured with environment variables. If a variable is
 not defined, a sensible default value will be used.
 
-Environment variable | Default value | Description
+| Environment variable | Default value | Description                                             |
---- | --- | ---
+|----------------------|---------------|---------------------------------------------------------|
-PBOTVM_DOMAIN | `pbot-vm` | The libvirt domain identifier
+| PBOTVM_DOMAIN        | `pbot-vm`     | The libvirt domain identifier                           |
-PBOTVM_SERVER | `9000` | `vm-server` port for incoming `vm-client` commands
+| PBOTVM_HOST          | `127.0.0.1`   | Where `vm-client` connects to `vm-server`               |
-PBOTVM_SERIAL | `5555` | TCP port for serial communication
+| PBOTVM_SERVER        | `9000`        | `vm-server` port for incoming `vm-client` commands      |
-PBOTVM_HEART  | `5556` | TCP port for serial heartbeats
+| PBOTVM_SERIAL        | `5555`        | TCP port for serial communication                       |
-PBOTVM_CID    | `7` | Context ID for VM socket (if using VSOCK)
+| PBOTVM_HEART         | `5556`        | TCP port for serial heartbeats                          |
-PBOTVM_VPORT  | `5555` | VM socket service port (if using VSOCK)
+| PBOTVM_CID           | `7`           | Context ID for VM socket (if using VSOCK)               |
-PBOTVM_TIMEOUT | `10` | Duration before command times out (in seconds)
+| PBOTVM_VPORT         | `5555`        | VM socket service port (if using VSOCK)                 |
-PBOTVM_NOREVERT | not set | If set then the VM will not revert to previous snapshot
+| PBOTVM_TIMEOUT       | `10`          | Duration before command times out (in seconds)          |
+| PBOTVM_NOREVERT      | not set       | If set then the VM will not revert to previous snapshot |
 
 ## Initial virtual machine set-up
 These steps need to be done only once during the first-time set-up.

k8s/candide-pbot.yaml

@@ -1,24 +1,24 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: candide
+  name: candide-pbot
   namespace: candide
   labels:
-    app: candide
+    app: candide-pbot
 spec:
   selector:
     matchLabels:
-      app: candide
+      app: candide-pbot
   replicas: 1
   strategy:
     type: Recreate
   template:
     metadata:
       labels:
-        app: candide
+        app: candide-pbot
     spec:
       containers:
-        - name: candide
+        - name: candide-pbot
           image: gcr.io/PROJECT_ID/REPOSITORY/IMAGE:GITHUB_SHA
           workingDir: "/opt/pbot"
           command: ["/opt/pbot/bin/pbot"]

k8s/candide-vm.yaml

@@ -0,0 +1,27 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: candide-vm
+  namespace: candide
+  labels:
+    app: candide-vm
+spec:
+  selector:
+    matchLabels:
+      app: candide-vm
+  replicas: 3
+  template:
+    metadata:
+      labels:
+        app: candide-vm
+    spec:
+      containers:
+        - name: candide-vm
+          image: gcr.io/PROJECT_ID/REPOSITORY/IMAGE:GITHUB_SHA
+          resources:
+            requests:
+              cpu: "0.1"
+              memory: 100Mi
+            limits:
+              cpu: "0.5"
+              memory: 500Mi