diff --git a/.github/workflows/candide.yaml b/.github/workflows/candide.yaml index 8cff618e..e0d83c47 100644 --- a/.github/workflows/candide.yaml +++ b/.github/workflows/candide.yaml @@ -1,4 +1,4 @@ -name: candide +name: candide on Libera on: push: @@ -12,11 +12,12 @@ env: DEPLOYMENT_NAME: candide NAMESPACE: candide REPOSITORY: pbot - IMAGE: pbot + IMAGE_PBOT: pbot + IMAGE_VM: pbot-vm jobs: - build: - name: Build & Deploy + pbot: + name: PBot runs-on: ubuntu-latest permissions: @@ -41,21 +42,17 @@ jobs: cluster_name: ${{ env.GKE_CLUSTER }} location: ${{ env.GKE_ZONE }} - - name: Testing - run: | - kubectl get pods - - name: Authenticate Docker run: | gcloud auth configure-docker - name: Build image run: |- - docker build --tag "gcr.io/$PROJECT_ID/$REPOSITORY/$IMAGE:$GITHUB_SHA" . + docker build --tag "gcr.io/$PROJECT_ID/$REPOSITORY/$IMAGE_PBOT:$GITHUB_SHA" . - name: Push image run: |- - docker push "gcr.io/$PROJECT_ID/$REPOSITORY/$IMAGE:$GITHUB_SHA" + docker push "gcr.io/$PROJECT_ID/$REPOSITORY/$IMAGE_PBOT:$GITHUB_SHA" - name: Get all changed non-plugin files id: changed-non-plugin-files @@ -83,5 +80,43 @@ jobs: - name: Deploy with restart if: steps.changed-non-plugin-files.outputs.any_changed == 'true' run: |- - sed -i -e "s/PROJECT_ID/$PROJECT_ID/g" -e "s/REPOSITORY/$REPOSITORY/g" -e "s/IMAGE/$IMAGE/g" -e "s/GITHUB_SHA/$GITHUB_SHA/g" k8s/deployment.yaml - kubectl apply -f k8s/deployment.yaml \ No newline at end of file + sed -i -e "s/PROJECT_ID/$PROJECT_ID/g" -e "s/REPOSITORY/$REPOSITORY/g" -e "s/IMAGE/$IMAGE_PBOT/g" -e "s/GITHUB_SHA/$GITHUB_SHA/g" k8s/deployment.yaml + kubectl apply -f k8s/candide-pbot.yaml + + vms: + name: VMs + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: 'Authenticate to Google Cloud' + id: 'auth' + uses: 'google-github-actions/auth@v1' + with: + credentials_json: '${{ secrets.GCP_CREDENTIALS }}' + + - name: Set up GKE credentials + uses: google-github-actions/get-gke-credentials@v2 + with: + cluster_name: ${{ env.GKE_CLUSTER }} + location: ${{ env.GKE_ZONE }} + + - name: Authenticate Docker + run: | + gcloud auth configure-docker + + - name: Build image + run: |- + docker build --tag "gcr.io/$PROJECT_ID/$REPOSITORY/$IMAGE_VM:$GITHUB_SHA" -f applets/pbot-vm/Dockerfile . + + - name: Push image + run: |- + docker push "gcr.io/$PROJECT_ID/$REPOSITORY/$IMAGE_VM:$GITHUB_SHA" + + - name: Deploy + run: |- + sed -i -e "s/PROJECT_ID/$PROJECT_ID/g" -e "s/REPOSITORY/$REPOSITORY/g" -e "s/IMAGE/$IMAGE_VM/g" -e "s/GITHUB_SHA/$GITHUB_SHA/g" k8s/deployment.yaml + kubectl apply -f k8s/candide-vm.yaml diff --git a/Dockerfile b/Dockerfile index 094a04a3..706727c9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,14 +11,14 @@ RUN apt-get install -y libfribidi0 libfribidi-bin gawk libsigsegv2 translate-she RUN useradd -ms /bin/bash pbot # Location for perl libraries. -RUN cpanm --local-lib=/home/pbot/perl5 local::lib && eval $(perl -I /home/pbot/perl5/lib/perl5/ -Mlocal::lib) +RUN su -u pbot cpanm --local-lib=/home/pbot/perl5 local::lib && eval $(perl -I /home/pbot/perl5/lib/perl5/ -Mlocal::lib) -# Install pbot from sources and get dependencies (replace nitrix with pragma- eventually). -RUN cd /opt && git clone --recursive --depth=1 https://github.com/nitrix/pbot +# Install pbot from sources and get dependencies. +COPY . /opt/pbot RUN cd /opt/pbot && cpanm -n --installdeps . --with-all-features --without-feature=compiler_vm_win32 # Wiktionary parser. -RUN pip install git+https://github.com/pragma-/WiktionaryParser --break-system-packages +RUN pip install git+https://github.com/pragma-/WiktionaryParser # Mount point to persist the bot's data. RUN mkdir /mnt/persistent diff --git a/Dockerfile.guest b/Dockerfile.guest deleted file mode 100644 index 9b6fb655..00000000 --- a/Dockerfile.guest +++ /dev/null @@ -1,14 +0,0 @@ -FROM debian:bookworm - -# Install system dependencies. -RUN apt-get update && apt-get install -y clang gcc clang g++ tcl ghc - -# Prefer a non-root user to run the guest. -RUN useradd -ms /bin/bash dummy -USER dummy - -# Just in case files are created in the working directory. -WORKDIR /home/dummy - -# Executable. -ENTRYPOINT /usr/bin/bash \ No newline at end of file diff --git a/applets/pbot-vm/Dockerfile b/applets/pbot-vm/Dockerfile new file mode 100644 index 00000000..ced6626d --- /dev/null +++ b/applets/pbot-vm/Dockerfile @@ -0,0 +1,27 @@ +FROM perl:5.34 + +# Install system dependencies. +RUN apt-get update && apt-get install -y clang gcc clang g++ tcl ghc git cpanminus + +# Need the sources. +COPY . /opt/pbot +RUN cd /opt/pbot && cpanm -n --installdeps . --with-all-features --without-feature=compiler_vm_win32 + +# Serial emulation. +RUN apt-get install -y socat +RUN sed -i -e "s/\/dev\/ttyS2/\/tmp\/ttyS2/g" /opt/pbot/applets/pbot-vm/guest/bin/guest-server +RUN sed -i -e "s/\/dev\/ttyS3/\/tmp\/ttyS3/g" /opt/pbot/applets/pbot-vm/guest/bin/guest-server + +# Setup the guest server. +RUN cd /opt/pbot/applets/pbot-vm/guest/bin && ln -al +RUN cd /opt/pbot/applets/pbot-vm && ./guest/bin/setup-guest + +# Prefer a non-root user. +RUN useradd -ms /bin/bash vm +USER vm + +# Just in case files are created in the working directory. +WORKDIR /home/vm + +# Single entry point for both the host and the guest. +ENTRYPOINT ["/opt/pbot/applets/pbot-vm/entrypoint.sh"] \ No newline at end of file diff --git a/applets/pbot-vm/entrypoint.sh b/applets/pbot-vm/entrypoint.sh new file mode 100644 index 00000000..bacc78e9 --- /dev/null +++ b/applets/pbot-vm/entrypoint.sh @@ -0,0 +1,8 @@ +#/bin/bash + +socat -d -d pty,raw,link=/tmp/ttyS2,echo=0 pty,raw,link=/tmp/ttyS2x,echo=0 & +socat -d -d pty,raw,link=/tmp/ttyS3,echo=0 pty,raw,link=/tmp/ttyS3x,echo=0 & + +guest-server & +/opt/pbot/applets/pbot-vm/docker-server & +/opt/pbot/applets/pbot-vm/vm-server \ No newline at end of file diff --git a/applets/pbot-vm/host/bin/docker-server b/applets/pbot-vm/host/bin/docker-server new file mode 100644 index 00000000..74657db1 --- /dev/null +++ b/applets/pbot-vm/host/bin/docker-server @@ -0,0 +1,188 @@ +#!/usr/bin/env perl + +# File: docker-server +# +# Purpose: Unlike the `vm-server`, the `docker-server` does not manage virtual machines and leaves that task to +# orchestration (e.g. Kubernetes) outside of its own control. Meaning the host and guest are one and the same. +# The rest of the code remains similar, listening for incoming commands from vm-client and invoking `vm-exec` to +# send commands to the `guest-server`. + +# SPDX-FileCopyrightText: 2021 Pragmatic Software +# SPDX-FileContributor: 2024 Alex Belanger for Docker support +# SPDX-License-Identifier: MIT + +use 5.020; + +use warnings; +use strict; + +use feature qw(signatures); +no warnings qw(experimental::signatures); + +use IO::Socket; +use Net::hostent; +use IPC::Shareable; +use Time::HiRes qw/gettimeofday/; +use Encode; + +use constant { + SERVER_PORT => $ENV{SERVER_PORT} // 9000, + COMPILE_TIMEOUT => $ENV{COMPILE_TIMEOUT} // 10, +}; + +sub execute($command) { + print "execute ($command)\n"; + + # to get $? from pipe + local $SIG{CHLD} = 'DEFAULT'; + + my $pid = open(my $fh, '-|', split / /, encode('UTF-8', $command)); + + if (not defined $pid) { + print "Couldn't fork: $!\n"; + return (-13, "[Fatal error]"); + } + + my $result = eval { + my $output = ''; + local $SIG{ALRM} = sub { kill 9, $pid; die "Timed-out: $output\n"; }; + alarm(COMPILE_TIMEOUT); + + while (my $line = decode('UTF-8', <$fh>)) { + $output .= $line; + } + + return $output; + }; + + alarm 0; + close $fh; + + my $ret = $? >> 8; + + if (my $exception = $@) { + # handle time-out exception + if ($exception =~ /Timed-out: (.*)/) { + return (-13, "[Timed-out] $1"); + } + + # propagate unhandled exception + die $exception; + } + + return ($ret, $result); +} + +sub server_listen($port) { + my $server = IO::Socket::INET->new ( + Proto => 'tcp', + LocalPort => $port, + Listen => SOMAXCONN, + ReuseAddr => 1, + Reuse => 1, + ); + die "Can't setup server: $!" unless $server; + print "Server $0 accepting clients at :$port\n"; + return $server; +} + +sub do_server() { + print "Starting PBot VM Server on port " . SERVER_PORT . "\n"; + my $server = eval { server_listen(SERVER_PORT) }; + + if ($@) { + print STDERR $@; + return; + } + + while ($running and my $client = $server->accept) { + print '-' x 20, "\n"; + my $hostinfo = gethostbyaddr($client->peeraddr); + print "Connect from ", $client->peerhost, " at ", scalar localtime, "\n"; + handle_client($client); + } + + print "Shutting down server.\n"; +} + +sub handle_client($client) { + my ($timed_out, $killed) = (0, 0); + + my $r = fork; + + if (not defined $r) { + print "Could not fork to handle client: $!\n"; + print $client "Fatal error.\n"; + close $client; + return; + } + + if ($r > 0) { + # nothing for parent to do with client + close $client; + return; + } + + $client->autoflush(1); + + eval { + # give client 5 seconds to send a line + local $SIG{ALRM} = sub { die "Client I/O timed-out\n"; }; + alarm 5; + + while (my $line = decode('UTF-8', <$client>)) { + $line =~ s/[\r\n]+$//; + next if $line =~ m/^\s*$/; + + # give client 5 more seconds + alarm 5; + + print "[$$] Read [$line]\n"; + + # disable client time-out + alarm 0; + + my ($ret, $result) = execute("perl vm-exec $line"); + + $result =~ s/\s+$//; + print "Ret: $ret; result: [$result]\n"; + + if ($result =~ m/\[Killed\]$/) { + $killed = 1; + $ret = -14; + } + + if ($ret == -13 && $result =~ m/\[Timed-out\]/) { + $timed_out = 1; + } + + print $client encode('UTF-8', $result . "\n"); + last; + } + }; + + # print client time-out exception + print "[$$] $@" if $@; + + alarm 0; + close $client; + + print "[$$] timed out: $timed_out; killed: $killed\n"; + + # child done + print "[$$] client exiting\n"; + print "=" x 20, "\n"; + exit; +} + +sub main() { + binmode(STDOUT, ':utf8'); + binmode(STDERR, ':utf8'); + + # let OS clean-up child exits + $SIG{CHLD} = 'IGNORE'; + + do_server(); +} + +main(); diff --git a/applets/pbot-vm/host/bin/vm-client b/applets/pbot-vm/host/bin/vm-client index e14c6826..3dc7faa7 100755 --- a/applets/pbot-vm/host/bin/vm-client +++ b/applets/pbot-vm/host/bin/vm-client @@ -2,13 +2,14 @@ # File: vm-client # -# Purpose: Interfaces with the PBot VM Host server hosted by `vm-server` -# at PeerAddr/PeerPort defined below. This allows us to host instances -# of virtual machines on remote servers. +# Purpose: Interfaces with the PBot VM Host server hosted by `vm-server` or `docker-server` +# at PeerAddr/PeerPort defined below. This allows us to host the virtual machines on remote servers +# and to abstract the choice of virtualization. # # This script is intended to be invoked by a PBot command such as `cc`. # SPDX-FileCopyrightText: 2021 Pragmatic Software +# SPDX-FileContributor: 2024 Alex Belanger for Docker support # SPDX-License-Identifier: MIT use warnings; @@ -17,12 +18,13 @@ use strict; use IO::Socket; use constant { + SERVER_HOST => $ENV{PBOTVM_HOST} // '127.0.0.1', SERVER_PORT => $ENV{PBOTVM_SERVER} // 9000, }; # TODO: extend to take a list of server/ports to cycle for load-balancing my $sock = IO::Socket::INET->new( - PeerAddr => '127.0.0.1', + PeerAddr => SERVER_HOST, PeerPort => SERVER_PORT, Proto => 'tcp' ); diff --git a/doc/VirtualMachine.md b/doc/VirtualMachine.md index b1291163..6d07bca3 100644 --- a/doc/VirtualMachine.md +++ b/doc/VirtualMachine.md @@ -20,16 +20,17 @@ the command should be executed. Many commands can be configured with environment variables. If a variable is not defined, a sensible default value will be used. -Environment variable | Default value | Description ---- | --- | --- -PBOTVM_DOMAIN | `pbot-vm` | The libvirt domain identifier -PBOTVM_SERVER | `9000` | `vm-server` port for incoming `vm-client` commands -PBOTVM_SERIAL | `5555` | TCP port for serial communication -PBOTVM_HEART | `5556` | TCP port for serial heartbeats -PBOTVM_CID | `7` | Context ID for VM socket (if using VSOCK) -PBOTVM_VPORT | `5555` | VM socket service port (if using VSOCK) -PBOTVM_TIMEOUT | `10` | Duration before command times out (in seconds) -PBOTVM_NOREVERT | not set | If set then the VM will not revert to previous snapshot +| Environment variable | Default value | Description | +|----------------------|---------------|---------------------------------------------------------| +| PBOTVM_DOMAIN | `pbot-vm` | The libvirt domain identifier | +| PBOTVM_HOST | `127.0.0.1` | Where `vm-client` connects to `vm-server` | +| PBOTVM_SERVER | `9000` | `vm-server` port for incoming `vm-client` commands | +| PBOTVM_SERIAL | `5555` | TCP port for serial communication | +| PBOTVM_HEART | `5556` | TCP port for serial heartbeats | +| PBOTVM_CID | `7` | Context ID for VM socket (if using VSOCK) | +| PBOTVM_VPORT | `5555` | VM socket service port (if using VSOCK) | +| PBOTVM_TIMEOUT | `10` | Duration before command times out (in seconds) | +| PBOTVM_NOREVERT | not set | If set then the VM will not revert to previous snapshot | ## Initial virtual machine set-up These steps need to be done only once during the first-time set-up. diff --git a/k8s/deployment.yaml b/k8s/candide-pbot.yaml similarity index 88% rename from k8s/deployment.yaml rename to k8s/candide-pbot.yaml index b301ead9..7567d8d3 100644 --- a/k8s/deployment.yaml +++ b/k8s/candide-pbot.yaml @@ -1,24 +1,24 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: candide + name: candide-pbot namespace: candide labels: - app: candide + app: candide-pbot spec: selector: matchLabels: - app: candide + app: candide-pbot replicas: 1 strategy: type: Recreate template: metadata: labels: - app: candide + app: candide-pbot spec: containers: - - name: candide + - name: candide-pbot image: gcr.io/PROJECT_ID/REPOSITORY/IMAGE:GITHUB_SHA workingDir: "/opt/pbot" command: ["/opt/pbot/bin/pbot"] diff --git a/k8s/candide-vm.yaml b/k8s/candide-vm.yaml new file mode 100644 index 00000000..da9d15f9 --- /dev/null +++ b/k8s/candide-vm.yaml @@ -0,0 +1,27 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: candide-vm + namespace: candide + labels: + app: candide-vm +spec: + selector: + matchLabels: + app: candide-vm + replicas: 3 + template: + metadata: + labels: + app: candide-vm + spec: + containers: + - name: candide-vm + image: gcr.io/PROJECT_ID/REPOSITORY/IMAGE:GITHUB_SHA + resources: + requests: + cpu: "0.1" + memory: 100Mi + limits: + cpu: "0.5" + memory: 500Mi