3
0
mirror of https://github.com/pragma-/pbot.git synced 2025-07-06 05:27:26 +02:00

First stab at whole vm thing.

This commit is contained in:
Alex Belanger 2024-03-11 23:25:14 -04:00
parent 88fbe30945
commit f1d9570917
No known key found for this signature in database
GPG Key ID: 31736D2B10A5F158
10 changed files with 323 additions and 49 deletions

View File

@ -1,4 +1,4 @@
name: candide
name: candide on Libera
on:
push:
@ -12,11 +12,12 @@ env:
DEPLOYMENT_NAME: candide
NAMESPACE: candide
REPOSITORY: pbot
IMAGE: pbot
IMAGE_PBOT: pbot
IMAGE_VM: pbot-vm
jobs:
build:
name: Build & Deploy
pbot:
name: PBot
runs-on: ubuntu-latest
permissions:
@ -41,21 +42,17 @@ jobs:
cluster_name: ${{ env.GKE_CLUSTER }}
location: ${{ env.GKE_ZONE }}
- name: Testing
run: |
kubectl get pods
- name: Authenticate Docker
run: |
gcloud auth configure-docker
- name: Build image
run: |-
docker build --tag "gcr.io/$PROJECT_ID/$REPOSITORY/$IMAGE:$GITHUB_SHA" .
docker build --tag "gcr.io/$PROJECT_ID/$REPOSITORY/$IMAGE_PBOT:$GITHUB_SHA" .
- name: Push image
run: |-
docker push "gcr.io/$PROJECT_ID/$REPOSITORY/$IMAGE:$GITHUB_SHA"
docker push "gcr.io/$PROJECT_ID/$REPOSITORY/$IMAGE_PBOT:$GITHUB_SHA"
- name: Get all changed non-plugin files
id: changed-non-plugin-files
@ -83,5 +80,43 @@ jobs:
- name: Deploy with restart
if: steps.changed-non-plugin-files.outputs.any_changed == 'true'
run: |-
sed -i -e "s/PROJECT_ID/$PROJECT_ID/g" -e "s/REPOSITORY/$REPOSITORY/g" -e "s/IMAGE/$IMAGE/g" -e "s/GITHUB_SHA/$GITHUB_SHA/g" k8s/deployment.yaml
kubectl apply -f k8s/deployment.yaml
sed -i -e "s/PROJECT_ID/$PROJECT_ID/g" -e "s/REPOSITORY/$REPOSITORY/g" -e "s/IMAGE/$IMAGE_PBOT/g" -e "s/GITHUB_SHA/$GITHUB_SHA/g" k8s/deployment.yaml
kubectl apply -f k8s/candide-pbot.yaml
vms:
name: VMs
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
submodules: recursive
- name: 'Authenticate to Google Cloud'
id: 'auth'
uses: 'google-github-actions/auth@v1'
with:
credentials_json: '${{ secrets.GCP_CREDENTIALS }}'
- name: Set up GKE credentials
uses: google-github-actions/get-gke-credentials@v2
with:
cluster_name: ${{ env.GKE_CLUSTER }}
location: ${{ env.GKE_ZONE }}
- name: Authenticate Docker
run: |
gcloud auth configure-docker
- name: Build image
run: |-
docker build --tag "gcr.io/$PROJECT_ID/$REPOSITORY/$IMAGE_VM:$GITHUB_SHA" -f applets/pbot-vm/Dockerfile .
- name: Push image
run: |-
docker push "gcr.io/$PROJECT_ID/$REPOSITORY/$IMAGE_VM:$GITHUB_SHA"
- name: Deploy
run: |-
sed -i -e "s/PROJECT_ID/$PROJECT_ID/g" -e "s/REPOSITORY/$REPOSITORY/g" -e "s/IMAGE/$IMAGE_VM/g" -e "s/GITHUB_SHA/$GITHUB_SHA/g" k8s/deployment.yaml
kubectl apply -f k8s/candide-vm.yaml

View File

@ -11,14 +11,14 @@ RUN apt-get install -y libfribidi0 libfribidi-bin gawk libsigsegv2 translate-she
RUN useradd -ms /bin/bash pbot
# Location for perl libraries.
RUN cpanm --local-lib=/home/pbot/perl5 local::lib && eval $(perl -I /home/pbot/perl5/lib/perl5/ -Mlocal::lib)
RUN su -u pbot cpanm --local-lib=/home/pbot/perl5 local::lib && eval $(perl -I /home/pbot/perl5/lib/perl5/ -Mlocal::lib)
# Install pbot from sources and get dependencies (replace nitrix with pragma- eventually).
RUN cd /opt && git clone --recursive --depth=1 https://github.com/nitrix/pbot
# Install pbot from sources and get dependencies.
COPY . /opt/pbot
RUN cd /opt/pbot && cpanm -n --installdeps . --with-all-features --without-feature=compiler_vm_win32
# Wiktionary parser.
RUN pip install git+https://github.com/pragma-/WiktionaryParser --break-system-packages
RUN pip install git+https://github.com/pragma-/WiktionaryParser
# Mount point to persist the bot's data.
RUN mkdir /mnt/persistent

View File

@ -1,14 +0,0 @@
FROM debian:bookworm
# Install system dependencies.
RUN apt-get update && apt-get install -y clang gcc clang g++ tcl ghc
# Prefer a non-root user to run the guest.
RUN useradd -ms /bin/bash dummy
USER dummy
# Just in case files are created in the working directory.
WORKDIR /home/dummy
# Executable.
ENTRYPOINT /usr/bin/bash

View File

@ -0,0 +1,27 @@
FROM perl:5.34
# Install system dependencies.
RUN apt-get update && apt-get install -y clang gcc clang g++ tcl ghc git cpanminus
# Need the sources.
COPY . /opt/pbot
RUN cd /opt/pbot && cpanm -n --installdeps . --with-all-features --without-feature=compiler_vm_win32
# Serial emulation.
RUN apt-get install -y socat
RUN sed -i -e "s/\/dev\/ttyS2/\/tmp\/ttyS2/g" /opt/pbot/applets/pbot-vm/guest/bin/guest-server
RUN sed -i -e "s/\/dev\/ttyS3/\/tmp\/ttyS3/g" /opt/pbot/applets/pbot-vm/guest/bin/guest-server
# Setup the guest server.
RUN cd /opt/pbot/applets/pbot-vm/guest/bin && ln -al
RUN cd /opt/pbot/applets/pbot-vm && ./guest/bin/setup-guest
# Prefer a non-root user.
RUN useradd -ms /bin/bash vm
USER vm
# Just in case files are created in the working directory.
WORKDIR /home/vm
# Single entry point for both the host and the guest.
ENTRYPOINT ["/opt/pbot/applets/pbot-vm/entrypoint.sh"]

View File

@ -0,0 +1,8 @@
#/bin/bash
socat -d -d pty,raw,link=/tmp/ttyS2,echo=0 pty,raw,link=/tmp/ttyS2x,echo=0 &
socat -d -d pty,raw,link=/tmp/ttyS3,echo=0 pty,raw,link=/tmp/ttyS3x,echo=0 &
guest-server &
/opt/pbot/applets/pbot-vm/docker-server &
/opt/pbot/applets/pbot-vm/vm-server

View File

@ -0,0 +1,188 @@
#!/usr/bin/env perl
# File: docker-server
#
# Purpose: Unlike the `vm-server`, the `docker-server` does not manage virtual machines and leaves that task to
# orchestration (e.g. Kubernetes) outside of its own control. Meaning the host and guest are one and the same.
# The rest of the code remains similar, listening for incoming commands from vm-client and invoking `vm-exec` to
# send commands to the `guest-server`.
# SPDX-FileCopyrightText: 2021 Pragmatic Software <pragma78@gmail.com>
# SPDX-FileContributor: 2024 Alex Belanger for Docker support
# SPDX-License-Identifier: MIT
use 5.020;
use warnings;
use strict;
use feature qw(signatures);
no warnings qw(experimental::signatures);
use IO::Socket;
use Net::hostent;
use IPC::Shareable;
use Time::HiRes qw/gettimeofday/;
use Encode;
use constant {
SERVER_PORT => $ENV{SERVER_PORT} // 9000,
COMPILE_TIMEOUT => $ENV{COMPILE_TIMEOUT} // 10,
};
sub execute($command) {
print "execute ($command)\n";
# to get $? from pipe
local $SIG{CHLD} = 'DEFAULT';
my $pid = open(my $fh, '-|', split / /, encode('UTF-8', $command));
if (not defined $pid) {
print "Couldn't fork: $!\n";
return (-13, "[Fatal error]");
}
my $result = eval {
my $output = '';
local $SIG{ALRM} = sub { kill 9, $pid; die "Timed-out: $output\n"; };
alarm(COMPILE_TIMEOUT);
while (my $line = decode('UTF-8', <$fh>)) {
$output .= $line;
}
return $output;
};
alarm 0;
close $fh;
my $ret = $? >> 8;
if (my $exception = $@) {
# handle time-out exception
if ($exception =~ /Timed-out: (.*)/) {
return (-13, "[Timed-out] $1");
}
# propagate unhandled exception
die $exception;
}
return ($ret, $result);
}
sub server_listen($port) {
my $server = IO::Socket::INET->new (
Proto => 'tcp',
LocalPort => $port,
Listen => SOMAXCONN,
ReuseAddr => 1,
Reuse => 1,
);
die "Can't setup server: $!" unless $server;
print "Server $0 accepting clients at :$port\n";
return $server;
}
sub do_server() {
print "Starting PBot VM Server on port " . SERVER_PORT . "\n";
my $server = eval { server_listen(SERVER_PORT) };
if ($@) {
print STDERR $@;
return;
}
while ($running and my $client = $server->accept) {
print '-' x 20, "\n";
my $hostinfo = gethostbyaddr($client->peeraddr);
print "Connect from ", $client->peerhost, " at ", scalar localtime, "\n";
handle_client($client);
}
print "Shutting down server.\n";
}
sub handle_client($client) {
my ($timed_out, $killed) = (0, 0);
my $r = fork;
if (not defined $r) {
print "Could not fork to handle client: $!\n";
print $client "Fatal error.\n";
close $client;
return;
}
if ($r > 0) {
# nothing for parent to do with client
close $client;
return;
}
$client->autoflush(1);
eval {
# give client 5 seconds to send a line
local $SIG{ALRM} = sub { die "Client I/O timed-out\n"; };
alarm 5;
while (my $line = decode('UTF-8', <$client>)) {
$line =~ s/[\r\n]+$//;
next if $line =~ m/^\s*$/;
# give client 5 more seconds
alarm 5;
print "[$$] Read [$line]\n";
# disable client time-out
alarm 0;
my ($ret, $result) = execute("perl vm-exec $line");
$result =~ s/\s+$//;
print "Ret: $ret; result: [$result]\n";
if ($result =~ m/\[Killed\]$/) {
$killed = 1;
$ret = -14;
}
if ($ret == -13 && $result =~ m/\[Timed-out\]/) {
$timed_out = 1;
}
print $client encode('UTF-8', $result . "\n");
last;
}
};
# print client time-out exception
print "[$$] $@" if $@;
alarm 0;
close $client;
print "[$$] timed out: $timed_out; killed: $killed\n";
# child done
print "[$$] client exiting\n";
print "=" x 20, "\n";
exit;
}
sub main() {
binmode(STDOUT, ':utf8');
binmode(STDERR, ':utf8');
# let OS clean-up child exits
$SIG{CHLD} = 'IGNORE';
do_server();
}
main();

View File

@ -2,13 +2,14 @@
# File: vm-client
#
# Purpose: Interfaces with the PBot VM Host server hosted by `vm-server`
# at PeerAddr/PeerPort defined below. This allows us to host instances
# of virtual machines on remote servers.
# Purpose: Interfaces with the PBot VM Host server hosted by `vm-server` or `docker-server`
# at PeerAddr/PeerPort defined below. This allows us to host the virtual machines on remote servers
# and to abstract the choice of virtualization.
#
# This script is intended to be invoked by a PBot command such as `cc`.
# SPDX-FileCopyrightText: 2021 Pragmatic Software <pragma78@gmail.com>
# SPDX-FileContributor: 2024 Alex Belanger for Docker support
# SPDX-License-Identifier: MIT
use warnings;
@ -17,12 +18,13 @@ use strict;
use IO::Socket;
use constant {
SERVER_HOST => $ENV{PBOTVM_HOST} // '127.0.0.1',
SERVER_PORT => $ENV{PBOTVM_SERVER} // 9000,
};
# TODO: extend to take a list of server/ports to cycle for load-balancing
my $sock = IO::Socket::INET->new(
PeerAddr => '127.0.0.1',
PeerAddr => SERVER_HOST,
PeerPort => SERVER_PORT,
Proto => 'tcp'
);

View File

@ -20,16 +20,17 @@ the command should be executed.
Many commands can be configured with environment variables. If a variable is
not defined, a sensible default value will be used.
Environment variable | Default value | Description
--- | --- | ---
PBOTVM_DOMAIN | `pbot-vm` | The libvirt domain identifier
PBOTVM_SERVER | `9000` | `vm-server` port for incoming `vm-client` commands
PBOTVM_SERIAL | `5555` | TCP port for serial communication
PBOTVM_HEART | `5556` | TCP port for serial heartbeats
PBOTVM_CID | `7` | Context ID for VM socket (if using VSOCK)
PBOTVM_VPORT | `5555` | VM socket service port (if using VSOCK)
PBOTVM_TIMEOUT | `10` | Duration before command times out (in seconds)
PBOTVM_NOREVERT | not set | If set then the VM will not revert to previous snapshot
| Environment variable | Default value | Description |
|----------------------|---------------|---------------------------------------------------------|
| PBOTVM_DOMAIN | `pbot-vm` | The libvirt domain identifier |
| PBOTVM_HOST | `127.0.0.1` | Where `vm-client` connects to `vm-server` |
| PBOTVM_SERVER | `9000` | `vm-server` port for incoming `vm-client` commands |
| PBOTVM_SERIAL | `5555` | TCP port for serial communication |
| PBOTVM_HEART | `5556` | TCP port for serial heartbeats |
| PBOTVM_CID | `7` | Context ID for VM socket (if using VSOCK) |
| PBOTVM_VPORT | `5555` | VM socket service port (if using VSOCK) |
| PBOTVM_TIMEOUT | `10` | Duration before command times out (in seconds) |
| PBOTVM_NOREVERT | not set | If set then the VM will not revert to previous snapshot |
## Initial virtual machine set-up
These steps need to be done only once during the first-time set-up.

View File

@ -1,24 +1,24 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: candide
name: candide-pbot
namespace: candide
labels:
app: candide
app: candide-pbot
spec:
selector:
matchLabels:
app: candide
app: candide-pbot
replicas: 1
strategy:
type: Recreate
template:
metadata:
labels:
app: candide
app: candide-pbot
spec:
containers:
- name: candide
- name: candide-pbot
image: gcr.io/PROJECT_ID/REPOSITORY/IMAGE:GITHUB_SHA
workingDir: "/opt/pbot"
command: ["/opt/pbot/bin/pbot"]

27
k8s/candide-vm.yaml Normal file
View File

@ -0,0 +1,27 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: candide-vm
namespace: candide
labels:
app: candide-vm
spec:
selector:
matchLabels:
app: candide-vm
replicas: 3
template:
metadata:
labels:
app: candide-vm
spec:
containers:
- name: candide-vm
image: gcr.io/PROJECT_ID/REPOSITORY/IMAGE:GITHUB_SHA
resources:
requests:
cpu: "0.1"
memory: 100Mi
limits:
cpu: "0.5"
memory: 500Mi