Merge pull request 'Only restart network once' (#1) from network-restart into main

Reviewed-on: #1

.gitignore

@@ -0,0 +1,3 @@
+.vagrant/
+__pycache__/
+.scullery_*

README.md

@@ -2,4 +2,83 @@
 
 This is intended to be a replacement for [KitchenCI](https://test-kitchen.chef.io/) and [kitchen-salt](https://github.com/saltstack/kitchen-salt), focussing on testing SaltStack states using Vagrant and Pytest/Testinfra.
 
+## Installation
 
+So far, a packaged installation is possible on openSUSE Tumbleweed:
+
+```
+$ zypper ar -f https://download.opensuse.org/repositories/home:/crameleon:/Scullery/openSUSE_Tumbleweed/home:crameleon:Scullery.repo
+$ zypper in scullery
+```
+
+This will pull in `vagrant`, `vagrant-libvirt`, `python3-vagrant`, `python3-pytest` and `python3-pytest-testinfra` as dependencies.
+
+Installation on Leap 15.4 is possible, however will currently cause dependency issues with the needed `python-vagrant` library.
+
+It is possible to run `scullery.py` from a Git checkout of this repository if the dependencies are resolved manually.
+
+Currently the script behaves purely as a console application and imports as a Python module are not supported.
+
+## Basic usage
+
+```
+scullery -h
+usage: scullery [-h] [--debug] [--config CONFIG] [--env] --suite SUITE [--stop | --test | --status | --refresh] [--force-stop]
+
+options:
+  -h, --help       show this help message and exit
+  --debug          Print extremely verbose output
+  --config CONFIG  Specify the configuration file to use
+  --env            Write environment file for direct use of Vagrant
+  --suite SUITE    Specify the suite to run
+  --stop           Stop running machines
+  --test           Start machines and run tests
+  --status         Get Vagrant deployment status
+  --refresh        Re-sync files and re-run bootstrap scripts
+  --force-stop     Invoke Vagrant destruction without having detected any running VM's
+```
+
+## Command line examples
+
+* To run a complete test suite:  
+  `$ scullery --suite <name of suite> --test`
+
+* To create Vagrant VM's without invoking the tests:
+  `$ scullery --suite <name of suite>`
+
+* To refresh the VM's Salt files:  
+  `$ scullery --suite <name of suite> --refresh`
+
+* To stop the VM's and tidy up:  
+  `$ scullery --suite <name of suite> --stop`
+
+## Interacting with Vagrant
+
+It is possible to directly interact with the Vagrant environment templated by Scullery. To do so, generate an environment file by adding `--env` to any command line call - for example:
+
+`$ scullery --suite <name of suite> --env`
+
+This will write a `.scullery_env` file to the current working directory. Export the environment variables in the file using your Shell, afterwards use `vagrant` like you normally would. Example for `bash`/`sh`:
+
+```
+$ set -a; . .scullery_env; set +a
+$ vagrant ssh # or any other Vagrant command
+```
+
+Do not forget to unset the exported variables before you call Scullery with a different `--suite`!
+
+## Hacking
+
+To test Scullery itself, call Pytest with your favourite arguments from the repository root:
+
+```
+$ pytest -v -rx -x tests/*.py
+```
+
+Make sure to not have _any_ other Scullery based Vagrant VM's running in the same Libvirt session and no `SCULLERY_*` or `VAGRANT_*` variables exported in your shell.
+
+The complete test suite takes about 30 minutes to execute on an Intel i7-6820HQ CPU with SSHD storage. If you want to test only certain functionality, simply add `-k <name of test function>`.
+
+## License
+
+This project is licensed under the European Union Public License found in the file `LICENSE`.

TODO.md

@@ -0,0 +1,7 @@
+# To-Do's
+
+- Introduce `--apply` argument to execute `state.apply` on VM's without chaining to test executions
+- Do not restart the network stack on `--refresh` (the `systemctl restart network` call is needed on the first boot to get Avahi DNS resolution to work - there is currently no logic to skip the command on subsequent executions of the bootstrap script)
+- Add example configurations
+- Use `pyproject.toml` style packaging to allow for installation as a Python module
+- Resolve Pytest warnings

Vagrantfile-Template

@@ -14,7 +14,11 @@ Vagrant.configure("2") do |config|
   config.vm.box = ENV['SCULLERY_BOX_NAME']
   config.vm.box_url = ENV['SCULLERY_BOX_IMAGE']
   config.vm.provision "shell", inline: <<-SHELL
-  systemctl restart network
+  if [ ! -f '/var/adm/scullery.configured' ]
+  then
+    systemctl restart network
+    touch /var/adm/scullery.configured
+  fi
   SHELL
   if ENV['SCULLERY_MASTERS']
     ENV['SCULLERY_MASTERS'].split(',').each do |vmname|
@@ -23,14 +27,14 @@ Vagrant.configure("2") do |config|
           master_config.vm.provider :libvirt do |libvirt|
             libvirt.memory = 768
           end
-          if salt_bootstrap
-            master_config.vm.provision "shell", path: salt_bootstrap
-          end
           master_config.vm.provision "shell", env: {'SALT_FILE_ROOTS': salt_file_roots}, inline: <<-SHELL
           printf 'auto_accept: True\n' > /etc/salt/master.d/notsecure.conf
           printf "$SALT_FILE_ROOTS" > /etc/salt/master.d/roots.conf
           systemctl enable --now salt-master
           SHELL
+          if salt_bootstrap
+            master_config.vm.provision "shell", path: salt_bootstrap
+          end
         end
     end
   end

scullery.py

@@ -17,6 +17,7 @@ from configparser import ConfigParser
 import logging
 import os
 import sys
+from pytest import ExitCode
 
 argparser = ArgumentParser()
 config = ConfigParser()
@@ -47,7 +48,11 @@ is_packaged = False
 if is_packaged:
     vagfile='/usr/share/scullery/{}'.format(vagfile)
 else:
-    vagfile='{}/{}'.format(os.path.abspath(os.path.dirname(__file__)), vagfile)
+    try:
+        me = __file__
+    except NameError:
+        me = sys.argv[0]
+    vagfile='{}/{}'.format(os.path.abspath(os.path.dirname(me)), vagfile)
 
 def _abort(msg):
     log.error(msg)
@@ -55,7 +60,7 @@ def _abort(msg):
 
 def _config():
     configmap = {'boxes': {}, 'suites': {}, 'tests': {}}
-    if not config.options('box'):
+    if not 'box' in config.sections():
         _abort('No "box" section found in the configuration file')
     multis = {'boxes': {'prefix': 'box.', 'singular': 'box'}, 'suites': {'prefix': 'suite.', 'singular': 'suite'}, 'tests': {'prefix': 'test.', 'singular': 'test'}}
     for multi, multiconf in multis.items():
@@ -73,7 +78,8 @@ def _config():
         if onemulti in config.sections():
             for option in config.options(onemulti):
                 for collection in configmap[multi]:
-                    configmap[multi][collection][option] = config.get(onemulti, option)
+                    if not option in configmap[multi][collection]:
+                        configmap[multi][collection][option] = config.get(onemulti, option)
         if multi in ['boxes', 'suites']:
             if not len(lowconf):
                 _abort('No {} configured'.format(multi))
@@ -100,6 +106,8 @@ def _setenv(envmap, dump=False):
             env[variable] = value
             if dump:
                 fh.write(f'{variable}={value}\n')
+        elif variable in env:
+            del env[variable]
     if dump:
         fh.close()
 
@@ -136,24 +144,42 @@ def vagrant_sshconfig(outfile):
     with open(outfile, 'w') as fh:
         fh.write(ssh_config)
 
-def runapply(state, target):
+def _saltcmd(target):
     if target == 'local':
         saltcmd = 'salt-call --local'
     else:
-        saltcmd = 'salt {}'.format(target)
+        saltcmd = 'salt -t10 {}'.format(target)
+    return saltcmd
+
+def runping(target):
+    saltcmd = _saltcmd(target)
+    sshout = v.ssh(command='sudo {} test.ping'.format(saltcmd))
+    log.info('\n{}\n'.format(str(sshout)))
+    return sshout
+
+def runapply(state, target):
+    saltcmd = _saltcmd(target)
     sshout = v.ssh(command='sudo {} state.apply {}'.format(saltcmd, state))
     log.info('\n{}\n'.format(str(sshout)))
 
 def runtests(payload, hosts):
-    if not os.path.isfile(sshfile):
-        vagrant_sshconfig(sshfile)
+    vagrant_sshconfig(sshfile)
     testresult = pytest.main(['--verbose', '--hosts={}'.format(','.join(hosts)), '--ssh-config={}'.format(sshfile), payload])
-    log.debug('Test result is {}'.format(str(testresult)))
-    if not testresult:
+    log.debug('Test result is {}'.format(str(testresult.value)))
+    if testresult == ExitCode.OK:
+        log.debug('Test succeeded')
+    else:
         log.warning('Tests failed')
         return False
     return True
 
+def _cleanup():
+    for file in [envfile, sshfile]:
+        if os.path.isfile(file):
+            log.debug('Removing {}'.format(file))
+            os.remove(file)
+
+
 def main_interactive():
     configmap = _config()
     boxes = configmap['boxes']
@@ -168,7 +194,7 @@ def main_interactive():
         _abort('Specified suite does not reference a box')
     boxconf = configmap['boxes'].get(box, None)
     if boxconf is None:
-        _abort('Suite referencs an undefined box')
+        _abort('Suite references an undefined box')
     box_name = boxconf.get('name', None)
     box_image = boxconf.get('image', None)
     box_file = boxconf.get('file', vagfile)
@@ -190,10 +216,6 @@ def main_interactive():
         if True in [args.stop, args.force_stop]:
             log.info('Destroying machines ...')
             v.destroy()
-            for file in [envfile, sshfile]:
-                if os.path.isfile(file):
-                    log.debug('Removing {}'.format(file))
-                    os.remove(file)
             if vagrant_isup(suite)[0] is False:
                 log.debug('OK')
             else:
@@ -202,7 +224,9 @@ def main_interactive():
             log.info('Deployment is already running')
         elif args.refresh:
             log.info('Deployment is running, initiating refresh ...')
+            _cleanup()
             v.provision()
+            vagrant_sshconfig(sshfile)
     elif status[0] is False:
         if status[1] is True:
             log.debug('Deployment is not running')
@@ -222,6 +246,9 @@ def main_interactive():
             else:
                 _abort('Start failed')
 
+    if args.stop:
+        _cleanup()
+
     if args.test:
         test = suiteconf.get('test', None)
         if test is None:
@@ -236,6 +263,11 @@ def main_interactive():
             log.debug('state.apply requested')
             if masters is not None:
                 target = 'scullery-*'
+                count = 0
+                while not runping(target):
+                    if count == 5:
+                        _abort('Unable to reach minions')
+                    count += 1
             else:
                 target = 'local'
             runapply(testconf['apply'], target)

tests/01_test_cli.py

@@ -0,0 +1,182 @@
+"""
+Copyright 2023, Georg Pfuetzenreuter
+
+Licensed under the EUPL, Version 1.2 or - as soon they will be approved by the European Commission - subsequent versions of the EUPL (the "Licence").
+You may not use this work except in compliance with the Licence.
+An English copy of the Licence is shipped in a file called LICENSE along with this applications source code.
+You may obtain copies of the Licence in any of the official languages at https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12.
+
+---
+
+Testing functions for Scullery - a SaltStack testing tool.
+"""
+
+import pytest
+import os
+import dotenv
+
+
+def test_no_arguments(script_runner, script):
+    result = script_runner.run(script)
+    assert result.success is False
+    assert 'the following arguments are required: --suite' in result.stderr
+
+
+def test_no_config(script_runner, script):
+    result = script_runner.run(script, '--suite', 'foo', print_result=True) 
+    assert result.success is False
+    assert 'Unable to locate configuration file' in result.stderr
+
+
+@pytest.mark.parametrize('section,message', [
+        ('box', 'No "box" section found in the configuration file'),
+        ('suites', 'No suites configured'),
+        ('boxes', 'No boxes configured'),
+        ('suite_box', 'Specified suite does not reference a box'),
+        ('box_name', 'Box configuration is incomplete'),
+        ('box_image', 'Box configuration is incomplete'),
+    ])
+def test_config_incomplete(script_runner, script, testbase, section, message):
+    configfile = '{}/configs/missing_{}.ini'.format(testbase, section)
+    result = script_runner.run(script, '--config', configfile, '--suite', 'one_minion')
+    assert result.success is False
+    assert message in result.stderr
+
+
+@pytest.mark.parametrize('section,message', [
+        ('suite_box', 'Suite references an undefined box')
+    ])
+def test_config_undefined(script_runner, script, testbase, section, message):
+    configfile = '{}/configs/undefined_{}.ini'.format(testbase, section)
+    result = script_runner.run(script, '--config', configfile, '--suite', 'one_minion')
+    assert result.success is False
+    assert message in result.stderr
+
+
+@pytest.mark.parametrize('config', ['complete'], indirect=True)
+@pytest.mark.parametrize('suite,report', [
+        ('one_minion', "[Status(name='scullery-minion0', state='not_created', provider='libvirt')]"),
+        ('two_minions', "[Status(name='scullery-minion0', state='not_created', provider='libvirt'), Status(name='scullery-minion1', state='not_created', provider='libvirt')]"),
+        ('one_master', "[Status(name='scullery-master0', state='not_created', provider='libvirt')]"),
+        ('one_minion_one_master', "[Status(name='scullery-master0', state='not_created', provider='libvirt'), Status(name='scullery-minion0', state='not_created', provider='libvirt')]"),
+        ('two_minions_one_master', "[Status(name='scullery-master0', state='not_created', provider='libvirt'), Status(name='scullery-minion0', state='not_created', provider='libvirt'), Status(name='scullery-minion1', state='not_created', provider='libvirt')]")
+    ])
+def test_status_report_not_running(script_runner, script, config, suite, report):
+    result = script_runner.run(script, '--config', config, '--suite', suite, '--status', '--debug', '--env')
+    assert result.success
+    assert result.stderr.endswith("INFO - main_interactive: Status report: {}\n".format(report))
+
+
+@pytest.mark.parametrize('config', ['complete'], indirect=True)
+@pytest.mark.parametrize('suite', [
+        'one_minion',
+        'two_minions',
+        'one_master',
+        'one_minion_one_master',
+        'two_minions_one_master'
+    ])
+def test_launch_stop(script_runner, script, virt, config, suite):
+    cmd = (script, '--config', config, '--suite', suite)
+    result = script_runner.run(*cmd)
+    assert result.success
+    assert 'Launching {} ...\n'.format(suite) in result.stderr
+    domains = []
+    print(virt.getURI())
+    print(virt.listDomainsID())
+    for domain in virt.listDomainsID():
+        print(domain)
+        domains.append(virt.lookupByID(domain).name())
+    print(str(domains))
+    # consider refining this
+    assert any('scullery' in domain for domain in domains)
+    result = script_runner.run(*cmd, '--stop')
+    assert result.success
+    assert 'Destroying machines ...\n' in result.stderr
+    domains = []
+    for domain in virt.listDomainsID():
+        domains.append(virt.lookupByID(domain).name())
+    assert not any('scullery' in domain for domain in domains)
+
+
+@pytest.mark.parametrize('config', ['complete'], indirect=True)
+@pytest.mark.parametrize('suite,masters,minions', [
+        ('one_minion', None, 'scullery-minion0'),
+        ('two_minions', None, 'scullery-minion0,scullery-minion1'),
+        ('one_master', 'scullery-master0', None),
+        ('one_minion_one_master', 'scullery-master0', 'scullery-minion0'),
+        ('two_minions_one_master', 'scullery-master0', 'scullery-minion0,scullery-minion1')
+    ])
+def test_envfile(script_runner, script, config, suite, masters, minions):
+    cmd = (script, '--config', config, '--suite', suite)
+    envfile = '.scullery_env'
+    result = script_runner.run(*cmd, '--env')
+    assert result.success
+    assert os.path.isfile(envfile)
+    envmap = dotenv.dotenv_values(envfile)
+    if masters is not None:
+        assert envmap['SCULLERY_MASTERS'] == masters 
+    if minions is not None:
+        assert envmap['SCULLERY_MINIONS'] == minions
+    script_runner.run(*cmd, '--stop')
+    assert os.path.isfile(envfile) is False
+
+
+@pytest.mark.parametrize('config', ['complete'], indirect=True)
+def test_test_undeclared(script_runner, script, config):
+    result = script_runner.run(script, '--config', config, '--suite', 'one_minion', '--test')
+    assert not result.success
+    assert result.stderr.endswith('Tests requested but not declared in suite configuration\n')
+
+
+@pytest.mark.parametrize('config', ['complete'], indirect=True)
+def test_test_undefined(script_runner, script, config):
+    result = script_runner.run(script, '--config', config, '--suite', 'one_minion_bogus_test', '--test')
+    assert not result.success
+    assert result.stderr.endswith('Specified test is not defined\n')
+
+
+@pytest.mark.parametrize('config', ['complete'], indirect=True)
+def test_test_incomplete(script_runner, script, config):
+    result = script_runner.run(script, '--config', config, '--suite', 'one_minion_bogus_test_2', '--test')
+    assert not result.success
+    assert result.stderr.endswith('Incomplete test configuration\n')
+
+
+@pytest.mark.parametrize('config', ['complete'], indirect=True)
+@pytest.mark.parametrize('suite', [
+        'one_minion',
+        'two_minions',
+        'one_master',
+        'one_minion_one_master',
+        'two_minions_one_master',
+        'one_minion_bootstrap',
+        'one_minion_one_master_bootstrap',
+        'one_minion_salt',
+        'two_minions_one_master_salt'
+    ])
+def test_stop(script_runner, script, config, suite):
+    cmd = (script, '--config', config, '--suite', suite, '--stop', '--force')
+    result = script_runner.run(*cmd)
+    assert result.success
+
+
+def test_stop_success(virt):
+    domains = []
+    for domain in virt.listDomainsID():
+        domains.append(virt.lookupByID(domain).name())
+    assert not any('scullery' in domain for domain in domains)
+
+
+@pytest.mark.parametrize('config', ['complete'], indirect=True)
+@pytest.mark.parametrize('suite', [
+        'one_minion',
+        'two_minions',
+        'one_master',
+        'one_minion_one_master',
+        'two_minions_one_master'
+    ])
+def test_already_stopped(script_runner, script, virt, config, suite):
+    cmd = (script, '--config', config, '--suite', suite, '--stop', '--debug')
+    result = script_runner.run(*cmd)
+    assert result.success
+    assert result.stderr.endswith('DEBUG - main_interactive: Deployment is not running\n')

tests/02_test_vagrant.py

@@ -0,0 +1,43 @@
+"""
+Copyright 2023, Georg Pfuetzenreuter
+
+Licensed under the EUPL, Version 1.2 or - as soon they will be approved by the European Commission - subsequent versions of the EUPL (the "Licence").
+You may not use this work except in compliance with the Licence.
+An English copy of the Licence is shipped in a file called LICENSE along with this applications source code.
+You may obtain copies of the Licence in any of the official languages at https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12.
+
+---
+
+Testing functions for Scullery - a SaltStack testing tool.
+"""
+
+import pytest
+import os
+from conftest import loadenv
+import sys
+
+@pytest.mark.parametrize('config', ['complete'], indirect=True)
+@pytest.mark.parametrize('suite', [
+        'one_minion_bootstrap',
+        'one_minion_one_master_bootstrap',
+    ])
+def test_bootstrap(script_runner, script, config, suite, vag, testbase):
+    bootstrap_script = '{}/scripts/bootstrap_hello_world.txt'.format(testbase)
+    do_digest = False
+    if sys.version_info[1] > 10:
+        do_digest = True
+        import hashlib
+        with open(bootstrap_script, 'rb') as fh:
+                  digest_local = hashlib.file_digest(fh, 'md5')
+    cmd = (script, '--config', config, '--suite', suite)
+    result = script_runner.run(*cmd, '--env')
+    assert result.success
+    assert result.stderr.endswith('main_interactive: Launching {} ...\n'.format(suite))
+    v = vag
+    v.env = loadenv()
+    assert v.ssh(command='cat /srv/hello_world.txt') == 'Hello world!\n'
+    if do_digest:
+        digest_remote = v.ssh(command='md5sum /srv/hello_world.txt | awk "{ print $1 }"')
+        assert digest_local == digest_remote
+    assert script_runner.run(*cmd, '--stop')
+

tests/03_test_salt.py

@@ -0,0 +1,44 @@
+"""
+Copyright 2023, Georg Pfuetzenreuter
+
+Licensed under the EUPL, Version 1.2 or - as soon they will be approved by the European Commission - subsequent versions of the EUPL (the "Licence").
+You may not use this work except in compliance with the Licence.
+An English copy of the Licence is shipped in a file called LICENSE along with this applications source code.
+You may obtain copies of the Licence in any of the official languages at https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12.
+
+---
+
+Testing functions for Scullery - a SaltStack testing tool.
+"""
+
+import pytest
+
+@pytest.mark.parametrize('config', ['complete'], indirect=True)
+@pytest.mark.parametrize('suite,count', [
+        ('one_minion_salt', 0),
+        ('two_minions_one_master_salt', 1)
+    ])
+def test_salt_test(script_runner, script, config, suite, count):
+    cmd = (script, '--config', config, '--suite', suite)
+    result = script_runner.run(*cmd, '--debug', '--env', '--test')
+    assert result.success
+    for message in [
+            'File /srv/hello_world.txt',
+            'Succeeded: 1',
+            'DEBUG - main_interactive: state.apply requested',
+            'INFO - main_interactive: Initiating tests ...',
+            'DEBUG - runtests: Test result is 0',
+            'DEBUG - runtests: Test succeeded'
+        ]:
+        assert message in result.stderr
+    assert not any(term in result.stderr for term in [
+            'FAILED', 'WARNING - runtests: Tests failed',
+            'DEBUG - runtests: Test result is 1', 'AssertionError'
+        ])
+    if count > 0:
+        minions = [0, 1]
+    else:
+        minions = [0]
+    for m in minions:
+        assert 'test_hello_world_content[paramiko://scullery-minion%i] PASSED' % m in result.stdout
+    assert script_runner.run(*cmd, '--stop')

tests/configs/complete.ini

@@ -0,0 +1,71 @@
+[box]
+bootstrap=tests/scripts/bootstrap.sh
+name=Tumbleweed.x86_64
+image=https://download.opensuse.org/repositories/home:/crameleon:/appliances/openSUSE_Tumbleweed/boxes/Tumbleweed.x86_64.json
+
+[box.tumbleweed]
+
+[box.tumbleweed_bootstrap]
+bootstrap=tests/scripts/bootstrap_hello_world.sh
+
+[box.tumbleweed_salt]
+bootstrap=tests/scripts/bootstrap_salt.sh
+
+[suite.one_minion]
+minions=1
+box=tumbleweed
+
+[suite.one_minion_bootstrap]
+minions=1
+box=tumbleweed_bootstrap
+
+[suite.one_minion_one_master_bootstrap]
+minions=1
+masters=1
+box=tumbleweed_bootstrap
+
+[suite.two_minions]
+minions=2
+box=tumbleweed
+
+[suite.one_minion_one_master]
+minions=1
+masters=1
+box=tumbleweed
+
+[suite.one_master]
+masters=1
+box=tumbleweed
+
+[suite.two_minions_one_master]
+minions=2
+masters=1
+box=tumbleweed
+
+[suite.one_minion_bogus_test]
+minions=1
+box=tumbleweed
+test=cashew
+
+[suite.one_minion_bogus_test_2]
+minions=1
+box=tumbleweed
+test=bogus
+
+[test.hello]
+apply=hello_world
+test=tests/salt/hello_world/tests/test_hello.py
+
+[test.bogus]
+apply=cashew
+
+[suite.one_minion_salt]
+minions=1
+box=tumbleweed_salt
+test=hello
+
+[suite.two_minions_one_master_salt]
+masters=1
+minions=2
+box=tumbleweed_salt
+test=hello

tests/configs/missing_box.ini

@@ -0,0 +1,5 @@
+[suite]
+foo=bar
+
+[test.foo]
+foo=bar

tests/configs/missing_box_image.ini

@@ -0,0 +1,9 @@
+[box]
+bootstrap=scripts/bootstrap.sh
+
+[box.tumbleweed]
+name=mybox
+
+[suite.one_minion]
+minions=1
+box=tumbleweed

tests/configs/missing_box_name.ini

@@ -0,0 +1,9 @@
+[box]
+bootstrap=scripts/bootstrap.sh
+
+[box.tumbleweed]
+image=https://download.opensuse.org/repositories/home:/crameleon:/appliances/openSUSE_Tumbleweed/boxes/Tumbleweed.x86_64.json
+
+[suite.one_minion]
+minions=1
+box=tumbleweed

tests/configs/missing_boxes.ini

@@ -0,0 +1,8 @@
+[box]
+image=https://example.com/example.box
+
+[suite.example_suite]
+minions=1
+
+[test.example]
+test=example.py

tests/configs/missing_suite_box.ini

@@ -0,0 +1,10 @@
+[box]
+bootstrap=scripts/bootstrap.sh
+
+[box.tumbleweed]
+name=Tumbleweed.x86_64
+image=https://download.opensuse.org/repositories/home:/crameleon:/appliances/openSUSE_Tumbleweed/boxes/Tumbleweed.x86_64.json
+
+[suite.one_minion]
+minions=1
+

tests/configs/missing_suites.ini

@@ -0,0 +1,9 @@
+[box]
+image=https://example.com/example.box
+
+[box.example]
+name=ExampleBox
+file=ExampleVagrantfile
+
+[test.example]
+test=example.py

tests/configs/salt.ini

@@ -0,0 +1,14 @@
+[box]
+[box.tumbleweed]
+bootstrap=tests/scripts/bootstrap_salt.sh
+name=Tumbleweed.x86_64
+image=https://download.opensuse.org/repositories/home:/crameleon:/appliances/openSUSE_Tumbleweed/boxes/Tumbleweed.x86_64.json
+
+[suite.one_minion]
+minions=1
+box=tumbleweed
+
+[suite.two_minions_one_master_bootstrap]
+minions=2
+masters=1
+box=tumbleweed

tests/configs/undefined_suite_box.ini

@@ -0,0 +1,10 @@
+[box]
+bootstrap=scripts/bootstrap.sh
+
+[box.tumbleweed]
+name=Tumbleweed.x86_64
+image=https://download.opensuse.org/repositories/home:/crameleon:/appliances/openSUSE_Tumbleweed/boxes/Tumbleweed.x86_64.json
+
+[suite.one_minion]
+minions=1
+box=leap

tests/conftest.py

@@ -0,0 +1,55 @@
+"""
+Copyright 2023, Georg Pfuetzenreuter
+
+Licensed under the EUPL, Version 1.2 or - as soon they will be approved by the European Commission - subsequent versions of the EUPL (the "Licence").
+You may not use this work except in compliance with the Licence.
+An English copy of the Licence is shipped in a file called LICENSE along with this applications source code.
+You may obtain copies of the Licence in any of the official languages at https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12.
+
+---
+
+Testing functions for Scullery - a SaltStack testing tool.
+"""
+
+import pytest
+import os
+import libvirt
+import vagrant
+import dotenv
+
+@pytest.fixture
+def script():
+    is_packaged = False
+    script = 'scullery'
+    if not is_packaged:
+        script = f'{script}.py'
+    return script
+
+
+@pytest.fixture
+def testbase():
+    return os.path.abspath(os.path.dirname(os.getenv('PYTEST_CURRENT_TEST')))
+
+
+@pytest.fixture
+def config(testbase, request):
+    return '{}/configs/{}.ini'.format(testbase, request.param)
+
+
+@pytest.fixture
+def virt():
+    return libvirt.openReadOnly('qemu:///system')
+
+@pytest.fixture
+def vag():
+    return vagrant.Vagrant(quiet_stderr=False, quiet_stdout=False)
+
+def loadenv():
+    env = os.environ.copy()
+    envmap = dotenv.dotenv_values('.scullery_env')
+    for variable, value in envmap.items():
+        if value is not None:
+            if isinstance(value, list):
+                value = ','.join(value)
+            env[variable] = value
+    return env

tests/salt/hello_world/pillar/hello_world.sls

@@ -0,0 +1 @@
+greeting: 'Hello salted world!'

tests/salt/hello_world/states/hello_world.sls

@@ -0,0 +1,3 @@
+/srv/hello_world.txt:
+  file.managed:
+    - contents_pillar: greeting

tests/salt/hello_world/tests/test_hello.py

@@ -0,0 +1,2 @@
+def test_hello_world_content(host):
+    assert host.file('/srv/hello_world.txt').content.decode('UTF-8') == 'Hello salted world!\n'

tests/scripts/bootstrap.sh

@@ -0,0 +1 @@
+ln -s /vagrant /tmp/mysources

tests/scripts/bootstrap_hello_world.sh

@@ -0,0 +1 @@
+echo 'Hello world!' > /srv/hello_world.txt

tests/scripts/bootstrap_salt.sh

@@ -0,0 +1,27 @@
+testbase='/vagrant/tests/salt/hello_world'
+sls='hello_world.sls'
+
+ln -s "$testbase/states/$sls" /srv/salt/
+ln -s "$testbase/pillar/$sls" /srv/pillar/
+
+tee /srv/pillar/top.sls >/dev/null <<EOF
+base:
+  '*':
+    - hello_world
+EOF
+
+if systemctl is-enabled salt-master
+then
+	count=0
+	until salt -t10 scullery-\* test.ping
+	do
+		if [ "$count" = 3 ]
+		then
+			logger -s 'No minions connected'
+			break
+		fi
+		echo 'Waiting for minions to connect ...'
+		sleep 5
+		count=$((count+1))
+	done
+fi