From b28e2f3667b1d659dd81cccb0b6fa35bbb210c7a Mon Sep 17 00:00:00 2001 From: Megan Date: Thu, 17 May 2018 18:29:00 -0500 Subject: [PATCH] implement test harness --- .gitignore | 10 +++ .travis.yml | 24 ++++++ Makefile | 77 +++++++++++++++++++ README.rst | 44 +++++++++++ tests/pytests/apply-all-tests/__init__.py | 0 .../apply-all-tests/test_000_apply_state.py | 23 ++++++ tests/srv/salt/top.sls | 3 + tools/filltmpl.py | 27 +++++++ tools/run-tests.sh | 21 +++++ tools/templates/Dockerfile.j2 | 14 ++++ 10 files changed, 243 insertions(+) create mode 100644 .gitignore create mode 100644 .travis.yml create mode 100644 Makefile create mode 100644 tests/pytests/apply-all-tests/__init__.py create mode 100644 tests/pytests/apply-all-tests/test_000_apply_state.py create mode 100644 tests/srv/salt/top.sls create mode 100644 tools/filltmpl.py create mode 100755 tools/run-tests.sh create mode 100644 tools/templates/Dockerfile.j2 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6ef3a4c --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +*.egg +*.egg-info/ +*.py[cod] +.env +.pytest_cache/ +__pycache__/ +Dockerfile.*_* +ignore/ +.venv/ +tmp/ diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..77cf70c --- /dev/null +++ b/.travis.yml @@ -0,0 +1,24 @@ +env: + matrix: + - OS_ID: centos_master_2017.7.2 + - OS_ID: debian_master_2017.7.2 + - OS_ID: opensuse_master_2017.7.2 + - OS_ID: ubuntu_master_2016.11.3 + - OS_ID: ubuntu_master_2017.7.2 + +sudo: required + +language: python + +services: +- docker + +before_install: +- pip install Jinja2 +- python ${TRAVIS_BUILD_DIR}/tools/filltmpl.py memcached ${OS_ID} + +install: +- docker build --force-rm -t "memcached:salt-testing-${OS_ID}" -f "Dockerfile.${OS_ID}" . + +script: +- ./tools/run-tests.sh memcached ${OS_ID} diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..292e452 --- /dev/null +++ b/Makefile @@ -0,0 +1,77 @@ +FORMULA_NAME = "memcached" +PWD = $(shell pwd) + +# --------------------------------------------------------------- +define render_dockerfile + python $(PWD)/tools/filltmpl.py $(FORMULA_NAME) $(1) +endef + +define docker_build + docker build --force-rm -t $(FORMULA_NAME):salt-testing-$(1) -f Dockerfile.$(1) . +endef + +define docker_run_local + docker run --rm -v $(PWD):/opt/$(FORMULA_NAME)-formula --env=STAGE=TEST -h salt-testing-$(1) --name salt-testing-$(1) -it $(FORMULA_NAME):salt-testing-$(1) /bin/bash +endef + +define run_tests + ./tools/run-tests.sh $(FORMULA_NAME) $(1) +endef + +# --- convenience functions ------------------------------------- +define build_thing + $(call render_dockerfile,$(1)) && $(call docker_build,$(1)) +endef + +define run_local_tests + $(call build_thing,$(1)) && $(call run_tests,$(1)) +endef + +define run_local + $(call build_thing,$(1)) && $(call docker_run_local,$(1)) +endef + +# --------------------------------------------------------------- +setup: + pip install Jinja2 + +clean: + find . -name '*.pyc' -exec rm '{}' ';' + find . -name '__pycache__' -type d -prune -exec rm -rf '{}' '+' + find . -name '.pytest_cache' -type d -prune -exec rm -rf '{}' '+' + rm -rf tests/Dockerfile* + +# --- centos_master_2017.7.2 ------------------------------------ +test-centos_master_2017.7.2: clean + $(call run_local_tests,centos_master_2017.7.2) + +local-centos_master_2017.7.2: clean + $(call run_local,centos_master_2017.7.2) + +# --- debian_master_2017.7.2 ------------------------------------ +test-debian_master_2017.7.2: clean + $(call run_local_tests,debian_master_2017.7.2) + +local-debian_master_2017.7.2: clean + $(call run_local,debian_master_2017.7.2) + +# --- opensuse_master_2017.7.2 ------------------------------------ +test-opensuse_master_2017.7.2: clean + $(call run_local_tests,opensuse_master_2017.7.2) + +local-opensuse_master_2017.7.2: clean + $(call run_local,opensuse_master_2017.7.2) + +# --- ubuntu_master_2016.11.3 ------------------------------------ +test-ubuntu_master_2016.11.3: clean + $(call run_local_tests,ubuntu_master_2016.11.3) + +local-ubuntu_master_2016.11.3: clean + $(call run_local,ubuntu_master_2016.11.3) + +# --- ubuntu_master_2017.7.2 ------------------------------------ +test-ubuntu_master_2017.7.2: clean + $(call run_local_tests,ubuntu_master_2017.7.2) + +local-ubuntu_master_2017.7.2: clean + $(call run_local,ubuntu_master_2017.7.2) diff --git a/README.rst b/README.rst index f198283..57069da 100644 --- a/README.rst +++ b/README.rst @@ -110,4 +110,48 @@ Instructions } } + +Running Tests +============= + +This test runner was implemented using the formula-test-harness_ project. + +Tests will be run on the following base images: + +* ``simplyadrian/allsalt:centos_master_2017.7.2`` +* ``simplyadrian/allsalt:debian_master_2017.7.2`` +* ``simplyadrian/allsalt:opensuse_master_2017.7.2`` +* ``simplyadrian/allsalt:ubuntu_master_2016.11.3`` +* ``simplyadrian/allsalt:ubuntu_master_2017.7.2`` + +Local Setup +----------- + +.. code-block:: shell + + pip install -U virtualenv + virtualenv .venv + source .venv/bin/activate + make setup + +Run tests +--------- + +* ``make test-centos_master_2017.7.2`` +* ``make test-debian_master_2017.7.2`` +* ``make test-opensuse_master_2017.7.2`` +* ``make test-ubuntu_master_2016.11.3`` +* ``make test-ubuntu_master_2017.7.2`` + +Run Containers +-------------- + +* ``make local-centos_master_2017.7.2`` +* ``make local-debian_master_2017.7.2`` +* ``make local-opensuse_master_2017.7.2`` +* ``make local-ubuntu_master_2016.11.3`` +* ``make local-ubuntu_master_2017.7.2`` + + +.. _formula-test-harness: https://github.com/intuitivetechnologygroup/formula-test-harness .. _`GitFS backend`: http://docs.saltstack.com/topics/tutorials/gitfs.html diff --git a/tests/pytests/apply-all-tests/__init__.py b/tests/pytests/apply-all-tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/pytests/apply-all-tests/test_000_apply_state.py b/tests/pytests/apply-all-tests/test_000_apply_state.py new file mode 100644 index 0000000..cf0b620 --- /dev/null +++ b/tests/pytests/apply-all-tests/test_000_apply_state.py @@ -0,0 +1,23 @@ +from subprocess import check_output +from unittest import TestCase + + +class ApplyStateTest(TestCase): + + def test_000_apply(self): + state_apply_response = check_output(["salt-call", "--local", "state.apply"]) + print('') + print('-' * 50) + print('state_apply_response:') + print(state_apply_response) + print('-' * 50) + print('') + + state_apply_response = state_apply_response.split('\n') + summary = state_apply_response[-8:] + failed = 0 + for line in summary: + if line.startswith('Failed:'): + failed = int(line.split(':').pop().strip()) + + self.assertEqual(failed, 0) diff --git a/tests/srv/salt/top.sls b/tests/srv/salt/top.sls new file mode 100644 index 0000000..6ea5fcf --- /dev/null +++ b/tests/srv/salt/top.sls @@ -0,0 +1,3 @@ +base: + '*': + - memcached diff --git a/tools/filltmpl.py b/tools/filltmpl.py new file mode 100644 index 0000000..0bbeace --- /dev/null +++ b/tools/filltmpl.py @@ -0,0 +1,27 @@ +import os +import sys + +from jinja2 import Template + +# base/tests +dir_path = os.path.dirname(os.path.realpath(__file__)) + +# base +base_path = os.path.dirname(dir_path) + + +if __name__ == '__main__': + formula_name = sys.argv[1] + image_tag = sys.argv[2] + + template = Template( + open(os.path.join(dir_path, 'templates', 'Dockerfile.j2')).read() + ) + + dockerfile = template.render({ + 'formula_name': formula_name, + 'image_tag': image_tag + }) + + with open(os.path.join(base_path, 'Dockerfile.{}'.format(image_tag)), 'w') as fh: + fh.write(dockerfile) diff --git a/tools/run-tests.sh b/tools/run-tests.sh new file mode 100755 index 0000000..550fa58 --- /dev/null +++ b/tools/run-tests.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -ev + +test -z $2 && echo "Usage: ${0} FORMULA_NAME OS_ID" && exit 1 +export FORMULA_NAME=$1 +export OS_ID=$2 + + +function docker-run-pytest() { + docker run --rm \ + -v "$@":/opt/tests \ + --env=STAGE=TEST \ + -h "salt-testing-${OS_ID}" \ + --name "salt-testing-${OS_ID}" \ + -it ${FORMULA_NAME}:"salt-testing-${OS_ID}" \ + pytest -sv /opt/tests +} + +for i in $(find $PWD/tests/pytests/* -maxdepth 0 -type d); do + docker-run-pytest $i; +done diff --git a/tools/templates/Dockerfile.j2 b/tools/templates/Dockerfile.j2 new file mode 100644 index 0000000..5686069 --- /dev/null +++ b/tools/templates/Dockerfile.j2 @@ -0,0 +1,14 @@ +FROM simplyadrian/allsalt:{{ image_tag }} + +{% if 'debian' in image_tag or 'ubuntu' in image_tag -%} +RUN apt-get update && \ + apt-get install -y python-pip +{% endif %} + +RUN pip install pytest && \ + sed -i "s/#master: salt/master: localhost/g" /etc/salt/minion + +ADD tests/srv /srv +ADD {{ formula_name }} /srv/salt/{{ formula_name }} + +WORKDIR /srv/salt