Implement tests/state.apply + improve config logic

- consolidate config parsing (boxes, suites, tests)
- implement state.apply execution
- implement test execution
- minor "refresh" logic improvement

Signed-off-by: Georg Pfuetzenreuter <mail@georg-pfuetzenreuter.net>

scullery.py

@@ -37,23 +37,22 @@ args = argparser.parse_args()
 configfile = args.config
 
 vmprefix = 'scullery'
+cwd = os.getcwd()
+sshfile='{}/.scullery_ssh'.format(cwd)
+envfile='{}/.scullery_env'.format(cwd)
 
 def _abort(msg):
     log.error(msg)
     sys.exit(1)
 
 def _config():
-    configmap = {'boxes': {}, 'suites': {}}
+    configmap = {'boxes': {}, 'suites': {}, 'tests': {}}
     if not config.options('box'):
         _abort('No "box" section found in the configuration file')
-    boxes = [section for section in config.sections() if section.startswith('box.')]
+    multis = {'boxes': {'prefix': 'box.', 'singular': 'box'}, 'suites': {'prefix': 'suite.', 'singular': 'suite'}, 'tests': {'prefix': 'test.', 'singular': 'test'}}
-    suites = [section for section in config.sections() if section.startswith('suite.')]
-    if not len(suites):
-        _abort('No suites configured')
-    log.debug('Suites: {}'.format(str(suites)))
-    multis = {'boxes': {'conf': boxes, 'prefix': 'box.'}, 'suites': {'conf': suites, 'prefix': 'suite.'}}
     for multi, multiconf in multis.items():
-        for section in multiconf['conf']:
+        lowconf = [section for section in config.sections() if section.startswith(multiconf['prefix'])]
+        for section in lowconf:
             collection = section.replace(multiconf['prefix'], '')
             configmap[multi][collection] = {}
             for option in config.options(section):
@@ -62,14 +61,14 @@ def _config():
                 else:
                     value = config.get(section, option)
                 configmap[multi][collection][option] = value
-    # a bit of an ugly alternative to the "DEFAULT" section
+        onemulti = multiconf['singular']
-    multis = {'boxes': {'singular': 'box'}, 'suites': {'singular': 'suite'}}
+        if onemulti in config.sections():
-    for multis, multiconf in multis.items():
+            for option in config.options(onemulti):
-        multi = multiconf['singular']
+                for collection in configmap[multi]:
-        if multi in config.sections():
+                    configmap[multi][collection][option] = config.get(onemulti, option)
-            for option in config.options(multi):
+        if multi in ['boxes', 'suites']:
-                for collection in configmap[multis]:
+            if not len(lowconf):
-                    configmap[multis][collection][option] = config.get(multi, option)
+                _abort('No {} configured'.format(multi))
     log.debug('Config map: {}'.format(str(configmap)))
     return configmap
 
@@ -85,7 +84,7 @@ def genvms(flavor, amount):
 def _setenv(envmap, dump=False):
     if dump:
         log.debug('Writing environment variable file')
-        fh = open('.scullery_env', 'w')
+        fh = open(envfile, 'w')
     for variable, value in envmap.items():
         if value is not None:
             if isinstance(value, list):
@@ -120,10 +119,38 @@ def vagrant_isup(suite):
     else:
         return False, False
 
+def vagrant_sshconfig(outfile):
+    try:
+        ssh_config = v.ssh_config()
+    except Exception as myerror:
+        log.exception(myerror)
+        log.error('Unable to fetch SSH configuration')
+    with open(outfile, 'w') as fh:
+        fh.write(ssh_config)
+
+def runapply(state, target):
+    if target == 'local':
+        saltcmd = 'salt-call --local'
+    else:
+        saltcmd = 'salt {}'.format(target)
+    sshout = v.ssh(command='sudo {} state.apply {}'.format(saltcmd, state))
+    log.info('\n{}\n'.format(str(sshout)))
+
+def runtests(payload, hosts):
+    if not os.path.isfile(sshfile):
+        vagrant_sshconfig(sshfile)
+    testresult = pytest.main(['--verbose', '--hosts={}'.format(','.join(hosts)), '--ssh-config={}'.format(sshfile), payload])
+    log.debug('Test result is {}'.format(str(testresult)))
+    if not testresult:
+        log.warning('Tests failed')
+        return False
+    return True
+
 def main_interactive():
     configmap = _config()
     boxes = configmap['boxes']
     suites = configmap['suites']
+    tests = configmap['tests']
     suite = args.suite
     if suite not in suites:
         _abort('No suite named {}'.format(suite))
@@ -151,15 +178,19 @@ def main_interactive():
         log.info('Status report: {}'.format(v.status()))
         return True
     status = vagrant_isup(suite)
-    if status[0] is True and status[1] is None or args.force_stop or args.refresh:
+    if status[0] is True and status[1] is None or args.force_stop:
         if True in [args.stop, args.force_stop]:
             log.info('Destroying machines ...')
             v.destroy()
+            for file in [envfile, sshfile]:
+                if os.path.isfile(file):
+                    log.debug('Removing {}'.format(file))
+                    os.remove(file)
             if vagrant_isup(suite)[0] is False:
                 log.debug('OK')
             else:
                 _abort('Destruction failed')
-        elif not args.refresh:
+        elif not args.refresh and not args.test:
             log.info('Deployment is already running')
         elif args.refresh:
             log.info('Deployment is running, initiating refresh ...')
@@ -183,6 +214,29 @@ def main_interactive():
             else:
                 _abort('Start failed')
 
+    if args.test:
+        test = suiteconf.get('test', None)
+        if test is None:
+            _abort('Tests requested but not declared in suite configuration')
+        if not test in tests:
+            _abort('Specified test is not defined')
+        testconf = tests[test]
+        if not 'test' in testconf:
+            _abort('Incomplete test configuration')
+
+        if 'apply' in testconf:
+            log.debug('state.apply requested')
+            if masters is not None:
+                target = masters[0]
+            else:
+                target = 'local'
+            runapply(testconf['apply'], target)
+        else:
+            log.warning('No state.apply requested')
+
+        log.info('Initiating tests ...')
+        runtests(testconf['test'], minions)
+
 logging.basicConfig(format='%(asctime)s %(levelname)s - %(funcName)s: %(message)s', datefmt='%H:%M:%S')
 log = logging.getLogger('scullery')
 
@@ -200,6 +254,12 @@ try:
 except ImportError as myerror:
     _abort('Could not load python-vagrant')
 
+if args.test:
+    try:
+        import pytest
+    except ImportError as myerror:
+        _abort('Could not load pytest')
+
 if os.path.isfile(configfile):
     config.read(configfile)
 else: