diff --git a/scullery.py b/scullery.py
index b014ede..8718481 100755
--- a/scullery.py
+++ b/scullery.py
@@ -106,6 +106,8 @@ def _setenv(envmap, dump=False):
             env[variable] = value
             if dump:
                 fh.write(f'{variable}={value}\n')
+        elif variable in env:
+            del env[variable]
     if dump:
         fh.close()
 
@@ -161,8 +163,7 @@ def runapply(state, target):
     log.info('\n{}\n'.format(str(sshout)))
 
 def runtests(payload, hosts):
-    if not os.path.isfile(sshfile):
-        vagrant_sshconfig(sshfile)
+    vagrant_sshconfig(sshfile)
     testresult = pytest.main(['--verbose', '--hosts={}'.format(','.join(hosts)), '--ssh-config={}'.format(sshfile), payload])
     log.debug('Test result is {}'.format(str(testresult.value)))
     if testresult == ExitCode.OK:
@@ -172,6 +173,13 @@ def runtests(payload, hosts):
         return False
     return True
 
+def _cleanup():
+    for file in [envfile, sshfile]:
+        if os.path.isfile(file):
+            log.debug('Removing {}'.format(file))
+            os.remove(file)
+
+
 def main_interactive():
     configmap = _config()
     boxes = configmap['boxes']
@@ -208,10 +216,6 @@ def main_interactive():
         if True in [args.stop, args.force_stop]:
             log.info('Destroying machines ...')
             v.destroy()
-            for file in [envfile, sshfile]:
-                if os.path.isfile(file):
-                    log.debug('Removing {}'.format(file))
-                    os.remove(file)
             if vagrant_isup(suite)[0] is False:
                 log.debug('OK')
             else:
@@ -220,7 +224,9 @@ def main_interactive():
             log.info('Deployment is already running')
         elif args.refresh:
             log.info('Deployment is running, initiating refresh ...')
+            _cleanup()
             v.provision()
+            vagrant_sshconfig(sshfile)
     elif status[0] is False:
         if status[1] is True:
             log.debug('Deployment is not running')
@@ -240,6 +246,9 @@ def main_interactive():
             else:
                 _abort('Start failed')
 
+    if args.stop:
+        _cleanup()
+
     if args.test:
         test = suiteconf.get('test', None)
         if test is None: