diff --git a/scullery.py b/scullery.py
index 2ccaef3..b014ede 100755
--- a/scullery.py
+++ b/scullery.py
@@ -17,6 +17,7 @@ from configparser import ConfigParser
 import logging
 import os
 import sys
+from pytest import ExitCode
 
 argparser = ArgumentParser()
 config = ConfigParser()
@@ -163,8 +164,10 @@ def runtests(payload, hosts):
     if not os.path.isfile(sshfile):
         vagrant_sshconfig(sshfile)
     testresult = pytest.main(['--verbose', '--hosts={}'.format(','.join(hosts)), '--ssh-config={}'.format(sshfile), payload])
-    log.debug('Test result is {}'.format(str(testresult)))
-    if not testresult:
+    log.debug('Test result is {}'.format(str(testresult.value)))
+    if testresult == ExitCode.OK:
+        log.debug('Test succeeded')
+    else:
         log.warning('Tests failed')
         return False
     return True