Merge pull request #4496 from clebergnu/simple_test_failure_presentation

SIMPLE Tests: improvements on failure presentation

avocado/core/__init__.py

@@ -66,6 +66,14 @@ def register_core_options():
                          default='all',
                          help_msg=help_msg)
 
+    help_msg = ('Fields to include in the presentation of SIMPLE test '
+                'failures.  Accepted values: status, stdout, stderr.')
+    stgs.register_option(section='simpletests.status',
+                         key='failure_fields',
+                         key_type=list,
+                         default=['status', 'stdout', 'stderr'],
+                         help_msg=help_msg)
+
     help_msg = ('The amount of time to give to the test process after '
                 'it it has been interrupted (such as with CTRL+C)')
     stgs.register_option(section='runner.timeout',

avocado/core/test.py

@@ -261,7 +261,7 @@ def record_and_warn(*args, **kwargs):
 
         self.__tags = tags
 
-        self.__config = config or settings.as_dict()
+        self._config = config or settings.as_dict()
 
         self.__base_logdir_tmp = None
         if base_logdir is None:
@@ -285,8 +285,8 @@ def record_and_warn(*args, **kwargs):
 
         self.__outputdir = utils_path.init_dir(self.logdir, 'data')
 
-        self.__sysinfo_enabled = self.__config.get('sysinfo.collect.per_test',
-                                                   False)
+        self.__sysinfo_enabled = self._config.get('sysinfo.collect.per_test',
+                                                  False)
 
         if self.__sysinfo_enabled:
             self.__sysinfodir = utils_path.init_dir(self.logdir, 'sysinfo')
@@ -786,8 +786,8 @@ def _run_avocado(self):
         whiteboard_file = os.path.join(self.logdir, 'whiteboard')
         genio.write_file(whiteboard_file, self.whiteboard)
 
-        output_check_record = self.__config.get('run.output_check_record')
-        output_check = self.__config.get('run.output_check')
+        output_check_record = self._config.get('run.output_check_record')
+        output_check = self._config.get('run.output_check')
 
         # record the output if the modes are valid
         if output_check_record == 'combined':
@@ -1038,7 +1038,7 @@ def _cleanup(self):
         if self.__base_logdir_tmp is not None:
             self.__base_logdir_tmp.cleanup()
             self.__base_logdir_tmp = None
-        if not self.__config.get('run.keep_tmp') and os.path.exists(
+        if not self._config.get('run.keep_tmp') and os.path.exists(
                 self.__base_tmpdir):
             shutil.rmtree(self.__base_tmpdir)
 
@@ -1076,7 +1076,6 @@ def __init__(self, name, params=None, base_logdir=None, config=None,
         self._command = None
         if self.filename is not None:
             self._command = pipes.quote(self.filename)
-        self._config = settings.as_dict()
 
     @property
     def filename(self):
@@ -1091,14 +1090,22 @@ def _log_detailed_cmd_info(self, result):
 
         :param result: :class:`avocado.utils.process.CmdResult` instance.
         """
-        self.log.info("Exit status: %s", result.exit_status)
-        self.log.info("Duration: %s", result.duration)
-
-    @staticmethod
-    def _cmd_error_to_test_failure(cmd_error):
-        return ("Exited with status: '%u', stdout: %r stderr: %r" %
-                (cmd_error.result.exit_status, cmd_error.result.stdout_text,
-                 cmd_error.result.stderr_text))
+        self.log.info("Detailed information about the executed command:")
+        self.log.info("  Exit status: %s", result.exit_status)
+        self.log.info("  Duration: %s", result.duration)
+        self.log.info("  STDOUT: %s", result.stdout_text.strip())
+        self.log.info("  STDERR: %s", result.stderr_text.strip())
+
+    def _cmd_error_to_test_failure(self, cmd_error):
+        failure_fields = self._config.get('simpletests.status.failure_fields')
+        msgs = []
+        if 'status' in failure_fields:
+            msgs.append("Exited with status: '%u'" % cmd_error.result.exit_status)
+        if 'stdout' in failure_fields:
+            msgs.append("stdout: %r" % cmd_error.result.stdout_text)
+        if 'stderr' in failure_fields:
+            msgs.append("stderr: %r" % cmd_error.result.stdout_text)
+        return ", ".join(msgs)
 
     def _execute_cmd(self):
         """

avocado/plugins/testlogs.py

@@ -21,6 +21,15 @@ def initialize(self):
                                  default=[],
                                  help_msg=help_msg)
 
+        help_msg = ('The specific log files that will be shown for tests '
+                    'whose exit status match the ones defined in the '
+                    '"job.output.testlogs.statuses" configuration. ')
+        settings.register_option(section='job.output.testlogs',
+                                 key='logfiles',
+                                 key_type=list,
+                                 default=['debug.log'],
+                                 help_msg=help_msg)
+
 
 class TestLogs(JobPre, JobPost):
 
@@ -40,9 +49,17 @@ def post(self, job):
         except FileNotFoundError:
             return
 
+        logfiles = job.config.get('job.output.testlogs.logfiles')
         for test in results['tests']:
             if test['status'] not in statuses:
                 continue
-            LOG_UI.info('Log content for test "%s" (%s)', test['id'], test['status'])
-            with open(test['logfile']) as log:
-                LOG_UI.debug(log.read())
+            for logfile in logfiles:
+                path = os.path.join(test['logdir'], logfile)
+                try:
+                    with open(path) as log:
+                        LOG_UI.info('Log file "%s" content for test "%s" (%s):',
+                                    logfile, test['id'], test['status'])
+                        LOG_UI.debug(log.read())
+                except (FileNotFoundError, PermissionError) as error:
+                    LOG_UI.error('Failure to access log file "%s": %s',
+                                 path, error)

selftests/functional/plugin/test_logs.py

@@ -0,0 +1,63 @@
+import os
+
+from avocado.core import exit_codes
+from avocado.utils import process, script
+
+from ... import AVOCADO, BASEDIR, TestCaseTmpDir
+
+CONFIG = """[job.output.testlogs]
+statuses = ["FAIL", "CANCEL"]"""
+
+
+class TestLogs(TestCaseTmpDir):
+
+    def setUp(self):
+        super(TestLogs, self).setUp()
+        with open(os.path.join(self.tmpdir.name, 'config'), 'w') as config:
+            config.write(CONFIG)
+
+    def test(self):
+        cmd_line = ("%s --config=%s run -- examples/tests/passtest.py "
+                    "examples/tests/failtest.py examples/tests/canceltest.py ")
+        cmd_line %= (AVOCADO, os.path.join(self.tmpdir.name, 'config'))
+        result = process.run(cmd_line, ignore_status=True)
+        self.assertEqual(result.exit_status, exit_codes.AVOCADO_TESTS_FAIL,
+                         "Avocado did not return rc %d:\n%s"
+                         % (exit_codes.AVOCADO_ALL_OK, result))
+        stdout_lines = result.stdout_text.splitlines()
+        self.assertNotIn('Log file "debug.log" content for test "1-examples/tests/passtest.py'
+                         ':PassTest.test" (PASS)', stdout_lines)
+        self.assertIn('Log file "debug.log" content for test "2-examples/tests/failtest.py:FailTest.test" (FAIL):', stdout_lines)
+        self.assertIn('Log file "debug.log" content for test "3-examples/tests/canceltest.py'
+                      ':CancelTest.test" (CANCEL):', stdout_lines)
+
+
+class TestLogsFiles(TestCaseTmpDir):
+
+    def setUp(self):
+        super(TestLogsFiles, self).setUp()
+        self.config_file = script.TemporaryScript(
+            'avocado.conf',
+            "[job.output.testlogs]\n"
+            "statuses = ['FAIL']\n"
+            "logfiles = ['stdout', 'stderr', 'DOES_NOT_EXIST']\n")
+        self.config_file.save()
+
+    def test_simpletest_logfiles(self):
+        fail_test = os.path.join(BASEDIR, 'examples', 'tests', 'failtest.sh')
+        cmd_line = ('%s --config %s run --job-results-dir %s --disable-sysinfo'
+                    ' -- %s' % (AVOCADO, self.config_file.path,
+                                self.tmpdir.name, fail_test))
+        result = process.run(cmd_line, ignore_status=True)
+        expected_rc = exit_codes.AVOCADO_TESTS_FAIL
+        self.assertEqual(result.exit_status, expected_rc,
+                         "Avocado did not return rc %d:\n%s" % (expected_rc, result))
+        self.assertNotIn('Log file "debug.log" content', result.stdout_text)
+        self.assertIn('Log file "stdout" content', result.stdout_text)
+        self.assertIn('Log file "stderr" content', result.stdout_text)
+        self.assertRegex(result.stderr_text,
+                         r'Failure to access log file.*DOES_NOT_EXIST"')
+
+    def tearDown(self):
+        super(TestLogsFiles, self).tearDown()
+        self.config_file.remove()

selftests/functional/test_basic.py

@@ -942,6 +942,32 @@ def tearDown(self):
         self.config_file.remove()
 
 
+class RunnerSimpleTestFailureFields(TestCaseTmpDir):
+
+    def setUp(self):
+        super(RunnerSimpleTestFailureFields, self).setUp()
+        self.config_file = script.TemporaryScript(
+            'avocado.conf',
+            "[simpletests.status]\n"
+            "failure_fields = ['stdout', 'stderr']\n")
+        self.config_file.save()
+
+    def test_simpletest_failure_fields(self):
+        fail_test = os.path.join(BASEDIR, 'examples', 'tests', 'failtest.sh')
+        cmd_line = ('%s --config %s run --job-results-dir %s --disable-sysinfo'
+                    ' -- %s' % (AVOCADO, self.config_file.path,
+                                self.tmpdir.name, fail_test))
+        result = process.run(cmd_line, ignore_status=True)
+        expected_rc = exit_codes.AVOCADO_TESTS_FAIL
+        self.assertEqual(result.exit_status, expected_rc,
+                         "Avocado did not return rc %d:\n%s" % (expected_rc, result))
+        self.assertNotIn("Exited with status: '1'", result.stdout_text)
+
+    def tearDown(self):
+        super(RunnerSimpleTestFailureFields, self).tearDown()
+        self.config_file.remove()
+
+
 class ExternalRunnerTest(TestCaseTmpDir):
 
     def setUp(self):