From 68919826fe4d4db4582223569c5a939a9c2f9c72 Mon Sep 17 00:00:00 2001 From: "Mardegan, Alberto" Date: Thu, 25 Apr 2019 08:03:28 +0000 Subject: oeqa/core/runner: dump stdout and stderr of each test case Some CI pipelines might perform further processing of the test output (for instance, to plot some metrics into a chart). However, Since `thud` we switched away from the XML-based jUnit reporting, and at the same time we lost the ability of collecting the stdout and stderr of the various tests. We now restore this functionality by adding `stdout` and `stderr` keys to the JSON reports. This behavior is off by default; in order to enable it, one must set the `TESTREPORT_FULLLOGS` variable in the bitbake configuration. Signed-off-by: Alberto Mardegan Signed-off-by: Richard Purdie Signed-off-by: Armin Kuster --- meta/classes/testimage.bbclass | 3 ++- meta/lib/oeqa/core/runner.py | 20 ++++++++++++++++---- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/meta/classes/testimage.bbclass b/meta/classes/testimage.bbclass index cb8c12acce..3479228307 100644 --- a/meta/classes/testimage.bbclass +++ b/meta/classes/testimage.bbclass @@ -314,7 +314,8 @@ def testimage_main(d): configuration = get_testimage_configuration(d, 'runtime', machine) results.logDetails(get_testimage_json_result_dir(d), configuration, - get_testimage_result_id(configuration)) + get_testimage_result_id(configuration), + dump_streams=d.getVar('TESTREPORT_FULLLOGS')) results.logSummary(pn) if not results.wasSuccessful(): bb.fatal('%s - FAILED - check the task log and the ssh log' % pn, forcelog=True) diff --git a/meta/lib/oeqa/core/runner.py b/meta/lib/oeqa/core/runner.py index df88b85f1c..478b7b6683 100644 --- a/meta/lib/oeqa/core/runner.py +++ b/meta/lib/oeqa/core/runner.py @@ -7,6 +7,7 @@ import unittest import logging import re import json +import sys from unittest import TextTestResult as _TestResult from unittest import TextTestRunner as _TestRunner @@ -45,6 +46,9 @@ class OETestResult(_TestResult): self.tc = tc + # stdout and stderr for each test case + self.logged_output = {} + def startTest(self, test): # May have been set by concurrencytest if test.id() not in self.starttime: @@ -53,6 +57,9 @@ class OETestResult(_TestResult): def stopTest(self, test): self.endtime[test.id()] = time.time() + if self.buffer: + self.logged_output[test.id()] = ( + sys.stdout.getvalue(), sys.stderr.getvalue()) super(OETestResult, self).stopTest(test) if test.id() in self.progressinfo: self.tc.logger.info(self.progressinfo[test.id()]) @@ -118,7 +125,8 @@ class OETestResult(_TestResult): self.successes.append((test, None)) super(OETestResult, self).addSuccess(test) - def logDetails(self, json_file_dir=None, configuration=None, result_id=None): + def logDetails(self, json_file_dir=None, configuration=None, result_id=None, + dump_streams=False): self.tc.logger.info("RESULTS:") result = {} @@ -144,10 +152,14 @@ class OETestResult(_TestResult): if status not in logs: logs[status] = [] logs[status].append("RESULTS - %s - Testcase %s: %s%s" % (case.id(), oeid, status, t)) + report = {'status': status} if log: - result[case.id()] = {'status': status, 'log': log} - else: - result[case.id()] = {'status': status} + report['log'] = log + if dump_streams and case.id() in self.logged_output: + (stdout, stderr) = self.logged_output[case.id()] + report['stdout'] = stdout + report['stderr'] = stderr + result[case.id()] = report for i in ['PASSED', 'SKIPPED', 'EXPECTEDFAIL', 'ERROR', 'FAILED', 'UNKNOWN']: if i not in logs: -- cgit 1.2.3-korg