From 7ba5a0af31605eaf91eb75b5f673f4caf81361dc Mon Sep 17 00:00:00 2001 From: Jon Mason Date: Sun, 2 Jun 2019 14:29:12 -0400 Subject: resulttool: modify to be multi-machine Currently, the code will sum all of the different machine results into a single report of the tests results. This can lead to confusion as to which machine may be experiencing issues. Modify the code to store the results in a per machine basis and report them accordingly. Signed-off-by: Jon Mason Signed-off-by: Richard Purdie Signed-off-by: Armin Kuster --- meta/lib/oeqa/selftest/cases/resulttooltests.py | 2 +- scripts/lib/resulttool/report.py | 104 +++++++++++---------- .../resulttool/template/test_report_full_text.txt | 44 +++++---- 3 files changed, 82 insertions(+), 68 deletions(-) diff --git a/meta/lib/oeqa/selftest/cases/resulttooltests.py b/meta/lib/oeqa/selftest/cases/resulttooltests.py index 10eb9c12d9..dac5c46801 100644 --- a/meta/lib/oeqa/selftest/cases/resulttooltests.py +++ b/meta/lib/oeqa/selftest/cases/resulttooltests.py @@ -56,7 +56,7 @@ class ResultToolTests(OESelftestTestCase): 'test4': {'status': 'ERROR'}, 'test5': {'status': 'SKIPPED'}}} report = ResultsTextReport() - result_report = report.get_aggregated_test_result(None, result_data) + result_report = report.get_aggregated_test_result(None, result_data, 'DummyMachine') self.assertTrue(result_report['passed'] == 2, msg="Passed count not correct:%s" % result_report['passed']) self.assertTrue(result_report['failed'] == 2, msg="Failed count not correct:%s" % result_report['failed']) self.assertTrue(result_report['skipped'] == 1, msg="Skipped count not correct:%s" % result_report['skipped']) diff --git a/scripts/lib/resulttool/report.py b/scripts/lib/resulttool/report.py index cb6b1cf948..a48c59f632 100644 --- a/scripts/lib/resulttool/report.py +++ b/scripts/lib/resulttool/report.py @@ -24,16 +24,19 @@ class ResultsTextReport(object): 'skipped': ['SKIPPED', 'skipped']} - def handle_ptest_result(self, k, status, result): + def handle_ptest_result(self, k, status, result, machine): + if machine not in self.ptests: + self.ptests[machine] = {} + if k == 'ptestresult.sections': # Ensure tests without any test results still show up on the report for suite in result['ptestresult.sections']: - if suite not in self.ptests: - self.ptests[suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []} + if suite not in self.ptests[machine]: + self.ptests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []} if 'duration' in result['ptestresult.sections'][suite]: - self.ptests[suite]['duration'] = result['ptestresult.sections'][suite]['duration'] + self.ptests[machine][suite]['duration'] = result['ptestresult.sections'][suite]['duration'] if 'timeout' in result['ptestresult.sections'][suite]: - self.ptests[suite]['duration'] += " T" + self.ptests[machine][suite]['duration'] += " T" return try: _, suite, test = k.split(".", 2) @@ -47,22 +50,25 @@ class ResultsTextReport(object): suite = suite + "." + suite1 except ValueError: pass - if suite not in self.ptests: - self.ptests[suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []} + if suite not in self.ptests[machine]: + self.ptests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []} for tk in self.result_types: if status in self.result_types[tk]: - self.ptests[suite][tk] += 1 + self.ptests[machine][suite][tk] += 1 + + def handle_ltptest_result(self, k, status, result, machine): + if machine not in self.ltptests: + self.ltptests[machine] = {} - def handle_ltptest_result(self, k, status, result): if k == 'ltpresult.sections': # Ensure tests without any test results still show up on the report for suite in result['ltpresult.sections']: - if suite not in self.ltptests: - self.ltptests[suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []} + if suite not in self.ltptests[machine]: + self.ltptests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []} if 'duration' in result['ltpresult.sections'][suite]: - self.ltptests[suite]['duration'] = result['ltpresult.sections'][suite]['duration'] + self.ltptests[machine][suite]['duration'] = result['ltpresult.sections'][suite]['duration'] if 'timeout' in result['ltpresult.sections'][suite]: - self.ltptests[suite]['duration'] += " T" + self.ltptests[machine][suite]['duration'] += " T" return try: _, suite, test = k.split(".", 2) @@ -77,20 +83,23 @@ class ResultsTextReport(object): suite = suite + "." + suite1 except ValueError: pass - if suite not in self.ltptests: - self.ltptests[suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []} + if suite not in self.ltptests[machine]: + self.ltptests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []} for tk in self.result_types: if status in self.result_types[tk]: - self.ltptests[suite][tk] += 1 + self.ltptests[machine][suite][tk] += 1 + + def handle_ltpposixtest_result(self, k, status, result, machine): + if machine not in self.ltpposixtests: + self.ltpposixtests[machine] = {} - def handle_ltpposixtest_result(self, k, status, result): if k == 'ltpposixresult.sections': # Ensure tests without any test results still show up on the report for suite in result['ltpposixresult.sections']: - if suite not in self.ltpposixtests: - self.ltpposixtests[suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []} + if suite not in self.ltpposixtests[machine]: + self.ltpposixtests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []} if 'duration' in result['ltpposixresult.sections'][suite]: - self.ltpposixtests[suite]['duration'] = result['ltpposixresult.sections'][suite]['duration'] + self.ltpposixtests[machine][suite]['duration'] = result['ltpposixresult.sections'][suite]['duration'] return try: _, suite, test = k.split(".", 2) @@ -104,19 +113,13 @@ class ResultsTextReport(object): suite = suite + "." + suite1 except ValueError: pass - if suite not in self.ltpposixtests: - self.ltpposixtests[suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []} + if suite not in self.ltpposixtests[machine]: + self.ltpposixtests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []} for tk in self.result_types: if status in self.result_types[tk]: - self.ltpposixtests[suite][tk] += 1 + self.ltpposixtests[machine][suite][tk] += 1 - def get_aggregated_test_result(self, logger, testresult): - test_count_report = {'passed': 0, 'failed': 0, 'skipped': 0, 'failed_testcases': []} - def get_aggregated_test_result(self, logger, testresult): - test_count_report = {'passed': 0, 'failed': 0, 'skipped': 0, 'failed_testcases': []} - def get_aggregated_test_result(self, logger, testresult): - test_count_report = {'passed': 0, 'failed': 0, 'skipped': 0, 'failed_testcases': []} - def get_aggregated_test_result(self, logger, testresult): + def get_aggregated_test_result(self, logger, testresult, machine): test_count_report = {'passed': 0, 'failed': 0, 'skipped': 0, 'failed_testcases': []} result = testresult.get('result', []) for k in result: @@ -127,11 +130,11 @@ class ResultsTextReport(object): if test_status in self.result_types['failed']: test_count_report['failed_testcases'].append(k) if k.startswith("ptestresult."): - self.handle_ptest_result(k, test_status, result) + self.handle_ptest_result(k, test_status, result, machine) if k.startswith("ltpresult."): - self.handle_ltptest_result(k, test_status, result) + self.handle_ltptest_result(k, test_status, result, machine) if k.startswith("ltpposixresult."): - self.handle_ltpposixtest_result(k, test_status, result) + self.handle_ltpposixtest_result(k, test_status, result, machine) return test_count_report def print_test_report(self, template_file_name, test_count_reports): @@ -141,10 +144,8 @@ class ResultsTextReport(object): env = Environment(loader=file_loader, trim_blocks=True) template = env.get_template(template_file_name) havefailed = False - haveptest = bool(self.ptests) - haveltp = bool(self.ltptests) - haveltpposix = bool(self.ltpposixtests) reportvalues = [] + machines = [] cols = ['passed', 'failed', 'skipped'] maxlen = {'passed' : 0, 'failed' : 0, 'skipped' : 0, 'result_id': 0, 'testseries' : 0, 'ptest' : 0 ,'ltptest': 0, 'ltpposixtest': 0} for line in test_count_reports: @@ -162,21 +163,24 @@ class ResultsTextReport(object): reportvalues.append(vals) if line['failed_testcases']: havefailed = True - for ptest in self.ptests: - if len(ptest) > maxlen['ptest']: - maxlen['ptest'] = len(ptest) - for ltptest in self.ltptests: - if len(ltptest) > maxlen['ltptest']: - maxlen['ltptest'] = len(ltptest) - for ltpposixtest in self.ltpposixtests: - if len(ltpposixtest) > maxlen['ltpposixtest']: - maxlen['ltpposixtest'] = len(ltpposixtest) + if line['machine'] not in machines: + machines.append(line['machine']) + for (machine, report) in self.ptests.items(): + for ptest in self.ptests[machine]: + if len(ptest) > maxlen['ptest']: + maxlen['ptest'] = len(ptest) + for (machine, report) in self.ltptests.items(): + for ltptest in self.ltptests[machine]: + if len(ltptest) > maxlen['ltptest']: + maxlen['ltptest'] = len(ltptest) + for (machine, report) in self.ltpposixtests.items(): + for ltpposixtest in self.ltpposixtests[machine]: + if len(ltpposixtest) > maxlen['ltpposixtest']: + maxlen['ltpposixtest'] = len(ltpposixtest) output = template.render(reportvalues=reportvalues, havefailed=havefailed, - haveptest=haveptest, + machines=machines, ptests=self.ptests, - haveltp=haveltp, - haveltpposix=haveltpposix, ltptests=self.ltptests, ltpposixtests=self.ltpposixtests, maxlen=maxlen) @@ -200,7 +204,9 @@ class ResultsTextReport(object): for testsuite in testresults: for resultid in testresults[testsuite]: result = testresults[testsuite][resultid] - test_count_report = self.get_aggregated_test_result(logger, result) + machine = result['configuration']['MACHINE'] + test_count_report = self.get_aggregated_test_result(logger, result, machine) + test_count_report['machine'] = machine test_count_report['testseries'] = result['configuration']['TESTSERIES'] test_count_report['result_id'] = resultid test_count_reports.append(test_count_report) diff --git a/scripts/lib/resulttool/template/test_report_full_text.txt b/scripts/lib/resulttool/template/test_report_full_text.txt index d2725b8d01..6a5a081c6d 100644 --- a/scripts/lib/resulttool/template/test_report_full_text.txt +++ b/scripts/lib/resulttool/template/test_report_full_text.txt @@ -9,54 +9,62 @@ Test Result Status Summary (Counts/Percentages sorted by testseries, ID) {% endfor %} -------------------------------------------------------------------------------------------------------------- -{% if haveptest %} + +{% for machine in machines %} +{% if ptests[machine] %} ============================================================================================================== -PTest Result Summary +{{ machine }} PTest Result Summary ============================================================================================================== -------------------------------------------------------------------------------------------------------------- {{ 'Recipe'.ljust(maxlen['ptest']) }} | {{ 'Passed'.ljust(maxlen['passed']) }} | {{ 'Failed'.ljust(maxlen['failed']) }} | {{ 'Skipped'.ljust(maxlen['skipped']) }} | {{ 'Time(s)'.ljust(10) }} -------------------------------------------------------------------------------------------------------------- -{% for ptest in ptests |sort %} -{{ ptest.ljust(maxlen['ptest']) }} | {{ (ptests[ptest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ptests[ptest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ptests[ptest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ptests[ptest]['duration']|string) }} +{% for ptest in ptests[machine] |sort %} +{{ ptest.ljust(maxlen['ptest']) }} | {{ (ptests[machine][ptest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ptests[machine][ptest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ptests[machine][ptest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ptests[machine][ptest]['duration']|string) }} {% endfor %} -------------------------------------------------------------------------------------------------------------- - {% else %} -There was no ptest data +There was no ptest data for {{ machine }} {% endif %} -{% if haveltp %} +{% endfor %} + + +{% for machine in machines %} +{% if ltptests[machine] %} ============================================================================================================== -Ltp Test Result Summary +{{ machine }} Ltp Test Result Summary ============================================================================================================== -------------------------------------------------------------------------------------------------------------- {{ 'Recipe'.ljust(maxlen['ltptest']) }} | {{ 'Passed'.ljust(maxlen['passed']) }} | {{ 'Failed'.ljust(maxlen['failed']) }} | {{ 'Skipped'.ljust(maxlen['skipped']) }} | {{ 'Time(s)'.ljust(10) }} -------------------------------------------------------------------------------------------------------------- -{% for ltptest in ltptests |sort %} -{{ ltptest.ljust(maxlen['ltptest']) }} | {{ (ltptests[ltptest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ltptests[ltptest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ltptests[ltptest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ltptests[ltptest]['duration']|string) }} +{% for ltptest in ltptests[machine] |sort %} +{{ ltptest.ljust(maxlen['ltptest']) }} | {{ (ltptests[machine][ltptest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ltptests[machine][ltptest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ltptests[machine][ltptest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ltptests[machine][ltptest]['duration']|string) }} {% endfor %} -------------------------------------------------------------------------------------------------------------- - {% else %} -There was no LTP Test data +There was no LTP Test data for {{ machine }} {% endif %} -{% if haveltpposix %} +{% endfor %} + + +{% for machine in machines %} +{% if ltpposixtests[machine] %} ============================================================================================================== -Ltp Posix Result Summary +{{ machine }} Ltp Posix Result Summary ============================================================================================================== -------------------------------------------------------------------------------------------------------------- {{ 'Recipe'.ljust(maxlen['ltpposixtest']) }} | {{ 'Passed'.ljust(maxlen['passed']) }} | {{ 'Failed'.ljust(maxlen['failed']) }} | {{ 'Skipped'.ljust(maxlen['skipped']) }} | {{ 'Time(s)'.ljust(10) }} -------------------------------------------------------------------------------------------------------------- -{% for ltpposixtest in ltpposixtests |sort %} -{{ ltpposixtest.ljust(maxlen['ltpposixtest']) }} | {{ (ltpposixtests[ltpposixtest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ltpposixtests[ltpposixtest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ltpposixtests[ltpposixtest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ltpposixtests[ltpposixtest]['duration']|string) }} +{% for ltpposixtest in ltpposixtests[machine] |sort %} +{{ ltpposixtest.ljust(maxlen['ltpposixtest']) }} | {{ (ltpposixtests[machine][ltpposixtest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ltpposixtests[machine][ltpposixtest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ltpposixtests[machine][ltpposixtest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ltpposixtests[machine][ltpposixtest]['duration']|string) }} {% endfor %} -------------------------------------------------------------------------------------------------------------- - {% else %} -There was no LTP Posix Test data +There was no LTP Posix Test data for {{ machine }} {% endif %} +{% endfor %} ============================================================================================================== -- cgit 1.2.3-korg