diff options
Diffstat (limited to 'scripts/lib/resulttool')
| -rw-r--r-- | scripts/lib/resulttool/__init__.py | 0 | ||||
| -rw-r--r-- | scripts/lib/resulttool/log.py | 107 | ||||
| -rwxr-xr-x | scripts/lib/resulttool/manualexecution.py | 235 | ||||
| -rw-r--r-- | scripts/lib/resulttool/merge.py | 46 | ||||
| -rw-r--r-- | scripts/lib/resulttool/regression.py | 447 | ||||
| -rw-r--r-- | scripts/lib/resulttool/report.py | 315 | ||||
| -rw-r--r-- | scripts/lib/resulttool/resultutils.py | 228 | ||||
| -rw-r--r-- | scripts/lib/resulttool/store.py | 104 | ||||
| -rw-r--r-- | scripts/lib/resulttool/template/test_report_full_text.txt | 79 |
9 files changed, 1561 insertions, 0 deletions
diff --git a/scripts/lib/resulttool/__init__.py b/scripts/lib/resulttool/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/scripts/lib/resulttool/__init__.py diff --git a/scripts/lib/resulttool/log.py b/scripts/lib/resulttool/log.py new file mode 100644 index 0000000000..15148ca288 --- /dev/null +++ b/scripts/lib/resulttool/log.py @@ -0,0 +1,107 @@ +# resulttool - Show logs +# +# Copyright (c) 2019 Garmin International +# +# SPDX-License-Identifier: GPL-2.0-only +# +import os +import resulttool.resultutils as resultutils + +def show_ptest(result, ptest, logger): + logdata = resultutils.ptestresult_get_log(result, ptest) + if logdata is not None: + print(logdata) + return 0 + + print("ptest '%s' log not found" % ptest) + return 1 + +def show_reproducible(result, reproducible, logger): + try: + print(result['reproducible'][reproducible]['diffoscope.text']) + return 0 + + except KeyError: + print("reproducible '%s' not found" % reproducible) + return 1 + +def log(args, logger): + results = resultutils.load_resultsdata(args.source) + + for _, run_name, _, r in resultutils.test_run_results(results): + if args.list_ptest: + print('\n'.join(sorted(r['ptestresult.sections'].keys()))) + + if args.dump_ptest: + for sectname in ['ptestresult.sections', 'ltpposixresult.sections', 'ltpresult.sections']: + if sectname in r: + for name, ptest in r[sectname].items(): + logdata = resultutils.generic_get_log(sectname, r, name) + if logdata is not None: + dest_dir = args.dump_ptest + if args.prepend_run: + dest_dir = os.path.join(dest_dir, run_name) + if not sectname.startswith("ptest"): + dest_dir = os.path.join(dest_dir, sectname.split(".")[0]) + + os.makedirs(dest_dir, exist_ok=True) + dest = os.path.join(dest_dir, '%s.log' % name) + if os.path.exists(dest): + print("Overlapping ptest logs found, skipping %s. The '--prepend-run' option would avoid this" % name) + continue + print(dest) + with open(dest, 'w') as f: + f.write(logdata) + + if args.raw_ptest: + found = False + for sectname in ['ptestresult.rawlogs', 'ltpposixresult.rawlogs', 'ltpresult.rawlogs']: + rawlog = resultutils.generic_get_rawlogs(sectname, r) + if rawlog is not None: + print(rawlog) + found = True + if not found: + print('Raw ptest logs not found') + return 1 + + if args.raw_reproducible: + if 'reproducible.rawlogs' in r: + print(r['reproducible.rawlogs']['log']) + else: + print('Raw reproducible logs not found') + return 1 + + for ptest in args.ptest: + if not show_ptest(r, ptest, logger): + return 1 + + for reproducible in args.reproducible: + if not show_reproducible(r, reproducible, logger): + return 1 + +def register_commands(subparsers): + """Register subcommands from this plugin""" + parser = subparsers.add_parser('log', help='show logs', + description='show the logs from test results', + group='analysis') + parser.set_defaults(func=log) + parser.add_argument('source', + help='the results file/directory/URL to import') + parser.add_argument('--list-ptest', action='store_true', + help='list the ptest test names') + parser.add_argument('--ptest', action='append', default=[], + help='show logs for a ptest') + parser.add_argument('--dump-ptest', metavar='DIR', + help='Dump all ptest log files to the specified directory.') + parser.add_argument('--reproducible', action='append', default=[], + help='show logs for a reproducible test') + parser.add_argument('--prepend-run', action='store_true', + help='''Dump ptest results to a subdirectory named after the test run when using --dump-ptest. + Required if more than one test run is present in the result file''') + parser.add_argument('--raw', action='store_true', + help='show raw (ptest) logs. Deprecated. Alias for "--raw-ptest"', dest='raw_ptest') + parser.add_argument('--raw-ptest', action='store_true', + help='show raw ptest log') + parser.add_argument('--raw-reproducible', action='store_true', + help='show raw reproducible build logs') + diff --git a/scripts/lib/resulttool/manualexecution.py b/scripts/lib/resulttool/manualexecution.py new file mode 100755 index 0000000000..ecb27c5933 --- /dev/null +++ b/scripts/lib/resulttool/manualexecution.py @@ -0,0 +1,235 @@ +# test case management tool - manual execution from testopia test cases +# +# Copyright (c) 2018, Intel Corporation. +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import argparse +import json +import os +import sys +import datetime +import re +import copy +from oeqa.core.runner import OETestResultJSONHelper + + +def load_json_file(f): + with open(f, "r") as filedata: + return json.load(filedata) + +def write_json_file(f, json_data): + os.makedirs(os.path.dirname(f), exist_ok=True) + with open(f, 'w') as filedata: + filedata.write(json.dumps(json_data, sort_keys=True, indent=4)) + +class ManualTestRunner(object): + + def _get_test_module(self, case_file): + return os.path.basename(case_file).split('.')[0] + + def _get_input(self, config): + while True: + output = input('{} = '.format(config)) + if re.match('^[a-z0-9-.]+$', output): + break + print('Only lowercase alphanumeric, hyphen and dot are allowed. Please try again') + return output + + def _get_available_config_options(self, config_options, test_module, target_config): + avail_config_options = None + if test_module in config_options: + avail_config_options = config_options[test_module].get(target_config) + return avail_config_options + + def _choose_config_option(self, options): + while True: + output = input('{} = '.format('Option index number')) + if output in options: + break + print('Only integer index inputs from above available configuration options are allowed. Please try again.') + return options[output] + + def _get_config(self, config_options, test_module): + from oeqa.utils.metadata import get_layers + from oeqa.utils.commands import get_bb_var + from resulttool.resultutils import store_map + + layers = get_layers(get_bb_var('BBLAYERS')) + configurations = {} + configurations['LAYERS'] = layers + configurations['STARTTIME'] = datetime.datetime.now().strftime('%Y%m%d%H%M%S') + configurations['TEST_TYPE'] = 'manual' + configurations['TEST_MODULE'] = test_module + + extra_config = set(store_map['manual']) - set(configurations) + for config in sorted(extra_config): + avail_config_options = self._get_available_config_options(config_options, test_module, config) + if avail_config_options: + print('---------------------------------------------') + print('These are available configuration #%s options:' % config) + print('---------------------------------------------') + for option, _ in sorted(avail_config_options.items(), key=lambda x: int(x[0])): + print('%s: %s' % (option, avail_config_options[option])) + print('Please select configuration option, enter the integer index number.') + value_conf = self._choose_config_option(avail_config_options) + print('---------------------------------------------\n') + else: + print('---------------------------------------------') + print('This is configuration #%s. Please provide configuration value(use "None" if not applicable).' % config) + print('---------------------------------------------') + value_conf = self._get_input('Configuration Value') + print('---------------------------------------------\n') + configurations[config] = value_conf + return configurations + + def _execute_test_steps(self, case): + test_result = {} + print('------------------------------------------------------------------------') + print('Executing test case: %s' % case['test']['@alias']) + print('------------------------------------------------------------------------') + print('You have total %s test steps to be executed.' % len(case['test']['execution'])) + print('------------------------------------------------------------------------\n') + for step, _ in sorted(case['test']['execution'].items(), key=lambda x: int(x[0])): + print('Step %s: %s' % (step, case['test']['execution'][step]['action'])) + expected_output = case['test']['execution'][step]['expected_results'] + if expected_output: + print('Expected output: %s' % expected_output) + while True: + done = input('\nPlease provide test results: (P)assed/(F)ailed/(B)locked/(S)kipped? \n').lower() + result_types = {'p':'PASSED', + 'f':'FAILED', + 'b':'BLOCKED', + 's':'SKIPPED'} + if done in result_types: + for r in result_types: + if done == r: + res = result_types[r] + if res == 'FAILED': + log_input = input('\nPlease enter the error and the description of the log: (Ex:log:211 Error Bitbake)\n') + test_result.update({case['test']['@alias']: {'status': '%s' % res, 'log': '%s' % log_input}}) + else: + test_result.update({case['test']['@alias']: {'status': '%s' % res}}) + break + print('Invalid input!') + return test_result + + def _get_write_dir(self): + return os.environ['BUILDDIR'] + '/tmp/log/manual/' + + def run_test(self, case_file, config_options_file, testcase_config_file): + test_module = self._get_test_module(case_file) + cases = load_json_file(case_file) + config_options = {} + if config_options_file: + config_options = load_json_file(config_options_file) + configurations = self._get_config(config_options, test_module) + result_id = 'manual_%s_%s' % (test_module, configurations['STARTTIME']) + test_results = {} + if testcase_config_file: + test_case_config = load_json_file(testcase_config_file) + test_case_to_execute = test_case_config['testcases'] + for case in copy.deepcopy(cases) : + if case['test']['@alias'] not in test_case_to_execute: + cases.remove(case) + + print('\nTotal number of test cases in this test suite: %s\n' % len(cases)) + for c in cases: + test_result = self._execute_test_steps(c) + test_results.update(test_result) + return configurations, result_id, self._get_write_dir(), test_results + + def _get_true_false_input(self, input_message): + yes_list = ['Y', 'YES'] + no_list = ['N', 'NO'] + while True: + more_config_option = input(input_message).upper() + if more_config_option in yes_list or more_config_option in no_list: + break + print('Invalid input!') + if more_config_option in no_list: + return False + return True + + def make_config_option_file(self, logger, case_file, config_options_file): + config_options = {} + if config_options_file: + config_options = load_json_file(config_options_file) + new_test_module = self._get_test_module(case_file) + print('Creating configuration options file for test module: %s' % new_test_module) + new_config_options = {} + + while True: + config_name = input('\nPlease provide test configuration to create:\n').upper() + new_config_options[config_name] = {} + while True: + config_value = self._get_input('Configuration possible option value') + config_option_index = len(new_config_options[config_name]) + 1 + new_config_options[config_name][config_option_index] = config_value + more_config_option = self._get_true_false_input('\nIs there more configuration option input: (Y)es/(N)o\n') + if not more_config_option: + break + more_config = self._get_true_false_input('\nIs there more configuration to create: (Y)es/(N)o\n') + if not more_config: + break + + if new_config_options: + config_options[new_test_module] = new_config_options + if not config_options_file: + config_options_file = os.path.join(self._get_write_dir(), 'manual_config_options.json') + write_json_file(config_options_file, config_options) + logger.info('Configuration option file created at %s' % config_options_file) + + def make_testcase_config_file(self, logger, case_file, testcase_config_file): + if testcase_config_file: + if os.path.exists(testcase_config_file): + print('\nTest configuration file with name %s already exists. Please provide a unique file name' % (testcase_config_file)) + return 0 + + if not testcase_config_file: + testcase_config_file = os.path.join(self._get_write_dir(), "testconfig_new.json") + + testcase_config = {} + cases = load_json_file(case_file) + new_test_module = self._get_test_module(case_file) + new_testcase_config = {} + new_testcase_config['testcases'] = [] + + print('\nAdd testcases for this configuration file:') + for case in cases: + print('\n' + case['test']['@alias']) + add_tc_config = self._get_true_false_input('\nDo you want to add this test case to test configuration : (Y)es/(N)o\n') + if add_tc_config: + new_testcase_config['testcases'].append(case['test']['@alias']) + write_json_file(testcase_config_file, new_testcase_config) + logger.info('Testcase Configuration file created at %s' % testcase_config_file) + +def manualexecution(args, logger): + testrunner = ManualTestRunner() + if args.make_config_options_file: + testrunner.make_config_option_file(logger, args.file, args.config_options_file) + return 0 + if args.make_testcase_config_file: + testrunner.make_testcase_config_file(logger, args.file, args.testcase_config_file) + return 0 + configurations, result_id, write_dir, test_results = testrunner.run_test(args.file, args.config_options_file, args.testcase_config_file) + resultjsonhelper = OETestResultJSONHelper() + resultjsonhelper.dump_testresult_file(write_dir, configurations, result_id, test_results) + return 0 + +def register_commands(subparsers): + """Register subcommands from this plugin""" + parser_build = subparsers.add_parser('manualexecution', help='helper script for results populating during manual test execution.', + description='helper script for results populating during manual test execution. You can find manual test case JSON file in meta/lib/oeqa/manual/', + group='manualexecution') + parser_build.set_defaults(func=manualexecution) + parser_build.add_argument('file', help='specify path to manual test case JSON file.Note: Please use \"\" to encapsulate the file path.') + parser_build.add_argument('-c', '--config-options-file', default='', + help='the config options file to import and used as available configuration option selection or make config option file') + parser_build.add_argument('-m', '--make-config-options-file', action='store_true', + help='make the configuration options file based on provided inputs') + parser_build.add_argument('-t', '--testcase-config-file', default='', + help='the testcase configuration file to enable user to run a selected set of test case or make a testcase configuration file') + parser_build.add_argument('-d', '--make-testcase-config-file', action='store_true', + help='make the testcase configuration file to run a set of test cases based on user selection')
\ No newline at end of file diff --git a/scripts/lib/resulttool/merge.py b/scripts/lib/resulttool/merge.py new file mode 100644 index 0000000000..18b4825a18 --- /dev/null +++ b/scripts/lib/resulttool/merge.py @@ -0,0 +1,46 @@ +# resulttool - merge multiple testresults.json files into a file or directory +# +# Copyright (c) 2019, Intel Corporation. +# Copyright (c) 2019, Linux Foundation +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import os +import json +import resulttool.resultutils as resultutils + +def merge(args, logger): + configvars = {} + if not args.not_add_testseries: + configvars = resultutils.extra_configvars.copy() + if args.executed_by: + configvars['EXECUTED_BY'] = args.executed_by + if resultutils.is_url(args.target_results) or os.path.isdir(args.target_results): + results = resultutils.load_resultsdata(args.target_results, configmap=resultutils.store_map, configvars=configvars) + resultutils.append_resultsdata(results, args.base_results, configmap=resultutils.store_map, configvars=configvars) + resultutils.save_resultsdata(results, args.target_results) + else: + results = resultutils.load_resultsdata(args.base_results, configmap=resultutils.flatten_map, configvars=configvars) + if os.path.exists(args.target_results): + resultutils.append_resultsdata(results, args.target_results, configmap=resultutils.flatten_map, configvars=configvars) + resultutils.save_resultsdata(results, os.path.dirname(args.target_results), fn=os.path.basename(args.target_results)) + + logger.info('Merged results to %s' % os.path.dirname(args.target_results)) + + return 0 + +def register_commands(subparsers): + """Register subcommands from this plugin""" + parser_build = subparsers.add_parser('merge', help='merge test result files/directories/URLs', + description='merge the results from multiple files/directories/URLs into the target file or directory', + group='setup') + parser_build.set_defaults(func=merge) + parser_build.add_argument('base_results', + help='the results file/directory/URL to import') + parser_build.add_argument('target_results', + help='the target file or directory to merge the base_results with') + parser_build.add_argument('-t', '--not-add-testseries', action='store_true', + help='do not add testseries configuration to results') + parser_build.add_argument('-x', '--executed-by', default='', + help='add executed-by configuration to each result file') diff --git a/scripts/lib/resulttool/regression.py b/scripts/lib/resulttool/regression.py new file mode 100644 index 0000000000..10e7d13841 --- /dev/null +++ b/scripts/lib/resulttool/regression.py @@ -0,0 +1,447 @@ +# resulttool - regression analysis +# +# Copyright (c) 2019, Intel Corporation. +# Copyright (c) 2019, Linux Foundation +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import resulttool.resultutils as resultutils + +from oeqa.utils.git import GitRepo +import oeqa.utils.gitarchive as gitarchive + +METADATA_MATCH_TABLE = { + "oeselftest": "OESELFTEST_METADATA" +} + +OESELFTEST_METADATA_GUESS_TABLE={ + "trigger-build-posttrigger": { + "run_all_tests": False, + "run_tests":["buildoptions.SourceMirroring.test_yocto_source_mirror"], + "skips": None, + "machine": None, + "select_tags":None, + "exclude_tags": None + }, + "reproducible": { + "run_all_tests": False, + "run_tests":["reproducible"], + "skips": None, + "machine": None, + "select_tags":None, + "exclude_tags": None + }, + "arch-qemu-quick": { + "run_all_tests": True, + "run_tests":None, + "skips": None, + "machine": None, + "select_tags":["machine"], + "exclude_tags": None + }, + "arch-qemu-full-x86-or-x86_64": { + "run_all_tests": True, + "run_tests":None, + "skips": None, + "machine": None, + "select_tags":["machine", "toolchain-system"], + "exclude_tags": None + }, + "arch-qemu-full-others": { + "run_all_tests": True, + "run_tests":None, + "skips": None, + "machine": None, + "select_tags":["machine", "toolchain-user"], + "exclude_tags": None + }, + "selftest": { + "run_all_tests": True, + "run_tests":None, + "skips": ["distrodata.Distrodata.test_checkpkg", "buildoptions.SourceMirroring.test_yocto_source_mirror", "reproducible"], + "machine": None, + "select_tags":None, + "exclude_tags": ["machine", "toolchain-system", "toolchain-user"] + }, + "bringup": { + "run_all_tests": True, + "run_tests":None, + "skips": ["distrodata.Distrodata.test_checkpkg", "buildoptions.SourceMirroring.test_yocto_source_mirror"], + "machine": None, + "select_tags":None, + "exclude_tags": ["machine", "toolchain-system", "toolchain-user"] + } +} + +STATUS_STRINGS = { + "None": "No matching test result" +} + +REGRESSIONS_DISPLAY_LIMIT=50 + +MISSING_TESTS_BANNER = "-------------------------- Missing tests --------------------------" +ADDITIONAL_DATA_BANNER = "--------------------- Matches and improvements --------------------" + +def test_has_at_least_one_matching_tag(test, tag_list): + return "oetags" in test and any(oetag in tag_list for oetag in test["oetags"]) + +def all_tests_have_at_least_one_matching_tag(results, tag_list): + return all(test_has_at_least_one_matching_tag(test_result, tag_list) or test_name.startswith("ptestresult") for (test_name, test_result) in results.items()) + +def any_test_have_any_matching_tag(results, tag_list): + return any(test_has_at_least_one_matching_tag(test, tag_list) for test in results.values()) + +def have_skipped_test(result, test_prefix): + return all( result[test]['status'] == "SKIPPED" for test in result if test.startswith(test_prefix)) + +def have_all_tests_skipped(result, test_prefixes_list): + return all(have_skipped_test(result, test_prefix) for test_prefix in test_prefixes_list) + +def guess_oeselftest_metadata(results): + """ + When an oeselftest test result is lacking OESELFTEST_METADATA, we can try to guess it based on results content. + Check results for specific values (absence/presence of oetags, number and name of executed tests...), + and if it matches one of known configuration from autobuilder configuration, apply guessed OSELFTEST_METADATA + to it to allow proper test filtering. + This guessing process is tightly coupled to config.json in autobuilder. It should trigger less and less, + as new tests will have OESELFTEST_METADATA properly appended at test reporting time + """ + + if len(results) == 1 and "buildoptions.SourceMirroring.test_yocto_source_mirror" in results: + return OESELFTEST_METADATA_GUESS_TABLE['trigger-build-posttrigger'] + elif all(result.startswith("reproducible") for result in results): + return OESELFTEST_METADATA_GUESS_TABLE['reproducible'] + elif all_tests_have_at_least_one_matching_tag(results, ["machine"]): + return OESELFTEST_METADATA_GUESS_TABLE['arch-qemu-quick'] + elif all_tests_have_at_least_one_matching_tag(results, ["machine", "toolchain-system"]): + return OESELFTEST_METADATA_GUESS_TABLE['arch-qemu-full-x86-or-x86_64'] + elif all_tests_have_at_least_one_matching_tag(results, ["machine", "toolchain-user"]): + return OESELFTEST_METADATA_GUESS_TABLE['arch-qemu-full-others'] + elif not any_test_have_any_matching_tag(results, ["machine", "toolchain-user", "toolchain-system"]): + if have_all_tests_skipped(results, ["distrodata.Distrodata.test_checkpkg", "buildoptions.SourceMirroring.test_yocto_source_mirror", "reproducible"]): + return OESELFTEST_METADATA_GUESS_TABLE['selftest'] + elif have_all_tests_skipped(results, ["distrodata.Distrodata.test_checkpkg", "buildoptions.SourceMirroring.test_yocto_source_mirror"]): + return OESELFTEST_METADATA_GUESS_TABLE['bringup'] + + return None + + +def metadata_matches(base_configuration, target_configuration): + """ + For passed base and target, check test type. If test type matches one of + properties described in METADATA_MATCH_TABLE, compare metadata if it is + present in base. Return true if metadata matches, or if base lacks some + data (either TEST_TYPE or the corresponding metadata) + """ + test_type = base_configuration.get('TEST_TYPE') + if test_type not in METADATA_MATCH_TABLE: + return True + + metadata_key = METADATA_MATCH_TABLE.get(test_type) + if target_configuration.get(metadata_key) != base_configuration.get(metadata_key): + return False + + return True + + +def machine_matches(base_configuration, target_configuration): + return base_configuration.get('MACHINE') == target_configuration.get('MACHINE') + + +def can_be_compared(logger, base, target): + """ + Some tests are not relevant to be compared, for example some oeselftest + run with different tests sets or parameters. Return true if tests can be + compared + """ + ret = True + base_configuration = base['configuration'] + target_configuration = target['configuration'] + + # Older test results lack proper OESELFTEST_METADATA: if not present, try to guess it based on tests results. + if base_configuration.get('TEST_TYPE') == 'oeselftest' and 'OESELFTEST_METADATA' not in base_configuration: + guess = guess_oeselftest_metadata(base['result']) + if guess is None: + logger.error(f"ERROR: did not manage to guess oeselftest metadata for {base_configuration['STARTTIME']}") + else: + logger.debug(f"Enriching {base_configuration['STARTTIME']} with {guess}") + base_configuration['OESELFTEST_METADATA'] = guess + if target_configuration.get('TEST_TYPE') == 'oeselftest' and 'OESELFTEST_METADATA' not in target_configuration: + guess = guess_oeselftest_metadata(target['result']) + if guess is None: + logger.error(f"ERROR: did not manage to guess oeselftest metadata for {target_configuration['STARTTIME']}") + else: + logger.debug(f"Enriching {target_configuration['STARTTIME']} with {guess}") + target_configuration['OESELFTEST_METADATA'] = guess + + # Test runs with LTP results in should only be compared with other runs with LTP tests in them + if base_configuration.get('TEST_TYPE') == 'runtime' and any(result.startswith("ltpresult") for result in base['result']): + ret = target_configuration.get('TEST_TYPE') == 'runtime' and any(result.startswith("ltpresult") for result in target['result']) + + return ret and metadata_matches(base_configuration, target_configuration) \ + and machine_matches(base_configuration, target_configuration) + +def get_status_str(raw_status): + raw_status_lower = raw_status.lower() if raw_status else "None" + return STATUS_STRINGS.get(raw_status_lower, raw_status) + +def get_additional_info_line(new_pass_count, new_tests): + result=[] + if new_tests: + result.append(f'+{new_tests} test(s) present') + if new_pass_count: + result.append(f'+{new_pass_count} test(s) now passing') + + if not result: + return "" + + return ' -> ' + ', '.join(result) + '\n' + +def compare_result(logger, base_name, target_name, base_result, target_result, display_limit=None): + base_result = base_result.get('result') + target_result = target_result.get('result') + result = {} + new_tests = 0 + regressions = {} + resultstring = "" + new_tests = 0 + new_pass_count = 0 + + display_limit = int(display_limit) if display_limit else REGRESSIONS_DISPLAY_LIMIT + + if base_result and target_result: + for k in base_result: + base_testcase = base_result[k] + base_status = base_testcase.get('status') + if base_status: + target_testcase = target_result.get(k, {}) + target_status = target_testcase.get('status') + if base_status != target_status: + result[k] = {'base': base_status, 'target': target_status} + else: + logger.error('Failed to retrieved base test case status: %s' % k) + + # Also count new tests that were not present in base results: it + # could be newly added tests, but it could also highlights some tests + # renames or fixed faulty ptests + for k in target_result: + if k not in base_result: + new_tests += 1 + if result: + new_pass_count = sum(test['target'] is not None and test['target'].startswith("PASS") for test in result.values()) + # Print a regression report only if at least one test has a regression status (FAIL, SKIPPED, absent...) + if new_pass_count < len(result): + resultstring = "Regression: %s\n %s\n" % (base_name, target_name) + for k in sorted(result): + if not result[k]['target'] or not result[k]['target'].startswith("PASS"): + # Differentiate each ptest kind when listing regressions + key_parts = k.split('.') + key = '.'.join(key_parts[:2]) if k.startswith('ptest') else key_parts[0] + # Append new regression to corresponding test family + regressions[key] = regressions.setdefault(key, []) + [' %s: %s -> %s\n' % (k, get_status_str(result[k]['base']), get_status_str(result[k]['target']))] + resultstring += f" Total: {sum([len(regressions[r]) for r in regressions])} new regression(s):\n" + for k in regressions: + resultstring += f" {len(regressions[k])} regression(s) for {k}\n" + count_to_print=min([display_limit, len(regressions[k])]) if display_limit > 0 else len(regressions[k]) + resultstring += ''.join(regressions[k][:count_to_print]) + if count_to_print < len(regressions[k]): + resultstring+=' [...]\n' + if new_pass_count > 0: + resultstring += f' Additionally, {new_pass_count} previously failing test(s) is/are now passing\n' + if new_tests > 0: + resultstring += f' Additionally, {new_tests} new test(s) is/are present\n' + else: + resultstring = "%s\n%s\n" % (base_name, target_name) + result = None + else: + resultstring = "%s\n%s\n" % (base_name, target_name) + + if not result: + additional_info = get_additional_info_line(new_pass_count, new_tests) + if additional_info: + resultstring += additional_info + + return result, resultstring + +def get_results(logger, source): + return resultutils.load_resultsdata(source, configmap=resultutils.regression_map) + +def regression(args, logger): + base_results = get_results(logger, args.base_result) + target_results = get_results(logger, args.target_result) + + regression_common(args, logger, base_results, target_results) + +# Some test case naming is poor and contains random strings, particularly lttng/babeltrace. +# Truncating the test names works since they contain file and line number identifiers +# which allows us to match them without the random components. +def fixup_ptest_names(results, logger): + for r in results: + for i in results[r]: + tests = list(results[r][i]['result'].keys()) + for test in tests: + new = None + if test.startswith(("ptestresult.lttng-tools.", "ptestresult.babeltrace.", "ptestresult.babeltrace2")) and "_-_" in test: + new = test.split("_-_")[0] + elif test.startswith(("ptestresult.curl.")) and "__" in test: + new = test.split("__")[0] + elif test.startswith(("ptestresult.dbus.")) and "__" in test: + new = test.split("__")[0] + elif test.startswith("ptestresult.binutils") and "build-st-" in test: + new = test.split(" ")[0] + elif test.startswith("ptestresult.gcc") and "/tmp/runtest." in test: + new = ".".join(test.split(".")[:2]) + if new: + results[r][i]['result'][new] = results[r][i]['result'][test] + del results[r][i]['result'][test] + +def regression_common(args, logger, base_results, target_results): + if args.base_result_id: + base_results = resultutils.filter_resultsdata(base_results, args.base_result_id) + if args.target_result_id: + target_results = resultutils.filter_resultsdata(target_results, args.target_result_id) + + fixup_ptest_names(base_results, logger) + fixup_ptest_names(target_results, logger) + + matches = [] + regressions = [] + notfound = [] + + for a in base_results: + if a in target_results: + base = list(base_results[a].keys()) + target = list(target_results[a].keys()) + # We may have multiple base/targets which are for different configurations. Start by + # removing any pairs which match + for c in base.copy(): + for b in target.copy(): + if not can_be_compared(logger, base_results[a][c], target_results[a][b]): + continue + res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b], args.limit) + if not res: + matches.append(resstr) + base.remove(c) + target.remove(b) + break + # Should only now see regressions, we may not be able to match multiple pairs directly + for c in base: + for b in target: + if not can_be_compared(logger, base_results[a][c], target_results[a][b]): + continue + res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b], args.limit) + if res: + regressions.append(resstr) + else: + notfound.append("%s not found in target" % a) + print("\n".join(sorted(regressions))) + print("\n" + MISSING_TESTS_BANNER + "\n") + print("\n".join(sorted(notfound))) + print("\n" + ADDITIONAL_DATA_BANNER + "\n") + print("\n".join(sorted(matches))) + return 0 + +def regression_git(args, logger): + base_results = {} + target_results = {} + + tag_name = "{branch}/{commit_number}-g{commit}/{tag_number}" + repo = GitRepo(args.repo) + + revs = gitarchive.get_test_revs(logger, repo, tag_name, branch=args.branch) + + if args.branch2: + revs2 = gitarchive.get_test_revs(logger, repo, tag_name, branch=args.branch2) + if not len(revs2): + logger.error("No revisions found to compare against") + return 1 + if not len(revs): + logger.error("No revision to report on found") + return 1 + else: + if len(revs) < 2: + logger.error("Only %d tester revisions found, unable to generate report" % len(revs)) + return 1 + + # Pick revisions + if args.commit: + if args.commit_number: + logger.warning("Ignoring --commit-number as --commit was specified") + index1 = gitarchive.rev_find(revs, 'commit', args.commit) + elif args.commit_number: + index1 = gitarchive.rev_find(revs, 'commit_number', args.commit_number) + else: + index1 = len(revs) - 1 + + if args.branch2: + revs2.append(revs[index1]) + index1 = len(revs2) - 1 + revs = revs2 + + if args.commit2: + if args.commit_number2: + logger.warning("Ignoring --commit-number2 as --commit2 was specified") + index2 = gitarchive.rev_find(revs, 'commit', args.commit2) + elif args.commit_number2: + index2 = gitarchive.rev_find(revs, 'commit_number', args.commit_number2) + else: + if index1 > 0: + index2 = index1 - 1 + # Find the closest matching commit number for comparision + # In future we could check the commit is a common ancestor and + # continue back if not but this good enough for now + while index2 > 0 and revs[index2].commit_number > revs[index1].commit_number: + index2 = index2 - 1 + else: + logger.error("Unable to determine the other commit, use " + "--commit2 or --commit-number2 to specify it") + return 1 + + logger.info("Comparing:\n%s\nto\n%s\n" % (revs[index1], revs[index2])) + + base_results = resultutils.git_get_result(repo, revs[index1][2]) + target_results = resultutils.git_get_result(repo, revs[index2][2]) + + regression_common(args, logger, base_results, target_results) + + return 0 + +def register_commands(subparsers): + """Register subcommands from this plugin""" + + parser_build = subparsers.add_parser('regression', help='regression file/directory analysis', + description='regression analysis comparing the base set of results to the target results', + group='analysis') + parser_build.set_defaults(func=regression) + parser_build.add_argument('base_result', + help='base result file/directory/URL for the comparison') + parser_build.add_argument('target_result', + help='target result file/directory/URL to compare with') + parser_build.add_argument('-b', '--base-result-id', default='', + help='(optional) filter the base results to this result ID') + parser_build.add_argument('-t', '--target-result-id', default='', + help='(optional) filter the target results to this result ID') + + parser_build = subparsers.add_parser('regression-git', help='regression git analysis', + description='regression analysis comparing base result set to target ' + 'result set', + group='analysis') + parser_build.set_defaults(func=regression_git) + parser_build.add_argument('repo', + help='the git repository containing the data') + parser_build.add_argument('-b', '--base-result-id', default='', + help='(optional) default select regression based on configurations unless base result ' + 'id was provided') + parser_build.add_argument('-t', '--target-result-id', default='', + help='(optional) default select regression based on configurations unless target result ' + 'id was provided') + + parser_build.add_argument('--branch', '-B', default='master', help="Branch to find commit in") + parser_build.add_argument('--branch2', help="Branch to find comparision revisions in") + parser_build.add_argument('--commit', help="Revision to search for") + parser_build.add_argument('--commit-number', help="Revision number to search for, redundant if --commit is specified") + parser_build.add_argument('--commit2', help="Revision to compare with") + parser_build.add_argument('--commit-number2', help="Revision number to compare with, redundant if --commit2 is specified") + parser_build.add_argument('-l', '--limit', default=REGRESSIONS_DISPLAY_LIMIT, help="Maximum number of changes to display per test. Can be set to 0 to print all changes") + diff --git a/scripts/lib/resulttool/report.py b/scripts/lib/resulttool/report.py new file mode 100644 index 0000000000..a349510ab8 --- /dev/null +++ b/scripts/lib/resulttool/report.py @@ -0,0 +1,315 @@ +# test result tool - report text based test results +# +# Copyright (c) 2019, Intel Corporation. +# Copyright (c) 2019, Linux Foundation +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import os +import glob +import json +import resulttool.resultutils as resultutils +from oeqa.utils.git import GitRepo +import oeqa.utils.gitarchive as gitarchive + + +class ResultsTextReport(object): + def __init__(self): + self.ptests = {} + self.ltptests = {} + self.ltpposixtests = {} + self.result_types = {'passed': ['PASSED', 'passed', 'PASS', 'XFAIL'], + 'failed': ['FAILED', 'failed', 'FAIL', 'ERROR', 'error', 'UNKNOWN', 'XPASS'], + 'skipped': ['SKIPPED', 'skipped', 'UNSUPPORTED', 'UNTESTED', 'UNRESOLVED']} + + + def handle_ptest_result(self, k, status, result, machine): + if machine not in self.ptests: + self.ptests[machine] = {} + + if k == 'ptestresult.sections': + # Ensure tests without any test results still show up on the report + for suite in result['ptestresult.sections']: + if suite not in self.ptests[machine]: + self.ptests[machine][suite] = { + 'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', + 'failed_testcases': [], "testcases": set(), + } + if 'duration' in result['ptestresult.sections'][suite]: + self.ptests[machine][suite]['duration'] = result['ptestresult.sections'][suite]['duration'] + if 'timeout' in result['ptestresult.sections'][suite]: + self.ptests[machine][suite]['duration'] += " T" + return True + + # process test result + try: + _, suite, test = k.split(".", 2) + except ValueError: + return True + + # Handle 'glib-2.0' + if 'ptestresult.sections' in result and suite not in result['ptestresult.sections']: + try: + _, suite, suite1, test = k.split(".", 3) + if suite + "." + suite1 in result['ptestresult.sections']: + suite = suite + "." + suite1 + except ValueError: + pass + + if suite not in self.ptests[machine]: + self.ptests[machine][suite] = { + 'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', + 'failed_testcases': [], "testcases": set(), + } + + # do not process duplicate results + if test in self.ptests[machine][suite]["testcases"]: + print("Warning duplicate ptest result '{}.{}' for {}".format(suite, test, machine)) + return False + + for tk in self.result_types: + if status in self.result_types[tk]: + self.ptests[machine][suite][tk] += 1 + self.ptests[machine][suite]["testcases"].add(test) + return True + + def handle_ltptest_result(self, k, status, result, machine): + if machine not in self.ltptests: + self.ltptests[machine] = {} + + if k == 'ltpresult.sections': + # Ensure tests without any test results still show up on the report + for suite in result['ltpresult.sections']: + if suite not in self.ltptests[machine]: + self.ltptests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []} + if 'duration' in result['ltpresult.sections'][suite]: + self.ltptests[machine][suite]['duration'] = result['ltpresult.sections'][suite]['duration'] + if 'timeout' in result['ltpresult.sections'][suite]: + self.ltptests[machine][suite]['duration'] += " T" + return + try: + _, suite, test = k.split(".", 2) + except ValueError: + return + # Handle 'glib-2.0' + if 'ltpresult.sections' in result and suite not in result['ltpresult.sections']: + try: + _, suite, suite1, test = k.split(".", 3) + if suite + "." + suite1 in result['ltpresult.sections']: + suite = suite + "." + suite1 + except ValueError: + pass + if suite not in self.ltptests[machine]: + self.ltptests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []} + for tk in self.result_types: + if status in self.result_types[tk]: + self.ltptests[machine][suite][tk] += 1 + + def handle_ltpposixtest_result(self, k, status, result, machine): + if machine not in self.ltpposixtests: + self.ltpposixtests[machine] = {} + + if k == 'ltpposixresult.sections': + # Ensure tests without any test results still show up on the report + for suite in result['ltpposixresult.sections']: + if suite not in self.ltpposixtests[machine]: + self.ltpposixtests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []} + if 'duration' in result['ltpposixresult.sections'][suite]: + self.ltpposixtests[machine][suite]['duration'] = result['ltpposixresult.sections'][suite]['duration'] + return + try: + _, suite, test = k.split(".", 2) + except ValueError: + return + # Handle 'glib-2.0' + if 'ltpposixresult.sections' in result and suite not in result['ltpposixresult.sections']: + try: + _, suite, suite1, test = k.split(".", 3) + if suite + "." + suite1 in result['ltpposixresult.sections']: + suite = suite + "." + suite1 + except ValueError: + pass + if suite not in self.ltpposixtests[machine]: + self.ltpposixtests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []} + for tk in self.result_types: + if status in self.result_types[tk]: + self.ltpposixtests[machine][suite][tk] += 1 + + def get_aggregated_test_result(self, logger, testresult, machine): + test_count_report = {'passed': 0, 'failed': 0, 'skipped': 0, 'failed_testcases': []} + result = testresult.get('result', []) + for k in result: + test_status = result[k].get('status', []) + if k.startswith("ptestresult."): + if not self.handle_ptest_result(k, test_status, result, machine): + continue + elif k.startswith("ltpresult."): + self.handle_ltptest_result(k, test_status, result, machine) + elif k.startswith("ltpposixresult."): + self.handle_ltpposixtest_result(k, test_status, result, machine) + + # process result if it was not skipped by a handler + for tk in self.result_types: + if test_status in self.result_types[tk]: + test_count_report[tk] += 1 + if test_status in self.result_types['failed']: + test_count_report['failed_testcases'].append(k) + return test_count_report + + def print_test_report(self, template_file_name, test_count_reports): + from jinja2 import Environment, FileSystemLoader + script_path = os.path.dirname(os.path.realpath(__file__)) + file_loader = FileSystemLoader(script_path + '/template') + env = Environment(loader=file_loader, trim_blocks=True) + template = env.get_template(template_file_name) + havefailed = False + reportvalues = [] + machines = [] + cols = ['passed', 'failed', 'skipped'] + maxlen = {'passed' : 0, 'failed' : 0, 'skipped' : 0, 'result_id': 0, 'testseries' : 0, 'ptest' : 0 ,'ltptest': 0, 'ltpposixtest': 0} + for line in test_count_reports: + total_tested = line['passed'] + line['failed'] + line['skipped'] + vals = {} + vals['result_id'] = line['result_id'] + vals['testseries'] = line['testseries'] + vals['sort'] = line['testseries'] + "_" + line['result_id'] + vals['failed_testcases'] = line['failed_testcases'] + for k in cols: + if total_tested: + vals[k] = "%d (%s%%)" % (line[k], format(line[k] / total_tested * 100, '.0f')) + else: + vals[k] = "0 (0%)" + for k in maxlen: + if k in vals and len(vals[k]) > maxlen[k]: + maxlen[k] = len(vals[k]) + reportvalues.append(vals) + if line['failed_testcases']: + havefailed = True + if line['machine'] not in machines: + machines.append(line['machine']) + reporttotalvalues = {} + for k in cols: + reporttotalvalues[k] = '%s' % sum([line[k] for line in test_count_reports]) + reporttotalvalues['count'] = '%s' % len(test_count_reports) + for (machine, report) in self.ptests.items(): + for ptest in self.ptests[machine]: + if len(ptest) > maxlen['ptest']: + maxlen['ptest'] = len(ptest) + for (machine, report) in self.ltptests.items(): + for ltptest in self.ltptests[machine]: + if len(ltptest) > maxlen['ltptest']: + maxlen['ltptest'] = len(ltptest) + for (machine, report) in self.ltpposixtests.items(): + for ltpposixtest in self.ltpposixtests[machine]: + if len(ltpposixtest) > maxlen['ltpposixtest']: + maxlen['ltpposixtest'] = len(ltpposixtest) + output = template.render(reportvalues=reportvalues, + reporttotalvalues=reporttotalvalues, + havefailed=havefailed, + machines=machines, + ptests=self.ptests, + ltptests=self.ltptests, + ltpposixtests=self.ltpposixtests, + maxlen=maxlen) + print(output) + + def view_test_report(self, logger, source_dir, branch, commit, tag, use_regression_map, raw_test, selected_test_case_only): + def print_selected_testcase_result(testresults, selected_test_case_only): + for testsuite in testresults: + for resultid in testresults[testsuite]: + result = testresults[testsuite][resultid]['result'] + test_case_result = result.get(selected_test_case_only, {}) + if test_case_result.get('status'): + print('Found selected test case result for %s from %s' % (selected_test_case_only, + resultid)) + print(test_case_result['status']) + else: + print('Could not find selected test case result for %s from %s' % (selected_test_case_only, + resultid)) + if test_case_result.get('log'): + print(test_case_result['log']) + test_count_reports = [] + configmap = resultutils.store_map + if use_regression_map: + configmap = resultutils.regression_map + if commit: + if tag: + logger.warning("Ignoring --tag as --commit was specified") + tag_name = "{branch}/{commit_number}-g{commit}/{tag_number}" + repo = GitRepo(source_dir) + revs = gitarchive.get_test_revs(logger, repo, tag_name, branch=branch) + rev_index = gitarchive.rev_find(revs, 'commit', commit) + testresults = resultutils.git_get_result(repo, revs[rev_index][2], configmap=configmap) + elif tag: + repo = GitRepo(source_dir) + testresults = resultutils.git_get_result(repo, [tag], configmap=configmap) + else: + testresults = resultutils.load_resultsdata(source_dir, configmap=configmap) + if raw_test: + raw_results = {} + for testsuite in testresults: + result = testresults[testsuite].get(raw_test, {}) + if result: + raw_results[testsuite] = {raw_test: result} + if raw_results: + if selected_test_case_only: + print_selected_testcase_result(raw_results, selected_test_case_only) + else: + print(json.dumps(raw_results, sort_keys=True, indent=4)) + else: + print('Could not find raw test result for %s' % raw_test) + return 0 + if selected_test_case_only: + print_selected_testcase_result(testresults, selected_test_case_only) + return 0 + for testsuite in testresults: + for resultid in testresults[testsuite]: + skip = False + result = testresults[testsuite][resultid] + machine = result['configuration']['MACHINE'] + + # Check to see if there is already results for these kinds of tests for the machine + for key in result['result'].keys(): + testtype = str(key).split('.')[0] + if ((machine in self.ltptests and testtype == "ltpiresult" and self.ltptests[machine]) or + (machine in self.ltpposixtests and testtype == "ltpposixresult" and self.ltpposixtests[machine])): + print("Already have test results for %s on %s, skipping %s" %(str(key).split('.')[0], machine, resultid)) + skip = True + break + if skip: + break + + test_count_report = self.get_aggregated_test_result(logger, result, machine) + test_count_report['machine'] = machine + test_count_report['testseries'] = result['configuration']['TESTSERIES'] + test_count_report['result_id'] = resultid + test_count_reports.append(test_count_report) + self.print_test_report('test_report_full_text.txt', test_count_reports) + +def report(args, logger): + report = ResultsTextReport() + report.view_test_report(logger, args.source_dir, args.branch, args.commit, args.tag, args.use_regression_map, + args.raw_test_only, args.selected_test_case_only) + return 0 + +def register_commands(subparsers): + """Register subcommands from this plugin""" + parser_build = subparsers.add_parser('report', help='summarise test results', + description='print a text-based summary of the test results', + group='analysis') + parser_build.set_defaults(func=report) + parser_build.add_argument('source_dir', + help='source file/directory/URL that contain the test result files to summarise') + parser_build.add_argument('--branch', '-B', default='master', help="Branch to find commit in") + parser_build.add_argument('--commit', help="Revision to report") + parser_build.add_argument('-t', '--tag', default='', + help='source_dir is a git repository, report on the tag specified from that repository') + parser_build.add_argument('-m', '--use_regression_map', action='store_true', + help='instead of the default "store_map", use the "regression_map" for report') + parser_build.add_argument('-r', '--raw_test_only', default='', + help='output raw test result only for the user provided test result id') + parser_build.add_argument('-s', '--selected_test_case_only', default='', + help='output selected test case result for the user provided test case id, if both test ' + 'result id and test case id are provided then output the selected test case result ' + 'from the provided test result id') diff --git a/scripts/lib/resulttool/resultutils.py b/scripts/lib/resulttool/resultutils.py new file mode 100644 index 0000000000..c5521d81bd --- /dev/null +++ b/scripts/lib/resulttool/resultutils.py @@ -0,0 +1,228 @@ +# resulttool - common library/utility functions +# +# Copyright (c) 2019, Intel Corporation. +# Copyright (c) 2019, Linux Foundation +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import os +import base64 +import zlib +import json +import scriptpath +import copy +import urllib.request +import posixpath +scriptpath.add_oe_lib_path() + +flatten_map = { + "oeselftest": [], + "runtime": [], + "sdk": [], + "sdkext": [], + "manual": [] +} +regression_map = { + "oeselftest": ['TEST_TYPE', 'MACHINE'], + "runtime": ['TESTSERIES', 'TEST_TYPE', 'IMAGE_BASENAME', 'MACHINE', 'IMAGE_PKGTYPE', 'DISTRO'], + "sdk": ['TESTSERIES', 'TEST_TYPE', 'IMAGE_BASENAME', 'MACHINE', 'SDKMACHINE'], + "sdkext": ['TESTSERIES', 'TEST_TYPE', 'IMAGE_BASENAME', 'MACHINE', 'SDKMACHINE'], + "manual": ['TEST_TYPE', 'TEST_MODULE', 'IMAGE_BASENAME', 'MACHINE'] +} +store_map = { + "oeselftest": ['TEST_TYPE'], + "runtime": ['TEST_TYPE', 'DISTRO', 'MACHINE', 'IMAGE_BASENAME'], + "sdk": ['TEST_TYPE', 'MACHINE', 'SDKMACHINE', 'IMAGE_BASENAME'], + "sdkext": ['TEST_TYPE', 'MACHINE', 'SDKMACHINE', 'IMAGE_BASENAME'], + "manual": ['TEST_TYPE', 'TEST_MODULE', 'MACHINE', 'IMAGE_BASENAME'] +} + +def is_url(p): + """ + Helper for determining if the given path is a URL + """ + return p.startswith('http://') or p.startswith('https://') + +extra_configvars = {'TESTSERIES': ''} + +# +# Load the json file and append the results data into the provided results dict +# +def append_resultsdata(results, f, configmap=store_map, configvars=extra_configvars): + if type(f) is str: + if is_url(f): + with urllib.request.urlopen(f) as response: + data = json.loads(response.read().decode('utf-8')) + url = urllib.parse.urlparse(f) + testseries = posixpath.basename(posixpath.dirname(url.path)) + else: + with open(f, "r") as filedata: + try: + data = json.load(filedata) + except json.decoder.JSONDecodeError: + print("Cannot decode {}. Possible corruption. Skipping.".format(f)) + data = "" + testseries = os.path.basename(os.path.dirname(f)) + else: + data = f + for res in data: + if "configuration" not in data[res] or "result" not in data[res]: + raise ValueError("Test results data without configuration or result section?") + for config in configvars: + if config == "TESTSERIES" and "TESTSERIES" not in data[res]["configuration"]: + data[res]["configuration"]["TESTSERIES"] = testseries + continue + if config not in data[res]["configuration"]: + data[res]["configuration"][config] = configvars[config] + testtype = data[res]["configuration"].get("TEST_TYPE") + if testtype not in configmap: + raise ValueError("Unknown test type %s" % testtype) + testpath = "/".join(data[res]["configuration"].get(i) for i in configmap[testtype]) + if testpath not in results: + results[testpath] = {} + results[testpath][res] = data[res] + +# +# Walk a directory and find/load results data +# or load directly from a file +# +def load_resultsdata(source, configmap=store_map, configvars=extra_configvars): + results = {} + if is_url(source) or os.path.isfile(source): + append_resultsdata(results, source, configmap, configvars) + return results + for root, dirs, files in os.walk(source): + for name in files: + f = os.path.join(root, name) + if name == "testresults.json": + append_resultsdata(results, f, configmap, configvars) + return results + +def filter_resultsdata(results, resultid): + newresults = {} + for r in results: + for i in results[r]: + if i == resultsid: + newresults[r] = {} + newresults[r][i] = results[r][i] + return newresults + +def strip_ptestresults(results): + newresults = copy.deepcopy(results) + #for a in newresults2: + # newresults = newresults2[a] + for res in newresults: + if 'result' not in newresults[res]: + continue + if 'ptestresult.rawlogs' in newresults[res]['result']: + del newresults[res]['result']['ptestresult.rawlogs'] + if 'ptestresult.sections' in newresults[res]['result']: + for i in newresults[res]['result']['ptestresult.sections']: + if 'log' in newresults[res]['result']['ptestresult.sections'][i]: + del newresults[res]['result']['ptestresult.sections'][i]['log'] + return newresults + +def decode_log(logdata): + if isinstance(logdata, str): + return logdata + elif isinstance(logdata, dict): + if "compressed" in logdata: + data = logdata.get("compressed") + data = base64.b64decode(data.encode("utf-8")) + data = zlib.decompress(data) + return data.decode("utf-8", errors='ignore') + return None + +def generic_get_log(sectionname, results, section): + if sectionname not in results: + return None + if section not in results[sectionname]: + return None + + ptest = results[sectionname][section] + if 'log' not in ptest: + return None + return decode_log(ptest['log']) + +def ptestresult_get_log(results, section): + return generic_get_log('ptestresult.sections', results, section) + +def generic_get_rawlogs(sectname, results): + if sectname not in results: + return None + if 'log' not in results[sectname]: + return None + return decode_log(results[sectname]['log']) + +def ptestresult_get_rawlogs(results): + return generic_get_rawlogs('ptestresult.rawlogs', results) + +def save_resultsdata(results, destdir, fn="testresults.json", ptestjson=False, ptestlogs=False): + for res in results: + if res: + dst = destdir + "/" + res + "/" + fn + else: + dst = destdir + "/" + fn + os.makedirs(os.path.dirname(dst), exist_ok=True) + resultsout = results[res] + if not ptestjson: + resultsout = strip_ptestresults(results[res]) + with open(dst, 'w') as f: + f.write(json.dumps(resultsout, sort_keys=True, indent=4)) + for res2 in results[res]: + if ptestlogs and 'result' in results[res][res2]: + seriesresults = results[res][res2]['result'] + rawlogs = ptestresult_get_rawlogs(seriesresults) + if rawlogs is not None: + with open(dst.replace(fn, "ptest-raw.log"), "w+") as f: + f.write(rawlogs) + if 'ptestresult.sections' in seriesresults: + for i in seriesresults['ptestresult.sections']: + sectionlog = ptestresult_get_log(seriesresults, i) + if sectionlog is not None: + with open(dst.replace(fn, "ptest-%s.log" % i), "w+") as f: + f.write(sectionlog) + +def git_get_result(repo, tags, configmap=store_map): + git_objs = [] + for tag in tags: + files = repo.run_cmd(['ls-tree', "--name-only", "-r", tag]).splitlines() + git_objs.extend([tag + ':' + f for f in files if f.endswith("testresults.json")]) + + def parse_json_stream(data): + """Parse multiple concatenated JSON objects""" + objs = [] + json_d = "" + for line in data.splitlines(): + if line == '}{': + json_d += '}' + objs.append(json.loads(json_d)) + json_d = '{' + else: + json_d += line + objs.append(json.loads(json_d)) + return objs + + # Optimize by reading all data with one git command + results = {} + for obj in parse_json_stream(repo.run_cmd(['show'] + git_objs + ['--'])): + append_resultsdata(results, obj, configmap=configmap) + + return results + +def test_run_results(results): + """ + Convenient generator function that iterates over all test runs that have a + result section. + + Generates a tuple of: + (result json file path, test run name, test run (dict), test run "results" (dict)) + for each test run that has a "result" section + """ + for path in results: + for run_name, test_run in results[path].items(): + if not 'result' in test_run: + continue + yield path, run_name, test_run, test_run['result'] + diff --git a/scripts/lib/resulttool/store.py b/scripts/lib/resulttool/store.py new file mode 100644 index 0000000000..e0951f0a8f --- /dev/null +++ b/scripts/lib/resulttool/store.py @@ -0,0 +1,104 @@ +# resulttool - store test results +# +# Copyright (c) 2019, Intel Corporation. +# Copyright (c) 2019, Linux Foundation +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import tempfile +import os +import subprocess +import json +import shutil +import scriptpath +scriptpath.add_bitbake_lib_path() +scriptpath.add_oe_lib_path() +import resulttool.resultutils as resultutils +import oeqa.utils.gitarchive as gitarchive + + +def store(args, logger): + tempdir = tempfile.mkdtemp(prefix='testresults.') + try: + configvars = resultutils.extra_configvars.copy() + if args.executed_by: + configvars['EXECUTED_BY'] = args.executed_by + if args.extra_test_env: + configvars['EXTRA_TEST_ENV'] = args.extra_test_env + results = {} + logger.info('Reading files from %s' % args.source) + if resultutils.is_url(args.source) or os.path.isfile(args.source): + resultutils.append_resultsdata(results, args.source, configvars=configvars) + else: + for root, dirs, files in os.walk(args.source): + for name in files: + f = os.path.join(root, name) + if name == "testresults.json": + resultutils.append_resultsdata(results, f, configvars=configvars) + elif args.all: + dst = f.replace(args.source, tempdir + "/") + os.makedirs(os.path.dirname(dst), exist_ok=True) + shutil.copyfile(f, dst) + + revisions = {} + + if not results and not args.all: + if args.allow_empty: + logger.info("No results found to store") + return 0 + logger.error("No results found to store") + return 1 + + # Find the branch/commit/commit_count and ensure they all match + for suite in results: + for result in results[suite]: + config = results[suite][result]['configuration']['LAYERS']['meta'] + revision = (config['commit'], config['branch'], str(config['commit_count'])) + if revision not in revisions: + revisions[revision] = {} + if suite not in revisions[revision]: + revisions[revision][suite] = {} + revisions[revision][suite][result] = results[suite][result] + + logger.info("Found %d revisions to store" % len(revisions)) + + for r in revisions: + results = revisions[r] + keywords = {'commit': r[0], 'branch': r[1], "commit_count": r[2]} + subprocess.check_call(["find", tempdir, "!", "-path", "./.git/*", "-delete"]) + resultutils.save_resultsdata(results, tempdir, ptestlogs=True) + + logger.info('Storing test result into git repository %s' % args.git_dir) + + gitarchive.gitarchive(tempdir, args.git_dir, False, False, + "Results of {branch}:{commit}", "branch: {branch}\ncommit: {commit}", "{branch}", + False, "{branch}/{commit_count}-g{commit}/{tag_number}", + 'Test run #{tag_number} of {branch}:{commit}', '', + [], [], False, keywords, logger) + + finally: + subprocess.check_call(["rm", "-rf", tempdir]) + + return 0 + +def register_commands(subparsers): + """Register subcommands from this plugin""" + parser_build = subparsers.add_parser('store', help='store test results into a git repository', + description='takes a results file or directory of results files and stores ' + 'them into the destination git repository, splitting out the results ' + 'files as configured', + group='setup') + parser_build.set_defaults(func=store) + parser_build.add_argument('source', + help='source file/directory/URL that contain the test result files to be stored') + parser_build.add_argument('git_dir', + help='the location of the git repository to store the results in') + parser_build.add_argument('-a', '--all', action='store_true', + help='include all files, not just testresults.json files') + parser_build.add_argument('-e', '--allow-empty', action='store_true', + help='don\'t error if no results to store are found') + parser_build.add_argument('-x', '--executed-by', default='', + help='add executed-by configuration to each result file') + parser_build.add_argument('-t', '--extra-test-env', default='', + help='add extra test environment data to each result file configuration') diff --git a/scripts/lib/resulttool/template/test_report_full_text.txt b/scripts/lib/resulttool/template/test_report_full_text.txt new file mode 100644 index 0000000000..2efba2ef6f --- /dev/null +++ b/scripts/lib/resulttool/template/test_report_full_text.txt @@ -0,0 +1,79 @@ +============================================================================================================== +Test Result Status Summary (Counts/Percentages sorted by testseries, ID) +============================================================================================================== +-------------------------------------------------------------------------------------------------------------- +{{ 'Test Series'.ljust(maxlen['testseries']) }} | {{ 'ID'.ljust(maxlen['result_id']) }} | {{ 'Passed'.ljust(maxlen['passed']) }} | {{ 'Failed'.ljust(maxlen['failed']) }} | {{ 'Skipped'.ljust(maxlen['skipped']) }} +-------------------------------------------------------------------------------------------------------------- +{% for report in reportvalues |sort(attribute='sort') %} +{{ report.testseries.ljust(maxlen['testseries']) }} | {{ report.result_id.ljust(maxlen['result_id']) }} | {{ (report.passed|string).ljust(maxlen['passed']) }} | {{ (report.failed|string).ljust(maxlen['failed']) }} | {{ (report.skipped|string).ljust(maxlen['skipped']) }} +{% endfor %} +-------------------------------------------------------------------------------------------------------------- +{{ 'Total'.ljust(maxlen['testseries']) }} | {{ reporttotalvalues['count'].ljust(maxlen['result_id']) }} | {{ reporttotalvalues['passed'].ljust(maxlen['passed']) }} | {{ reporttotalvalues['failed'].ljust(maxlen['failed']) }} | {{ reporttotalvalues['skipped'].ljust(maxlen['skipped']) }} +-------------------------------------------------------------------------------------------------------------- + +{% for machine in machines %} +{% if ptests[machine] %} +============================================================================================================== +{{ machine }} PTest Result Summary +============================================================================================================== +-------------------------------------------------------------------------------------------------------------- +{{ 'Recipe'.ljust(maxlen['ptest']) }} | {{ 'Passed'.ljust(maxlen['passed']) }} | {{ 'Failed'.ljust(maxlen['failed']) }} | {{ 'Skipped'.ljust(maxlen['skipped']) }} | {{ 'Time(s)'.ljust(10) }} +-------------------------------------------------------------------------------------------------------------- +{% for ptest in ptests[machine] |sort %} +{{ ptest.ljust(maxlen['ptest']) }} | {{ (ptests[machine][ptest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ptests[machine][ptest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ptests[machine][ptest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ptests[machine][ptest]['duration']|string) }} +{% endfor %} +-------------------------------------------------------------------------------------------------------------- + +{% endif %} +{% endfor %} + +{% for machine in machines %} +{% if ltptests[machine] %} +============================================================================================================== +{{ machine }} Ltp Test Result Summary +============================================================================================================== +-------------------------------------------------------------------------------------------------------------- +{{ 'Recipe'.ljust(maxlen['ltptest']) }} | {{ 'Passed'.ljust(maxlen['passed']) }} | {{ 'Failed'.ljust(maxlen['failed']) }} | {{ 'Skipped'.ljust(maxlen['skipped']) }} | {{ 'Time(s)'.ljust(10) }} +-------------------------------------------------------------------------------------------------------------- +{% for ltptest in ltptests[machine] |sort %} +{{ ltptest.ljust(maxlen['ltptest']) }} | {{ (ltptests[machine][ltptest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ltptests[machine][ltptest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ltptests[machine][ltptest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ltptests[machine][ltptest]['duration']|string) }} +{% endfor %} +-------------------------------------------------------------------------------------------------------------- + +{% endif %} +{% endfor %} + +{% for machine in machines %} +{% if ltpposixtests[machine] %} +============================================================================================================== +{{ machine }} Ltp Posix Result Summary +============================================================================================================== +-------------------------------------------------------------------------------------------------------------- +{{ 'Recipe'.ljust(maxlen['ltpposixtest']) }} | {{ 'Passed'.ljust(maxlen['passed']) }} | {{ 'Failed'.ljust(maxlen['failed']) }} | {{ 'Skipped'.ljust(maxlen['skipped']) }} | {{ 'Time(s)'.ljust(10) }} +-------------------------------------------------------------------------------------------------------------- +{% for ltpposixtest in ltpposixtests[machine] |sort %} +{{ ltpposixtest.ljust(maxlen['ltpposixtest']) }} | {{ (ltpposixtests[machine][ltpposixtest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ltpposixtests[machine][ltpposixtest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ltpposixtests[machine][ltpposixtest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ltpposixtests[machine][ltpposixtest]['duration']|string) }} +{% endfor %} +-------------------------------------------------------------------------------------------------------------- + +{% endif %} +{% endfor %} + + +============================================================================================================== +Failed test cases (sorted by testseries, ID) +============================================================================================================== +{% if havefailed %} +-------------------------------------------------------------------------------------------------------------- +{% for report in reportvalues |sort(attribute='sort') %} +{% if report.failed_testcases %} +testseries | result_id : {{ report.testseries }} | {{ report.result_id }} +{% for testcase in report.failed_testcases %} + {{ testcase }} +{% endfor %} +{% endif %} +{% endfor %} +-------------------------------------------------------------------------------------------------------------- +{% else %} +There were no test failures +{% endif %} |