diff options
author | Markus Lehtonen <markus.lehtonen@linux.intel.com> | 2017-03-22 10:23:37 +0200 |
---|---|---|
committer | Markus Lehtonen <markus.lehtonen@linux.intel.com> | 2017-03-31 16:29:05 +0300 |
commit | 6fee5984a80cdc89414abb684a5d8b5a77c18f2c (patch) | |
tree | 9cdfea6bf0f97b3129ede104fb7791de0fb06eda | |
parent | f1cce62c1e2e0f10b4d48fbd55d90ad37624b5e1 (diff) | |
download | openembedded-core-contrib-6fee5984a80cdc89414abb684a5d8b5a77c18f2c.tar.gz |
build-perf-git-import.py: implement --buildstats=c option
Combine buildstats into one (optimized) JSON file. Commit this file
under refs/notes/buildstats in order to save space.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
-rwxr-xr-x | scripts/contrib/build-perf-git-import.py | 99 |
1 files changed, 84 insertions, 15 deletions
diff --git a/scripts/contrib/build-perf-git-import.py b/scripts/contrib/build-perf-git-import.py index 787de7aabc..9993f90e1e 100755 --- a/scripts/contrib/build-perf-git-import.py +++ b/scripts/contrib/build-perf-git-import.py @@ -46,6 +46,12 @@ log = logging.getLogger() TEST_STATUSES = ('SUCCESS', 'FAILURE', 'ERROR', 'SKIPPED', 'EXPECTED_FAILURE', 'UNEXPECTED_SUCCESS') +BS_RUSAGE_FIELDS = ('ru_utime', 'ru_stime', 'ru_maxrss', 'ru_minflt', + 'ru_majflt', 'ru_inblock', 'ru_oublock', 'ru_nvcsw', + 'ru_nivcsw') +BS_IOSTAT_FIELDS = ('rchar', 'wchar', 'syscr', 'syscw', 'read_bytes', + 'write_bytes', 'cancelled_write_bytes') + class CommitError(Exception): """Script's internal error handling""" @@ -305,37 +311,65 @@ def time_log_to_json(time_log): def optimize_buildstat_task(task_data): """Optimize JSON formatted buildstat task data""" - bs_rusage_fields = ('ru_utime', 'ru_stime', 'ru_maxrss', 'ru_minflt', - 'ru_majflt', 'ru_inblock', 'ru_oublock', 'ru_nvcsw', - 'ru_nivcsw') - bs_iostat_fields = ('rchar', 'wchar', 'syscr', 'syscw', 'read_bytes', - 'write_bytes', 'cancelled_write_bytes') - if 'iostat' in task_data: - optimized = [task_data['iostat'][k] for k in bs_iostat_fields] + optimized = [task_data['iostat'][k] for k in BS_IOSTAT_FIELDS] task_data['iostat'] = optimized if 'rusage' in task_data: - optimized = [task_data['rusage'][k] for k in bs_rusage_fields] + optimized = [task_data['rusage'][k] for k in BS_RUSAGE_FIELDS] task_data['rusage'] = optimized if 'child_rusage' in task_data: - optimized = [task_data['child_rusage'][k] for k in bs_rusage_fields] + optimized = [task_data['child_rusage'][k] for k in BS_RUSAGE_FIELDS] task_data['child_rusage'] = optimized +def optimize_buildstats(buildstats): + """Optimize buildstats data""" + for recipe in buildstats: + for task, data in recipe['tasks'].items(): + optimize_buildstat_task(data) + def optimize_buildstats_file(buildstats_file): """Optimize buildstats JSON file""" with open(buildstats_file) as fobj: buildstats = json.load(fobj, object_pairs_hook=OrderedDict) - for recipe in buildstats: - for task, data in recipe['tasks'].items(): - optimize_buildstat_task(data) + optimize_buildstats(buildstats) # Write buildstats back into json file with open(buildstats_file, 'w') as fobj: json.dump(buildstats, fobj) +def combine_buildstats_files(results_data, results_dir): + """Combine buildstats into one JSON file""" + buildstats = OrderedDict() + new_buildstats_file = os.path.join(results_dir, 'buildstats.json') + + if os.path.exists(new_buildstats_file): + # Do not overwrite an existing buildstats file + log.debug("Conbined buildstats file already exists") + return + + for test in results_data['tests'].values(): + for measurement in test['measurements'].values(): + if 'buildstats_file' in measurement['values']: + buildstats_file = os.path.join(results_dir, + measurement['values']['buildstats_file']) + with open(buildstats_file) as fobj: + meas_bs = json.load(fobj, object_pairs_hook=OrderedDict) + #optimize_buildstats(meas_bs) + + bs_key = test['name'] + '.' + measurement['name'] + buildstats[bs_key] = meas_bs + + # Remove separate buildstats file + del(measurement['values']['buildstats_file']) + os.unlink(buildstats_file) + + # Write-out combined buildstats + with open(os.path.join(results_dir, 'buildstats.json'), 'w') as fobj: + json.dump(buildstats, fobj) + def convert_buildstats(indir, outfile, optimize=False): """Convert buildstats into JSON format""" @@ -634,6 +668,10 @@ def convert_old_results(poky_repo, results_dir, tester_host, new_fmt, out_log.records[0].time)), ('tests', tests))) + # Combine buildstats + if buildstats == 'c': + combine_buildstats_files(results, results_dir) + # Create metadata dict metadata = create_metadata(tester_host, {'commit': git_rev, @@ -713,6 +751,10 @@ def convert_json_results(poky_repo, results_dir, new_fmt, metadata_override, elif buildstats == 'o': optimize_buildstats_file(buildstats_file) + # Combine buildstats + if buildstats == 'c': + combine_buildstats_files(results, results_dir) + # Remove old results file os.unlink(results_file) @@ -913,6 +955,17 @@ def git_commit_dir(data_repo, src_dir, branch, msg, tag=None, tag_msg="", data_repo.run_cmd(['tag', '-a', '-m', tag_msg, tag, 'HEAD'], env) +def git_notes_add(data_repo, fname, ref, obj, timestamp=None): + """Add git notes""" + env = {} + if timestamp: + env['GIT_COMMITTER_DATE'] = timestamp + env['GIT_AUTHOR_DATE'] = timestamp + + log.debug('Adding %s to git notes (%s)', fname, ref) + data_repo.run_cmd(['notes', '--ref', ref, 'add', '-F', fname, obj], env) + + def import_testrun(archive, data_repo, poky_repo, branch_fmt, tag_fmt, convert=False, metadata_override=None, buildstats='y'): """Import one testrun into Git""" @@ -1090,8 +1143,23 @@ hostname: {host} tag_msg = "Test run #{} of {}:{} on {}\n".format( tag_cnt, fmt_fields['branch'], fmt_fields['rev'], fmt_fields['host']) - git_commit_dir(data_repo, results_dir, git_branch, commit_msg, - git_tag, tag_msg, git_timestamp) + + # Store buildstas.json in Git notes, thus move the file away + bs_file = os.path.join(results_dir, 'buildstats.json') + bs_tmp = os.path.abspath('buildstats.json.' + str(datetime.now().timestamp())) + if os.path.exists(bs_file): + shutil.move(bs_file, bs_tmp) + + # Commit data + try: + git_commit_dir(data_repo, results_dir, git_branch, commit_msg, + git_tag, tag_msg, git_timestamp) + if os.path.exists(bs_tmp): + git_notes_add(data_repo, bs_tmp, 'buildstats/' + git_branch, + 'HEAD', git_timestamp) + finally: + if os.path.exists(bs_tmp): + os.unlink(bs_tmp) finally: shutil.rmtree(tmpdir) return True, "OK" @@ -1157,7 +1225,8 @@ def parse_args(argv=None): help="Convert results to new format") parser.add_argument('-M', '--metadata-override', type=os.path.abspath, help="Pre-filled test metadata in JSON format") - parser.add_argument('--buildstats', choices=('y', 'n', 'o'), default='y', + parser.add_argument('--buildstats', choices=('y', 'n', 'o', 'c'), + default='c', help="Import buildstats") parser.add_argument('-P', '--poky-git', type=os.path.abspath, help="Path to poky clone") |