diff options
Diffstat (limited to 'meta/lib')
297 files changed, 12870 insertions, 1891 deletions
diff --git a/meta/lib/bblayers/buildconf.py b/meta/lib/bblayers/buildconf.py new file mode 100644 index 0000000000..722cf0723c --- /dev/null +++ b/meta/lib/bblayers/buildconf.py @@ -0,0 +1,84 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import logging +import os +import sys + +from bblayers.common import LayerPlugin + +logger = logging.getLogger('bitbake-layers') + +sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) + +import oe.buildcfg + +def plugin_init(plugins): + return BuildConfPlugin() + +class BuildConfPlugin(LayerPlugin): + notes_fixme = """FIXME: Please place here the detailed instructions for using this build configuration. +They will be shown to the users when they set up their builds via TEMPLATECONF. +""" + summary_fixme = """FIXME: Please place here the short summary of what this build configuration is for. +It will be shown to the users when they set up their builds via TEMPLATECONF. +""" + + def _save_conf(self, templatename, templatepath, oecorepath, relpaths_to_oecore): + confdir = os.path.join(os.environ["BBPATH"], "conf") + destdir = os.path.join(templatepath, "conf", "templates", templatename) + os.makedirs(destdir, exist_ok=True) + + with open(os.path.join(confdir, "local.conf")) as src: + with open(os.path.join(destdir, "local.conf.sample"), 'w') as dest: + dest.write(src.read()) + + with open(os.path.join(confdir, "bblayers.conf")) as src: + with open(os.path.join(destdir, "bblayers.conf.sample"), 'w') as dest: + bblayers_data = src.read() + + for (abspath, relpath) in relpaths_to_oecore: + bblayers_data = bblayers_data.replace(abspath, "##OEROOT##/" + relpath) + dest.write(bblayers_data) + + with open(os.path.join(destdir, "conf-summary.txt"), 'w') as dest: + dest.write(self.summary_fixme) + with open(os.path.join(destdir, "conf-notes.txt"), 'w') as dest: + dest.write(self.notes_fixme) + + logger.info("""Configuration template placed into {} +Please review the files in there, and particularly provide a configuration summary in {} +and notes in {} +You can try out the configuration with +TEMPLATECONF={} . {}/oe-init-build-env build-try-{}""" +.format(destdir, os.path.join(destdir, "conf-summary.txt"), os.path.join(destdir, "conf-notes.txt"), destdir, oecorepath, templatename)) + + def do_save_build_conf(self, args): + """ Save the currently active build configuration (conf/local.conf, conf/bblayers.conf) as a template into a layer.\n This template can later be used for setting up builds via TEMPLATECONF. """ + layers = oe.buildcfg.get_layer_revisions(self.tinfoil.config_data) + targetlayer = None + oecore = None + + for l in layers: + if os.path.abspath(l[0]) == os.path.abspath(args.layerpath): + targetlayer = l[0] + if l[1] == 'meta': + oecore = os.path.dirname(l[0]) + + if not targetlayer: + logger.error("Layer {} not in one of the currently enabled layers:\n{}".format(args.layerpath, "\n".join([l[0] for l in layers]))) + elif not oecore: + logger.error("Openembedded-core not in one of the currently enabled layers:\n{}".format("\n".join([l[0] for l in layers]))) + else: + relpaths_to_oecore = [(l[0], os.path.relpath(l[0], start=oecore)) for l in layers] + self._save_conf(args.templatename, targetlayer, oecore, relpaths_to_oecore) + + def register_commands(self, sp): + parser_build_conf = self.add_command(sp, 'save-build-conf', self.do_save_build_conf, parserecipes=False) + parser_build_conf.add_argument('layerpath', + help='The path to the layer where the configuration template should be saved.') + parser_build_conf.add_argument('templatename', + help='The name of the configuration template.') diff --git a/meta/lib/bblayers/create.py b/meta/lib/bblayers/create.py index 7ddb777dc7..517554c587 100644 --- a/meta/lib/bblayers/create.py +++ b/meta/lib/bblayers/create.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # @@ -10,6 +12,7 @@ import shutil import bb.utils from bblayers.common import LayerPlugin +from bblayers.action import ActionPlugin logger = logging.getLogger('bitbake-layers') @@ -51,7 +54,7 @@ class CreatePlugin(LayerPlugin): shutil.copy(license_src, license_dst) # Get the compat value for core layer. - compat = self.tinfoil.config_data.getVar('LAYERSERIES_COMPAT_core') or "" + compat = self.tinfoil.config_data.getVar('LAYERSERIES_CORENAMES') or "" # Create the layer.conf from templates/layer.conf layerconf_template = read_template('layer.conf').format( @@ -67,11 +70,19 @@ class CreatePlugin(LayerPlugin): with open(os.path.join(example, args.examplerecipe + '_%s.bb') % args.version, 'w') as fd: fd.write(example_template) - logger.plain('Add your new layer with \'bitbake-layers add-layer %s\'' % args.layerdir) + if args.add_layer: + # Add the layer to bblayers.conf + args.layerdir = [layerdir] + ActionPlugin.do_add_layer(self, args) + logger.plain('Layer added %s' % args.layerdir) + + else: + logger.plain('Add your new layer with \'bitbake-layers add-layer %s\'' % args.layerdir) def register_commands(self, sp): parser_create_layer = self.add_command(sp, 'create-layer', self.do_create_layer, parserecipes=False) parser_create_layer.add_argument('layerdir', help='Layer directory to create') + parser_create_layer.add_argument('--add-layer', '-a', action='store_true', help='Add the layer to bblayers.conf after creation') parser_create_layer.add_argument('--layerid', '-i', help='Layer id to use if different from layername') parser_create_layer.add_argument('--priority', '-p', default=6, help='Priority of recipes in layer') parser_create_layer.add_argument('--example-recipe-name', '-e', dest='examplerecipe', default='example', help='Filename of the example recipe') diff --git a/meta/lib/bblayers/makesetup.py b/meta/lib/bblayers/makesetup.py new file mode 100644 index 0000000000..99d5973760 --- /dev/null +++ b/meta/lib/bblayers/makesetup.py @@ -0,0 +1,102 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import logging +import os +import sys + +import bb.utils + +from bblayers.common import LayerPlugin + +logger = logging.getLogger('bitbake-layers') + +sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) + +import oe.buildcfg + +def plugin_init(plugins): + return MakeSetupPlugin() + +class MakeSetupPlugin(LayerPlugin): + + def _get_remotes_with_url(self, repo_path): + remotes = {} + for r in oe.buildcfg.get_metadata_git_remotes(repo_path): + remotes[r] = {'uri':oe.buildcfg.get_metadata_git_remote_url(repo_path, r)} + return remotes + + def _is_submodule(self, repo_path): + # This is slightly brittle: git does not offer a way to tell whether + # a given repo dir is a submodule checkout, so we need to rely on .git + # being a file (rather than a dir like it is in standalone checkouts). + # The file typically contains a gitdir pointer to elsewhere. + return os.path.isfile(os.path.join(repo_path,".git")) + + def make_repo_config(self, destdir): + """ This is a helper function for the writer plugins that discovers currently configured layers. + The writers do not have to use it, but it can save a bit of work and avoid duplicated code, hence it is + available here. """ + repos = {} + layers = oe.buildcfg.get_layer_revisions(self.tinfoil.config_data) + destdir_repo = oe.buildcfg.get_metadata_git_toplevel(destdir) + + for (l_path, l_name, l_branch, l_rev, l_ismodified) in layers: + if l_name == 'workspace': + continue + if l_ismodified: + logger.error("Layer {name} in {path} has uncommitted modifications or is not in a git repository.".format(name=l_name,path=l_path)) + return + repo_path = oe.buildcfg.get_metadata_git_toplevel(l_path) + + if self._is_submodule(repo_path): + continue + if repo_path not in repos.keys(): + repos[repo_path] = {'path':os.path.basename(repo_path),'git-remote':{ + 'rev':l_rev, + 'branch':l_branch, + 'remotes':self._get_remotes_with_url(repo_path), + 'describe':oe.buildcfg.get_metadata_git_describe(repo_path)}} + if repo_path == destdir_repo: + repos[repo_path]['contains_this_file'] = True + if not repos[repo_path]['git-remote']['remotes'] and not repos[repo_path]['contains_this_file']: + logger.error("Layer repository in {path} does not have any remotes configured. Please add at least one with 'git remote add'.".format(path=repo_path)) + return + + top_path = os.path.commonpath([os.path.dirname(r) for r in repos.keys()]) + + repos_nopaths = {} + for r in repos.keys(): + r_nopath = os.path.basename(r) + repos_nopaths[r_nopath] = repos[r] + r_relpath = os.path.relpath(r, top_path) + repos_nopaths[r_nopath]['path'] = r_relpath + return repos_nopaths + + def do_make_setup(self, args): + """ Writes out a configuration file and/or a script that replicate the directory structure and revisions of the layers in a current build. """ + for p in self.plugins: + if str(p) == args.writer: + p.do_write(self, args) + + def register_commands(self, sp): + parser_setup_layers = self.add_command(sp, 'create-layers-setup', self.do_make_setup, parserecipes=False) + parser_setup_layers.add_argument('destdir', + help='Directory where to write the output\n(if it is inside one of the layers, the layer becomes a bootstrap repository and thus will be excluded from fetching).') + parser_setup_layers.add_argument('--output-prefix', '-o', + help='File name prefix for the output files, if the default (setup-layers) is undesirable.') + + self.plugins = [] + + for path in (self.tinfoil.config_data.getVar('BBPATH').split(':')): + pluginpath = os.path.join(path, 'lib', 'bblayers', 'setupwriters') + bb.utils.load_plugins(logger, self.plugins, pluginpath) + + parser_setup_layers.add_argument('--writer', '-w', choices=[str(p) for p in self.plugins], help="Choose the output format (defaults to oe-setup-layers).\n\nCurrently supported options are:\noe-setup-layers - a self-contained python script and a json config for it.\n\n", default="oe-setup-layers") + + for plugin in self.plugins: + if hasattr(plugin, 'register_arguments'): + plugin.register_arguments(parser_setup_layers) diff --git a/meta/lib/bblayers/setupwriters/oe-setup-layers.py b/meta/lib/bblayers/setupwriters/oe-setup-layers.py new file mode 100644 index 0000000000..59ca968ff3 --- /dev/null +++ b/meta/lib/bblayers/setupwriters/oe-setup-layers.py @@ -0,0 +1,117 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import logging +import os +import json +import stat + +logger = logging.getLogger('bitbake-layers') + +def plugin_init(plugins): + return OeSetupLayersWriter() + +class OeSetupLayersWriter(): + + def __str__(self): + return "oe-setup-layers" + + def _write_python(self, input, output): + with open(input) as f: + script = f.read() + with open(output, 'w') as f: + f.write(script) + st = os.stat(output) + os.chmod(output, st.st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) + + def _write_json(self, repos, output): + with open(output, 'w') as f: + json.dump(repos, f, sort_keys=True, indent=4) + + def _read_repo_config(self, json_path): + with open(json_path) as f: + json_config = json.load(f) + + supported_versions = ["1.0"] + if json_config["version"] not in supported_versions: + err = "File {} has version {}, which is not in supported versions: {}".format(json_path, json_config["version"], supported_versions) + logger.error(err) + raise Exception(err) + + return json_config + + def _modify_repo_config(self, json_config, args): + sources = json_config['sources'] + for pair in args.custom_references: + try: + repo, rev = pair.split(':', maxsplit=1) + except ValueError: + err = "Invalid custom reference specified: '{}'. Provide one using 'REPOSITORY:REFERENCE'.".format(pair) + logger.error(err) + raise Exception(err) + if not repo in sources.keys(): + err = "Repository {} does not exist in setup-layers config".format(repo) + logger.error(err) + raise Exception(err) + + layer_remote = json_config['sources'][repo]['git-remote'] + layer_remote['rev'] = rev + # Clear describe + layer_remote['describe'] = '' + + def do_write(self, parent, args): + """ Writes out a python script and a json config that replicate the directory structure and revisions of the layers in a current build. """ + output = args.output_prefix or "setup-layers" + output = os.path.join(os.path.abspath(args.destdir), output) + + if args.update: + # Modify existing layers setup + if args.custom_references is None: + err = "No custom reference specified. Please provide one using '--use-custom-reference REPOSITORY:REFERENCE'." + logger.error(err) + raise Exception(err) + + json = self._read_repo_config(output + ".json") + if not 'sources' in json.keys(): + err = "File {}.json does not contain valid layer sources.".format(output) + logger.error(err) + raise Exception(err) + + else: + # Create new layers setup + if not os.path.exists(args.destdir): + os.makedirs(args.destdir) + repos = parent.make_repo_config(args.destdir) + json = {"version":"1.0","sources":repos} + if not repos: + err = "Could not determine layer sources" + logger.error(err) + raise Exception(err) + + if args.custom_references is not None: + self._modify_repo_config(json, args) + + self._write_json(json, output + ".json") + logger.info('Created {}.json'.format(output)) + if not args.json_only: + self._write_python(os.path.join(os.path.dirname(__file__),'../../../../scripts/oe-setup-layers'), output) + logger.info('Created {}'.format(output)) + + def register_arguments(self, parser): + parser.add_argument('--json-only', action='store_true', + help='When using the oe-setup-layers writer, write only the layer configuruation in json format. Otherwise, also a copy of scripts/oe-setup-layers (from oe-core or poky) is provided, which is a self contained python script that fetches all the needed layers and sets them to correct revisions using the data from the json.') + + parser.add_argument('--update', '-u', + action='store_true', + help=("Instead of writing a new json file, update an existing layer setup json file with custom references provided via the '--use-custom-reference' option." + "\nThis will only update repositories for which a custom reference is specified, all other repositores will be left unchanged.")) + parser.add_argument('--use-custom-reference', '-r', + action='append', + dest='custom_references', + metavar='REPOSITORY:REFERENCE', + help=("A pair consisting of a repository and a custom reference to use for it (by default the currently checked out commit id would be written out)." + "\nThis value can be any reference that 'git checkout' would accept, and is not checked for validity." + "\nThis option can be used multiple times.")) diff --git a/meta/lib/buildstats.py b/meta/lib/buildstats.py index 99a8303d5e..1ffe679801 100644 --- a/meta/lib/buildstats.py +++ b/meta/lib/buildstats.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # # Implements system state sampling. Called by buildstats.bbclass. diff --git a/meta/lib/oe/__init__.py b/meta/lib/oe/__init__.py index 4e7c09da04..6eb536ad28 100644 --- a/meta/lib/oe/__init__.py +++ b/meta/lib/oe/__init__.py @@ -1,6 +1,12 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # from pkgutil import extend_path __path__ = extend_path(__path__, __name__) + +BBIMPORTS = ["data", "path", "utils", "types", "package", "packagedata", \ + "packagegroup", "sstatesig", "lsb", "cachedpath", "license", \ + "qa", "reproducible", "rust", "buildcfg", "go"] diff --git a/meta/lib/oe/buildcfg.py b/meta/lib/oe/buildcfg.py index 90f5e05715..27b059b834 100644 --- a/meta/lib/oe/buildcfg.py +++ b/meta/lib/oe/buildcfg.py @@ -1,32 +1,62 @@ +import os import subprocess import bb.process def detect_revision(d): path = get_scmbasepath(d) - return get_metadata_git_revision(path, d) + return get_metadata_git_revision(path) def detect_branch(d): path = get_scmbasepath(d) - return get_metadata_git_branch(path, d) + return get_metadata_git_branch(path) def get_scmbasepath(d): return os.path.join(d.getVar('COREBASE'), 'meta') -def get_metadata_git_branch(path, d): +def get_metadata_git_branch(path): try: rev, _ = bb.process.run('git rev-parse --abbrev-ref HEAD', cwd=path) except bb.process.ExecutionError: rev = '<unknown>' return rev.strip() -def get_metadata_git_revision(path, d): +def get_metadata_git_revision(path): try: rev, _ = bb.process.run('git rev-parse HEAD', cwd=path) except bb.process.ExecutionError: rev = '<unknown>' return rev.strip() +def get_metadata_git_toplevel(path): + try: + toplevel, _ = bb.process.run('git rev-parse --show-toplevel', cwd=path) + except bb.process.ExecutionError: + return "" + return toplevel.strip() + +def get_metadata_git_remotes(path): + try: + remotes_list, _ = bb.process.run('git remote', cwd=path) + remotes = remotes_list.split() + except bb.process.ExecutionError: + remotes = [] + return remotes + +def get_metadata_git_remote_url(path, remote): + try: + uri, _ = bb.process.run('git remote get-url {remote}'.format(remote=remote), cwd=path) + except bb.process.ExecutionError: + return "" + return uri.strip() + +def get_metadata_git_describe(path): + try: + describe, _ = bb.process.run('git describe --tags', cwd=path) + except bb.process.ExecutionError: + return "" + return describe.strip() + def is_layer_modified(path): try: subprocess.check_output("""cd %s; export PSEUDO_UNLOAD=1; set -e; @@ -45,5 +75,5 @@ def get_layer_revisions(d): layers = (d.getVar("BBLAYERS") or "").split() revisions = [] for i in layers: - revisions.append((i, os.path.basename(i), get_metadata_git_branch(i, None).strip(), get_metadata_git_revision(i, None), is_layer_modified(i))) + revisions.append((i, os.path.basename(i), get_metadata_git_branch(i).strip(), get_metadata_git_revision(i), is_layer_modified(i))) return revisions diff --git a/meta/lib/oe/buildhistory_analysis.py b/meta/lib/oe/buildhistory_analysis.py index b1856846b6..4edad01580 100644 --- a/meta/lib/oe/buildhistory_analysis.py +++ b/meta/lib/oe/buildhistory_analysis.py @@ -562,7 +562,7 @@ def compare_siglists(a_blob, b_blob, taskdiff=False): elif not hash2 in hashfiles: out.append("Unable to find matching sigdata for %s with hash %s" % (desc, hash2)) else: - out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb, collapsed=True) + out2 = bb.siggen.compare_sigfiles(hashfiles[hash1]['path'], hashfiles[hash2]['path'], recursecb, collapsed=True) for line in out2: m = hashlib.sha256() m.update(line.encode('utf-8')) diff --git a/meta/lib/oe/cachedpath.py b/meta/lib/oe/cachedpath.py index 254257a83f..0138b791d4 100644 --- a/meta/lib/oe/cachedpath.py +++ b/meta/lib/oe/cachedpath.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # # Based on standard python library functions but avoid diff --git a/meta/lib/oe/classextend.py b/meta/lib/oe/classextend.py index e08d788b75..5161d33d2d 100644 --- a/meta/lib/oe/classextend.py +++ b/meta/lib/oe/classextend.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # @@ -30,6 +32,9 @@ class ClassExtender(object): if name.endswith("-" + self.extname): name = name.replace("-" + self.extname, "") if name.startswith("virtual/"): + # Assume large numbers of dashes means a triplet is present and we don't need to convert + if name.count("-") >= 3 and name.endswith(("-go", "-binutils", "-gcc", "-g++")): + return name subs = name.split("/", 1)[1] if not subs.startswith(self.extname): return "virtual/" + self.extname + "-" + subs @@ -148,9 +153,7 @@ class NativesdkClassExtender(ClassExtender): def map_depends(self, dep): if dep.startswith(self.extname): return dep - if dep.endswith(("-gcc", "-g++")): - return dep + "-crosssdk" - elif dep.endswith(("-native", "-native-runtime")) or ('nativesdk-' in dep) or ('-cross-' in dep) or ('-crosssdk-' in dep): + if dep.endswith(("-native", "-native-runtime")) or ('nativesdk-' in dep) or ('-cross-' in dep) or ('-crosssdk-' in dep): return dep else: return self.extend_name(dep) diff --git a/meta/lib/oe/classutils.py b/meta/lib/oe/classutils.py index 08bb66b365..ec3f6ad720 100644 --- a/meta/lib/oe/classutils.py +++ b/meta/lib/oe/classutils.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/meta/lib/oe/copy_buildsystem.py b/meta/lib/oe/copy_buildsystem.py index 79642fd76a..81abfbf9e2 100644 --- a/meta/lib/oe/copy_buildsystem.py +++ b/meta/lib/oe/copy_buildsystem.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # # This class should provide easy access to the different aspects of the @@ -20,7 +22,7 @@ def _smart_copy(src, dest): mode = os.stat(src).st_mode if stat.S_ISDIR(mode): bb.utils.mkdirhier(dest) - cmd = "tar --exclude='.git' --exclude='__pycache__' --xattrs --xattrs-include='*' -chf - -C %s -p . \ + cmd = "tar --exclude='.git' --exclude='__pycache__' --xattrs --xattrs-include='*' -cf - -C %s -p . \ | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dest) subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) else: diff --git a/meta/lib/oe/cve_check.py b/meta/lib/oe/cve_check.py index aa06497727..ed5c714cb8 100644 --- a/meta/lib/oe/cve_check.py +++ b/meta/lib/oe/cve_check.py @@ -1,3 +1,9 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + import collections import re import itertools @@ -73,33 +79,33 @@ def get_patched_cves(d): import re import oe.patch - pn = d.getVar("PN") - cve_match = re.compile("CVE:( CVE\-\d{4}\-\d+)+") + cve_match = re.compile(r"CVE:( CVE-\d{4}-\d+)+") # Matches the last "CVE-YYYY-ID" in the file name, also if written # in lowercase. Possible to have multiple CVE IDs in a single # file name, but only the last one will be detected from the file name. # However, patch files contents addressing multiple CVE IDs are supported # (cve_match regular expression) - - cve_file_name_match = re.compile(".*([Cc][Vv][Ee]\-\d{4}\-\d+)") + cve_file_name_match = re.compile(r".*(CVE-\d{4}-\d+)", re.IGNORECASE) patched_cves = set() - bb.debug(2, "Looking for patches that solves CVEs for %s" % pn) - for url in oe.patch.src_patches(d): + patches = oe.patch.src_patches(d) + bb.debug(2, "Scanning %d patches for CVEs" % len(patches)) + for url in patches: patch_file = bb.fetch.decodeurl(url)[2] - # Remote compressed patches may not be unpacked, so silently ignore them - if not os.path.isfile(patch_file): - bb.warn("%s does not exist, cannot extract CVE list" % patch_file) - continue - # Check patch file name for CVE ID fname_match = cve_file_name_match.search(patch_file) if fname_match: cve = fname_match.group(1).upper() patched_cves.add(cve) - bb.debug(2, "Found CVE %s from patch file name %s" % (cve, patch_file)) + bb.debug(2, "Found %s from patch file name %s" % (cve, patch_file)) + + # Remote patches won't be present and compressed patches won't be + # unpacked, so say we're not scanning them + if not os.path.isfile(patch_file): + bb.note("%s is remote or compressed, not scanning content" % patch_file) + continue with open(patch_file, "r", encoding="utf-8") as f: try: @@ -124,6 +130,13 @@ def get_patched_cves(d): if not fname_match and not text_match: bb.debug(2, "Patch %s doesn't solve CVEs" % patch_file) + # Search for additional patched CVEs + for cve in (d.getVarFlags("CVE_STATUS") or {}): + decoded_status, _, _ = decode_cve_status(d, cve) + if decoded_status == "Patched": + bb.debug(2, "CVE %s is additionally patched" % cve) + patched_cves.add(cve) + return patched_cves @@ -143,7 +156,7 @@ def get_cpe_ids(cve_product, version): else: vendor = "*" - cpe_id = f'cpe:2.3:a:{vendor}:{product}:{version}:*:*:*:*:*:*:*' + cpe_id = 'cpe:2.3:*:{}:{}:{}:*:*:*:*:*:*:*'.format(vendor, product, version) cpe_ids.append(cpe_id) return cpe_ids @@ -159,7 +172,7 @@ def cve_check_merge_jsons(output, data): for product in output["package"]: if product["name"] == data["package"][0]["name"]: - bb.error("Error adding the same package twice") + bb.error("Error adding the same package %s twice" % product["name"]) return output["package"].append(data["package"][0]) @@ -173,3 +186,60 @@ def update_symlinks(target_path, link_path): if os.path.exists(os.path.realpath(link_path)): os.remove(link_path) os.symlink(os.path.basename(target_path), link_path) + + +def convert_cve_version(version): + """ + This function converts from CVE format to Yocto version format. + eg 8.3_p1 -> 8.3p1, 6.2_rc1 -> 6.2-rc1 + + Unless it is redefined using CVE_VERSION in the recipe, + cve_check uses the version in the name of the recipe (${PV}) + to check vulnerabilities against a CVE in the database downloaded from NVD. + + When the version has an update, i.e. + "p1" in OpenSSH 8.3p1, + "-rc1" in linux kernel 6.2-rc1, + the database stores the version as version_update (8.3_p1, 6.2_rc1). + Therefore, we must transform this version before comparing to the + recipe version. + + In this case, the parameter of the function is 8.3_p1. + If the version uses the Release Candidate format, "rc", + this function replaces the '_' by '-'. + If the version uses the Update format, "p", + this function removes the '_' completely. + """ + import re + + matches = re.match('^([0-9.]+)_((p|rc)[0-9]+)$', version) + + if not matches: + return version + + version = matches.group(1) + update = matches.group(2) + + if matches.group(3) == "rc": + return version + '-' + update + + return version + update + +def decode_cve_status(d, cve): + """ + Convert CVE_STATUS into status, detail and description. + """ + status = d.getVarFlag("CVE_STATUS", cve) + if not status: + return ("", "", "") + + status_split = status.split(':', 1) + detail = status_split[0] + description = status_split[1].strip() if (len(status_split) > 1) else "" + + status_mapping = d.getVarFlag("CVE_CHECK_STATUSMAP", detail) + if status_mapping is None: + bb.warn('Invalid detail "%s" for CVE_STATUS[%s] = "%s", fallback to Unpatched' % (detail, cve, status)) + status_mapping = "Unpatched" + + return (status_mapping, detail, description) diff --git a/meta/lib/oe/data.py b/meta/lib/oe/data.py index 602130a904..37121cfad2 100644 --- a/meta/lib/oe/data.py +++ b/meta/lib/oe/data.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/meta/lib/oe/distro_check.py b/meta/lib/oe/distro_check.py index 4b2a9bec01..3494520f40 100644 --- a/meta/lib/oe/distro_check.py +++ b/meta/lib/oe/distro_check.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/meta/lib/oe/elf.py b/meta/lib/oe/elf.py index 46c884a775..eab2349a4f 100644 --- a/meta/lib/oe/elf.py +++ b/meta/lib/oe/elf.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # @@ -19,6 +21,7 @@ def machine_dict(d): "x86_64": (62, 0, 0, True, 64), "epiphany": (4643, 0, 0, True, 32), "lm32": (138, 0, 0, False, 32), + "loongarch64":(258, 0, 0, True, 64), "mips": ( 8, 0, 0, False, 32), "mipsel": ( 8, 0, 0, True, 32), "microblaze": (189, 0, 0, False, 32), @@ -43,6 +46,7 @@ def machine_dict(d): "ia64": (50, 0, 0, True, 64), "alpha": (36902, 0, 0, True, 64), "hppa": (15, 3, 0, False, 32), + "loongarch64":(258, 0, 0, True, 64), "m68k": ( 4, 0, 0, False, 32), "mips": ( 8, 0, 0, False, 32), "mipsel": ( 8, 0, 0, True, 32), diff --git a/meta/lib/oe/go.py b/meta/lib/oe/go.py new file mode 100644 index 0000000000..dfd957d157 --- /dev/null +++ b/meta/lib/oe/go.py @@ -0,0 +1,34 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +import re + +def map_arch(a): + if re.match('i.86', a): + return '386' + elif a == 'x86_64': + return 'amd64' + elif re.match('arm.*', a): + return 'arm' + elif re.match('aarch64.*', a): + return 'arm64' + elif re.match('mips64el.*', a): + return 'mips64le' + elif re.match('mips64.*', a): + return 'mips64' + elif a == 'mips': + return 'mips' + elif a == 'mipsel': + return 'mipsle' + elif re.match('p(pc|owerpc)(64le)', a): + return 'ppc64le' + elif re.match('p(pc|owerpc)(64)', a): + return 'ppc64' + elif a == 'riscv64': + return 'riscv64' + elif a == 'loongarch64': + return 'loong64' + return '' diff --git a/meta/lib/oe/gpg_sign.py b/meta/lib/oe/gpg_sign.py index aa9bb49f2c..ede6186c84 100644 --- a/meta/lib/oe/gpg_sign.py +++ b/meta/lib/oe/gpg_sign.py @@ -1,13 +1,16 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # """Helper module for GPG signing""" -import os import bb -import subprocess +import os import shlex +import subprocess +import tempfile class LocalSigner(object): """Class for handling local (on the build host) signing""" @@ -71,8 +74,6 @@ class LocalSigner(object): cmd += ['--homedir', self.gpg_path] if armor: cmd += ['--armor'] - if output_suffix: - cmd += ['-o', input_file + "." + output_suffix] if use_sha256: cmd += ['--digest-algo', "SHA256"] @@ -81,19 +82,27 @@ class LocalSigner(object): if self.gpg_version > (2,1,): cmd += ['--pinentry-mode', 'loopback'] - cmd += [input_file] - try: if passphrase_file: with open(passphrase_file) as fobj: passphrase = fobj.readline(); - job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE) - (_, stderr) = job.communicate(passphrase.encode("utf-8")) + if not output_suffix: + output_suffix = 'asc' if armor else 'sig' + output_file = input_file + "." + output_suffix + with tempfile.TemporaryDirectory(dir=os.path.dirname(output_file)) as tmp_dir: + tmp_file = os.path.join(tmp_dir, os.path.basename(output_file)) + cmd += ['-o', tmp_file] + + cmd += [input_file] + + job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE) + (_, stderr) = job.communicate(passphrase.encode("utf-8")) - if job.returncode: - bb.fatal("GPG exited with code %d: %s" % (job.returncode, stderr.decode("utf-8"))) + if job.returncode: + bb.fatal("GPG exited with code %d: %s" % (job.returncode, stderr.decode("utf-8"))) + os.rename(tmp_file, output_file) except IOError as e: bb.error("IO error (%s): %s" % (e.errno, e.strerror)) raise Exception("Failed to sign '%s'" % input_file) diff --git a/meta/lib/oe/license.py b/meta/lib/oe/license.py index 99cfa5f733..d9c8d94da4 100644 --- a/meta/lib/oe/license.py +++ b/meta/lib/oe/license.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # """Code for parsing OpenEmbedded license strings""" diff --git a/meta/lib/oe/lsb.py b/meta/lib/oe/lsb.py index 43e46380d7..3ec03e5042 100644 --- a/meta/lib/oe/lsb.py +++ b/meta/lib/oe/lsb.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/meta/lib/oe/maketype.py b/meta/lib/oe/maketype.py index d36082c535..7a83bdf602 100644 --- a/meta/lib/oe/maketype.py +++ b/meta/lib/oe/maketype.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # """OpenEmbedded variable typing support diff --git a/meta/lib/oe/manifest.py b/meta/lib/oe/manifest.py index 1a058dcd73..61f18adc4a 100644 --- a/meta/lib/oe/manifest.py +++ b/meta/lib/oe/manifest.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/meta/lib/oe/npm_registry.py b/meta/lib/oe/npm_registry.py index 96c0affb45..d97ced7cda 100644 --- a/meta/lib/oe/npm_registry.py +++ b/meta/lib/oe/npm_registry.py @@ -1,3 +1,9 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + import bb import json import subprocess @@ -5,7 +11,7 @@ import subprocess _ALWAYS_SAFE = frozenset('ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' '0123456789' - '_.-~') + '_.-~()') MISSING_OK = object() diff --git a/meta/lib/oe/overlayfs.py b/meta/lib/oe/overlayfs.py index b5d5e88e80..8b88900f71 100644 --- a/meta/lib/oe/overlayfs.py +++ b/meta/lib/oe/overlayfs.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # # This file contains common functions for overlayfs and its QA check @@ -38,7 +40,11 @@ def unitFileList(d): bb.fatal("Missing required mount point for OVERLAYFS_MOUNT_POINT[%s] in your MACHINE configuration" % mountPoint) for mountPoint in overlayMountPoints: - for path in d.getVarFlag('OVERLAYFS_WRITABLE_PATHS', mountPoint).split(): + mountPointList = d.getVarFlag('OVERLAYFS_WRITABLE_PATHS', mountPoint) + if not mountPointList: + bb.debug(1, "No mount points defined for %s flag, don't add to file list", mountPoint) + continue + for path in mountPointList.split(): fileList.append(mountUnitName(path)) fileList.append(helperUnitName(path)) diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py index 7d387ee81d..1511ba47c4 100644 --- a/meta/lib/oe/package.py +++ b/meta/lib/oe/package.py @@ -1,11 +1,22 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # +import errno +import fnmatch +import itertools +import os +import shlex +import re +import glob import stat import mmap import subprocess +import oe.cachedpath + def runstrip(arg): # Function to strip a single file, called from split_and_strip_files below # A working 'file' (one which works on the target architecture) @@ -30,7 +41,7 @@ def runstrip(arg): stripcmd = [strip] skip_strip = False - # kernel module + # kernel module if elftype & 16: if is_kernel_module_signed(file): bb.debug(1, "Skip strip on signed module %s" % file) @@ -103,7 +114,7 @@ def is_static_lib(path): return start == magic return False -def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, d, qa_already_stripped=False): +def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, max_process, qa_already_stripped=False): """ Strip executable code (like executables, shared libraries) _in_place_ - Based on sysroot_strip in staging.bbclass @@ -111,6 +122,7 @@ def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, d, qa_already_stripp :param strip_cmd: Strip command (usually ${STRIP}) :param libdir: ${libdir} - strip .so files in this directory :param base_libdir: ${base_libdir} - strip .so files in this directory + :param max_process: number of stripping processes started in parallel :param qa_already_stripped: Set to True if already-stripped' in ${INSANE_SKIP} This is for proper logging and messages only. """ @@ -153,7 +165,7 @@ def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, d, qa_already_stripp # ...but is it ELF, and is it already stripped? checkelf.append(file) inodecache[file] = s.st_ino - results = oe.utils.multiprocess_launch(is_elf, checkelf, d) + results = oe.utils.multiprocess_launch_mp(is_elf, checkelf, max_process) for (file, elf_file) in results: #elf_file = is_elf(file) if elf_file & 1: @@ -181,7 +193,7 @@ def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, d, qa_already_stripp elf_file = int(elffiles[file]) sfiles.append((file, elf_file, strip_cmd)) - oe.utils.multiprocess_launch(runstrip, sfiles, d) + oe.utils.multiprocess_launch_mp(runstrip, sfiles, max_process) def file_translate(file): @@ -290,3 +302,1725 @@ def read_shlib_providers(d): shlib_provider[s[0]] = {} shlib_provider[s[0]][s[1]] = (dep_pkg, s[2]) return shlib_provider + +# We generate a master list of directories to process, we start by +# seeding this list with reasonable defaults, then load from +# the fs-perms.txt files +def fixup_perms(d): + import pwd, grp + + cpath = oe.cachedpath.CachedPath() + dvar = d.getVar('PKGD') + + # init using a string with the same format as a line as documented in + # the fs-perms.txt file + # <path> <mode> <uid> <gid> <walk> <fmode> <fuid> <fgid> + # <path> link <link target> + # + # __str__ can be used to print out an entry in the input format + # + # if fs_perms_entry.path is None: + # an error occurred + # if fs_perms_entry.link, you can retrieve: + # fs_perms_entry.path = path + # fs_perms_entry.link = target of link + # if not fs_perms_entry.link, you can retrieve: + # fs_perms_entry.path = path + # fs_perms_entry.mode = expected dir mode or None + # fs_perms_entry.uid = expected uid or -1 + # fs_perms_entry.gid = expected gid or -1 + # fs_perms_entry.walk = 'true' or something else + # fs_perms_entry.fmode = expected file mode or None + # fs_perms_entry.fuid = expected file uid or -1 + # fs_perms_entry_fgid = expected file gid or -1 + class fs_perms_entry(): + def __init__(self, line): + lsplit = line.split() + if len(lsplit) == 3 and lsplit[1].lower() == "link": + self._setlink(lsplit[0], lsplit[2]) + elif len(lsplit) == 8: + self._setdir(lsplit[0], lsplit[1], lsplit[2], lsplit[3], lsplit[4], lsplit[5], lsplit[6], lsplit[7]) + else: + msg = "Fixup Perms: invalid config line %s" % line + oe.qa.handle_error("perm-config", msg, d) + self.path = None + self.link = None + + def _setdir(self, path, mode, uid, gid, walk, fmode, fuid, fgid): + self.path = os.path.normpath(path) + self.link = None + self.mode = self._procmode(mode) + self.uid = self._procuid(uid) + self.gid = self._procgid(gid) + self.walk = walk.lower() + self.fmode = self._procmode(fmode) + self.fuid = self._procuid(fuid) + self.fgid = self._procgid(fgid) + + def _setlink(self, path, link): + self.path = os.path.normpath(path) + self.link = link + + def _procmode(self, mode): + if not mode or (mode and mode == "-"): + return None + else: + return int(mode,8) + + # Note uid/gid -1 has special significance in os.lchown + def _procuid(self, uid): + if uid is None or uid == "-": + return -1 + elif uid.isdigit(): + return int(uid) + else: + return pwd.getpwnam(uid).pw_uid + + def _procgid(self, gid): + if gid is None or gid == "-": + return -1 + elif gid.isdigit(): + return int(gid) + else: + return grp.getgrnam(gid).gr_gid + + # Use for debugging the entries + def __str__(self): + if self.link: + return "%s link %s" % (self.path, self.link) + else: + mode = "-" + if self.mode: + mode = "0%o" % self.mode + fmode = "-" + if self.fmode: + fmode = "0%o" % self.fmode + uid = self._mapugid(self.uid) + gid = self._mapugid(self.gid) + fuid = self._mapugid(self.fuid) + fgid = self._mapugid(self.fgid) + return "%s %s %s %s %s %s %s %s" % (self.path, mode, uid, gid, self.walk, fmode, fuid, fgid) + + def _mapugid(self, id): + if id is None or id == -1: + return "-" + else: + return "%d" % id + + # Fix the permission, owner and group of path + def fix_perms(path, mode, uid, gid, dir): + if mode and not os.path.islink(path): + #bb.note("Fixup Perms: chmod 0%o %s" % (mode, dir)) + os.chmod(path, mode) + # -1 is a special value that means don't change the uid/gid + # if they are BOTH -1, don't bother to lchown + if not (uid == -1 and gid == -1): + #bb.note("Fixup Perms: lchown %d:%d %s" % (uid, gid, dir)) + os.lchown(path, uid, gid) + + # Return a list of configuration files based on either the default + # files/fs-perms.txt or the contents of FILESYSTEM_PERMS_TABLES + # paths are resolved via BBPATH + def get_fs_perms_list(d): + str = "" + bbpath = d.getVar('BBPATH') + fs_perms_tables = d.getVar('FILESYSTEM_PERMS_TABLES') or "" + for conf_file in fs_perms_tables.split(): + confpath = bb.utils.which(bbpath, conf_file) + if confpath: + str += " %s" % bb.utils.which(bbpath, conf_file) + else: + bb.warn("cannot find %s specified in FILESYSTEM_PERMS_TABLES" % conf_file) + return str + + fs_perms_table = {} + fs_link_table = {} + + # By default all of the standard directories specified in + # bitbake.conf will get 0755 root:root. + target_path_vars = [ 'base_prefix', + 'prefix', + 'exec_prefix', + 'base_bindir', + 'base_sbindir', + 'base_libdir', + 'datadir', + 'sysconfdir', + 'servicedir', + 'sharedstatedir', + 'localstatedir', + 'infodir', + 'mandir', + 'docdir', + 'bindir', + 'sbindir', + 'libexecdir', + 'libdir', + 'includedir' ] + + for path in target_path_vars: + dir = d.getVar(path) or "" + if dir == "": + continue + fs_perms_table[dir] = fs_perms_entry(d.expand("%s 0755 root root false - - -" % (dir))) + + # Now we actually load from the configuration files + for conf in get_fs_perms_list(d).split(): + if not os.path.exists(conf): + continue + with open(conf) as f: + for line in f: + if line.startswith('#'): + continue + lsplit = line.split() + if len(lsplit) == 0: + continue + if len(lsplit) != 8 and not (len(lsplit) == 3 and lsplit[1].lower() == "link"): + msg = "Fixup perms: %s invalid line: %s" % (conf, line) + oe.qa.handle_error("perm-line", msg, d) + continue + entry = fs_perms_entry(d.expand(line)) + if entry and entry.path: + if entry.link: + fs_link_table[entry.path] = entry + if entry.path in fs_perms_table: + fs_perms_table.pop(entry.path) + else: + fs_perms_table[entry.path] = entry + if entry.path in fs_link_table: + fs_link_table.pop(entry.path) + + # Debug -- list out in-memory table + #for dir in fs_perms_table: + # bb.note("Fixup Perms: %s: %s" % (dir, str(fs_perms_table[dir]))) + #for link in fs_link_table: + # bb.note("Fixup Perms: %s: %s" % (link, str(fs_link_table[link]))) + + # We process links first, so we can go back and fixup directory ownership + # for any newly created directories + # Process in sorted order so /run gets created before /run/lock, etc. + for entry in sorted(fs_link_table.values(), key=lambda x: x.link): + link = entry.link + dir = entry.path + origin = dvar + dir + if not (cpath.exists(origin) and cpath.isdir(origin) and not cpath.islink(origin)): + continue + + if link[0] == "/": + target = dvar + link + ptarget = link + else: + target = os.path.join(os.path.dirname(origin), link) + ptarget = os.path.join(os.path.dirname(dir), link) + if os.path.exists(target): + msg = "Fixup Perms: Unable to correct directory link, target already exists: %s -> %s" % (dir, ptarget) + oe.qa.handle_error("perm-link", msg, d) + continue + + # Create path to move directory to, move it, and then setup the symlink + bb.utils.mkdirhier(os.path.dirname(target)) + #bb.note("Fixup Perms: Rename %s -> %s" % (dir, ptarget)) + bb.utils.rename(origin, target) + #bb.note("Fixup Perms: Link %s -> %s" % (dir, link)) + os.symlink(link, origin) + + for dir in fs_perms_table: + origin = dvar + dir + if not (cpath.exists(origin) and cpath.isdir(origin)): + continue + + fix_perms(origin, fs_perms_table[dir].mode, fs_perms_table[dir].uid, fs_perms_table[dir].gid, dir) + + if fs_perms_table[dir].walk == 'true': + for root, dirs, files in os.walk(origin): + for dr in dirs: + each_dir = os.path.join(root, dr) + fix_perms(each_dir, fs_perms_table[dir].mode, fs_perms_table[dir].uid, fs_perms_table[dir].gid, dir) + for f in files: + each_file = os.path.join(root, f) + fix_perms(each_file, fs_perms_table[dir].fmode, fs_perms_table[dir].fuid, fs_perms_table[dir].fgid, dir) + +# Get a list of files from file vars by searching files under current working directory +# The list contains symlinks, directories and normal files. +def files_from_filevars(filevars): + cpath = oe.cachedpath.CachedPath() + files = [] + for f in filevars: + if os.path.isabs(f): + f = '.' + f + if not f.startswith("./"): + f = './' + f + globbed = glob.glob(f, recursive=True) + if globbed: + if [ f ] != globbed: + files += globbed + continue + files.append(f) + + symlink_paths = [] + for ind, f in enumerate(files): + # Handle directory symlinks. Truncate path to the lowest level symlink + parent = '' + for dirname in f.split('/')[:-1]: + parent = os.path.join(parent, dirname) + if dirname == '.': + continue + if cpath.islink(parent): + bb.warn("FILES contains file '%s' which resides under a " + "directory symlink. Please fix the recipe and use the " + "real path for the file." % f[1:]) + symlink_paths.append(f) + files[ind] = parent + f = parent + break + + if not cpath.islink(f): + if cpath.isdir(f): + newfiles = [ os.path.join(f,x) for x in os.listdir(f) ] + if newfiles: + files += newfiles + + return files, symlink_paths + +# Called in package_<rpm,ipk,deb>.bbclass to get the correct list of configuration files +def get_conffiles(pkg, d): + pkgdest = d.getVar('PKGDEST') + root = os.path.join(pkgdest, pkg) + cwd = os.getcwd() + os.chdir(root) + + conffiles = d.getVar('CONFFILES:%s' % pkg); + if conffiles == None: + conffiles = d.getVar('CONFFILES') + if conffiles == None: + conffiles = "" + conffiles = conffiles.split() + conf_orig_list = files_from_filevars(conffiles)[0] + + # Remove links and directories from conf_orig_list to get conf_list which only contains normal files + conf_list = [] + for f in conf_orig_list: + if os.path.isdir(f): + continue + if os.path.islink(f): + continue + if not os.path.exists(f): + continue + conf_list.append(f) + + # Remove the leading './' + for i in range(0, len(conf_list)): + conf_list[i] = conf_list[i][1:] + + os.chdir(cwd) + return sorted(conf_list) + +def legitimize_package_name(s): + """ + Make sure package names are legitimate strings + """ + + def fixutf(m): + cp = m.group(1) + if cp: + return ('\\u%s' % cp).encode('latin-1').decode('unicode_escape') + + # Handle unicode codepoints encoded as <U0123>, as in glibc locale files. + s = re.sub(r'<U([0-9A-Fa-f]{1,4})>', fixutf, s) + + # Remaining package name validity fixes + return s.lower().replace('_', '-').replace('@', '+').replace(',', '+').replace('/', '-') + +def split_locales(d): + cpath = oe.cachedpath.CachedPath() + if (d.getVar('PACKAGE_NO_LOCALE') == '1'): + bb.debug(1, "package requested not splitting locales") + return + + packages = (d.getVar('PACKAGES') or "").split() + + dvar = d.getVar('PKGD') + pn = d.getVar('LOCALEBASEPN') + + try: + locale_index = packages.index(pn + '-locale') + packages.pop(locale_index) + except ValueError: + locale_index = len(packages) + + localepaths = [] + locales = set() + for localepath in (d.getVar('LOCALE_PATHS') or "").split(): + localedir = dvar + localepath + if not cpath.isdir(localedir): + bb.debug(1, 'No locale files in %s' % localepath) + continue + + localepaths.append(localepath) + with os.scandir(localedir) as it: + for entry in it: + if entry.is_dir(): + locales.add(entry.name) + + if len(locales) == 0: + bb.debug(1, "No locale files in this package") + return + + summary = d.getVar('SUMMARY') or pn + description = d.getVar('DESCRIPTION') or "" + locale_section = d.getVar('LOCALE_SECTION') + mlprefix = d.getVar('MLPREFIX') or "" + for l in sorted(locales): + ln = legitimize_package_name(l) + pkg = pn + '-locale-' + ln + packages.insert(locale_index, pkg) + locale_index += 1 + files = [] + for localepath in localepaths: + files.append(os.path.join(localepath, l)) + d.setVar('FILES:' + pkg, " ".join(files)) + d.setVar('RRECOMMENDS:' + pkg, '%svirtual-locale-%s' % (mlprefix, ln)) + d.setVar('RPROVIDES:' + pkg, '%s-locale %s%s-translation' % (pn, mlprefix, ln)) + d.setVar('SUMMARY:' + pkg, '%s - %s translations' % (summary, l)) + d.setVar('DESCRIPTION:' + pkg, '%s This package contains language translation files for the %s locale.' % (description, l)) + if locale_section: + d.setVar('SECTION:' + pkg, locale_section) + + d.setVar('PACKAGES', ' '.join(packages)) + + # Disabled by RP 18/06/07 + # Wildcards aren't supported in debian + # They break with ipkg since glibc-locale* will mean that + # glibc-localedata-translit* won't install as a dependency + # for some other package which breaks meta-toolchain + # Probably breaks since virtual-locale- isn't provided anywhere + #rdep = (d.getVar('RDEPENDS:%s' % pn) or "").split() + #rdep.append('%s-locale*' % pn) + #d.setVar('RDEPENDS:%s' % pn, ' '.join(rdep)) + +def package_debug_vars(d): + # We default to '.debug' style + if d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-file-directory': + # Single debug-file-directory style debug info + debug_vars = { + "append": ".debug", + "staticappend": "", + "dir": "", + "staticdir": "", + "libdir": "/usr/lib/debug", + "staticlibdir": "/usr/lib/debug-static", + "srcdir": "/usr/src/debug", + } + elif d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-without-src': + # Original OE-core, a.k.a. ".debug", style debug info, but without sources in /usr/src/debug + debug_vars = { + "append": "", + "staticappend": "", + "dir": "/.debug", + "staticdir": "/.debug-static", + "libdir": "", + "staticlibdir": "", + "srcdir": "", + } + elif d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-with-srcpkg': + debug_vars = { + "append": "", + "staticappend": "", + "dir": "/.debug", + "staticdir": "/.debug-static", + "libdir": "", + "staticlibdir": "", + "srcdir": "/usr/src/debug", + } + else: + # Original OE-core, a.k.a. ".debug", style debug info + debug_vars = { + "append": "", + "staticappend": "", + "dir": "/.debug", + "staticdir": "/.debug-static", + "libdir": "", + "staticlibdir": "", + "srcdir": "/usr/src/debug", + } + + return debug_vars + + +def parse_debugsources_from_dwarfsrcfiles_output(dwarfsrcfiles_output): + debugfiles = {} + + for line in dwarfsrcfiles_output.splitlines(): + if line.startswith("\t"): + debugfiles[os.path.normpath(line.split()[0])] = "" + + return debugfiles.keys() + +def source_info(file, d, fatal=True): + cmd = ["dwarfsrcfiles", file] + try: + output = subprocess.check_output(cmd, universal_newlines=True, stderr=subprocess.STDOUT) + retval = 0 + except subprocess.CalledProcessError as exc: + output = exc.output + retval = exc.returncode + + # 255 means a specific file wasn't fully parsed to get the debug file list, which is not a fatal failure + if retval != 0 and retval != 255: + msg = "dwarfsrcfiles failed with exit code %s (cmd was %s)%s" % (retval, cmd, ":\n%s" % output if output else "") + if fatal: + bb.fatal(msg) + bb.note(msg) + + debugsources = parse_debugsources_from_dwarfsrcfiles_output(output) + + return list(debugsources) + +def splitdebuginfo(file, dvar, dv, d): + # Function to split a single file into two components, one is the stripped + # target system binary, the other contains any debugging information. The + # two files are linked to reference each other. + # + # return a mapping of files:debugsources + + src = file[len(dvar):] + dest = dv["libdir"] + os.path.dirname(src) + dv["dir"] + "/" + os.path.basename(src) + dv["append"] + debugfile = dvar + dest + sources = [] + + if file.endswith(".ko") and file.find("/lib/modules/") != -1: + if oe.package.is_kernel_module_signed(file): + bb.debug(1, "Skip strip on signed module %s" % file) + return (file, sources) + + # Split the file... + bb.utils.mkdirhier(os.path.dirname(debugfile)) + #bb.note("Split %s -> %s" % (file, debugfile)) + # Only store off the hard link reference if we successfully split! + + dvar = d.getVar('PKGD') + objcopy = d.getVar("OBJCOPY") + + newmode = None + if not os.access(file, os.W_OK) or os.access(file, os.R_OK): + origmode = os.stat(file)[stat.ST_MODE] + newmode = origmode | stat.S_IWRITE | stat.S_IREAD + os.chmod(file, newmode) + + # We need to extract the debug src information here... + if dv["srcdir"]: + sources = source_info(file, d) + + bb.utils.mkdirhier(os.path.dirname(debugfile)) + + subprocess.check_output([objcopy, '--only-keep-debug', file, debugfile], stderr=subprocess.STDOUT) + + # Set the debuglink to have the view of the file path on the target + subprocess.check_output([objcopy, '--add-gnu-debuglink', debugfile, file], stderr=subprocess.STDOUT) + + if newmode: + os.chmod(file, origmode) + + return (file, sources) + +def splitstaticdebuginfo(file, dvar, dv, d): + # Unlike the function above, there is no way to split a static library + # two components. So to get similar results we will copy the unmodified + # static library (containing the debug symbols) into a new directory. + # We will then strip (preserving symbols) the static library in the + # typical location. + # + # return a mapping of files:debugsources + + src = file[len(dvar):] + dest = dv["staticlibdir"] + os.path.dirname(src) + dv["staticdir"] + "/" + os.path.basename(src) + dv["staticappend"] + debugfile = dvar + dest + sources = [] + + # Copy the file... + bb.utils.mkdirhier(os.path.dirname(debugfile)) + #bb.note("Copy %s -> %s" % (file, debugfile)) + + dvar = d.getVar('PKGD') + + newmode = None + if not os.access(file, os.W_OK) or os.access(file, os.R_OK): + origmode = os.stat(file)[stat.ST_MODE] + newmode = origmode | stat.S_IWRITE | stat.S_IREAD + os.chmod(file, newmode) + + # We need to extract the debug src information here... + if dv["srcdir"]: + sources = source_info(file, d) + + bb.utils.mkdirhier(os.path.dirname(debugfile)) + + # Copy the unmodified item to the debug directory + shutil.copy2(file, debugfile) + + if newmode: + os.chmod(file, origmode) + + return (file, sources) + +def inject_minidebuginfo(file, dvar, dv, d): + # Extract just the symbols from debuginfo into minidebuginfo, + # compress it with xz and inject it back into the binary in a .gnu_debugdata section. + # https://sourceware.org/gdb/onlinedocs/gdb/MiniDebugInfo.html + + readelf = d.getVar('READELF') + nm = d.getVar('NM') + objcopy = d.getVar('OBJCOPY') + + minidebuginfodir = d.expand('${WORKDIR}/minidebuginfo') + + src = file[len(dvar):] + dest = dv["libdir"] + os.path.dirname(src) + dv["dir"] + "/" + os.path.basename(src) + dv["append"] + debugfile = dvar + dest + minidebugfile = minidebuginfodir + src + '.minidebug' + bb.utils.mkdirhier(os.path.dirname(minidebugfile)) + + # If we didn't produce debuginfo for any reason, we can't produce minidebuginfo either + # so skip it. + if not os.path.exists(debugfile): + bb.debug(1, 'ELF file {} has no debuginfo, skipping minidebuginfo injection'.format(file)) + return + + # minidebuginfo does not make sense to apply to ELF objects other than + # executables and shared libraries, skip applying the minidebuginfo + # generation for objects like kernel modules. + for line in subprocess.check_output([readelf, '-h', debugfile], universal_newlines=True).splitlines(): + if not line.strip().startswith("Type:"): + continue + elftype = line.split(":")[1].strip() + if not any(elftype.startswith(i) for i in ["EXEC", "DYN"]): + bb.debug(1, 'ELF file {} is not executable/shared, skipping minidebuginfo injection'.format(file)) + return + break + + # Find non-allocated PROGBITS, NOTE, and NOBITS sections in the debuginfo. + # We will exclude all of these from minidebuginfo to save space. + remove_section_names = [] + for line in subprocess.check_output([readelf, '-W', '-S', debugfile], universal_newlines=True).splitlines(): + # strip the leading " [ 1]" section index to allow splitting on space + if ']' not in line: + continue + fields = line[line.index(']') + 1:].split() + if len(fields) < 7: + continue + name = fields[0] + type = fields[1] + flags = fields[6] + # .debug_ sections will be removed by objcopy -S so no need to explicitly remove them + if name.startswith('.debug_'): + continue + if 'A' not in flags and type in ['PROGBITS', 'NOTE', 'NOBITS']: + remove_section_names.append(name) + + # List dynamic symbols in the binary. We can exclude these from minidebuginfo + # because they are always present in the binary. + dynsyms = set() + for line in subprocess.check_output([nm, '-D', file, '--format=posix', '--defined-only'], universal_newlines=True).splitlines(): + dynsyms.add(line.split()[0]) + + # Find all function symbols from debuginfo which aren't in the dynamic symbols table. + # These are the ones we want to keep in minidebuginfo. + keep_symbols_file = minidebugfile + '.symlist' + found_any_symbols = False + with open(keep_symbols_file, 'w') as f: + for line in subprocess.check_output([nm, debugfile, '--format=sysv', '--defined-only'], universal_newlines=True).splitlines(): + fields = line.split('|') + if len(fields) < 7: + continue + name = fields[0].strip() + type = fields[3].strip() + if type == 'FUNC' and name not in dynsyms: + f.write('{}\n'.format(name)) + found_any_symbols = True + + if not found_any_symbols: + bb.debug(1, 'ELF file {} contains no symbols, skipping minidebuginfo injection'.format(file)) + return + + bb.utils.remove(minidebugfile) + bb.utils.remove(minidebugfile + '.xz') + + subprocess.check_call([objcopy, '-S'] + + ['--remove-section={}'.format(s) for s in remove_section_names] + + ['--keep-symbols={}'.format(keep_symbols_file), debugfile, minidebugfile]) + + subprocess.check_call(['xz', '--keep', minidebugfile]) + + subprocess.check_call([objcopy, '--add-section', '.gnu_debugdata={}.xz'.format(minidebugfile), file]) + +def copydebugsources(debugsrcdir, sources, d): + # The debug src information written out to sourcefile is further processed + # and copied to the destination here. + + cpath = oe.cachedpath.CachedPath() + + if debugsrcdir and sources: + sourcefile = d.expand("${WORKDIR}/debugsources.list") + bb.utils.remove(sourcefile) + + # filenames are null-separated - this is an artefact of the previous use + # of rpm's debugedit, which was writing them out that way, and the code elsewhere + # is still assuming that. + debuglistoutput = '\0'.join(sources) + '\0' + with open(sourcefile, 'a') as sf: + sf.write(debuglistoutput) + + dvar = d.getVar('PKGD') + strip = d.getVar("STRIP") + objcopy = d.getVar("OBJCOPY") + workdir = d.getVar("WORKDIR") + sdir = d.getVar("S") + cflags = d.expand("${CFLAGS}") + + prefixmap = {} + for flag in cflags.split(): + if not flag.startswith("-fdebug-prefix-map"): + continue + if "recipe-sysroot" in flag: + continue + flag = flag.split("=") + prefixmap[flag[1]] = flag[2] + + nosuchdir = [] + basepath = dvar + for p in debugsrcdir.split("/"): + basepath = basepath + "/" + p + if not cpath.exists(basepath): + nosuchdir.append(basepath) + bb.utils.mkdirhier(basepath) + cpath.updatecache(basepath) + + for pmap in prefixmap: + # Ignore files from the recipe sysroots (target and native) + cmd = "LC_ALL=C ; sort -z -u '%s' | egrep -v -z '((<internal>|<built-in>)$|/.*recipe-sysroot.*/)' | " % sourcefile + # We need to ignore files that are not actually ours + # we do this by only paying attention to items from this package + cmd += "fgrep -zw '%s' | " % prefixmap[pmap] + # Remove prefix in the source paths + cmd += "sed 's#%s/##g' | " % (prefixmap[pmap]) + cmd += "(cd '%s' ; cpio -pd0mlL --no-preserve-owner '%s%s' 2>/dev/null)" % (pmap, dvar, prefixmap[pmap]) + + try: + subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError: + # Can "fail" if internal headers/transient sources are attempted + pass + # cpio seems to have a bug with -lL together and symbolic links are just copied, not dereferenced. + # Work around this by manually finding and copying any symbolic links that made it through. + cmd = "find %s%s -type l -print0 -delete | sed s#%s%s/##g | (cd '%s' ; cpio -pd0mL --no-preserve-owner '%s%s')" % \ + (dvar, prefixmap[pmap], dvar, prefixmap[pmap], pmap, dvar, prefixmap[pmap]) + subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) + + # debugsources.list may be polluted from the host if we used externalsrc, + # cpio uses copy-pass and may have just created a directory structure + # matching the one from the host, if thats the case move those files to + # debugsrcdir to avoid host contamination. + # Empty dir structure will be deleted in the next step. + + # Same check as above for externalsrc + if workdir not in sdir: + if os.path.exists(dvar + debugsrcdir + sdir): + cmd = "mv %s%s%s/* %s%s" % (dvar, debugsrcdir, sdir, dvar,debugsrcdir) + subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) + + # The copy by cpio may have resulted in some empty directories! Remove these + cmd = "find %s%s -empty -type d -delete" % (dvar, debugsrcdir) + subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) + + # Also remove debugsrcdir if its empty + for p in nosuchdir[::-1]: + if os.path.exists(p) and not os.listdir(p): + os.rmdir(p) + + +def process_split_and_strip_files(d): + cpath = oe.cachedpath.CachedPath() + + dvar = d.getVar('PKGD') + pn = d.getVar('PN') + hostos = d.getVar('HOST_OS') + + oldcwd = os.getcwd() + os.chdir(dvar) + + dv = package_debug_vars(d) + + # + # First lets figure out all of the files we may have to process ... do this only once! + # + elffiles = {} + symlinks = {} + staticlibs = [] + inodes = {} + libdir = os.path.abspath(dvar + os.sep + d.getVar("libdir")) + baselibdir = os.path.abspath(dvar + os.sep + d.getVar("base_libdir")) + skipfiles = (d.getVar("INHIBIT_PACKAGE_STRIP_FILES") or "").split() + if (d.getVar('INHIBIT_PACKAGE_STRIP') != '1' or \ + d.getVar('INHIBIT_PACKAGE_DEBUG_SPLIT') != '1'): + checkelf = {} + checkelflinks = {} + for root, dirs, files in cpath.walk(dvar): + for f in files: + file = os.path.join(root, f) + + # Skip debug files + if dv["append"] and file.endswith(dv["append"]): + continue + if dv["dir"] and dv["dir"] in os.path.dirname(file[len(dvar):]): + continue + + if file in skipfiles: + continue + + if oe.package.is_static_lib(file): + staticlibs.append(file) + continue + + try: + ltarget = cpath.realpath(file, dvar, False) + s = cpath.lstat(ltarget) + except OSError as e: + (err, strerror) = e.args + if err != errno.ENOENT: + raise + # Skip broken symlinks + continue + if not s: + continue + # Check its an executable + if (s[stat.ST_MODE] & stat.S_IXUSR) or (s[stat.ST_MODE] & stat.S_IXGRP) \ + or (s[stat.ST_MODE] & stat.S_IXOTH) \ + or ((file.startswith(libdir) or file.startswith(baselibdir)) \ + and (".so" in f or ".node" in f)) \ + or (f.startswith('vmlinux') or ".ko" in f): + + if cpath.islink(file): + checkelflinks[file] = ltarget + continue + # Use a reference of device ID and inode number to identify files + file_reference = "%d_%d" % (s.st_dev, s.st_ino) + checkelf[file] = (file, file_reference) + + results = oe.utils.multiprocess_launch(oe.package.is_elf, checkelflinks.values(), d) + results_map = {} + for (ltarget, elf_file) in results: + results_map[ltarget] = elf_file + for file in checkelflinks: + ltarget = checkelflinks[file] + # If it's a symlink, and points to an ELF file, we capture the readlink target + if results_map[ltarget]: + target = os.readlink(file) + #bb.note("Sym: %s (%d)" % (ltarget, results_map[ltarget])) + symlinks[file] = target + + results = oe.utils.multiprocess_launch(oe.package.is_elf, checkelf.keys(), d) + + # Sort results by file path. This ensures that the files are always + # processed in the same order, which is important to make sure builds + # are reproducible when dealing with hardlinks + results.sort(key=lambda x: x[0]) + + for (file, elf_file) in results: + # It's a file (or hardlink), not a link + # ...but is it ELF, and is it already stripped? + if elf_file & 1: + if elf_file & 2: + if 'already-stripped' in (d.getVar('INSANE_SKIP:' + pn) or "").split(): + bb.note("Skipping file %s from %s for already-stripped QA test" % (file[len(dvar):], pn)) + else: + msg = "File '%s' from %s was already stripped, this will prevent future debugging!" % (file[len(dvar):], pn) + oe.qa.handle_error("already-stripped", msg, d) + continue + + # At this point we have an unstripped elf file. We need to: + # a) Make sure any file we strip is not hardlinked to anything else outside this tree + # b) Only strip any hardlinked file once (no races) + # c) Track any hardlinks between files so that we can reconstruct matching debug file hardlinks + + # Use a reference of device ID and inode number to identify files + file_reference = checkelf[file][1] + if file_reference in inodes: + os.unlink(file) + os.link(inodes[file_reference][0], file) + inodes[file_reference].append(file) + else: + inodes[file_reference] = [file] + # break hardlink + bb.utils.break_hardlinks(file) + elffiles[file] = elf_file + # Modified the file so clear the cache + cpath.updatecache(file) + + def strip_pkgd_prefix(f): + nonlocal dvar + + if f.startswith(dvar): + return f[len(dvar):] + + return f + + # + # First lets process debug splitting + # + if (d.getVar('INHIBIT_PACKAGE_DEBUG_SPLIT') != '1'): + results = oe.utils.multiprocess_launch(splitdebuginfo, list(elffiles), d, extraargs=(dvar, dv, d)) + + if dv["srcdir"] and not hostos.startswith("mingw"): + if (d.getVar('PACKAGE_DEBUG_STATIC_SPLIT') == '1'): + results = oe.utils.multiprocess_launch(splitstaticdebuginfo, staticlibs, d, extraargs=(dvar, dv, d)) + else: + for file in staticlibs: + results.append( (file,source_info(file, d)) ) + + d.setVar("PKGDEBUGSOURCES", {strip_pkgd_prefix(f): sorted(s) for f, s in results}) + + sources = set() + for r in results: + sources.update(r[1]) + + # Hardlink our debug symbols to the other hardlink copies + for ref in inodes: + if len(inodes[ref]) == 1: + continue + + target = inodes[ref][0][len(dvar):] + for file in inodes[ref][1:]: + src = file[len(dvar):] + dest = dv["libdir"] + os.path.dirname(src) + dv["dir"] + "/" + os.path.basename(target) + dv["append"] + fpath = dvar + dest + ftarget = dvar + dv["libdir"] + os.path.dirname(target) + dv["dir"] + "/" + os.path.basename(target) + dv["append"] + bb.utils.mkdirhier(os.path.dirname(fpath)) + # Only one hardlink of separated debug info file in each directory + if not os.access(fpath, os.R_OK): + #bb.note("Link %s -> %s" % (fpath, ftarget)) + os.link(ftarget, fpath) + + # Create symlinks for all cases we were able to split symbols + for file in symlinks: + src = file[len(dvar):] + dest = dv["libdir"] + os.path.dirname(src) + dv["dir"] + "/" + os.path.basename(src) + dv["append"] + fpath = dvar + dest + # Skip it if the target doesn't exist + try: + s = os.stat(fpath) + except OSError as e: + (err, strerror) = e.args + if err != errno.ENOENT: + raise + continue + + ltarget = symlinks[file] + lpath = os.path.dirname(ltarget) + lbase = os.path.basename(ltarget) + ftarget = "" + if lpath and lpath != ".": + ftarget += lpath + dv["dir"] + "/" + ftarget += lbase + dv["append"] + if lpath.startswith(".."): + ftarget = os.path.join("..", ftarget) + bb.utils.mkdirhier(os.path.dirname(fpath)) + #bb.note("Symlink %s -> %s" % (fpath, ftarget)) + os.symlink(ftarget, fpath) + + # Process the dv["srcdir"] if requested... + # This copies and places the referenced sources for later debugging... + copydebugsources(dv["srcdir"], sources, d) + # + # End of debug splitting + # + + # + # Now lets go back over things and strip them + # + if (d.getVar('INHIBIT_PACKAGE_STRIP') != '1'): + strip = d.getVar("STRIP") + sfiles = [] + for file in elffiles: + elf_file = int(elffiles[file]) + #bb.note("Strip %s" % file) + sfiles.append((file, elf_file, strip)) + if (d.getVar('PACKAGE_STRIP_STATIC') == '1' or d.getVar('PACKAGE_DEBUG_STATIC_SPLIT') == '1'): + for f in staticlibs: + sfiles.append((f, 16, strip)) + + oe.utils.multiprocess_launch(oe.package.runstrip, sfiles, d) + + # Build "minidebuginfo" and reinject it back into the stripped binaries + if bb.utils.contains('DISTRO_FEATURES', 'minidebuginfo', True, False, d): + oe.utils.multiprocess_launch(inject_minidebuginfo, list(elffiles), d, + extraargs=(dvar, dv, d)) + + # + # End of strip + # + os.chdir(oldcwd) + + +def populate_packages(d): + cpath = oe.cachedpath.CachedPath() + + workdir = d.getVar('WORKDIR') + outdir = d.getVar('DEPLOY_DIR') + dvar = d.getVar('PKGD') + packages = d.getVar('PACKAGES').split() + pn = d.getVar('PN') + + bb.utils.mkdirhier(outdir) + os.chdir(dvar) + + autodebug = not (d.getVar("NOAUTOPACKAGEDEBUG") or False) + + split_source_package = (d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-with-srcpkg') + + # If debug-with-srcpkg mode is enabled then add the source package if it + # doesn't exist and add the source file contents to the source package. + if split_source_package: + src_package_name = ('%s-src' % d.getVar('PN')) + if not src_package_name in packages: + packages.append(src_package_name) + d.setVar('FILES:%s' % src_package_name, '/usr/src/debug') + + # Sanity check PACKAGES for duplicates + # Sanity should be moved to sanity.bbclass once we have the infrastructure + package_dict = {} + + for i, pkg in enumerate(packages): + if pkg in package_dict: + msg = "%s is listed in PACKAGES multiple times, this leads to packaging errors." % pkg + oe.qa.handle_error("packages-list", msg, d) + # Ensure the source package gets the chance to pick up the source files + # before the debug package by ordering it first in PACKAGES. Whether it + # actually picks up any source files is controlled by + # PACKAGE_DEBUG_SPLIT_STYLE. + elif pkg.endswith("-src"): + package_dict[pkg] = (10, i) + elif autodebug and pkg.endswith("-dbg"): + package_dict[pkg] = (30, i) + else: + package_dict[pkg] = (50, i) + packages = sorted(package_dict.keys(), key=package_dict.get) + d.setVar('PACKAGES', ' '.join(packages)) + pkgdest = d.getVar('PKGDEST') + + seen = [] + + # os.mkdir masks the permissions with umask so we have to unset it first + oldumask = os.umask(0) + + debug = [] + for root, dirs, files in cpath.walk(dvar): + dir = root[len(dvar):] + if not dir: + dir = os.sep + for f in (files + dirs): + path = "." + os.path.join(dir, f) + if "/.debug/" in path or "/.debug-static/" in path or path.endswith("/.debug"): + debug.append(path) + + for pkg in packages: + root = os.path.join(pkgdest, pkg) + bb.utils.mkdirhier(root) + + filesvar = d.getVar('FILES:%s' % pkg) or "" + if "//" in filesvar: + msg = "FILES variable for package %s contains '//' which is invalid. Attempting to fix this but you should correct the metadata.\n" % pkg + oe.qa.handle_error("files-invalid", msg, d) + filesvar.replace("//", "/") + + origfiles = filesvar.split() + files, symlink_paths = oe.package.files_from_filevars(origfiles) + + if autodebug and pkg.endswith("-dbg"): + files.extend(debug) + + for file in files: + if (not cpath.islink(file)) and (not cpath.exists(file)): + continue + if file in seen: + continue + seen.append(file) + + def mkdir(src, dest, p): + src = os.path.join(src, p) + dest = os.path.join(dest, p) + fstat = cpath.stat(src) + os.mkdir(dest) + os.chmod(dest, fstat.st_mode) + os.chown(dest, fstat.st_uid, fstat.st_gid) + if p not in seen: + seen.append(p) + cpath.updatecache(dest) + + def mkdir_recurse(src, dest, paths): + if cpath.exists(dest + '/' + paths): + return + while paths.startswith("./"): + paths = paths[2:] + p = "." + for c in paths.split("/"): + p = os.path.join(p, c) + if not cpath.exists(os.path.join(dest, p)): + mkdir(src, dest, p) + + if cpath.isdir(file) and not cpath.islink(file): + mkdir_recurse(dvar, root, file) + continue + + mkdir_recurse(dvar, root, os.path.dirname(file)) + fpath = os.path.join(root,file) + if not cpath.islink(file): + os.link(file, fpath) + continue + ret = bb.utils.copyfile(file, fpath) + if ret is False or ret == 0: + bb.fatal("File population failed") + + # Check if symlink paths exist + for file in symlink_paths: + if not os.path.exists(os.path.join(root,file)): + bb.fatal("File '%s' cannot be packaged into '%s' because its " + "parent directory structure does not exist. One of " + "its parent directories is a symlink whose target " + "directory is not included in the package." % + (file, pkg)) + + os.umask(oldumask) + os.chdir(workdir) + + # Handle excluding packages with incompatible licenses + package_list = [] + for pkg in packages: + licenses = d.getVar('_exclude_incompatible-' + pkg) + if licenses: + msg = "Excluding %s from packaging as it has incompatible license(s): %s" % (pkg, licenses) + oe.qa.handle_error("incompatible-license", msg, d) + else: + package_list.append(pkg) + d.setVar('PACKAGES', ' '.join(package_list)) + + unshipped = [] + for root, dirs, files in cpath.walk(dvar): + dir = root[len(dvar):] + if not dir: + dir = os.sep + for f in (files + dirs): + path = os.path.join(dir, f) + if ('.' + path) not in seen: + unshipped.append(path) + + if unshipped != []: + msg = pn + ": Files/directories were installed but not shipped in any package:" + if "installed-vs-shipped" in (d.getVar('INSANE_SKIP:' + pn) or "").split(): + bb.note("Package %s skipping QA tests: installed-vs-shipped" % pn) + else: + for f in unshipped: + msg = msg + "\n " + f + msg = msg + "\nPlease set FILES such that these items are packaged. Alternatively if they are unneeded, avoid installing them or delete them within do_install.\n" + msg = msg + "%s: %d installed and not shipped files." % (pn, len(unshipped)) + oe.qa.handle_error("installed-vs-shipped", msg, d) + +def process_fixsymlinks(pkgfiles, d): + cpath = oe.cachedpath.CachedPath() + pkgdest = d.getVar('PKGDEST') + packages = d.getVar("PACKAGES", False).split() + + dangling_links = {} + pkg_files = {} + for pkg in packages: + dangling_links[pkg] = [] + pkg_files[pkg] = [] + inst_root = os.path.join(pkgdest, pkg) + for path in pkgfiles[pkg]: + rpath = path[len(inst_root):] + pkg_files[pkg].append(rpath) + rtarget = cpath.realpath(path, inst_root, True, assume_dir = True) + if not cpath.lexists(rtarget): + dangling_links[pkg].append(os.path.normpath(rtarget[len(inst_root):])) + + newrdepends = {} + for pkg in dangling_links: + for l in dangling_links[pkg]: + found = False + bb.debug(1, "%s contains dangling link %s" % (pkg, l)) + for p in packages: + if l in pkg_files[p]: + found = True + bb.debug(1, "target found in %s" % p) + if p == pkg: + break + if pkg not in newrdepends: + newrdepends[pkg] = [] + newrdepends[pkg].append(p) + break + if found == False: + bb.note("%s contains dangling symlink to %s" % (pkg, l)) + + for pkg in newrdepends: + rdepends = bb.utils.explode_dep_versions2(d.getVar('RDEPENDS:' + pkg) or "") + for p in newrdepends[pkg]: + if p not in rdepends: + rdepends[p] = [] + d.setVar('RDEPENDS:' + pkg, bb.utils.join_deps(rdepends, commasep=False)) + +def process_filedeps(pkgfiles, d): + """ + Collect perfile run-time dependency metadata + Output: + FILERPROVIDESFLIST:pkg - list of all files w/ deps + FILERPROVIDES:filepath:pkg - per file dep + + FILERDEPENDSFLIST:pkg - list of all files w/ deps + FILERDEPENDS:filepath:pkg - per file dep + """ + if d.getVar('SKIP_FILEDEPS') == '1': + return + + pkgdest = d.getVar('PKGDEST') + packages = d.getVar('PACKAGES') + rpmdeps = d.getVar('RPMDEPS') + + def chunks(files, n): + return [files[i:i+n] for i in range(0, len(files), n)] + + pkglist = [] + for pkg in packages.split(): + if d.getVar('SKIP_FILEDEPS:' + pkg) == '1': + continue + if pkg.endswith('-dbg') or pkg.endswith('-doc') or pkg.find('-locale-') != -1 or pkg.find('-localedata-') != -1 or pkg.find('-gconv-') != -1 or pkg.find('-charmap-') != -1 or pkg.startswith('kernel-module-') or pkg.endswith('-src'): + continue + for files in chunks(pkgfiles[pkg], 100): + pkglist.append((pkg, files, rpmdeps, pkgdest)) + + processed = oe.utils.multiprocess_launch(oe.package.filedeprunner, pkglist, d) + + provides_files = {} + requires_files = {} + + for result in processed: + (pkg, provides, requires) = result + + if pkg not in provides_files: + provides_files[pkg] = [] + if pkg not in requires_files: + requires_files[pkg] = [] + + for file in sorted(provides): + provides_files[pkg].append(file) + key = "FILERPROVIDES:" + file + ":" + pkg + d.appendVar(key, " " + " ".join(provides[file])) + + for file in sorted(requires): + requires_files[pkg].append(file) + key = "FILERDEPENDS:" + file + ":" + pkg + d.appendVar(key, " " + " ".join(requires[file])) + + for pkg in requires_files: + d.setVar("FILERDEPENDSFLIST:" + pkg, " ".join(sorted(requires_files[pkg]))) + for pkg in provides_files: + d.setVar("FILERPROVIDESFLIST:" + pkg, " ".join(sorted(provides_files[pkg]))) + +def process_shlibs(pkgfiles, d): + cpath = oe.cachedpath.CachedPath() + + exclude_shlibs = d.getVar('EXCLUDE_FROM_SHLIBS', False) + if exclude_shlibs: + bb.note("not generating shlibs") + return + + lib_re = re.compile(r"^.*\.so") + libdir_re = re.compile(r".*/%s$" % d.getVar('baselib')) + + packages = d.getVar('PACKAGES') + + shlib_pkgs = [] + exclusion_list = d.getVar("EXCLUDE_PACKAGES_FROM_SHLIBS") + if exclusion_list: + for pkg in packages.split(): + if pkg not in exclusion_list.split(): + shlib_pkgs.append(pkg) + else: + bb.note("not generating shlibs for %s" % pkg) + else: + shlib_pkgs = packages.split() + + hostos = d.getVar('HOST_OS') + + workdir = d.getVar('WORKDIR') + + ver = d.getVar('PKGV') + if not ver: + msg = "PKGV not defined" + oe.qa.handle_error("pkgv-undefined", msg, d) + return + + pkgdest = d.getVar('PKGDEST') + + shlibswork_dir = d.getVar('SHLIBSWORKDIR') + + def linux_so(file, pkg, pkgver, d): + needs_ldconfig = False + needed = set() + sonames = set() + renames = [] + ldir = os.path.dirname(file).replace(pkgdest + "/" + pkg, '') + cmd = d.getVar('OBJDUMP') + " -p " + shlex.quote(file) + " 2>/dev/null" + fd = os.popen(cmd) + lines = fd.readlines() + fd.close() + rpath = tuple() + for l in lines: + m = re.match(r"\s+RPATH\s+([^\s]*)", l) + if m: + rpaths = m.group(1).replace("$ORIGIN", ldir).split(":") + rpath = tuple(map(os.path.normpath, rpaths)) + for l in lines: + m = re.match(r"\s+NEEDED\s+([^\s]*)", l) + if m: + dep = m.group(1) + if dep not in needed: + needed.add((dep, file, rpath)) + m = re.match(r"\s+SONAME\s+([^\s]*)", l) + if m: + this_soname = m.group(1) + prov = (this_soname, ldir, pkgver) + if not prov in sonames: + # if library is private (only used by package) then do not build shlib for it + if not private_libs or len([i for i in private_libs if fnmatch.fnmatch(this_soname, i)]) == 0: + sonames.add(prov) + if libdir_re.match(os.path.dirname(file)): + needs_ldconfig = True + if needs_ldconfig and snap_symlinks and (os.path.basename(file) != this_soname): + renames.append((file, os.path.join(os.path.dirname(file), this_soname))) + return (needs_ldconfig, needed, sonames, renames) + + def darwin_so(file, needed, sonames, renames, pkgver): + if not os.path.exists(file): + return + ldir = os.path.dirname(file).replace(pkgdest + "/" + pkg, '') + + def get_combinations(base): + # + # Given a base library name, find all combinations of this split by "." and "-" + # + combos = [] + options = base.split(".") + for i in range(1, len(options) + 1): + combos.append(".".join(options[0:i])) + options = base.split("-") + for i in range(1, len(options) + 1): + combos.append("-".join(options[0:i])) + return combos + + if (file.endswith('.dylib') or file.endswith('.so')) and not pkg.endswith('-dev') and not pkg.endswith('-dbg') and not pkg.endswith('-src'): + # Drop suffix + name = os.path.basename(file).rsplit(".",1)[0] + # Find all combinations + combos = get_combinations(name) + for combo in combos: + if not combo in sonames: + prov = (combo, ldir, pkgver) + sonames.add(prov) + if file.endswith('.dylib') or file.endswith('.so'): + rpath = [] + p = subprocess.Popen([d.expand("${HOST_PREFIX}otool"), '-l', file], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + out, err = p.communicate() + # If returned successfully, process stdout for results + if p.returncode == 0: + for l in out.split("\n"): + l = l.strip() + if l.startswith('path '): + rpath.append(l.split()[1]) + + p = subprocess.Popen([d.expand("${HOST_PREFIX}otool"), '-L', file], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + out, err = p.communicate() + # If returned successfully, process stdout for results + if p.returncode == 0: + for l in out.split("\n"): + l = l.strip() + if not l or l.endswith(":"): + continue + if "is not an object file" in l: + continue + name = os.path.basename(l.split()[0]).rsplit(".", 1)[0] + if name and name not in needed[pkg]: + needed[pkg].add((name, file, tuple())) + + def mingw_dll(file, needed, sonames, renames, pkgver): + if not os.path.exists(file): + return + + if file.endswith(".dll"): + # assume all dlls are shared objects provided by the package + sonames.add((os.path.basename(file), os.path.dirname(file).replace(pkgdest + "/" + pkg, ''), pkgver)) + + if (file.endswith(".dll") or file.endswith(".exe")): + # use objdump to search for "DLL Name: .*\.dll" + p = subprocess.Popen([d.expand("${OBJDUMP}"), "-p", file], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + # process the output, grabbing all .dll names + if p.returncode == 0: + for m in re.finditer(r"DLL Name: (.*?\.dll)$", out.decode(), re.MULTILINE | re.IGNORECASE): + dllname = m.group(1) + if dllname: + needed[pkg].add((dllname, file, tuple())) + + if d.getVar('PACKAGE_SNAP_LIB_SYMLINKS') == "1": + snap_symlinks = True + else: + snap_symlinks = False + + needed = {} + + shlib_provider = oe.package.read_shlib_providers(d) + + for pkg in shlib_pkgs: + private_libs = d.getVar('PRIVATE_LIBS:' + pkg) or d.getVar('PRIVATE_LIBS') or "" + private_libs = private_libs.split() + needs_ldconfig = False + bb.debug(2, "calculating shlib provides for %s" % pkg) + + pkgver = d.getVar('PKGV:' + pkg) + if not pkgver: + pkgver = d.getVar('PV_' + pkg) + if not pkgver: + pkgver = ver + + needed[pkg] = set() + sonames = set() + renames = [] + linuxlist = [] + for file in pkgfiles[pkg]: + soname = None + if cpath.islink(file): + continue + if hostos.startswith("darwin"): + darwin_so(file, needed, sonames, renames, pkgver) + elif hostos.startswith("mingw"): + mingw_dll(file, needed, sonames, renames, pkgver) + elif os.access(file, os.X_OK) or lib_re.match(file): + linuxlist.append(file) + + if linuxlist: + results = oe.utils.multiprocess_launch(linux_so, linuxlist, d, extraargs=(pkg, pkgver, d)) + for r in results: + ldconfig = r[0] + needed[pkg] |= r[1] + sonames |= r[2] + renames.extend(r[3]) + needs_ldconfig = needs_ldconfig or ldconfig + + for (old, new) in renames: + bb.note("Renaming %s to %s" % (old, new)) + bb.utils.rename(old, new) + pkgfiles[pkg].remove(old) + + shlibs_file = os.path.join(shlibswork_dir, pkg + ".list") + if len(sonames): + with open(shlibs_file, 'w') as fd: + for s in sorted(sonames): + if s[0] in shlib_provider and s[1] in shlib_provider[s[0]]: + (old_pkg, old_pkgver) = shlib_provider[s[0]][s[1]] + if old_pkg != pkg: + bb.warn('%s-%s was registered as shlib provider for %s, changing it to %s-%s because it was built later' % (old_pkg, old_pkgver, s[0], pkg, pkgver)) + bb.debug(1, 'registering %s-%s as shlib provider for %s' % (pkg, pkgver, s[0])) + fd.write(s[0] + ':' + s[1] + ':' + s[2] + '\n') + if s[0] not in shlib_provider: + shlib_provider[s[0]] = {} + shlib_provider[s[0]][s[1]] = (pkg, pkgver) + if needs_ldconfig: + bb.debug(1, 'adding ldconfig call to postinst for %s' % pkg) + postinst = d.getVar('pkg_postinst:%s' % pkg) + if not postinst: + postinst = '#!/bin/sh\n' + postinst += d.getVar('ldconfig_postinst_fragment') + d.setVar('pkg_postinst:%s' % pkg, postinst) + bb.debug(1, 'LIBNAMES: pkg %s sonames %s' % (pkg, sonames)) + + assumed_libs = d.getVar('ASSUME_SHLIBS') + if assumed_libs: + libdir = d.getVar("libdir") + for e in assumed_libs.split(): + l, dep_pkg = e.split(":") + lib_ver = None + dep_pkg = dep_pkg.rsplit("_", 1) + if len(dep_pkg) == 2: + lib_ver = dep_pkg[1] + dep_pkg = dep_pkg[0] + if l not in shlib_provider: + shlib_provider[l] = {} + shlib_provider[l][libdir] = (dep_pkg, lib_ver) + + libsearchpath = [d.getVar('libdir'), d.getVar('base_libdir')] + + for pkg in shlib_pkgs: + bb.debug(2, "calculating shlib requirements for %s" % pkg) + + private_libs = d.getVar('PRIVATE_LIBS:' + pkg) or d.getVar('PRIVATE_LIBS') or "" + private_libs = private_libs.split() + + deps = list() + for n in needed[pkg]: + # if n is in private libraries, don't try to search provider for it + # this could cause problem in case some abc.bb provides private + # /opt/abc/lib/libfoo.so.1 and contains /usr/bin/abc depending on system library libfoo.so.1 + # but skipping it is still better alternative than providing own + # version and then adding runtime dependency for the same system library + if private_libs and len([i for i in private_libs if fnmatch.fnmatch(n[0], i)]) > 0: + bb.debug(2, '%s: Dependency %s covered by PRIVATE_LIBS' % (pkg, n[0])) + continue + if n[0] in shlib_provider.keys(): + shlib_provider_map = shlib_provider[n[0]] + matches = set() + for p in itertools.chain(list(n[2]), sorted(shlib_provider_map.keys()), libsearchpath): + if p in shlib_provider_map: + matches.add(p) + if len(matches) > 1: + matchpkgs = ', '.join([shlib_provider_map[match][0] for match in matches]) + bb.error("%s: Multiple shlib providers for %s: %s (used by files: %s)" % (pkg, n[0], matchpkgs, n[1])) + elif len(matches) == 1: + (dep_pkg, ver_needed) = shlib_provider_map[matches.pop()] + + bb.debug(2, '%s: Dependency %s requires package %s (used by files: %s)' % (pkg, n[0], dep_pkg, n[1])) + + if dep_pkg == pkg: + continue + + if ver_needed: + dep = "%s (>= %s)" % (dep_pkg, ver_needed) + else: + dep = dep_pkg + if not dep in deps: + deps.append(dep) + continue + bb.note("Couldn't find shared library provider for %s, used by files: %s" % (n[0], n[1])) + + deps_file = os.path.join(pkgdest, pkg + ".shlibdeps") + if os.path.exists(deps_file): + os.remove(deps_file) + if deps: + with open(deps_file, 'w') as fd: + for dep in sorted(deps): + fd.write(dep + '\n') + +def process_pkgconfig(pkgfiles, d): + packages = d.getVar('PACKAGES') + workdir = d.getVar('WORKDIR') + pkgdest = d.getVar('PKGDEST') + + shlibs_dirs = d.getVar('SHLIBSDIRS').split() + shlibswork_dir = d.getVar('SHLIBSWORKDIR') + + pc_re = re.compile(r'(.*)\.pc$') + var_re = re.compile(r'(.*)=(.*)') + field_re = re.compile(r'(.*): (.*)') + + pkgconfig_provided = {} + pkgconfig_needed = {} + for pkg in packages.split(): + pkgconfig_provided[pkg] = [] + pkgconfig_needed[pkg] = [] + for file in sorted(pkgfiles[pkg]): + m = pc_re.match(file) + if m: + pd = bb.data.init() + name = m.group(1) + pkgconfig_provided[pkg].append(os.path.basename(name)) + if not os.access(file, os.R_OK): + continue + with open(file, 'r') as f: + lines = f.readlines() + for l in lines: + m = field_re.match(l) + if m: + hdr = m.group(1) + exp = pd.expand(m.group(2)) + if hdr == 'Requires': + pkgconfig_needed[pkg] += exp.replace(',', ' ').split() + continue + m = var_re.match(l) + if m: + name = m.group(1) + val = m.group(2) + pd.setVar(name, pd.expand(val)) + + for pkg in packages.split(): + pkgs_file = os.path.join(shlibswork_dir, pkg + ".pclist") + if pkgconfig_provided[pkg] != []: + with open(pkgs_file, 'w') as f: + for p in sorted(pkgconfig_provided[pkg]): + f.write('%s\n' % p) + + # Go from least to most specific since the last one found wins + for dir in reversed(shlibs_dirs): + if not os.path.exists(dir): + continue + for file in sorted(os.listdir(dir)): + m = re.match(r'^(.*)\.pclist$', file) + if m: + pkg = m.group(1) + with open(os.path.join(dir, file)) as fd: + lines = fd.readlines() + pkgconfig_provided[pkg] = [] + for l in lines: + pkgconfig_provided[pkg].append(l.rstrip()) + + for pkg in packages.split(): + deps = [] + for n in pkgconfig_needed[pkg]: + found = False + for k in pkgconfig_provided.keys(): + if n in pkgconfig_provided[k]: + if k != pkg and not (k in deps): + deps.append(k) + found = True + if found == False: + bb.note("couldn't find pkgconfig module '%s' in any package" % n) + deps_file = os.path.join(pkgdest, pkg + ".pcdeps") + if len(deps): + with open(deps_file, 'w') as fd: + for dep in deps: + fd.write(dep + '\n') + +def read_libdep_files(d): + pkglibdeps = {} + packages = d.getVar('PACKAGES').split() + for pkg in packages: + pkglibdeps[pkg] = {} + for extension in ".shlibdeps", ".pcdeps", ".clilibdeps": + depsfile = d.expand("${PKGDEST}/" + pkg + extension) + if os.access(depsfile, os.R_OK): + with open(depsfile) as fd: + lines = fd.readlines() + for l in lines: + l.rstrip() + deps = bb.utils.explode_dep_versions2(l) + for dep in deps: + if not dep in pkglibdeps[pkg]: + pkglibdeps[pkg][dep] = deps[dep] + return pkglibdeps + +def process_depchains(pkgfiles, d): + """ + For a given set of prefix and postfix modifiers, make those packages + RRECOMMENDS on the corresponding packages for its RDEPENDS. + + Example: If package A depends upon package B, and A's .bb emits an + A-dev package, this would make A-dev Recommends: B-dev. + + If only one of a given suffix is specified, it will take the RRECOMMENDS + based on the RDEPENDS of *all* other packages. If more than one of a given + suffix is specified, its will only use the RDEPENDS of the single parent + package. + """ + + packages = d.getVar('PACKAGES') + postfixes = (d.getVar('DEPCHAIN_POST') or '').split() + prefixes = (d.getVar('DEPCHAIN_PRE') or '').split() + + def pkg_adddeprrecs(pkg, base, suffix, getname, depends, d): + + #bb.note('depends for %s is %s' % (base, depends)) + rreclist = bb.utils.explode_dep_versions2(d.getVar('RRECOMMENDS:' + pkg) or "") + + for depend in sorted(depends): + if depend.find('-native') != -1 or depend.find('-cross') != -1 or depend.startswith('virtual/'): + #bb.note("Skipping %s" % depend) + continue + if depend.endswith('-dev'): + depend = depend[:-4] + if depend.endswith('-dbg'): + depend = depend[:-4] + pkgname = getname(depend, suffix) + #bb.note("Adding %s for %s" % (pkgname, depend)) + if pkgname not in rreclist and pkgname != pkg: + rreclist[pkgname] = [] + + #bb.note('setting: RRECOMMENDS:%s=%s' % (pkg, ' '.join(rreclist))) + d.setVar('RRECOMMENDS:%s' % pkg, bb.utils.join_deps(rreclist, commasep=False)) + + def pkg_addrrecs(pkg, base, suffix, getname, rdepends, d): + + #bb.note('rdepends for %s is %s' % (base, rdepends)) + rreclist = bb.utils.explode_dep_versions2(d.getVar('RRECOMMENDS:' + pkg) or "") + + for depend in sorted(rdepends): + if depend.find('virtual-locale-') != -1: + #bb.note("Skipping %s" % depend) + continue + if depend.endswith('-dev'): + depend = depend[:-4] + if depend.endswith('-dbg'): + depend = depend[:-4] + pkgname = getname(depend, suffix) + #bb.note("Adding %s for %s" % (pkgname, depend)) + if pkgname not in rreclist and pkgname != pkg: + rreclist[pkgname] = [] + + #bb.note('setting: RRECOMMENDS:%s=%s' % (pkg, ' '.join(rreclist))) + d.setVar('RRECOMMENDS:%s' % pkg, bb.utils.join_deps(rreclist, commasep=False)) + + def add_dep(list, dep): + if dep not in list: + list.append(dep) + + depends = [] + for dep in bb.utils.explode_deps(d.getVar('DEPENDS') or ""): + add_dep(depends, dep) + + rdepends = [] + for pkg in packages.split(): + for dep in bb.utils.explode_deps(d.getVar('RDEPENDS:' + pkg) or ""): + add_dep(rdepends, dep) + + #bb.note('rdepends is %s' % rdepends) + + def post_getname(name, suffix): + return '%s%s' % (name, suffix) + def pre_getname(name, suffix): + return '%s%s' % (suffix, name) + + pkgs = {} + for pkg in packages.split(): + for postfix in postfixes: + if pkg.endswith(postfix): + if not postfix in pkgs: + pkgs[postfix] = {} + pkgs[postfix][pkg] = (pkg[:-len(postfix)], post_getname) + + for prefix in prefixes: + if pkg.startswith(prefix): + if not prefix in pkgs: + pkgs[prefix] = {} + pkgs[prefix][pkg] = (pkg[:-len(prefix)], pre_getname) + + if "-dbg" in pkgs: + pkglibdeps = read_libdep_files(d) + pkglibdeplist = [] + for pkg in pkglibdeps: + for k in pkglibdeps[pkg]: + add_dep(pkglibdeplist, k) + dbgdefaultdeps = ((d.getVar('DEPCHAIN_DBGDEFAULTDEPS') == '1') or (bb.data.inherits_class('packagegroup', d))) + + for suffix in pkgs: + for pkg in pkgs[suffix]: + if d.getVarFlag('RRECOMMENDS:' + pkg, 'nodeprrecs'): + continue + (base, func) = pkgs[suffix][pkg] + if suffix == "-dev": + pkg_adddeprrecs(pkg, base, suffix, func, depends, d) + elif suffix == "-dbg": + if not dbgdefaultdeps: + pkg_addrrecs(pkg, base, suffix, func, pkglibdeplist, d) + continue + if len(pkgs[suffix]) == 1: + pkg_addrrecs(pkg, base, suffix, func, rdepends, d) + else: + rdeps = [] + for dep in bb.utils.explode_deps(d.getVar('RDEPENDS:' + base) or ""): + add_dep(rdeps, dep) + pkg_addrrecs(pkg, base, suffix, func, rdeps, d) diff --git a/meta/lib/oe/package_manager/__init__.py b/meta/lib/oe/package_manager/__init__.py index d3b45705ec..6774cdb794 100644 --- a/meta/lib/oe/package_manager/__init__.py +++ b/meta/lib/oe/package_manager/__init__.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # @@ -120,7 +122,8 @@ def generate_locale_archive(d, rootfs, target_arch, localedir): "riscv32": ["--uint32-align=4", "--little-endian"], "i586": ["--uint32-align=4", "--little-endian"], "i686": ["--uint32-align=4", "--little-endian"], - "x86_64": ["--uint32-align=4", "--little-endian"] + "x86_64": ["--uint32-align=4", "--little-endian"], + "loongarch64": ["--uint32-align=4", "--little-endian"] } if target_arch in locale_arch_options: arch_options = locale_arch_options[target_arch] @@ -467,7 +470,10 @@ def create_packages_dir(d, subrepo_dir, deploydir, taskname, filterbydependencie # Detect bitbake -b usage nodeps = d.getVar("BB_LIMITEDDEPS") or False if nodeps or not filterbydependencies: - oe.path.symlink(deploydir, subrepo_dir, True) + for arch in d.getVar("ALL_MULTILIB_PACKAGE_ARCHS").split() + d.getVar("ALL_MULTILIB_PACKAGE_ARCHS").replace("-", "_").split(): + target = os.path.join(deploydir + "/" + arch) + if os.path.exists(target): + oe.path.symlink(target, subrepo_dir + "/" + arch, True) return start = None diff --git a/meta/lib/oe/package_manager/deb/__init__.py b/meta/lib/oe/package_manager/deb/__init__.py index b96ea0bad4..0c23c884c1 100644 --- a/meta/lib/oe/package_manager/deb/__init__.py +++ b/meta/lib/oe/package_manager/deb/__init__.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # @@ -80,15 +82,15 @@ class DpkgIndexer(Indexer): return oe.utils.multiprocess_launch(create_index, index_cmds, self.d) - if self.d.getVar('PACKAGE_FEED_SIGN', True) == '1': - signer = get_signer(self.d, self.d.getVar('PACKAGE_FEED_GPG_BACKEND', True)) + if self.d.getVar('PACKAGE_FEED_SIGN') == '1': + signer = get_signer(self.d, self.d.getVar('PACKAGE_FEED_GPG_BACKEND')) else: signer = None if signer: for f in index_sign_files: signer.detach_sign(f, - self.d.getVar('PACKAGE_FEED_GPG_NAME', True), - self.d.getVar('PACKAGE_FEED_GPG_PASSPHRASE_FILE', True), + self.d.getVar('PACKAGE_FEED_GPG_NAME'), + self.d.getVar('PACKAGE_FEED_GPG_PASSPHRASE_FILE'), output_suffix="gpg", use_sha256=True) diff --git a/meta/lib/oe/package_manager/deb/manifest.py b/meta/lib/oe/package_manager/deb/manifest.py index d8eab24a06..72983bae98 100644 --- a/meta/lib/oe/package_manager/deb/manifest.py +++ b/meta/lib/oe/package_manager/deb/manifest.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/meta/lib/oe/package_manager/deb/rootfs.py b/meta/lib/oe/package_manager/deb/rootfs.py index 8fbaca11d6..1e25b64ed9 100644 --- a/meta/lib/oe/package_manager/deb/rootfs.py +++ b/meta/lib/oe/package_manager/deb/rootfs.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/meta/lib/oe/package_manager/deb/sdk.py b/meta/lib/oe/package_manager/deb/sdk.py index f4b0b6510a..6f3005053e 100644 --- a/meta/lib/oe/package_manager/deb/sdk.py +++ b/meta/lib/oe/package_manager/deb/sdk.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # @@ -67,7 +69,12 @@ class PkgSdk(Sdk): self.target_pm.run_pre_post_installs() + env_bkp = os.environ.copy() + os.environ['PATH'] = self.d.expand("${COREBASE}/scripts/nativesdk-intercept") + \ + os.pathsep + os.environ["PATH"] + self.target_pm.run_intercepts(populate_sdk='target') + os.environ.update(env_bkp) execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_TARGET_COMMAND")) diff --git a/meta/lib/oe/package_manager/ipk/__init__.py b/meta/lib/oe/package_manager/ipk/__init__.py index 7cbea0fa80..8cc9953a02 100644 --- a/meta/lib/oe/package_manager/ipk/__init__.py +++ b/meta/lib/oe/package_manager/ipk/__init__.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # @@ -14,6 +16,7 @@ class OpkgIndexer(Indexer): ] opkg_index_cmd = bb.utils.which(os.getenv('PATH'), "opkg-make-index") + opkg_index_cmd_extra_params = self.d.getVar('OPKG_MAKE_INDEX_EXTRA_PARAMS') or "" if self.d.getVar('PACKAGE_FEED_SIGN') == '1': signer = get_signer(self.d, self.d.getVar('PACKAGE_FEED_GPG_BACKEND')) else: @@ -39,8 +42,8 @@ class OpkgIndexer(Indexer): if not os.path.exists(pkgs_file): open(pkgs_file, "w").close() - index_cmds.add('%s --checksum md5 --checksum sha256 -r %s -p %s -m %s' % - (opkg_index_cmd, pkgs_file, pkgs_file, pkgs_dir)) + index_cmds.add('%s --checksum md5 --checksum sha256 -r %s -p %s -m %s %s' % + (opkg_index_cmd, pkgs_file, pkgs_file, pkgs_dir, opkg_index_cmd_extra_params)) index_sign_files.add(pkgs_file) @@ -131,7 +134,7 @@ class OpkgDpkgPM(PackageManager): tmp_dir = tempfile.mkdtemp() current_dir = os.getcwd() os.chdir(tmp_dir) - data_tar = 'data.tar.xz' + data_tar = 'data.tar.zst' try: cmd = [ar_cmd, 'x', pkg_path] @@ -245,7 +248,7 @@ class OpkgPM(OpkgDpkgPM): """ if (self.d.getVar('FEED_DEPLOYDIR_BASE_URI') or "") != "": for arch in self.pkg_archs.split(): - cfg_file_name = os.path.join(self.target_rootfs, + cfg_file_name = oe.path.join(self.target_rootfs, self.d.getVar("sysconfdir"), "opkg", "local-%s-feed.conf" % arch) @@ -503,6 +506,6 @@ class OpkgPM(OpkgDpkgPM): "trying to extract the package." % pkg) tmp_dir = super(OpkgPM, self).extract(pkg, pkg_info) - bb.utils.remove(os.path.join(tmp_dir, "data.tar.xz")) + bb.utils.remove(os.path.join(tmp_dir, "data.tar.zst")) return tmp_dir diff --git a/meta/lib/oe/package_manager/ipk/manifest.py b/meta/lib/oe/package_manager/ipk/manifest.py index ae451c5c70..3549d7428d 100644 --- a/meta/lib/oe/package_manager/ipk/manifest.py +++ b/meta/lib/oe/package_manager/ipk/manifest.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # @@ -62,7 +64,7 @@ class PkgManifest(Manifest): if len(pkgs_to_install) == 0: return - output = pm.dummy_install(pkgs_to_install).decode('utf-8') + output = pm.dummy_install(pkgs_to_install) with open(self.full_manifest, 'w+') as manifest: pkg_re = re.compile('^Installing ([^ ]+) [^ ].*') diff --git a/meta/lib/oe/package_manager/ipk/rootfs.py b/meta/lib/oe/package_manager/ipk/rootfs.py index 10a831994e..ba93eb62ea 100644 --- a/meta/lib/oe/package_manager/ipk/rootfs.py +++ b/meta/lib/oe/package_manager/ipk/rootfs.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # @@ -163,7 +165,7 @@ class PkgRootfs(DpkgOpkgRootfs): """ def _multilib_sanity_test(self, dirs): - allow_replace = self.d.getVar("MULTILIBRE_ALLOW_REP") + allow_replace = "|".join((self.d.getVar("MULTILIBRE_ALLOW_REP") or "").split()) if allow_replace is None: allow_replace = "" diff --git a/meta/lib/oe/package_manager/ipk/sdk.py b/meta/lib/oe/package_manager/ipk/sdk.py index e2ca415c8e..3acd55f548 100644 --- a/meta/lib/oe/package_manager/ipk/sdk.py +++ b/meta/lib/oe/package_manager/ipk/sdk.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # @@ -61,12 +63,19 @@ class PkgSdk(Sdk): self.target_pm.install_complementary(self.d.getVar('SDKIMAGE_INSTALL_COMPLEMENTARY')) + env_bkp = os.environ.copy() + os.environ['PATH'] = self.d.expand("${COREBASE}/scripts/nativesdk-intercept") + \ + os.pathsep + os.environ["PATH"] + self.target_pm.run_intercepts(populate_sdk='target') + os.environ.update(env_bkp) execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_TARGET_COMMAND")) if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d): self.target_pm.remove_packaging_data() + else: + self.target_pm.remove_lists() bb.note("Installing NATIVESDK packages") self._populate_sysroot(self.host_pm, self.host_manifest) @@ -78,6 +87,8 @@ class PkgSdk(Sdk): if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d): self.host_pm.remove_packaging_data() + else: + self.host_pm.remove_lists() target_sysconfdir = os.path.join(self.sdk_target_sysroot, self.sysconfdir) host_sysconfdir = os.path.join(self.sdk_host_sysroot, self.sysconfdir) diff --git a/meta/lib/oe/package_manager/rpm/__init__.py b/meta/lib/oe/package_manager/rpm/__init__.py index d97dab3293..f40c880af4 100644 --- a/meta/lib/oe/package_manager/rpm/__init__.py +++ b/meta/lib/oe/package_manager/rpm/__init__.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # @@ -96,11 +98,15 @@ class RpmPM(PackageManager): archs = ["sdk_provides_dummy_target"] + archs confdir = "%s/%s" %(self.target_rootfs, "etc/dnf/vars/") bb.utils.mkdirhier(confdir) - open(confdir + "arch", 'w').write(":".join(archs)) + with open(confdir + "arch", 'w') as f: + f.write(":".join(archs)) + distro_codename = self.d.getVar('DISTRO_CODENAME') - open(confdir + "releasever", 'w').write(distro_codename if distro_codename is not None else '') + with open(confdir + "releasever", 'w') as f: + f.write(distro_codename if distro_codename is not None else '') - open(oe.path.join(self.target_rootfs, "etc/dnf/dnf.conf"), 'w').write("") + with open(oe.path.join(self.target_rootfs, "etc/dnf/dnf.conf"), 'w') as f: + f.write("") def _configure_rpm(self): @@ -110,14 +116,17 @@ class RpmPM(PackageManager): platformconfdir = "%s/%s" %(self.target_rootfs, "etc/rpm/") rpmrcconfdir = "%s/%s" %(self.target_rootfs, "etc/") bb.utils.mkdirhier(platformconfdir) - open(platformconfdir + "platform", 'w').write("%s-pc-linux" % self.primary_arch) + with open(platformconfdir + "platform", 'w') as f: + f.write("%s-pc-linux" % self.primary_arch) with open(rpmrcconfdir + "rpmrc", 'w') as f: f.write("arch_compat: %s: %s\n" % (self.primary_arch, self.archs if len(self.archs) > 0 else self.primary_arch)) f.write("buildarch_compat: %s: noarch\n" % self.primary_arch) - open(platformconfdir + "macros", 'w').write("%_transaction_color 7\n") + with open(platformconfdir + "macros", 'w') as f: + f.write("%_transaction_color 7\n") if self.d.getVar('RPM_PREFER_ELF_ARCH'): - open(platformconfdir + "macros", 'a').write("%%_prefer_color %s" % (self.d.getVar('RPM_PREFER_ELF_ARCH'))) + with open(platformconfdir + "macros", 'a') as f: + f.write("%%_prefer_color %s" % (self.d.getVar('RPM_PREFER_ELF_ARCH'))) if self.d.getVar('RPM_SIGN_PACKAGES') == '1': signer = get_signer(self.d, self.d.getVar('RPM_GPG_BACKEND')) @@ -164,13 +173,13 @@ class RpmPM(PackageManager): repo_uri = uri + "/" + arch repo_id = "oe-remote-repo" + "-".join(urlparse(repo_uri).path.split("/")) repo_name = "OE Remote Repo:" + " ".join(urlparse(repo_uri).path.split("/")) - open(oe.path.join(self.target_rootfs, "etc", "yum.repos.d", repo_base + ".repo"), 'a').write( - "[%s]\nname=%s\nbaseurl=%s\n%s\n" % (repo_id, repo_name, repo_uri, gpg_opts)) + with open(oe.path.join(self.target_rootfs, "etc", "yum.repos.d", repo_base + ".repo"), 'a') as f: + f.write("[%s]\nname=%s\nbaseurl=%s\n%s\n" % (repo_id, repo_name, repo_uri, gpg_opts)) else: repo_name = "OE Remote Repo:" + " ".join(urlparse(uri).path.split("/")) repo_uri = uri - open(oe.path.join(self.target_rootfs, "etc", "yum.repos.d", repo_base + ".repo"), 'w').write( - "[%s]\nname=%s\nbaseurl=%s\n%s" % (repo_base, repo_name, repo_uri, gpg_opts)) + with open(oe.path.join(self.target_rootfs, "etc", "yum.repos.d", repo_base + ".repo"), 'w') as f: + f.write("[%s]\nname=%s\nbaseurl=%s\n%s" % (repo_base, repo_name, repo_uri, gpg_opts)) def _prepare_pkg_transaction(self): os.environ['D'] = self.target_rootfs @@ -329,7 +338,8 @@ class RpmPM(PackageManager): return e.output.decode("utf-8") def dump_install_solution(self, pkgs): - open(self.solution_manifest, 'w').write(" ".join(pkgs)) + with open(self.solution_manifest, 'w') as f: + f.write(" ".join(pkgs)) return pkgs def load_old_install_solution(self): @@ -363,7 +373,8 @@ class RpmPM(PackageManager): bb.utils.mkdirhier(target_path) num = self._script_num_prefix(target_path) saved_script_name = oe.path.join(target_path, "%d-%s" % (num, pkg)) - open(saved_script_name, 'w').write(output) + with open(saved_script_name, 'w') as f: + f.write(output) os.chmod(saved_script_name, 0o755) def _handle_intercept_failure(self, registered_pkgs): @@ -375,11 +386,12 @@ class RpmPM(PackageManager): self.save_rpmpostinst(pkg) def extract(self, pkg): - output = self._invoke_dnf(["repoquery", "--queryformat", "%{location}", pkg]) + output = self._invoke_dnf(["repoquery", "--location", pkg]) pkg_name = output.splitlines()[-1] if not pkg_name.endswith(".rpm"): bb.fatal("dnf could not find package %s in repository: %s" %(pkg, output)) - pkg_path = oe.path.join(self.rpm_repo_dir, pkg_name) + # Strip file: prefix + pkg_path = pkg_name[5:] cpio_cmd = bb.utils.which(os.getenv("PATH"), "cpio") rpm2cpio_cmd = bb.utils.which(os.getenv("PATH"), "rpm2cpio") diff --git a/meta/lib/oe/package_manager/rpm/manifest.py b/meta/lib/oe/package_manager/rpm/manifest.py index e6604b301f..6ee7c329f0 100644 --- a/meta/lib/oe/package_manager/rpm/manifest.py +++ b/meta/lib/oe/package_manager/rpm/manifest.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/meta/lib/oe/package_manager/rpm/rootfs.py b/meta/lib/oe/package_manager/rpm/rootfs.py index 00d07cd9cc..3ba5396320 100644 --- a/meta/lib/oe/package_manager/rpm/rootfs.py +++ b/meta/lib/oe/package_manager/rpm/rootfs.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # @@ -108,7 +110,7 @@ class PkgRootfs(Rootfs): if self.progress_reporter: self.progress_reporter.next_stage() - self._setup_dbg_rootfs(['/etc', '/var/lib/rpm', '/var/cache/dnf', '/var/lib/dnf']) + self._setup_dbg_rootfs(['/etc/rpm', '/etc/rpmrc', '/etc/dnf', '/var/lib/rpm', '/var/cache/dnf', '/var/lib/dnf']) execute_pre_post_process(self.d, rpm_post_process_cmds) diff --git a/meta/lib/oe/package_manager/rpm/sdk.py b/meta/lib/oe/package_manager/rpm/sdk.py index c5f232431f..ea79fe050b 100644 --- a/meta/lib/oe/package_manager/rpm/sdk.py +++ b/meta/lib/oe/package_manager/rpm/sdk.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # @@ -65,7 +67,12 @@ class PkgSdk(Sdk): self.target_pm.install_complementary(self.d.getVar('SDKIMAGE_INSTALL_COMPLEMENTARY')) + env_bkp = os.environ.copy() + os.environ['PATH'] = self.d.expand("${COREBASE}/scripts/nativesdk-intercept") + \ + os.pathsep + os.environ["PATH"] + self.target_pm.run_intercepts(populate_sdk='target') + os.environ.update(env_bkp) execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_TARGET_COMMAND")) @@ -110,5 +117,6 @@ class PkgSdk(Sdk): for f in glob.glob(os.path.join(self.sdk_output, "etc", "rpm*")): self.movefile(f, native_sysconf_dir) for f in glob.glob(os.path.join(self.sdk_output, "etc", "dnf", "*")): - self.movefile(f, native_sysconf_dir) + self.mkdirhier(native_sysconf_dir + "/dnf") + self.movefile(f, native_sysconf_dir + "/dnf") self.remove(os.path.join(self.sdk_output, "etc"), True) diff --git a/meta/lib/oe/packagedata.py b/meta/lib/oe/packagedata.py index 212f048bc6..2d1d6ddeb7 100644 --- a/meta/lib/oe/packagedata.py +++ b/meta/lib/oe/packagedata.py @@ -1,9 +1,16 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # import codecs import os +import json +import bb.compress.zstd +import oe.path + +from glob import glob def packaged(pkg, d): return os.access(get_subpkgedata_fn(pkg, d) + '.packaged', os.R_OK) @@ -108,3 +115,252 @@ def recipename(pkg, d): """Return the recipe name for the given binary package name.""" return pkgmap(d).get(pkg) + +def foreach_runtime_provider_pkgdata(d, rdep, include_rdep=False): + pkgdata_dir = d.getVar("PKGDATA_DIR") + possibles = set() + try: + possibles |= set(os.listdir("%s/runtime-rprovides/%s/" % (pkgdata_dir, rdep))) + except OSError: + pass + + if include_rdep: + possibles.add(rdep) + + for p in sorted(list(possibles)): + rdep_data = read_subpkgdata(p, d) + yield p, rdep_data + +def get_package_mapping(pkg, basepkg, d, depversions=None): + import oe.packagedata + + data = oe.packagedata.read_subpkgdata(pkg, d) + key = "PKG:%s" % pkg + + if key in data: + if bb.data.inherits_class('allarch', d) and bb.data.inherits_class('packagegroup', d) and pkg != data[key]: + bb.error("An allarch packagegroup shouldn't depend on packages which are dynamically renamed (%s to %s)" % (pkg, data[key])) + # Have to avoid undoing the write_extra_pkgs(global_variants...) + if bb.data.inherits_class('allarch', d) and not d.getVar('MULTILIB_VARIANTS') \ + and data[key] == basepkg: + return pkg + if depversions == []: + # Avoid returning a mapping if the renamed package rprovides its original name + rprovkey = "RPROVIDES:%s" % pkg + if rprovkey in data: + if pkg in bb.utils.explode_dep_versions2(data[rprovkey]): + bb.note("%s rprovides %s, not replacing the latter" % (data[key], pkg)) + return pkg + # Do map to rewritten package name + return data[key] + + return pkg + +def get_package_additional_metadata(pkg_type, d): + base_key = "PACKAGE_ADD_METADATA" + for key in ("%s_%s" % (base_key, pkg_type.upper()), base_key): + if d.getVar(key, False) is None: + continue + d.setVarFlag(key, "type", "list") + if d.getVarFlag(key, "separator") is None: + d.setVarFlag(key, "separator", "\\n") + metadata_fields = [field.strip() for field in oe.data.typed_value(key, d)] + return "\n".join(metadata_fields).strip() + +def runtime_mapping_rename(varname, pkg, d): + #bb.note("%s before: %s" % (varname, d.getVar(varname))) + + new_depends = {} + deps = bb.utils.explode_dep_versions2(d.getVar(varname) or "") + for depend, depversions in deps.items(): + new_depend = get_package_mapping(depend, pkg, d, depversions) + if depend != new_depend: + bb.note("package name mapping done: %s -> %s" % (depend, new_depend)) + new_depends[new_depend] = deps[depend] + + d.setVar(varname, bb.utils.join_deps(new_depends, commasep=False)) + + #bb.note("%s after: %s" % (varname, d.getVar(varname))) + +def emit_pkgdata(pkgfiles, d): + def process_postinst_on_target(pkg, mlprefix): + pkgval = d.getVar('PKG:%s' % pkg) + if pkgval is None: + pkgval = pkg + + defer_fragment = """ +if [ -n "$D" ]; then + $INTERCEPT_DIR/postinst_intercept delay_to_first_boot %s mlprefix=%s + exit 0 +fi +""" % (pkgval, mlprefix) + + postinst = d.getVar('pkg_postinst:%s' % pkg) + postinst_ontarget = d.getVar('pkg_postinst_ontarget:%s' % pkg) + + if postinst_ontarget: + bb.debug(1, 'adding deferred pkg_postinst_ontarget() to pkg_postinst() for %s' % pkg) + if not postinst: + postinst = '#!/bin/sh\n' + postinst += defer_fragment + postinst += postinst_ontarget + d.setVar('pkg_postinst:%s' % pkg, postinst) + + def add_set_e_to_scriptlets(pkg): + for scriptlet_name in ('pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm'): + scriptlet = d.getVar('%s:%s' % (scriptlet_name, pkg)) + if scriptlet: + scriptlet_split = scriptlet.split('\n') + if scriptlet_split[0].startswith("#!"): + scriptlet = scriptlet_split[0] + "\nset -e\n" + "\n".join(scriptlet_split[1:]) + else: + scriptlet = "set -e\n" + "\n".join(scriptlet_split[0:]) + d.setVar('%s:%s' % (scriptlet_name, pkg), scriptlet) + + def write_if_exists(f, pkg, var): + def encode(str): + import codecs + c = codecs.getencoder("unicode_escape") + return c(str)[0].decode("latin1") + + val = d.getVar('%s:%s' % (var, pkg)) + if val: + f.write('%s:%s: %s\n' % (var, pkg, encode(val))) + return val + val = d.getVar('%s' % (var)) + if val: + f.write('%s: %s\n' % (var, encode(val))) + return val + + def write_extra_pkgs(variants, pn, packages, pkgdatadir): + for variant in variants: + with open("%s/%s-%s" % (pkgdatadir, variant, pn), 'w') as fd: + fd.write("PACKAGES: %s\n" % ' '.join( + map(lambda pkg: '%s-%s' % (variant, pkg), packages.split()))) + + def write_extra_runtime_pkgs(variants, packages, pkgdatadir): + for variant in variants: + for pkg in packages.split(): + ml_pkg = "%s-%s" % (variant, pkg) + subdata_file = "%s/runtime/%s" % (pkgdatadir, ml_pkg) + with open(subdata_file, 'w') as fd: + fd.write("PKG:%s: %s" % (ml_pkg, pkg)) + + packages = d.getVar('PACKAGES') + pkgdest = d.getVar('PKGDEST') + pkgdatadir = d.getVar('PKGDESTWORK') + + data_file = pkgdatadir + d.expand("/${PN}") + with open(data_file, 'w') as fd: + fd.write("PACKAGES: %s\n" % packages) + + pkgdebugsource = d.getVar("PKGDEBUGSOURCES") or [] + + pn = d.getVar('PN') + global_variants = (d.getVar('MULTILIB_GLOBAL_VARIANTS') or "").split() + variants = (d.getVar('MULTILIB_VARIANTS') or "").split() + + if bb.data.inherits_class('kernel', d) or bb.data.inherits_class('module-base', d): + write_extra_pkgs(variants, pn, packages, pkgdatadir) + + if bb.data.inherits_class('allarch', d) and not variants \ + and not bb.data.inherits_class('packagegroup', d): + write_extra_pkgs(global_variants, pn, packages, pkgdatadir) + + workdir = d.getVar('WORKDIR') + + for pkg in packages.split(): + pkgval = d.getVar('PKG:%s' % pkg) + if pkgval is None: + pkgval = pkg + d.setVar('PKG:%s' % pkg, pkg) + + extended_data = { + "files_info": {} + } + + pkgdestpkg = os.path.join(pkgdest, pkg) + files = {} + files_extra = {} + total_size = 0 + seen = set() + for f in pkgfiles[pkg]: + fpath = os.sep + os.path.relpath(f, pkgdestpkg) + + fstat = os.lstat(f) + files[fpath] = fstat.st_size + + extended_data["files_info"].setdefault(fpath, {}) + extended_data["files_info"][fpath]['size'] = fstat.st_size + + if fstat.st_ino not in seen: + seen.add(fstat.st_ino) + total_size += fstat.st_size + + if fpath in pkgdebugsource: + extended_data["files_info"][fpath]['debugsrc'] = pkgdebugsource[fpath] + del pkgdebugsource[fpath] + + d.setVar('FILES_INFO:' + pkg , json.dumps(files, sort_keys=True)) + + process_postinst_on_target(pkg, d.getVar("MLPREFIX")) + add_set_e_to_scriptlets(pkg) + + subdata_file = pkgdatadir + "/runtime/%s" % pkg + with open(subdata_file, 'w') as sf: + for var in (d.getVar('PKGDATA_VARS') or "").split(): + val = write_if_exists(sf, pkg, var) + + write_if_exists(sf, pkg, 'FILERPROVIDESFLIST') + for dfile in sorted((d.getVar('FILERPROVIDESFLIST:' + pkg) or "").split()): + write_if_exists(sf, pkg, 'FILERPROVIDES:' + dfile) + + write_if_exists(sf, pkg, 'FILERDEPENDSFLIST') + for dfile in sorted((d.getVar('FILERDEPENDSFLIST:' + pkg) or "").split()): + write_if_exists(sf, pkg, 'FILERDEPENDS:' + dfile) + + sf.write('%s:%s: %d\n' % ('PKGSIZE', pkg, total_size)) + + subdata_extended_file = pkgdatadir + "/extended/%s.json.zstd" % pkg + num_threads = int(d.getVar("BB_NUMBER_THREADS")) + with bb.compress.zstd.open(subdata_extended_file, "wt", encoding="utf-8", num_threads=num_threads) as f: + json.dump(extended_data, f, sort_keys=True, separators=(",", ":")) + + # Symlinks needed for rprovides lookup + rprov = d.getVar('RPROVIDES:%s' % pkg) or d.getVar('RPROVIDES') + if rprov: + for p in bb.utils.explode_deps(rprov): + subdata_sym = pkgdatadir + "/runtime-rprovides/%s/%s" % (p, pkg) + bb.utils.mkdirhier(os.path.dirname(subdata_sym)) + oe.path.relsymlink(subdata_file, subdata_sym, True) + + allow_empty = d.getVar('ALLOW_EMPTY:%s' % pkg) + if not allow_empty: + allow_empty = d.getVar('ALLOW_EMPTY') + root = "%s/%s" % (pkgdest, pkg) + os.chdir(root) + g = glob('*') + if g or allow_empty == "1": + # Symlinks needed for reverse lookups (from the final package name) + subdata_sym = pkgdatadir + "/runtime-reverse/%s" % pkgval + oe.path.relsymlink(subdata_file, subdata_sym, True) + + packagedfile = pkgdatadir + '/runtime/%s.packaged' % pkg + open(packagedfile, 'w').close() + + if bb.data.inherits_class('kernel', d) or bb.data.inherits_class('module-base', d): + write_extra_runtime_pkgs(variants, packages, pkgdatadir) + + if bb.data.inherits_class('allarch', d) and not variants \ + and not bb.data.inherits_class('packagegroup', d): + write_extra_runtime_pkgs(global_variants, packages, pkgdatadir) + +def mapping_rename_hook(d): + """ + Rewrite variables to account for package renaming in things + like debian.bbclass or manual PKG variable name changes + """ + pkg = d.getVar("PKG") + oe.packagedata.runtime_mapping_rename("RDEPENDS", pkg, d) + oe.packagedata.runtime_mapping_rename("RRECOMMENDS", pkg, d) + oe.packagedata.runtime_mapping_rename("RSUGGESTS", pkg, d) diff --git a/meta/lib/oe/packagegroup.py b/meta/lib/oe/packagegroup.py index 8fcaecde82..7b7594751a 100644 --- a/meta/lib/oe/packagegroup.py +++ b/meta/lib/oe/packagegroup.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/meta/lib/oe/patch.py b/meta/lib/oe/patch.py index 4ec9caed45..60a0cc8291 100644 --- a/meta/lib/oe/patch.py +++ b/meta/lib/oe/patch.py @@ -1,10 +1,14 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # +import os +import shlex +import subprocess import oe.path import oe.types -import subprocess class NotFoundError(bb.BBHandledException): def __init__(self, path): @@ -25,8 +29,6 @@ class CmdError(bb.BBHandledException): def runcmd(args, dir = None): - import pipes - if dir: olddir = os.path.abspath(os.curdir) if not os.path.exists(dir): @@ -35,7 +37,7 @@ def runcmd(args, dir = None): # print("cwd: %s -> %s" % (olddir, dir)) try: - args = [ pipes.quote(str(arg)) for arg in args ] + args = [ shlex.quote(str(arg)) for arg in args ] cmd = " ".join(args) # print("cmd: %s" % cmd) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) @@ -215,7 +217,7 @@ class PatchTree(PatchSet): with open(self.seriespath, 'w') as f: for p in patches: f.write(p) - + def Import(self, patch, force = None): """""" PatchSet.Import(self, patch, force) @@ -292,8 +294,9 @@ class PatchTree(PatchSet): self.Pop(all=True) class GitApplyTree(PatchTree): - patch_line_prefix = '%% original patch' - ignore_commit_prefix = '%% ignore' + notes_ref = "refs/notes/devtool" + original_patch = 'original patch' + ignore_commit = 'ignore' def __init__(self, dir, d): PatchTree.__init__(self, dir, d) @@ -450,7 +453,7 @@ class GitApplyTree(PatchTree): # Prepare git command cmd = ["git"] GitApplyTree.gitCommandUserOptions(cmd, commituser, commitemail) - cmd += ["commit", "-F", tmpfile] + cmd += ["commit", "-F", tmpfile, "--no-verify"] # git doesn't like plain email addresses as authors if author and '<' in author: cmd.append('--author="%s"' % author) @@ -459,44 +462,131 @@ class GitApplyTree(PatchTree): return (tmpfile, cmd) @staticmethod - def extractPatches(tree, startcommit, outdir, paths=None): + def addNote(repo, ref, key, value=None): + note = key + (": %s" % value if value else "") + notes_ref = GitApplyTree.notes_ref + runcmd(["git", "config", "notes.rewriteMode", "ignore"], repo) + runcmd(["git", "config", "notes.displayRef", notes_ref, notes_ref], repo) + runcmd(["git", "config", "notes.rewriteRef", notes_ref, notes_ref], repo) + runcmd(["git", "notes", "--ref", notes_ref, "append", "-m", note, ref], repo) + + @staticmethod + def removeNote(repo, ref, key): + notes = GitApplyTree.getNotes(repo, ref) + notes = {k: v for k, v in notes.items() if k != key and not k.startswith(key + ":")} + runcmd(["git", "notes", "--ref", GitApplyTree.notes_ref, "remove", "--ignore-missing", ref], repo) + for note, value in notes.items(): + GitApplyTree.addNote(repo, ref, note, value) + + @staticmethod + def getNotes(repo, ref): + import re + + note = None + try: + note = runcmd(["git", "notes", "--ref", GitApplyTree.notes_ref, "show", ref], repo) + prefix = "" + except CmdError: + note = runcmd(['git', 'show', '-s', '--format=%B', ref], repo) + prefix = "%% " + + note_re = re.compile(r'^%s(.*?)(?::\s*(.*))?$' % prefix) + notes = dict() + for line in note.splitlines(): + m = note_re.match(line) + if m: + notes[m.group(1)] = m.group(2) + + return notes + + @staticmethod + def commitIgnored(subject, dir=None, files=None, d=None): + if files: + runcmd(['git', 'add'] + files, dir) + cmd = ["git"] + GitApplyTree.gitCommandUserOptions(cmd, d=d) + cmd += ["commit", "-m", subject, "--no-verify"] + runcmd(cmd, dir) + GitApplyTree.addNote(dir, "HEAD", GitApplyTree.ignore_commit) + + @staticmethod + def extractPatches(tree, startcommits, outdir, paths=None): import tempfile import shutil tempdir = tempfile.mkdtemp(prefix='oepatch') try: - shellcmd = ["git", "format-patch", "--no-signature", "--no-numbered", startcommit, "-o", tempdir] - if paths: - shellcmd.append('--') - shellcmd.extend(paths) - out = runcmd(["sh", "-c", " ".join(shellcmd)], tree) - if out: - for srcfile in out.split(): - for encoding in ['utf-8', 'latin-1']: - patchlines = [] - outfile = None - try: - with open(srcfile, 'r', encoding=encoding) as f: - for line in f: - if line.startswith(GitApplyTree.patch_line_prefix): - outfile = line.split()[-1].strip() - continue - if line.startswith(GitApplyTree.ignore_commit_prefix): - continue - patchlines.append(line) - except UnicodeDecodeError: + for name, rev in startcommits.items(): + shellcmd = ["git", "format-patch", "--no-signature", "--no-numbered", rev, "-o", tempdir] + if paths: + shellcmd.append('--') + shellcmd.extend(paths) + out = runcmd(["sh", "-c", " ".join(shellcmd)], os.path.join(tree, name)) + if out: + for srcfile in out.split(): + # This loop, which is used to remove any line that + # starts with "%% original patch", is kept for backwards + # compatibility. If/when that compatibility is dropped, + # it can be replaced with code to just read the first + # line of the patch file to get the SHA-1, and the code + # below that writes the modified patch file can be + # replaced with a simple file move. + for encoding in ['utf-8', 'latin-1']: + patchlines = [] + try: + with open(srcfile, 'r', encoding=encoding, newline='') as f: + for line in f: + if line.startswith("%% " + GitApplyTree.original_patch): + continue + patchlines.append(line) + except UnicodeDecodeError: + continue + break + else: + raise PatchError('Unable to find a character encoding to decode %s' % srcfile) + + sha1 = patchlines[0].split()[1] + notes = GitApplyTree.getNotes(os.path.join(tree, name), sha1) + if GitApplyTree.ignore_commit in notes: continue - break - else: - raise PatchError('Unable to find a character encoding to decode %s' % srcfile) - - if not outfile: - outfile = os.path.basename(srcfile) - with open(os.path.join(outdir, outfile), 'w') as of: - for line in patchlines: - of.write(line) + outfile = notes.get(GitApplyTree.original_patch, os.path.basename(srcfile)) + + bb.utils.mkdirhier(os.path.join(outdir, name)) + with open(os.path.join(outdir, name, outfile), 'w') as of: + for line in patchlines: + of.write(line) finally: shutil.rmtree(tempdir) + def _need_dirty_check(self): + fetch = bb.fetch2.Fetch([], self.d) + check_dirtyness = False + for url in fetch.urls: + url_data = fetch.ud[url] + parm = url_data.parm + # a git url with subpath param will surely be dirty + # since the git tree from which we clone will be emptied + # from all files that are not in the subpath + if url_data.type == 'git' and parm.get('subpath'): + check_dirtyness = True + return check_dirtyness + + def _commitpatch(self, patch, patchfilevar): + output = "" + # Add all files + shellcmd = ["git", "add", "-f", "-A", "."] + output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) + # Exclude the patches directory + shellcmd = ["git", "reset", "HEAD", self.patchdir] + output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) + # Commit the result + (tmpfile, shellcmd) = self.prepareCommit(patch['file'], self.commituser, self.commitemail) + try: + shellcmd.insert(0, patchfilevar) + output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) + finally: + os.remove(tmpfile) + return output + def _applypatch(self, patch, force = False, reverse = False, run = True): import shutil @@ -511,27 +601,26 @@ class GitApplyTree(PatchTree): return runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) - # Add hooks which add a pointer to the original patch file name in the commit message reporoot = (runcmd("git rev-parse --show-toplevel".split(), self.dir) or '').strip() if not reporoot: raise Exception("Cannot get repository root for directory %s" % self.dir) - hooks_dir = os.path.join(reporoot, '.git', 'hooks') - hooks_dir_backup = hooks_dir + '.devtool-orig' - if os.path.lexists(hooks_dir_backup): - raise Exception("Git hooks backup directory already exists: %s" % hooks_dir_backup) - if os.path.lexists(hooks_dir): - shutil.move(hooks_dir, hooks_dir_backup) - os.mkdir(hooks_dir) - commithook = os.path.join(hooks_dir, 'commit-msg') - applyhook = os.path.join(hooks_dir, 'applypatch-msg') - with open(commithook, 'w') as f: - # NOTE: the formatting here is significant; if you change it you'll also need to - # change other places which read it back - f.write('echo "\n%s: $PATCHFILE" >> $1' % GitApplyTree.patch_line_prefix) - os.chmod(commithook, 0o755) - shutil.copy2(commithook, applyhook) + + patch_applied = True try: patchfilevar = 'PATCHFILE="%s"' % os.path.basename(patch['file']) + if self._need_dirty_check(): + # Check dirtyness of the tree + try: + output = runcmd(["git", "--work-tree=%s" % reporoot, "status", "--short"]) + except CmdError: + pass + else: + if output: + # The tree is dirty, no need to try to apply patches with git anymore + # since they fail, fallback directly to patch + output = PatchTree._applypatch(self, patch, force, reverse, run) + output += self._commitpatch(patch, patchfilevar) + return output try: shellcmd = [patchfilevar, "git", "--work-tree=%s" % reporoot] self.gitCommandUserOptions(shellcmd, self.commituser, self.commitemail) @@ -558,24 +647,14 @@ class GitApplyTree(PatchTree): except CmdError: # Fall back to patch output = PatchTree._applypatch(self, patch, force, reverse, run) - # Add all files - shellcmd = ["git", "add", "-f", "-A", "."] - output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) - # Exclude the patches directory - shellcmd = ["git", "reset", "HEAD", self.patchdir] - output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) - # Commit the result - (tmpfile, shellcmd) = self.prepareCommit(patch['file'], self.commituser, self.commitemail) - try: - shellcmd.insert(0, patchfilevar) - output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) - finally: - os.remove(tmpfile) + output += self._commitpatch(patch, patchfilevar) return output + except: + patch_applied = False + raise finally: - shutil.rmtree(hooks_dir) - if os.path.lexists(hooks_dir_backup): - shutil.move(hooks_dir_backup, hooks_dir) + if patch_applied: + GitApplyTree.addNote(self.dir, "HEAD", GitApplyTree.original_patch, os.path.basename(patch['file'])) class QuiltTree(PatchSet): @@ -736,8 +815,9 @@ class NOOPResolver(Resolver): self.patchset.Push() except Exception: import sys - os.chdir(olddir) raise + finally: + os.chdir(olddir) # Patch resolver which relies on the user doing all the work involved in the # resolution, with the exception of refreshing the remote copy of the patch @@ -797,9 +877,9 @@ class UserResolver(Resolver): # User did not fix the problem. Abort. raise PatchError("Patch application failed, and user did not fix and refresh the patch.") except Exception: - os.chdir(olddir) raise - os.chdir(olddir) + finally: + os.chdir(olddir) def patch_path(url, fetch, workdir, expand=True): @@ -919,4 +999,3 @@ def should_apply(parm, d): return False, "applies to later version" return True, None - diff --git a/meta/lib/oe/path.py b/meta/lib/oe/path.py index c8d8ad05b9..5d21cdcbdf 100644 --- a/meta/lib/oe/path.py +++ b/meta/lib/oe/path.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # @@ -123,7 +125,8 @@ def copyhardlinktree(src, dst): if os.path.isdir(src): if len(glob.glob('%s/.??*' % src)) > 0: source = './.??* ' - source += './*' + if len(glob.glob('%s/**' % src)) > 0: + source += './*' s_dir = src else: source = src @@ -169,6 +172,9 @@ def symlink(source, destination, force=False): if e.errno != errno.EEXIST or os.readlink(destination) != source: raise +def relsymlink(target, name, force=False): + symlink(os.path.relpath(target, os.path.dirname(name)), name, force=force) + def find(dir, **walkoptions): """ Given a directory, recurses into that directory, returning all files as absolute paths. """ diff --git a/meta/lib/oe/prservice.py b/meta/lib/oe/prservice.py index 339f7aebca..c41242c878 100644 --- a/meta/lib/oe/prservice.py +++ b/meta/lib/oe/prservice.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # @@ -76,8 +78,7 @@ def prserv_export_tofile(d, metainfo, datainfo, lockdown, nomax=False): bb.utils.mkdirhier(d.getVar('PRSERV_DUMPDIR')) df = d.getVar('PRSERV_DUMPFILE') #write data - lf = bb.utils.lockfile("%s.lock" % df) - with open(df, "a") as f: + with open(df, "a") as f, bb.utils.fileslocked(["%s.lock" % df]) as locks: if metainfo: #dump column info f.write("#PR_core_ver = \"%s\"\n\n" % metainfo['core_ver']); @@ -111,7 +112,6 @@ def prserv_export_tofile(d, metainfo, datainfo, lockdown, nomax=False): if not nomax: for i in idx: f.write("PRAUTO_%s_%s = \"%s\"\n" % (str(datainfo[idx[i]]['version']),str(datainfo[idx[i]]['pkgarch']),str(datainfo[idx[i]]['value']))) - bb.utils.unlockfile(lf) def prserv_check_avail(d): host_params = list([_f for _f in (d.getVar("PRSERV_HOST") or '').split(':') if _f]) diff --git a/meta/lib/oe/qa.py b/meta/lib/oe/qa.py index 89acd3ead0..f8ae3c743f 100644 --- a/meta/lib/oe/qa.py +++ b/meta/lib/oe/qa.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # @@ -211,6 +213,23 @@ def exit_with_message_if_errors(message, d): def exit_if_errors(d): exit_with_message_if_errors("Fatal QA errors were found, failing task.", d) +def check_upstream_status(fullpath): + import re + kinda_status_re = re.compile(r"^.*upstream.*status.*$", re.IGNORECASE | re.MULTILINE) + strict_status_re = re.compile(r"^Upstream-Status: (Pending|Submitted|Denied|Inappropriate|Backport|Inactive-Upstream)( .+)?$", re.MULTILINE) + guidelines = "https://docs.yoctoproject.org/contributor-guide/recipe-style-guide.html#patch-upstream-status" + + with open(fullpath, encoding='utf-8', errors='ignore') as f: + file_content = f.read() + match_kinda = kinda_status_re.search(file_content) + match_strict = strict_status_re.search(file_content) + + if not match_strict: + if match_kinda: + return "Malformed Upstream-Status in patch\n%s\nPlease correct according to %s :\n%s" % (fullpath, guidelines, match_kinda.group(0)) + else: + return "Missing Upstream-Status in patch\n%s\nPlease add according to %s ." % (fullpath, guidelines) + if __name__ == "__main__": import sys diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py index b04992c66d..de1fbdd3a8 100644 --- a/meta/lib/oe/recipeutils.py +++ b/meta/lib/oe/recipeutils.py @@ -24,9 +24,9 @@ from collections import OrderedDict, defaultdict from bb.utils import vercmp_string # Help us to find places to insert values -recipe_progression = ['SUMMARY', 'DESCRIPTION', 'AUTHOR', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LICENSE_FLAGS', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRCPV', 'SRC_URI', 'S', 'do_fetch()', 'do_unpack()', 'do_patch()', 'EXTRA_OECONF', 'EXTRA_OECMAKE', 'EXTRA_OESCONS', 'do_configure()', 'EXTRA_OEMAKE', 'do_compile()', 'do_install()', 'do_populate_sysroot()', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'populate_packages()', 'do_package()', 'do_deploy()', 'BBCLASSEXTEND'] +recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LICENSE_FLAGS', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRC_URI', 'S', 'do_fetch()', 'do_unpack()', 'do_patch()', 'EXTRA_OECONF', 'EXTRA_OECMAKE', 'EXTRA_OESCONS', 'do_configure()', 'EXTRA_OEMAKE', 'do_compile()', 'do_install()', 'do_populate_sysroot()', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'populate_packages()', 'do_package()', 'do_deploy()', 'BBCLASSEXTEND'] # Variables that sometimes are a bit long but shouldn't be wrapped -nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER', r'SRC_URI\[(.+\.)?md5sum\]', r'SRC_URI\[(.+\.)?sha256sum\]'] +nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER', r'SRC_URI\[(.+\.)?md5sum\]', r'SRC_URI\[(.+\.)?sha[0-9]+sum\]'] list_vars = ['SRC_URI', 'LIC_FILES_CHKSUM'] meta_vars = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION'] @@ -421,8 +421,6 @@ def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True, all_variants=F # Ensure we handle class-target if we're dealing with one of the variants variants.append('target') for variant in variants: - if variant.startswith("devupstream"): - localdata.setVar('SRCPV', 'git') localdata.setVar('CLASSOVERRIDE', 'class-%s' % variant) fetch_urls(localdata) @@ -666,19 +664,23 @@ def get_bbappend_path(d, destlayerdir, wildcardver=False): return (appendpath, pathok) -def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, machine=None, extralines=None, removevalues=None, redirect_output=None, params=None): +def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, machine=None, extralines=None, removevalues=None, redirect_output=None, params=None, update_original_recipe=False): """ Writes a bbappend file for a recipe Parameters: rd: data dictionary for the recipe destlayerdir: base directory of the layer to place the bbappend in (subdirectory path from there will be determined automatically) - srcfiles: dict of source files to add to SRC_URI, where the value - is the full path to the file to be added, and the value is the - original filename as it would appear in SRC_URI or None if it - isn't already present. You may pass None for this parameter if - you simply want to specify your own content via the extralines - parameter. + srcfiles: dict of source files to add to SRC_URI, where the key + is the full path to the file to be added, and the value is a + dict with following optional keys: + path: the original filename as it would appear in SRC_URI + or None if it isn't already present. + patchdir: the patchdir parameter + newname: the name to give to the new added file. None to use + the default value: basename(path) + You may pass None for this parameter if you simply want to specify + your own content via the extralines parameter. install: dict mapping entries in srcfiles to a tuple of two elements: install path (*without* ${D} prefix) and permission value (as a string, e.g. '0644'). @@ -699,18 +701,29 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, params: Parameters to use when adding entries to SRC_URI. If specified, should be a list of dicts with the same length as srcfiles. + update_original_recipe: + Force to update the original recipe instead of creating/updating + a bbapend. destlayerdir must contain the original recipe """ if not removevalues: removevalues = {} - # Determine how the bbappend should be named - appendpath, pathok = get_bbappend_path(rd, destlayerdir, wildcardver) - if not appendpath: - bb.error('Unable to determine layer directory containing %s' % recipefile) - return (None, None) - if not pathok: - bb.warn('Unable to determine correct subdirectory path for bbappend file - check that what %s adds to BBFILES also matches .bbappend files. Using %s for now, but until you fix this the bbappend will not be applied.' % (os.path.join(destlayerdir, 'conf', 'layer.conf'), os.path.dirname(appendpath))) + recipefile = rd.getVar('FILE') + if update_original_recipe: + if destlayerdir not in recipefile: + bb.error("destlayerdir %s doesn't contain the original recipe (%s), cannot update it" % (destlayerdir, recipefile)) + return (None, None) + + appendpath = recipefile + else: + # Determine how the bbappend should be named + appendpath, pathok = get_bbappend_path(rd, destlayerdir, wildcardver) + if not appendpath: + bb.error('Unable to determine layer directory containing %s' % recipefile) + return (None, None) + if not pathok: + bb.warn('Unable to determine correct subdirectory path for bbappend file - check that what %s adds to BBFILES also matches .bbappend files. Using %s for now, but until you fix this the bbappend will not be applied.' % (os.path.join(destlayerdir, 'conf', 'layer.conf'), os.path.dirname(appendpath))) appenddir = os.path.dirname(appendpath) if not redirect_output: @@ -755,7 +768,7 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, bbappendlines.append((varname, op, value)) destsubdir = rd.getVar('PN') - if srcfiles: + if not update_original_recipe and srcfiles: bbappendlines.append(('FILESEXTRAPATHS:prepend', ':=', '${THISDIR}/${PN}:')) appendoverride = '' @@ -765,22 +778,38 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, copyfiles = {} if srcfiles: instfunclines = [] - for i, (newfile, origsrcfile) in enumerate(srcfiles.items()): - srcfile = origsrcfile + for i, (newfile, param) in enumerate(srcfiles.items()): srcurientry = None - if not srcfile: - srcfile = os.path.basename(newfile) + if not 'path' in param or not param['path']: + if 'newname' in param and param['newname']: + srcfile = param['newname'] + else: + srcfile = os.path.basename(newfile) srcurientry = 'file://%s' % srcfile + oldentry = None + for uri in rd.getVar('SRC_URI').split(): + if srcurientry in uri: + oldentry = uri if params and params[i]: srcurientry = '%s;%s' % (srcurientry, ';'.join('%s=%s' % (k,v) for k,v in params[i].items())) # Double-check it's not there already # FIXME do we care if the entry is added by another bbappend that might go away? if not srcurientry in rd.getVar('SRC_URI').split(): if machine: + if oldentry: + appendline('SRC_URI:remove%s' % appendoverride, '=', ' ' + oldentry) appendline('SRC_URI:append%s' % appendoverride, '=', ' ' + srcurientry) else: + if oldentry: + if update_original_recipe: + removevalues['SRC_URI'] = oldentry + else: + appendline('SRC_URI:remove', '=', oldentry) appendline('SRC_URI', '+=', srcurientry) - copyfiles[newfile] = srcfile + param['path'] = srcfile + else: + srcfile = param['path'] + copyfiles[newfile] = param if install: institem = install.pop(newfile, None) if institem: @@ -800,6 +829,8 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, # multiple times per operation when we're handling overrides) if os.path.exists(appendpath) and not os.path.exists(outfile): shutil.copy2(appendpath, outfile) + elif update_original_recipe: + outfile = recipefile else: bb.note('Writing append file %s' % appendpath) outfile = appendpath @@ -903,7 +934,12 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, outdir = redirect_output else: outdir = appenddir - for newfile, srcfile in copyfiles.items(): + for newfile, param in copyfiles.items(): + srcfile = param['path'] + patchdir = param.get('patchdir', ".") + + if patchdir != ".": + newfile = os.path.join(os.path.split(newfile)[0], patchdir, os.path.split(newfile)[1]) filedest = os.path.join(outdir, destsubdir, os.path.basename(srcfile)) if os.path.abspath(newfile) != os.path.abspath(filedest): if newfile.startswith(tempfile.gettempdir()): @@ -947,10 +983,9 @@ def replace_dir_vars(path, d): path = path.replace(dirpath, '${%s}' % dirvars[dirpath]) return path -def get_recipe_pv_without_srcpv(pv, uri_type): +def get_recipe_pv_with_pfx_sfx(pv, uri_type): """ - Get PV without SRCPV common in SCM's for now only - support git. + Get PV separating prefix and suffix components. Returns tuple with pv, prefix and suffix. """ @@ -958,7 +993,7 @@ def get_recipe_pv_without_srcpv(pv, uri_type): sfx = '' if uri_type == 'git': - git_regex = re.compile(r"(?P<pfx>v?)(?P<ver>.*?)(?P<sfx>\+[^\+]*(git)?r?(AUTOINC\+))(?P<rev>.*)") + git_regex = re.compile(r"(?P<pfx>v?)(?P<ver>.*?)(?P<sfx>\+[^\+]*(git)?r?(AUTOINC\+)?)(?P<rev>.*)") m = git_regex.match(pv) if m: @@ -1010,7 +1045,7 @@ def get_recipe_upstream_version(rd): src_uri = src_uris.split()[0] uri_type, _, _, _, _, _ = decodeurl(src_uri) - (pv, pfx, sfx) = get_recipe_pv_without_srcpv(rd.getVar('PV'), uri_type) + (pv, pfx, sfx) = get_recipe_pv_with_pfx_sfx(rd.getVar('PV'), uri_type) ru['current_version'] = pv manual_upstream_version = rd.getVar("RECIPE_UPSTREAM_VERSION") @@ -1038,7 +1073,7 @@ def get_recipe_upstream_version(rd): revision = ud.method.latest_revision(ud, rd, 'default') upversion = pv if revision != rd.getVar("SRCREV"): - upversion = upversion + "-new-commits-available" + upversion = upversion + "-new-commits-available" else: pupver = ud.method.latest_versionstring(ud, rd) (upversion, revision) = pupver diff --git a/meta/lib/oe/reproducible.py b/meta/lib/oe/reproducible.py index 2e815df190..448befce33 100644 --- a/meta/lib/oe/reproducible.py +++ b/meta/lib/oe/reproducible.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # import os @@ -113,7 +115,8 @@ def get_source_date_epoch_from_git(d, sourcedir): return None bb.debug(1, "git repository: %s" % gitpath) - p = subprocess.run(['git', '--git-dir', gitpath, 'log', '-1', '--pretty=%ct'], check=True, stdout=subprocess.PIPE) + p = subprocess.run(['git', '-c', 'log.showSignature=false', '--git-dir', gitpath, 'log', '-1', '--pretty=%ct'], + check=True, stdout=subprocess.PIPE) return int(p.stdout.decode('utf-8')) def get_source_date_epoch_from_youngest_file(d, sourcedir): @@ -128,6 +131,9 @@ def get_source_date_epoch_from_youngest_file(d, sourcedir): files = [f for f in files if not f[0] == '.'] for fname in files: + if fname == "singletask.lock": + # Ignore externalsrc/devtool lockfile [YOCTO #14921] + continue filename = os.path.join(root, fname) try: mtime = int(os.lstat(filename).st_mtime) diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py index 9e6b411fb6..8cd48f9450 100644 --- a/meta/lib/oe/rootfs.py +++ b/meta/lib/oe/rootfs.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # from abc import ABCMeta, abstractmethod @@ -104,7 +106,7 @@ class Rootfs(object, metaclass=ABCMeta): def _cleanup(self): pass - def _setup_dbg_rootfs(self, dirs): + def _setup_dbg_rootfs(self, package_paths): gen_debugfs = self.d.getVar('IMAGE_GEN_DEBUGFS') or '0' if gen_debugfs != '1': return @@ -120,11 +122,12 @@ class Rootfs(object, metaclass=ABCMeta): bb.utils.mkdirhier(self.image_rootfs) bb.note(" Copying back package database...") - for dir in dirs: - if not os.path.isdir(self.image_rootfs + '-orig' + dir): - continue - bb.utils.mkdirhier(self.image_rootfs + os.path.dirname(dir)) - shutil.copytree(self.image_rootfs + '-orig' + dir, self.image_rootfs + dir, symlinks=True) + for path in package_paths: + bb.utils.mkdirhier(self.image_rootfs + os.path.dirname(path)) + if os.path.isdir(self.image_rootfs + '-orig' + path): + shutil.copytree(self.image_rootfs + '-orig' + path, self.image_rootfs + path, symlinks=True) + elif os.path.isfile(self.image_rootfs + '-orig' + path): + shutil.copyfile(self.image_rootfs + '-orig' + path, self.image_rootfs + path) # Copy files located in /usr/lib/debug or /usr/src/debug for dir in ["/usr/lib/debug", "/usr/src/debug"]: @@ -160,6 +163,13 @@ class Rootfs(object, metaclass=ABCMeta): bb.note(" Install extra debug packages...") self.pm.install(extra_debug_pkgs.split(), True) + bb.note(" Removing package database...") + for path in package_paths: + if os.path.isdir(self.image_rootfs + path): + shutil.rmtree(self.image_rootfs + path) + elif os.path.isfile(self.image_rootfs + path): + os.remove(self.image_rootfs + path) + bb.note(" Rename debug rootfs...") try: shutil.rmtree(self.image_rootfs + '-dbg') @@ -171,14 +181,8 @@ class Rootfs(object, metaclass=ABCMeta): bb.utils.rename(self.image_rootfs + '-orig', self.image_rootfs) def _exec_shell_cmd(self, cmd): - fakerootcmd = self.d.getVar('FAKEROOT') - if fakerootcmd is not None: - exec_cmd = [fakerootcmd, cmd] - else: - exec_cmd = cmd - try: - subprocess.check_output(exec_cmd, stderr=subprocess.STDOUT) + subprocess.check_output(cmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: return("Command '%s' returned %d:\n%s" % (e.cmd, e.returncode, e.output)) @@ -190,6 +194,18 @@ class Rootfs(object, metaclass=ABCMeta): post_process_cmds = self.d.getVar("ROOTFS_POSTPROCESS_COMMAND") rootfs_post_install_cmds = self.d.getVar('ROOTFS_POSTINSTALL_COMMAND') + def make_last(command, commands): + commands = commands.split() + if command in commands: + commands.remove(command) + commands.append(command) + return "".join(commands) + + # We want this to run as late as possible, in particular after + # systemd_sysusers_create and set_user_group. Using :append is not enough + make_last("tidy_shadowutils_files", post_process_cmds) + make_last("rootfs_reproducible", post_process_cmds) + execute_pre_post_process(self.d, pre_process_cmds) if self.progress_reporter: @@ -323,19 +339,30 @@ class Rootfs(object, metaclass=ABCMeta): bb.note("No Kernel Modules found, not running depmod") return - kernel_abi_ver_file = oe.path.join(self.d.getVar('PKGDATA_DIR'), "kernel-depmod", - 'kernel-abiversion') - if not os.path.exists(kernel_abi_ver_file): - bb.fatal("No kernel-abiversion file found (%s), cannot run depmod, aborting" % kernel_abi_ver_file) + pkgdatadir = self.d.getVar('PKGDATA_DIR') + + # PKGDATA_DIR can include multiple kernels so we run depmod for each + # one of them. + for direntry in os.listdir(pkgdatadir): + match = re.match('(.*)-depmod', direntry) + if not match: + continue + kernel_package_name = match.group(1) - with open(kernel_abi_ver_file) as f: - kernel_ver = f.read().strip(' \n') + kernel_abi_ver_file = oe.path.join(pkgdatadir, direntry, kernel_package_name + '-abiversion') + if not os.path.exists(kernel_abi_ver_file): + bb.fatal("No kernel-abiversion file found (%s), cannot run depmod, aborting" % kernel_abi_ver_file) - versioned_modules_dir = os.path.join(self.image_rootfs, modules_dir, kernel_ver) + with open(kernel_abi_ver_file) as f: + kernel_ver = f.read().strip(' \n') - bb.utils.mkdirhier(versioned_modules_dir) + versioned_modules_dir = os.path.join(self.image_rootfs, modules_dir, kernel_ver) - self._exec_shell_cmd(['depmodwrapper', '-a', '-b', self.image_rootfs, kernel_ver]) + bb.utils.mkdirhier(versioned_modules_dir) + + bb.note("Running depmodwrapper for %s ..." % versioned_modules_dir) + if self._exec_shell_cmd(['depmodwrapper', '-a', '-b', self.image_rootfs, kernel_ver, kernel_package_name]): + bb.fatal("Kernel modules dependency generation failed") """ Create devfs: @@ -384,6 +411,10 @@ def create_rootfs(d, manifest_dir=None, progress_reporter=None, logcatcher=None) def image_list_installed_packages(d, rootfs_dir=None): + # Theres no rootfs for baremetal images + if bb.data.inherits_class('baremetal-image', d): + return "" + if not rootfs_dir: rootfs_dir = d.getVar('IMAGE_ROOTFS') diff --git a/meta/lib/oe/rust.py b/meta/lib/oe/rust.py index ec70b34805..185553eeeb 100644 --- a/meta/lib/oe/rust.py +++ b/meta/lib/oe/rust.py @@ -1,5 +1,13 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + # Handle mismatches between `uname -m`-style output and Rust's arch names def arch_to_rust_arch(arch): if arch == "ppc64le": return "powerpc64le" + if arch in ('riscv32', 'riscv64'): + return arch + 'gc' return arch diff --git a/meta/lib/oe/sbom.py b/meta/lib/oe/sbom.py index 52bf51440e..fd4b6895d8 100644 --- a/meta/lib/oe/sbom.py +++ b/meta/lib/oe/sbom.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # @@ -12,6 +14,10 @@ def get_recipe_spdxid(d): return "SPDXRef-%s-%s" % ("Recipe", d.getVar("PN")) +def get_download_spdxid(d, idx): + return "SPDXRef-Download-%s-%d" % (d.getVar("PN"), idx) + + def get_package_spdxid(pkg): return "SPDXRef-Package-%s" % pkg @@ -32,18 +38,54 @@ def get_sdk_spdxid(sdk): return "SPDXRef-SDK-%s" % sdk -def write_doc(d, spdx_doc, subdir, spdx_deploy=None, indent=None): +def _doc_path_by_namespace(spdx_deploy, arch, doc_namespace): + return spdx_deploy / "by-namespace" / arch / doc_namespace.replace("/", "_") + + +def doc_find_by_namespace(spdx_deploy, search_arches, doc_namespace): + for pkgarch in search_arches: + p = _doc_path_by_namespace(spdx_deploy, pkgarch, doc_namespace) + if os.path.exists(p): + return p + return None + + +def _doc_path_by_hashfn(spdx_deploy, arch, doc_name, hashfn): + return ( + spdx_deploy / "by-hash" / arch / hashfn.split()[1] / (doc_name + ".spdx.json") + ) + + +def doc_find_by_hashfn(spdx_deploy, search_arches, doc_name, hashfn): + for pkgarch in search_arches: + p = _doc_path_by_hashfn(spdx_deploy, pkgarch, doc_name, hashfn) + if os.path.exists(p): + return p + return None + + +def doc_path(spdx_deploy, doc_name, arch, subdir): + return spdx_deploy / arch / subdir / (doc_name + ".spdx.json") + + +def write_doc(d, spdx_doc, arch, subdir, spdx_deploy=None, indent=None): from pathlib import Path if spdx_deploy is None: spdx_deploy = Path(d.getVar("SPDXDEPLOY")) - dest = spdx_deploy / subdir / (spdx_doc.name + ".spdx.json") + dest = doc_path(spdx_deploy, spdx_doc.name, arch, subdir) dest.parent.mkdir(exist_ok=True, parents=True) with dest.open("wb") as f: doc_sha1 = spdx_doc.to_json(f, sort_keys=True, indent=indent) - l = spdx_deploy / "by-namespace" / spdx_doc.documentNamespace.replace("/", "_") + l = _doc_path_by_namespace(spdx_deploy, arch, spdx_doc.documentNamespace) + l.parent.mkdir(exist_ok=True, parents=True) + l.symlink_to(os.path.relpath(dest, l.parent)) + + l = _doc_path_by_hashfn( + spdx_deploy, arch, spdx_doc.name, d.getVar("BB_HASHFILENAME") + ) l.parent.mkdir(exist_ok=True, parents=True) l.symlink_to(os.path.relpath(dest, l.parent)) diff --git a/meta/lib/oe/sdk.py b/meta/lib/oe/sdk.py index 27347667e8..3dc3672210 100644 --- a/meta/lib/oe/sdk.py +++ b/meta/lib/oe/sdk.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # @@ -68,7 +70,7 @@ class Sdk(object, metaclass=ABCMeta): #FIXME: using umbrella exc catching because bb.utils method raises it except Exception as e: bb.debug(1, "printing the stack trace\n %s" %traceback.format_exc()) - bb.error("unable to place %s in final SDK location" % sourcefile) + bb.fatal("unable to place %s in final SDK location" % sourcefile) def mkdirhier(self, dirpath): try: diff --git a/meta/lib/oe/spdx.py b/meta/lib/oe/spdx.py index 6d56ed90df..7aaf2af5ed 100644 --- a/meta/lib/oe/spdx.py +++ b/meta/lib/oe/spdx.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # @@ -214,6 +216,18 @@ class SPDXPackageVerificationCode(SPDXObject): class SPDXPackage(SPDXObject): + ALLOWED_CHECKSUMS = [ + "SHA1", + "SHA224", + "SHA256", + "SHA384", + "SHA512", + "MD2", + "MD4", + "MD5", + "MD6", + ] + name = _String() SPDXID = _String() versionInfo = _String() @@ -232,6 +246,7 @@ class SPDXPackage(SPDXObject): hasFiles = _StringList() packageFileName = _String() annotations = _ObjectList(SPDXAnnotation) + checksums = _ObjectList(SPDXChecksum) class SPDXFile(SPDXObject): diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py index de65244932..a46e5502ab 100644 --- a/meta/lib/oe/sstatesig.py +++ b/meta/lib/oe/sstatesig.py @@ -1,9 +1,12 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # import bb.siggen import bb.runqueue import oe +import netrc def sstate_rundepfilter(siggen, fn, recipename, task, dep, depname, dataCaches): # Return True if we should keep the dependency, False to drop it @@ -24,18 +27,15 @@ def sstate_rundepfilter(siggen, fn, recipename, task, dep, depname, dataCaches): return "/allarch.bbclass" in inherits def isImage(mc, fn): return "/image.bbclass" in " ".join(dataCaches[mc].inherits[fn]) - def isSPDXTask(task): - return task in ("do_create_spdx", "do_create_runtime_spdx") depmc, _, deptaskname, depmcfn = bb.runqueue.split_tid_mcfn(dep) mc, _ = bb.runqueue.split_mc(fn) - # Keep all dependencies between SPDX tasks in the signature. SPDX documents - # are linked together by hashes, which means if a dependent document changes, - # all downstream documents must be re-written (even if they are "safe" - # dependencies). - if isSPDXTask(task) and isSPDXTask(deptaskname): - return True + # We can skip the rm_work task signature to avoid running the task + # when we remove some tasks from the dependencie chain + # i.e INHERIT:remove = "create-spdx" will trigger the do_rm_work + if task == "do_rm_work": + return False # (Almost) always include our own inter-task dependencies (unless it comes # from a mcdepends). The exception is the special @@ -93,15 +93,6 @@ def sstate_lockedsigs(d): sigs[pn][task] = [h, siggen_lockedsigs_var] return sigs -class SignatureGeneratorOEBasic(bb.siggen.SignatureGeneratorBasic): - name = "OEBasic" - def init_rundepcheck(self, data): - self.abisaferecipes = (data.getVar("SIGGEN_EXCLUDERECIPES_ABISAFE") or "").split() - self.saferecipedeps = (data.getVar("SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS") or "").split() - pass - def rundep_check(self, fn, recipename, task, dep, depname, dataCaches = None): - return sstate_rundepfilter(self, fn, recipename, task, dep, depname, dataCaches) - class SignatureGeneratorOEBasicHashMixIn(object): supports_multiconfig_datacaches = True @@ -114,6 +105,8 @@ class SignatureGeneratorOEBasicHashMixIn(object): self.lockedhashfn = {} self.machine = data.getVar("MACHINE") self.mismatch_msgs = [] + self.mismatch_number = 0 + self.lockedsigs_msgs = "" self.unlockedrecipes = (data.getVar("SIGGEN_UNLOCKED_RECIPES") or "").split() self.unlockedrecipes = { k: "" for k in self.unlockedrecipes } @@ -150,9 +143,10 @@ class SignatureGeneratorOEBasicHashMixIn(object): super().set_taskdata(data[3:]) def dump_sigs(self, dataCache, options): - sigfile = os.getcwd() + "/locked-sigs.inc" - bb.plain("Writing locked sigs to %s" % sigfile) - self.dump_lockedsigs(sigfile) + if 'lockedsigs' in options: + sigfile = os.getcwd() + "/locked-sigs.inc" + bb.plain("Writing locked sigs to %s" % sigfile) + self.dump_lockedsigs(sigfile) return super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigs(dataCache, options) @@ -197,6 +191,7 @@ class SignatureGeneratorOEBasicHashMixIn(object): #bb.warn("Using %s %s %s" % (recipename, task, h)) if h != h_locked and h_locked != unihash: + self.mismatch_number += 1 self.mismatch_msgs.append('The %s:%s sig is computed to be %s, but the sig is locked to %s in %s' % (recipename, task, h, h_locked, var)) @@ -211,10 +206,10 @@ class SignatureGeneratorOEBasicHashMixIn(object): return self.lockedhashes[tid] return super().get_stampfile_hash(tid) - def get_unihash(self, tid): + def get_cached_unihash(self, tid): if tid in self.lockedhashes and self.lockedhashes[tid] and not self._internal: return self.lockedhashes[tid] - return super().get_unihash(tid) + return super().get_cached_unihash(tid) def dump_sigtask(self, fn, task, stampbase, runtime): tid = fn + ":" + task @@ -225,6 +220,9 @@ class SignatureGeneratorOEBasicHashMixIn(object): def dump_lockedsigs(self, sigfile, taskfilter=None): types = {} for tid in self.runtaskdeps: + # Bitbake changed this to a tuple in newer versions + if isinstance(tid, tuple): + tid = tid[1] if taskfilter: if not tid in taskfilter: continue @@ -274,6 +272,15 @@ class SignatureGeneratorOEBasicHashMixIn(object): warn_msgs = [] error_msgs = [] sstate_missing_msgs = [] + info_msgs = None + + if self.lockedsigs: + if len(self.lockedsigs) > 10: + self.lockedsigs_msgs = "There are %s recipes with locked tasks (%s task(s) have non matching signature)" % (len(self.lockedsigs), self.mismatch_number) + else: + self.lockedsigs_msgs = "The following recipes have locked tasks:" + for pn in self.lockedsigs: + self.lockedsigs_msgs += " %s" % (pn) for tid in sq_data['hash']: if tid not in found: @@ -286,7 +293,9 @@ class SignatureGeneratorOEBasicHashMixIn(object): % (pn, taskname, sq_data['hash'][tid])) checklevel = d.getVar("SIGGEN_LOCKEDSIGS_TASKSIG_CHECK") - if checklevel == 'warn': + if checklevel == 'info': + info_msgs = self.lockedsigs_msgs + if checklevel == 'warn' or checklevel == 'info': warn_msgs += self.mismatch_msgs elif checklevel == 'error': error_msgs += self.mismatch_msgs @@ -297,6 +306,8 @@ class SignatureGeneratorOEBasicHashMixIn(object): elif checklevel == 'error': error_msgs += sstate_missing_msgs + if info_msgs: + bb.note(info_msgs) if warn_msgs: bb.warn("\n".join(warn_msgs)) if error_msgs: @@ -316,9 +327,21 @@ class SignatureGeneratorOEEquivHash(SignatureGeneratorOEBasicHashMixIn, bb.sigge self.method = data.getVar('SSTATE_HASHEQUIV_METHOD') if not self.method: bb.fatal("OEEquivHash requires SSTATE_HASHEQUIV_METHOD to be set") + self.max_parallel = int(data.getVar('BB_HASHSERVE_MAX_PARALLEL') or 1) + self.username = data.getVar("BB_HASHSERVE_USERNAME") + self.password = data.getVar("BB_HASHSERVE_PASSWORD") + if not self.username or not self.password: + try: + n = netrc.netrc() + auth = n.authenticators(self.server) + if auth is not None: + self.username, _, self.password = auth + except FileNotFoundError: + pass + except netrc.NetrcParseError as e: + bb.warn("Error parsing %s:%s: %s" % (e.filename, str(e.lineno), e.msg)) # Insert these classes into siggen's namespace so it can see and select them -bb.siggen.SignatureGeneratorOEBasic = SignatureGeneratorOEBasic bb.siggen.SignatureGeneratorOEBasicHash = SignatureGeneratorOEBasicHash bb.siggen.SignatureGeneratorOEEquivHash = SignatureGeneratorOEEquivHash @@ -332,14 +355,14 @@ def find_siginfo(pn, taskname, taskhashlist, d): if not taskname: # We have to derive pn and taskname key = pn - splitit = key.split('.bb:') - taskname = splitit[1] - pn = os.path.basename(splitit[0]).split('_')[0] - if key.startswith('virtual:native:'): - pn = pn + '-native' + if key.startswith("mc:"): + # mc:<mc>:<pn>:<task> + _, _, pn, taskname = key.split(':', 3) + else: + # <pn>:<task> + pn, taskname = key.split(':', 1) hashfiles = {} - filedates = {} def get_hashval(siginfo): if siginfo.endswith('.siginfo'): @@ -347,6 +370,9 @@ def find_siginfo(pn, taskname, taskhashlist, d): else: return siginfo.rpartition('.')[2] + def get_time(fullpath): + return os.stat(fullpath).st_mtime + # First search in stamps dir localdata = d.createCopy() localdata.setVar('MULTIMACH_TARGET_SYS', '*') @@ -362,24 +388,21 @@ def find_siginfo(pn, taskname, taskhashlist, d): filespec = '%s.%s.sigdata.*' % (stamp, taskname) foundall = False import glob + bb.debug(1, "Calling glob.glob on {}".format(filespec)) for fullpath in glob.glob(filespec): match = False if taskhashlist: for taskhash in taskhashlist: if fullpath.endswith('.%s' % taskhash): - hashfiles[taskhash] = fullpath + hashfiles[taskhash] = {'path':fullpath, 'sstate':False, 'time':get_time(fullpath)} if len(hashfiles) == len(taskhashlist): foundall = True break else: - try: - filedates[fullpath] = os.stat(fullpath).st_mtime - except OSError: - continue hashval = get_hashval(fullpath) - hashfiles[hashval] = fullpath + hashfiles[hashval] = {'path':fullpath, 'sstate':False, 'time':get_time(fullpath)} - if not taskhashlist or (len(filedates) < 2 and not foundall): + if not taskhashlist or (len(hashfiles) < 2 and not foundall): # That didn't work, look in sstate-cache hashes = taskhashlist or ['?' * 64] localdata = bb.data.createCopy(d) @@ -388,6 +411,9 @@ def find_siginfo(pn, taskname, taskhashlist, d): localdata.setVar('TARGET_VENDOR', '*') localdata.setVar('TARGET_OS', '*') localdata.setVar('PN', pn) + # gcc-source is a special case, same as with local stamps above + if pn.startswith("gcc-source"): + localdata.setVar('PN', "gcc") localdata.setVar('PV', '*') localdata.setVar('PR', '*') localdata.setVar('BB_TASKHASH', hashval) @@ -399,24 +425,18 @@ def find_siginfo(pn, taskname, taskhashlist, d): localdata.setVar('SSTATE_EXTRAPATH', "${NATIVELSBSTRING}/") filespec = '%s.siginfo' % localdata.getVar('SSTATE_PKG') + bb.debug(1, "Calling glob.glob on {}".format(filespec)) matchedfiles = glob.glob(filespec) for fullpath in matchedfiles: actual_hashval = get_hashval(fullpath) if actual_hashval in hashfiles: continue - hashfiles[hashval] = fullpath - if not taskhashlist: - try: - filedates[fullpath] = os.stat(fullpath).st_mtime - except: - continue + hashfiles[actual_hashval] = {'path':fullpath, 'sstate':True, 'time':get_time(fullpath)} - if taskhashlist: - return hashfiles - else: - return filedates + return hashfiles bb.siggen.find_siginfo = find_siginfo +bb.siggen.find_siginfo_version = 2 def sstate_get_manifest_filename(task, d): @@ -461,11 +481,15 @@ def find_sstate_manifest(taskdata, taskdata2, taskname, d, multilibcache): pkgarchs.append('allarch') pkgarchs.append('${SDK_ARCH}_${SDK_ARCH}-${SDKPKGSUFFIX}') + searched_manifests = [] + for pkgarch in pkgarchs: manifest = d2.expand("${SSTATE_MANIFESTS}/manifest-%s-%s.%s" % (pkgarch, taskdata, taskname)) if os.path.exists(manifest): return manifest, d2 - bb.fatal("Manifest %s not found in %s (variant '%s')?" % (manifest, d2.expand(" ".join(pkgarchs)), variant)) + searched_manifests.append(manifest) + bb.fatal("The sstate manifest for task '%s:%s' (multilib variant '%s') could not be found.\nThe pkgarchs considered were: %s.\nBut none of these manifests exists:\n %s" + % (taskdata, taskname, variant, d2.expand(", ".join(pkgarchs)),"\n ".join(searched_manifests))) return None, d2 def OEOuthashBasic(path, sigfile, task, d): @@ -585,9 +609,9 @@ def OEOuthashBasic(path, sigfile, task, d): update_hash(" %10s" % pwd.getpwuid(s.st_uid).pw_name) update_hash(" %10s" % grp.getgrgid(s.st_gid).gr_name) except KeyError as e: - bb.warn("KeyError in %s" % path) msg = ("KeyError: %s\nPath %s is owned by uid %d, gid %d, which doesn't match " - "any user/group on target. This may be due to host contamination." % (e, path, s.st_uid, s.st_gid)) + "any user/group on target. This may be due to host contamination." % + (e, os.path.abspath(path), s.st_uid, s.st_gid)) raise Exception(msg).with_traceback(e.__traceback__) if include_timestamps: @@ -650,6 +674,10 @@ def OEOuthashBasic(path, sigfile, task, d): if f == 'fixmepath': continue process(os.path.join(root, f)) + + for dir in dirs: + if os.path.islink(os.path.join(root, dir)): + process(os.path.join(root, dir)) finally: os.chdir(prev_dir) diff --git a/meta/lib/oe/terminal.py b/meta/lib/oe/terminal.py index de8dcebf94..4412bc14c1 100644 --- a/meta/lib/oe/terminal.py +++ b/meta/lib/oe/terminal.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # import logging @@ -102,6 +104,10 @@ class Rxvt(XTerminal): command = 'rxvt -T "{title}" -e {command}' priority = 1 +class URxvt(XTerminal): + command = 'urxvt -T "{title}" -e {command}' + priority = 1 + class Screen(Terminal): command = 'screen -D -m -t "{title}" -S devshell {command}' diff --git a/meta/lib/oe/types.py b/meta/lib/oe/types.py index bbbabafbf6..b929afb1f3 100644 --- a/meta/lib/oe/types.py +++ b/meta/lib/oe/types.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/meta/lib/oe/useradd.py b/meta/lib/oe/useradd.py index 3caa3f851a..54aa86feb5 100644 --- a/meta/lib/oe/useradd.py +++ b/meta/lib/oe/useradd.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # import argparse diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py index 1ee947d584..14a7d07ef0 100644 --- a/meta/lib/oe/utils.py +++ b/meta/lib/oe/utils.py @@ -1,10 +1,13 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # import subprocess import multiprocessing import traceback +import errno def read_file(filename): try: @@ -256,16 +259,23 @@ def execute_pre_post_process(d, cmds): if cmds is None: return - for cmd in cmds.strip().split(';'): - cmd = cmd.strip() - if cmd != '': - bb.note("Executing %s ..." % cmd) - bb.build.exec_func(cmd, d) + cmds = cmds.replace(";", " ") + + for cmd in cmds.split(): + bb.note("Executing %s ..." % cmd) + bb.build.exec_func(cmd, d) + +def get_bb_number_threads(d): + return int(d.getVar("BB_NUMBER_THREADS") or os.cpu_count() or 1) + +def multiprocess_launch(target, items, d, extraargs=None): + max_process = get_bb_number_threads(d) + return multiprocess_launch_mp(target, items, max_process, extraargs) -# For each item in items, call the function 'target' with item as the first +# For each item in items, call the function 'target' with item as the first # argument, extraargs as the other arguments and handle any exceptions in the # parent thread -def multiprocess_launch(target, items, d, extraargs=None): +def multiprocess_launch_mp(target, items, max_process, extraargs=None): class ProcessLaunch(multiprocessing.Process): def __init__(self, *args, **kwargs): @@ -300,7 +310,6 @@ def multiprocess_launch(target, items, d, extraargs=None): self.update() return self._result - max_process = int(d.getVar("BB_NUMBER_THREADS") or os.cpu_count() or 1) launched = [] errors = [] results = [] @@ -520,3 +529,14 @@ def directory_size(root, blocksize=4096): total += sum(roundup(getsize(os.path.join(root, name))) for name in files) total += roundup(getsize(root)) return total + +# Update the mtime of a file, skip if permission/read-only issues +def touch(filename): + try: + os.utime(filename, None) + except PermissionError: + pass + except OSError as e: + # Handle read-only file systems gracefully + if e.errno != errno.EROFS: + raise e diff --git a/meta/lib/oeqa/controllers/__init__.py b/meta/lib/oeqa/controllers/__init__.py index cc3836c4bf..0fc905be9a 100644 --- a/meta/lib/oeqa/controllers/__init__.py +++ b/meta/lib/oeqa/controllers/__init__.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # # Enable other layers to have modules in the same named directory diff --git a/meta/lib/oeqa/controllers/testtargetloader.py b/meta/lib/oeqa/controllers/testtargetloader.py index 23101c7371..209ff7061a 100644 --- a/meta/lib/oeqa/controllers/testtargetloader.py +++ b/meta/lib/oeqa/controllers/testtargetloader.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/meta/lib/oeqa/core/context.py b/meta/lib/oeqa/core/context.py index 2abe353d27..9313271f58 100644 --- a/meta/lib/oeqa/core/context.py +++ b/meta/lib/oeqa/core/context.py @@ -81,7 +81,7 @@ class OETestContext(object): def runTests(self, processes=None, skips=[]): self.runner = self.runnerClass(self, descriptions=False, verbosity=2) - # Dinamically skip those tests specified though arguments + # Dynamically skip those tests specified though arguments self.skipTests(skips) self._run_start_time = time.time() diff --git a/meta/lib/oeqa/core/decorator/data.py b/meta/lib/oeqa/core/decorator/data.py index 3ce10e5499..5444b2cb75 100644 --- a/meta/lib/oeqa/core/decorator/data.py +++ b/meta/lib/oeqa/core/decorator/data.py @@ -186,6 +186,16 @@ class skipIfNotQemu(OETestDecorator): self.case.skipTest('Test only runs on qemu machines') @registerDecorator +class skipIfNotQemuUsermode(OETestDecorator): + """ + Skip test if MACHINE_FEATURES does not contain qemu-usermode + """ + def setUpDecorator(self): + self.logger.debug("Checking if MACHINE_FEATURES does not contain qemu-usermode") + if 'qemu-usermode' not in self.case.td.get('MACHINE_FEATURES', '').split(): + self.case.skipTest('Test requires qemu-usermode in MACHINE_FEATURES') + +@registerDecorator class skipIfQemu(OETestDecorator): """ Skip test if MACHINE is qemu* @@ -194,3 +204,27 @@ class skipIfQemu(OETestDecorator): self.logger.debug("Checking if qemu MACHINE") if self.case.td.get('MACHINE', '').startswith('qemu'): self.case.skipTest('Test only runs on real hardware') + +@registerDecorator +class skipIfArch(OETestDecorator): + """ + Skip test if HOST_ARCH is present in the tuple specified. + """ + + attrs = ('archs',) + def setUpDecorator(self): + arch = self.case.td['HOST_ARCH'] + if arch in self.archs: + self.case.skipTest('Test skipped on %s' % arch) + +@registerDecorator +class skipIfNotArch(OETestDecorator): + """ + Skip test if HOST_ARCH is not present in the tuple specified. + """ + + attrs = ('archs',) + def setUpDecorator(self): + arch = self.case.td['HOST_ARCH'] + if arch not in self.archs: + self.case.skipTest('Test skipped on %s' % arch) diff --git a/meta/lib/oeqa/core/loader.py b/meta/lib/oeqa/core/loader.py index 11978213b8..d12d5a055c 100644 --- a/meta/lib/oeqa/core/loader.py +++ b/meta/lib/oeqa/core/loader.py @@ -37,7 +37,7 @@ def _find_duplicated_modules(suite, directory): if path: raise ImportError("Duplicated %s module found in %s" % (module, path)) -def _built_modules_dict(modules): +def _built_modules_dict(modules, logger): modules_dict = {} if modules == None: @@ -48,6 +48,9 @@ def _built_modules_dict(modules): # characters, whereas class names do m = re.match(r'^([0-9a-z_.]+)(?:\.(\w[^.]*)(?:\.([^.]+))?)?$', module, flags=re.ASCII) if not m: + logger.warn("module '%s' was skipped from selected modules, "\ + "because it doesn't match with module name assumptions: "\ + "package and module names do not contain upper case characters, whereas class names do" % module) continue module_name, class_name, test_name = m.groups() @@ -58,6 +61,8 @@ def _built_modules_dict(modules): modules_dict[module_name][class_name] = [] if test_name and test_name not in modules_dict[module_name][class_name]: modules_dict[module_name][class_name].append(test_name) + if modules and not modules_dict: + raise OEQATestNotFound("All selected modules were skipped, this would trigger selftest with all tests and -r ignored.") return modules_dict @@ -71,7 +76,7 @@ class OETestLoader(unittest.TestLoader): *args, **kwargs): self.tc = tc - self.modules = _built_modules_dict(modules) + self.modules = _built_modules_dict(modules, tc.logger) self.tests = tests self.modules_required = modules_required @@ -311,6 +316,9 @@ class OETestLoader(unittest.TestLoader): module_name_small in self.modules) \ else False + if any(c.isupper() for c in module.__name__): + raise SystemExit("Module '%s' contains uppercase characters and this isn't supported. Please fix the module name." % module.__name__) + return (load_module, load_underscore) diff --git a/meta/lib/oeqa/core/runner.py b/meta/lib/oeqa/core/runner.py index d50690ab37..a86a706bd9 100644 --- a/meta/lib/oeqa/core/runner.py +++ b/meta/lib/oeqa/core/runner.py @@ -44,6 +44,7 @@ class OETestResult(_TestResult): self.endtime = {} self.progressinfo = {} self.extraresults = {} + self.shownmsg = [] # Inject into tc so that TestDepends decorator can see results tc.results = self @@ -74,6 +75,7 @@ class OETestResult(_TestResult): for (scase, msg) in getattr(self, t): if test.id() == scase.id(): self.tc.logger.info(str(msg)) + self.shownmsg.append(test.id()) break def logSummary(self, component, context_msg=''): @@ -169,7 +171,6 @@ class OETestResult(_TestResult): def logDetails(self, json_file_dir=None, configuration=None, result_id=None, dump_streams=False): - self.tc.logger.info("RESULTS:") result = self.extraresults logs = {} @@ -193,6 +194,10 @@ class OETestResult(_TestResult): report = {'status': status} if log: report['log'] = log + # Class setup failures wouldn't enter stopTest so would never display + if case.id() not in self.shownmsg: + self.tc.logger.info("Failure (%s) for %s:\n" % (status, case.id()) + log) + if duration: report['duration'] = duration @@ -215,6 +220,7 @@ class OETestResult(_TestResult): report['stderr'] = stderr result[case.id()] = report + self.tc.logger.info("RESULTS:") for i in ['PASSED', 'SKIPPED', 'EXPECTEDFAIL', 'ERROR', 'FAILED', 'UNKNOWN']: if i not in logs: continue @@ -229,6 +235,10 @@ class OETestResult(_TestResult): # Override as we unexpected successes aren't failures for us return (len(self.failures) == len(self.errors) == 0) + def hasAnyFailingTest(self): + # Account for expected failures + return not self.wasSuccessful() or len(self.expectedFailures) + class OEListTestsResult(object): def wasSuccessful(self): return True diff --git a/meta/lib/oeqa/core/target/qemu.py b/meta/lib/oeqa/core/target/qemu.py index 79fd724f7d..d93b3ac94a 100644 --- a/meta/lib/oeqa/core/target/qemu.py +++ b/meta/lib/oeqa/core/target/qemu.py @@ -14,15 +14,13 @@ from collections import defaultdict from .ssh import OESSHTarget from oeqa.utils.qemurunner import QemuRunner -from oeqa.utils.dump import MonitorDumper -from oeqa.utils.dump import TargetDumper supported_fstypes = ['ext3', 'ext4', 'cpio.gz', 'wic'] class OEQemuTarget(OESSHTarget): def __init__(self, logger, server_ip, timeout=300, user='root', port=None, machine='', rootfs='', kernel='', kvm=False, slirp=False, - dump_dir='', dump_host_cmds='', display='', bootlog='', + dump_dir='', display='', bootlog='', tmpdir='', dir_image='', boottime=60, serial_ports=2, boot_patterns = defaultdict(str), ovmf=False, tmpfsdir=None, **kwargs): @@ -44,18 +42,9 @@ class OEQemuTarget(OESSHTarget): self.runner = QemuRunner(machine=machine, rootfs=rootfs, tmpdir=tmpdir, deploy_dir_image=dir_image, display=display, logfile=bootlog, boottime=boottime, - use_kvm=kvm, use_slirp=slirp, dump_dir=dump_dir, - dump_host_cmds=dump_host_cmds, logger=logger, + use_kvm=kvm, use_slirp=slirp, dump_dir=dump_dir, logger=logger, serial_ports=serial_ports, boot_patterns = boot_patterns, use_ovmf=ovmf, tmpfsdir=tmpfsdir) - dump_monitor_cmds = kwargs.get("testimage_dump_monitor") - self.monitor_dumper = MonitorDumper(dump_monitor_cmds, dump_dir, self.runner) - if self.monitor_dumper: - self.monitor_dumper.create_dir("qmp") - - dump_target_cmds = kwargs.get("testimage_dump_target") - self.target_dumper = TargetDumper(dump_target_cmds, dump_dir, self.runner) - self.target_dumper.create_dir("qemu") def start(self, params=None, extra_bootparams=None, runqemuparams=''): if self.use_slirp and not self.server_ip: diff --git a/meta/lib/oeqa/core/target/ssh.py b/meta/lib/oeqa/core/target/ssh.py index f956a7744f..09cdd14c75 100644 --- a/meta/lib/oeqa/core/target/ssh.py +++ b/meta/lib/oeqa/core/target/ssh.py @@ -34,17 +34,20 @@ class OESSHTarget(OETarget): self.timeout = timeout self.user = user ssh_options = [ + '-o', 'ServerAliveCountMax=2', + '-o', 'ServerAliveInterval=30', '-o', 'UserKnownHostsFile=/dev/null', '-o', 'StrictHostKeyChecking=no', '-o', 'LogLevel=ERROR' ] + scp_options = [ + '-r' + ] self.ssh = ['ssh', '-l', self.user ] + ssh_options - self.scp = ['scp'] + ssh_options + self.scp = ['scp'] + ssh_options + scp_options if port: self.ssh = self.ssh + [ '-p', port ] self.scp = self.scp + [ '-P', port ] - self._monitor_dumper = None - self.target_dumper = None def start(self, **kwargs): pass @@ -52,15 +55,6 @@ class OESSHTarget(OETarget): def stop(self, **kwargs): pass - @property - def monitor_dumper(self): - return self._monitor_dumper - - @monitor_dumper.setter - def monitor_dumper(self, dumper): - self._monitor_dumper = dumper - self.monitor_dumper.dump_monitor() - def _run(self, command, timeout=None, ignore_status=True): """ Runs command in target using SSHProcess. @@ -78,7 +72,7 @@ class OESSHTarget(OETarget): return (status, output) - def run(self, command, timeout=None): + def run(self, command, timeout=None, ignore_status=True): """ Runs command in target. @@ -97,16 +91,9 @@ class OESSHTarget(OETarget): else: processTimeout = self.timeout - status, output = self._run(sshCmd, processTimeout, True) + status, output = self._run(sshCmd, processTimeout, ignore_status) self.logger.debug('Command: %s\nStatus: %d Output: %s\n' % (command, status, output)) - if (status == 255) and (('No route to host') in output): - if self.monitor_dumper: - self.monitor_dumper.dump_monitor() - if status == 255: - if self.target_dumper: - self.target_dumper.dump_target() - if self.monitor_dumper: - self.monitor_dumper.dump_monitor() + return (status, output) def copyTo(self, localSrc, remoteDst): @@ -224,27 +211,41 @@ def SSHCall(command, logger, timeout=None, **opts): def run(): nonlocal output nonlocal process + output_raw = b'' starttime = time.time() process = subprocess.Popen(command, **options) + has_timeout = False if timeout: endtime = starttime + timeout eof = False - while time.time() < endtime and not eof: - logger.debug('time: %s, endtime: %s' % (time.time(), endtime)) + os.set_blocking(process.stdout.fileno(), False) + while not has_timeout and not eof: try: + logger.debug('Waiting for process output: time: %s, endtime: %s' % (time.time(), endtime)) if select.select([process.stdout], [], [], 5)[0] != []: - reader = codecs.getreader('utf-8')(process.stdout, 'ignore') - data = reader.read(1024, 4096) + # wait a bit for more data, tries to avoid reading single characters + time.sleep(0.2) + data = process.stdout.read() if not data: - process.stdout.close() eof = True else: - output += data - logger.debug('Partial data from SSH call: %s' % data) + output_raw += data + # ignore errors to capture as much as possible + logger.debug('Partial data from SSH call:\n%s' % data.decode('utf-8', errors='ignore')) endtime = time.time() + timeout except InterruptedError: + logger.debug('InterruptedError') + continue + except BlockingIOError: + logger.debug('BlockingIOError') continue + if time.time() >= endtime: + logger.debug('SSHCall has timeout! Time: %s, endtime: %s' % (time.time(), endtime)) + has_timeout = True + + process.stdout.close() + # process hasn't returned yet if not eof: process.terminate() @@ -252,16 +253,42 @@ def SSHCall(command, logger, timeout=None, **opts): try: process.kill() except OSError: + logger.debug('OSError when killing process') pass endtime = time.time() - starttime lastline = ("\nProcess killed - no output for %d seconds. Total" " running time: %d seconds." % (timeout, endtime)) - logger.debug('Received data from SSH call %s ' % lastline) + logger.debug('Received data from SSH call:\n%s ' % lastline) output += lastline + process.wait() else: - output = process.communicate()[0].decode('utf-8', errors='ignore') - logger.debug('Data from SSH call: %s' % output.rstrip()) + output_raw = process.communicate()[0] + + output = output_raw.decode('utf-8', errors='ignore') + logger.debug('Data from SSH call:\n%s' % output.rstrip()) + + # timout or not, make sure process exits and is not hanging + if process.returncode == None: + try: + process.wait(timeout=5) + except TimeoutExpired: + try: + process.kill() + except OSError: + logger.debug('OSError') + pass + process.wait() + + if has_timeout: + # Version of openssh before 8.6_p1 returns error code 0 when killed + # by a signal, when the timeout occurs we will receive a 0 error + # code because the process is been terminated and it's wrong because + # that value means success, but the process timed out. + # Afterwards, from version 8.6_p1 onwards, the returned code is 255. + # Fix this behaviour by checking the return code + if process.returncode == 0: + process.returncode = 255 options = { "stdout": subprocess.PIPE, @@ -288,6 +315,9 @@ def SSHCall(command, logger, timeout=None, **opts): # whilst running and ensure we don't leave a process behind. if process.poll() is None: process.kill() + if process.returncode == None: + process.wait() logger.debug('Something went wrong, killing SSH process') raise - return (process.wait(), output.rstrip()) + + return (process.returncode, output.rstrip()) diff --git a/meta/lib/oeqa/core/utils/concurrencytest.py b/meta/lib/oeqa/core/utils/concurrencytest.py index 161a2f6e90..d10f8f7f04 100644 --- a/meta/lib/oeqa/core/utils/concurrencytest.py +++ b/meta/lib/oeqa/core/utils/concurrencytest.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-or-later # # Modified for use in OE by Richard Purdie, 2018 @@ -57,6 +59,7 @@ class BBThreadsafeForwardingResult(ThreadsafeForwardingResult): self.outputbuf = output self.finalresult = finalresult self.finalresult.buffer = True + self.target = target def _add_result_with_semaphore(self, method, test, *args, **kwargs): self.semaphore.acquire() @@ -65,13 +68,14 @@ class BBThreadsafeForwardingResult(ThreadsafeForwardingResult): self.result.starttime[test.id()] = self._test_start.timestamp() self.result.threadprogress[self.threadnum].append(test.id()) totalprogress = sum(len(x) for x in self.result.threadprogress.values()) - self.result.progressinfo[test.id()] = "%s: %s/%s %s/%s (%ss) (%s)" % ( + self.result.progressinfo[test.id()] = "%s: %s/%s %s/%s (%ss) (%s failed) (%s)" % ( self.threadnum, len(self.result.threadprogress[self.threadnum]), self.totalinprocess, totalprogress, self.totaltests, "{0:.2f}".format(time.time()-self._test_start.timestamp()), + self.target.failed_tests, test.id()) finally: self.semaphore.release() @@ -189,11 +193,12 @@ class dummybuf(object): # class ConcurrentTestSuite(unittest.TestSuite): - def __init__(self, suite, processes, setupfunc, removefunc): + def __init__(self, suite, processes, setupfunc, removefunc, bb_vars): super(ConcurrentTestSuite, self).__init__([suite]) self.processes = processes self.setupfunc = setupfunc self.removefunc = removefunc + self.bb_vars = bb_vars def run(self, result): testservers, totaltests = fork_for_tests(self.processes, self) @@ -239,7 +244,7 @@ class ConcurrentTestSuite(unittest.TestSuite): def fork_for_tests(concurrency_num, suite): testservers = [] if 'BUILDDIR' in os.environ: - selftestdir = get_test_layer() + selftestdir = get_test_layer(suite.bb_vars['BBLAYERS']) test_blocks = partition_tests(suite, concurrency_num) # Clear the tests from the original suite so it doesn't keep them alive @@ -259,7 +264,7 @@ def fork_for_tests(concurrency_num, suite): ourpid = os.getpid() try: newbuilddir = None - stream = os.fdopen(c2pwrite, 'wb', 1) + stream = os.fdopen(c2pwrite, 'wb') os.close(c2pread) (builddir, newbuilddir) = suite.setupfunc("-st-" + str(ourpid), selftestdir, process_suite) @@ -304,7 +309,7 @@ def fork_for_tests(concurrency_num, suite): os._exit(0) else: os.close(c2pwrite) - stream = os.fdopen(c2pread, 'rb', 1) + stream = os.fdopen(c2pread, 'rb') # Collect stdout/stderr into an io buffer output = io.BytesIO() testserver = ProtocolTestCase(stream, passthrough=output) diff --git a/meta/lib/oeqa/files/maturin/guessing-game/Cargo.toml b/meta/lib/oeqa/files/maturin/guessing-game/Cargo.toml new file mode 100644 index 0000000000..de95025e86 --- /dev/null +++ b/meta/lib/oeqa/files/maturin/guessing-game/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "guessing-game" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[lib] +name = "guessing_game" +# "cdylib" is necessary to produce a shared library for Python to import from. +crate-type = ["cdylib"] + +[dependencies] +rand = "0.8.4" + +[dependencies.pyo3] +version = "0.19.0" +# "abi3-py38" tells pyo3 (and maturin) to build using the stable ABI with minimum Python version 3.8 +features = ["abi3-py38"] + diff --git a/meta/lib/oeqa/files/maturin/guessing-game/LICENSE-APACHE b/meta/lib/oeqa/files/maturin/guessing-game/LICENSE-APACHE new file mode 100644 index 0000000000..16fe87b06e --- /dev/null +++ b/meta/lib/oeqa/files/maturin/guessing-game/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/meta/lib/oeqa/files/maturin/guessing-game/LICENSE-MIT b/meta/lib/oeqa/files/maturin/guessing-game/LICENSE-MIT new file mode 100644 index 0000000000..c4a9a58791 --- /dev/null +++ b/meta/lib/oeqa/files/maturin/guessing-game/LICENSE-MIT @@ -0,0 +1,25 @@ +Copyright (c) 2018 konstin + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/meta/lib/oeqa/files/maturin/guessing-game/pyproject.toml b/meta/lib/oeqa/files/maturin/guessing-game/pyproject.toml new file mode 100644 index 0000000000..ff35abc472 --- /dev/null +++ b/meta/lib/oeqa/files/maturin/guessing-game/pyproject.toml @@ -0,0 +1,8 @@ +[build-system] +requires = ["maturin>=1.0,<2.0"] +build-backend = "maturin" + +[tool.maturin] +# "extension-module" tells pyo3 we want to build an extension module (skips linking against libpython.so) +features = ["pyo3/extension-module"] + diff --git a/meta/lib/oeqa/files/maturin/guessing-game/src/lib.rs b/meta/lib/oeqa/files/maturin/guessing-game/src/lib.rs new file mode 100644 index 0000000000..6828466ed1 --- /dev/null +++ b/meta/lib/oeqa/files/maturin/guessing-game/src/lib.rs @@ -0,0 +1,48 @@ +use pyo3::prelude::*; +use rand::Rng; +use std::cmp::Ordering; +use std::io; + +#[pyfunction] +fn guess_the_number() { + println!("Guess the number!"); + + let secret_number = rand::thread_rng().gen_range(1..101); + + loop { + println!("Please input your guess."); + + let mut guess = String::new(); + + io::stdin() + .read_line(&mut guess) + .expect("Failed to read line"); + + let guess: u32 = match guess.trim().parse() { + Ok(num) => num, + Err(_) => continue, + }; + + println!("You guessed: {}", guess); + + match guess.cmp(&secret_number) { + Ordering::Less => println!("Too small!"), + Ordering::Greater => println!("Too big!"), + Ordering::Equal => { + println!("You win!"); + break; + } + } + } +} + +/// A Python module implemented in Rust. The name of this function must match +/// the `lib.name` setting in the `Cargo.toml`, else Python will not be able to +/// import the module. +#[pymodule] +fn guessing_game(_py: Python, m: &PyModule) -> PyResult<()> { + m.add_function(wrap_pyfunction!(guess_the_number, m)?)?; + + Ok(()) +} + diff --git a/meta/lib/oeqa/files/test.rs b/meta/lib/oeqa/files/test.rs new file mode 100644 index 0000000000..f79c691f08 --- /dev/null +++ b/meta/lib/oeqa/files/test.rs @@ -0,0 +1,2 @@ +fn main() { +} diff --git a/meta/lib/oeqa/oetest.py b/meta/lib/oeqa/oetest.py index 9c84466dd0..bcb6a878c7 100644 --- a/meta/lib/oeqa/oetest.py +++ b/meta/lib/oeqa/oetest.py @@ -28,7 +28,7 @@ try: import oeqa.sdkext except ImportError: pass -from oeqa.utils.decorators import LogResults, gettag, getResults +from oeqa.utils.decorators import LogResults, gettag logger = logging.getLogger("BitBake") @@ -57,7 +57,6 @@ def filterByTagExp(testsuite, tagexp): @LogResults class oeTest(unittest.TestCase): - pscmd = "ps" longMessage = True @classmethod @@ -110,20 +109,6 @@ class oeRuntimeTest(oeTest): def tearDown(self): # Uninstall packages in the DUT self.tc.install_uninstall_packages(self.id(), False) - - res = getResults() - # If a test fails or there is an exception dump - # for QemuTarget only - if (type(self.target).__name__ == "QemuTarget" and - (self.id() in res.getErrorList() or - self.id() in res.getFailList())): - self.tc.host_dumper.create_dir(self._testMethodName) - self.tc.host_dumper.dump_host() - self.target.target_dumper.dump_target( - self.tc.host_dumper.dump_dir) - print ("%s dump data stored in %s" % (self._testMethodName, - self.tc.host_dumper.dump_dir)) - self.tearDownLocal() # Method to be run after tearDown and implemented by child classes @@ -256,7 +241,7 @@ class TestContext(object): modules = [] for test in self.testslist: - if re.search("\w+\.\w+\.test_\S+", test): + if re.search(r"\w+\.\w+\.test_\S+", test): test = '.'.join(t.split('.')[:3]) module = pkgutil.get_loader(test) modules.append(module) @@ -398,11 +383,6 @@ class RuntimeTestContext(TestContext): def _get_test_suites_required(self): return [t for t in self.d.getVar("TEST_SUITES").split() if t != "auto"] - def loadTests(self): - super(RuntimeTestContext, self).loadTests() - if oeTest.hasPackage("procps"): - oeRuntimeTest.pscmd = "ps -ef" - def extract_packages(self): """ Find packages that will be needed during runtime. diff --git a/meta/lib/oeqa/runtime/cases/_qemutiny.py b/meta/lib/oeqa/runtime/cases/_qemutiny.py index 6886e36502..816fd4a7cb 100644 --- a/meta/lib/oeqa/runtime/cases/_qemutiny.py +++ b/meta/lib/oeqa/runtime/cases/_qemutiny.py @@ -1,12 +1,19 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.target.qemu import OEQemuTarget class QemuTinyTest(OERuntimeTestCase): def test_boot_tiny(self): - status, output = self.target.run_serial('uname -a') - msg = "Cannot detect poky tiny boot!" - self.assertTrue("yocto-tiny" in output, msg) + # Until the target has explicit run_serial support, check that the + # target is the qemu runner + if isinstance(self.target, OEQemuTarget): + status, output = self.target.runner.run_serial('uname -a') + self.assertIn("Linux", output) + else: + self.skipTest("Target %s is not OEQemuTarget" % self.target) diff --git a/meta/lib/oeqa/runtime/cases/apt.py b/meta/lib/oeqa/runtime/cases/apt.py index 574a34f148..8000645843 100644 --- a/meta/lib/oeqa/runtime/cases/apt.py +++ b/meta/lib/oeqa/runtime/cases/apt.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -37,9 +39,9 @@ class AptRepoTest(AptTest): self.target.run('cd %s; echo deb [ allow-insecure=yes ] %s/all ./ > sources.list' % (apt_get_sourceslist_dir, apt_get_source_server)) def setup_source_config_for_package_install_signed(self): - apt_get_source_server = 'http:\/\/%s:%s' % (self.tc.target.server_ip, self.repo_server.port) + apt_get_source_server = 'http://%s:%s' % (self.tc.target.server_ip, self.repo_server.port) apt_get_sourceslist_dir = '/etc/apt/' - self.target.run("cd %s; cp sources.list sources.list.bak; sed -i 's/\[trusted=yes\] http:\/\/bogus_ip:bogus_port/%s/g' sources.list" % (apt_get_sourceslist_dir, apt_get_source_server)) + self.target.run("cd %s; cp sources.list sources.list.bak; sed -i 's|\[trusted=yes\] http://bogus_ip:bogus_port|%s|g' sources.list" % (apt_get_sourceslist_dir, apt_get_source_server)) def cleanup_source_config_for_package_install(self): apt_get_sourceslist_dir = '/etc/apt/' diff --git a/meta/lib/oeqa/runtime/cases/boot.py b/meta/lib/oeqa/runtime/cases/boot.py index e1ad88a174..dcee3311f7 100644 --- a/meta/lib/oeqa/runtime/cases/boot.py +++ b/meta/lib/oeqa/runtime/cases/boot.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/runtime/cases/buildcpio.py b/meta/lib/oeqa/runtime/cases/buildcpio.py index e29bf16ccb..7be734cb4f 100644 --- a/meta/lib/oeqa/runtime/cases/buildcpio.py +++ b/meta/lib/oeqa/runtime/cases/buildcpio.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -12,7 +14,7 @@ class BuildCpioTest(OERuntimeTestCase): @classmethod def setUpClass(cls): - uri = 'https://downloads.yoctoproject.org/mirror/sources/cpio-2.13.tar.gz' + uri = 'https://downloads.yoctoproject.org/mirror/sources/cpio-2.15.tar.gz' cls.project = TargetBuildProject(cls.tc.target, uri, dl_dir = cls.tc.td['DL_DIR']) @@ -27,7 +29,6 @@ class BuildCpioTest(OERuntimeTestCase): @OEHasPackage(['autoconf']) def test_cpio(self): self.project.download_archive() - self.project.run_configure('--disable-maintainer-mode', - 'sed -i -e "/char \*program_name/d" src/global.c;') + self.project.run_configure() self.project.run_make() self.project.run_install() diff --git a/meta/lib/oeqa/runtime/cases/buildgalculator.py b/meta/lib/oeqa/runtime/cases/buildgalculator.py index e5cc3e2888..2cfb3243dc 100644 --- a/meta/lib/oeqa/runtime/cases/buildgalculator.py +++ b/meta/lib/oeqa/runtime/cases/buildgalculator.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/runtime/cases/buildlzip.py b/meta/lib/oeqa/runtime/cases/buildlzip.py index bc70b41461..44f4f1be71 100644 --- a/meta/lib/oeqa/runtime/cases/buildlzip.py +++ b/meta/lib/oeqa/runtime/cases/buildlzip.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/runtime/cases/connman.py b/meta/lib/oeqa/runtime/cases/connman.py index f0d15fac9b..a488752e3f 100644 --- a/meta/lib/oeqa/runtime/cases/connman.py +++ b/meta/lib/oeqa/runtime/cases/connman.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/runtime/cases/date.py b/meta/lib/oeqa/runtime/cases/date.py index bd6537400e..a2523de67a 100644 --- a/meta/lib/oeqa/runtime/cases/date.py +++ b/meta/lib/oeqa/runtime/cases/date.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/runtime/cases/df.py b/meta/lib/oeqa/runtime/cases/df.py index bb155c9cf9..43e0ebf9ea 100644 --- a/meta/lib/oeqa/runtime/cases/df.py +++ b/meta/lib/oeqa/runtime/cases/df.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/runtime/cases/dnf.py b/meta/lib/oeqa/runtime/cases/dnf.py index f40c63026e..3ccb18ce83 100644 --- a/meta/lib/oeqa/runtime/cases/dnf.py +++ b/meta/lib/oeqa/runtime/cases/dnf.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -73,48 +75,43 @@ class DnfRepoTest(DnfTest): def test_dnf_makecache(self): self.dnf_with_repo('makecache') - -# Does not work when repo is specified on the command line -# @OETestDepends(['dnf.DnfRepoTest.test_dnf_makecache']) -# def test_dnf_repolist(self): -# self.dnf_with_repo('repolist') - @OETestDepends(['dnf.DnfRepoTest.test_dnf_makecache']) def test_dnf_repoinfo(self): self.dnf_with_repo('repoinfo') @OETestDepends(['dnf.DnfRepoTest.test_dnf_makecache']) def test_dnf_install(self): - output = self.dnf_with_repo('list run-postinsts-dev') - if 'Installed Packages' in output: - self.dnf_with_repo('remove -y run-postinsts-dev') - self.dnf_with_repo('install -y run-postinsts-dev') + self.dnf_with_repo('remove -y dnf-test-*') + self.dnf_with_repo('install -y dnf-test-dep') @OETestDepends(['dnf.DnfRepoTest.test_dnf_install']) def test_dnf_install_dependency(self): - self.dnf_with_repo('remove -y run-postinsts') - self.dnf_with_repo('install -y run-postinsts-dev') + self.dnf_with_repo('remove -y dnf-test-*') + self.dnf_with_repo('install -y dnf-test-main') + output = self.dnf('list --installed dnf-test-*') + self.assertIn("dnf-test-main.", output) + self.assertIn("dnf-test-dep.", output) @OETestDepends(['dnf.DnfRepoTest.test_dnf_install_dependency']) def test_dnf_install_from_disk(self): - self.dnf_with_repo('remove -y run-postinsts-dev') - self.dnf_with_repo('install -y --downloadonly run-postinsts-dev') - status, output = self.target.run('find /var/cache/dnf -name run-postinsts-dev*rpm', 1500) + self.dnf_with_repo('remove -y dnf-test-dep') + self.dnf_with_repo('install -y --downloadonly dnf-test-dep') + status, output = self.target.run('find /var/cache/dnf -name dnf-test-dep*rpm') self.assertEqual(status, 0, output) self.dnf_with_repo('install -y %s' % output) @OETestDepends(['dnf.DnfRepoTest.test_dnf_install_from_disk']) def test_dnf_install_from_http(self): - output = subprocess.check_output('%s %s -name run-postinsts-dev*' % (bb.utils.which(os.getenv('PATH'), "find"), + output = subprocess.check_output('%s %s -name dnf-test-dep*' % (bb.utils.which(os.getenv('PATH'), "find"), os.path.join(self.tc.td['WORKDIR'], 'oe-testimage-repo')), shell=True).decode("utf-8") rpm_path = output.split("/")[-2] + "/" + output.split("/")[-1] url = 'http://%s:%s/%s' %(self.target.server_ip, self.repo_server.port, rpm_path) - self.dnf_with_repo('remove -y run-postinsts-dev') + self.dnf_with_repo('remove -y dnf-test-dep') self.dnf_with_repo('install -y %s' % url) @OETestDepends(['dnf.DnfRepoTest.test_dnf_install']) def test_dnf_reinstall(self): - self.dnf_with_repo('reinstall -y run-postinsts-dev') + self.dnf_with_repo('reinstall -y dnf-test-main') @OETestDepends(['dnf.DnfRepoTest.test_dnf_makecache']) @skipIfInDataVar('DISTRO_FEATURES', 'usrmerge', 'Test run when not enable usrmerge') @@ -137,55 +134,40 @@ class DnfRepoTest(DnfTest): self.target.run('cp -r /etc/dnf %s/etc' % rootpath, 1500) self.target.run('cp /bin/sh %s/bin' % rootpath, 1500) self.target.run('mount -o bind /dev %s/dev/' % rootpath, 1500) - self.dnf_with_repo('install --installroot=%s -v -y --rpmverbosity=debug busybox run-postinsts' % rootpath) + self.dnf_with_repo('install --installroot=%s -v -y --rpmverbosity=debug busybox' % rootpath) status, output = self.target.run('test -e %s/var/cache/dnf' % rootpath, 1500) self.assertEqual(0, status, output) status, output = self.target.run('test -e %s/bin/busybox' % rootpath, 1500) self.assertEqual(0, status, output) @OETestDepends(['dnf.DnfRepoTest.test_dnf_makecache']) - @skipIfNotInDataVar('DISTRO_FEATURES', 'usrmerge', 'Test run when enable usrmege') + @skipIfNotInDataVar('DISTRO_FEATURES', 'usrmerge', 'Test run when enable usrmerge') @OEHasPackage('busybox') def test_dnf_installroot_usrmerge(self): rootpath = '/home/root/chroot/test' #Copy necessary files to avoid errors with not yet installed tools on #installroot directory. - self.target.run('mkdir -p %s/etc' % rootpath, 1500) - self.target.run('mkdir -p %s/usr/bin %s/usr/sbin' % (rootpath, rootpath), 1500) - self.target.run('ln -sf -r %s/usr/bin %s/bin' % (rootpath, rootpath), 1500) - self.target.run('ln -sf -r %s/usr/sbin %s/sbin' % (rootpath, rootpath), 1500) - self.target.run('mkdir -p %s/dev' % rootpath, 1500) + self.target.run('mkdir -p %s/etc' % rootpath) + self.target.run('mkdir -p %s/usr/bin %s/usr/sbin' % (rootpath, rootpath)) + self.target.run('ln -sf usr/bin %s/bin' % (rootpath)) + self.target.run('ln -sf usr/sbin %s/sbin' % (rootpath)) + self.target.run('mkdir -p %s/dev' % rootpath) #Handle different architectures lib dirs - self.target.run('mkdir -p %s/usr/lib' % rootpath, 1500) - self.target.run('mkdir -p %s/usr/libx32' % rootpath, 1500) - self.target.run('mkdir -p %s/usr/lib64' % rootpath, 1500) - self.target.run('cp /lib/libtinfo.so.5 %s/usr/lib' % rootpath, 1500) - self.target.run('cp /libx32/libtinfo.so.5 %s/usr/libx32' % rootpath, 1500) - self.target.run('cp /lib64/libtinfo.so.5 %s/usr/lib64' % rootpath, 1500) - self.target.run('ln -sf -r %s/lib %s/usr/lib' % (rootpath,rootpath), 1500) - self.target.run('ln -sf -r %s/libx32 %s/usr/libx32' % (rootpath,rootpath), 1500) - self.target.run('ln -sf -r %s/lib64 %s/usr/lib64' % (rootpath,rootpath), 1500) - self.target.run('cp -r /etc/rpm %s/etc' % rootpath, 1500) - self.target.run('cp -r /etc/dnf %s/etc' % rootpath, 1500) - self.target.run('cp /bin/sh %s/bin' % rootpath, 1500) - self.target.run('mount -o bind /dev %s/dev/' % rootpath, 1500) - self.dnf_with_repo('install --installroot=%s -v -y --rpmverbosity=debug busybox run-postinsts' % rootpath) - status, output = self.target.run('test -e %s/var/cache/dnf' % rootpath, 1500) + self.target.run("for l in /lib*; do mkdir -p %s/usr/$l; ln -s usr/$l %s/$l; done" % (rootpath, rootpath)) + self.target.run('cp -r /etc/rpm %s/etc' % rootpath) + self.target.run('cp -r /etc/dnf %s/etc' % rootpath) + self.target.run('cp /bin/busybox %s/bin/sh' % rootpath) + self.target.run('mount -o bind /dev %s/dev/' % rootpath) + self.dnf_with_repo('install --installroot=%s -v -y --rpmverbosity=debug busybox' % rootpath) + status, output = self.target.run('test -e %s/var/cache/dnf' % rootpath) self.assertEqual(0, status, output) - status, output = self.target.run('test -e %s/bin/busybox' % rootpath, 1500) + status, output = self.target.run('test -e %s/bin/busybox' % rootpath) self.assertEqual(0, status, output) @OETestDepends(['dnf.DnfRepoTest.test_dnf_makecache']) def test_dnf_exclude(self): - excludepkg = 'curl-dev' - self.dnf_with_repo('install -y curl*') - self.dnf('list %s' % excludepkg, 0) - #Avoid remove dependencies to skip some errors on different archs and images - self.dnf_with_repo('remove --setopt=clean_requirements_on_remove=0 -y curl*') - #check curl-dev is not installed adter removing all curl occurrences - status, output = self.target.run('dnf list --installed | grep %s'% excludepkg, 1500) - self.assertEqual(1, status, "%s was not removed, is listed as installed"%excludepkg) - self.dnf_with_repo('install -y --exclude=%s --exclude=curl-staticdev curl*' % excludepkg) - #check curl-dev is not installed after being excluded - status, output = self.target.run('dnf list --installed | grep %s'% excludepkg , 1500) - self.assertEqual(1, status, "%s was not excluded, is listed as installed"%excludepkg) + self.dnf_with_repo('remove -y dnf-test-*') + self.dnf_with_repo('install -y --exclude=dnf-test-dep dnf-test-*') + output = self.dnf('list --installed dnf-test-*') + self.assertIn("dnf-test-main.", output) + self.assertNotIn("dnf-test-dev.", output) diff --git a/meta/lib/oeqa/runtime/cases/ethernet_ip_connman.py b/meta/lib/oeqa/runtime/cases/ethernet_ip_connman.py index b93ee29941..eac8f2d082 100644 --- a/meta/lib/oeqa/runtime/cases/ethernet_ip_connman.py +++ b/meta/lib/oeqa/runtime/cases/ethernet_ip_connman.py @@ -1,3 +1,8 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends from oeqa.core.decorator.data import skipIfQemu diff --git a/meta/lib/oeqa/runtime/cases/gcc.py b/meta/lib/oeqa/runtime/cases/gcc.py index 1b6e431bf4..17b1483e8d 100644 --- a/meta/lib/oeqa/runtime/cases/gcc.py +++ b/meta/lib/oeqa/runtime/cases/gcc.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/runtime/cases/gi.py b/meta/lib/oeqa/runtime/cases/gi.py index 42bd100a31..78c7ddda2c 100644 --- a/meta/lib/oeqa/runtime/cases/gi.py +++ b/meta/lib/oeqa/runtime/cases/gi.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/runtime/cases/go.py b/meta/lib/oeqa/runtime/cases/go.py index 89ba2c3ecb..39a80f4dca 100644 --- a/meta/lib/oeqa/runtime/cases/go.py +++ b/meta/lib/oeqa/runtime/cases/go.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -16,4 +18,4 @@ class GoHelloworldTest(OERuntimeTestCase): self.assertEqual(status, 0, msg=msg) msg = 'Incorrect output: %s' % output - self.assertEqual(output, "Hello, Go examples!", msg=msg) + self.assertEqual(output, "Hello, world!", msg=msg) diff --git a/meta/lib/oeqa/runtime/cases/gstreamer.py b/meta/lib/oeqa/runtime/cases/gstreamer.py index f735f82e3b..2295769cfd 100644 --- a/meta/lib/oeqa/runtime/cases/gstreamer.py +++ b/meta/lib/oeqa/runtime/cases/gstreamer.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/runtime/cases/kernelmodule.py b/meta/lib/oeqa/runtime/cases/kernelmodule.py index 47fd2f850c..9c42fcc586 100644 --- a/meta/lib/oeqa/runtime/cases/kernelmodule.py +++ b/meta/lib/oeqa/runtime/cases/kernelmodule.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/runtime/cases/ksample.py b/meta/lib/oeqa/runtime/cases/ksample.py index c69e3fe4ac..b6848762e3 100644 --- a/meta/lib/oeqa/runtime/cases/ksample.py +++ b/meta/lib/oeqa/runtime/cases/ksample.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/runtime/cases/ldd.py b/meta/lib/oeqa/runtime/cases/ldd.py index 9c2caa8f65..f6841c6675 100644 --- a/meta/lib/oeqa/runtime/cases/ldd.py +++ b/meta/lib/oeqa/runtime/cases/ldd.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/runtime/cases/login.py b/meta/lib/oeqa/runtime/cases/login.py new file mode 100644 index 0000000000..e1bc60d49b --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/login.py @@ -0,0 +1,116 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +import shutil +import subprocess +import tempfile +import time +import os +from datetime import datetime +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.runtime.decorator.package import OEHasPackage + +### Status of qemu images. +# - runqemu qemuppc64 comes up blank. (skip) +# - qemuarmv5 comes up with multiple heads but sending "head" to screendump. +# seems to create a png with a bad header? (skip for now, but come back to fix) +# - qemuriscv32 and qemuloongarch64 doesn't work with testimage apparently? (skip) +# - qemumips64 is missing mouse icon. +# - qemumips takes forever to render and is missing mouse icon. +# - qemuarm and qemuppc are odd as they don't resize so we need to just set width. +# - All images have home and screen flipper icons not always rendered fully at first. +# the sleep seems to help this out some, depending on machine load. +### + +class LoginTest(OERuntimeTestCase): + @OEHasPackage(['matchbox-desktop', 'dbus-wait']) + def test_screenshot(self): + if self.td.get('MACHINE') in ("qemuppc64", "qemuarmv5", "qemuriscv32", "qemuriscv64", "qemuloongarch64"): + self.skipTest("{0} is not currently supported.".format(self.td.get('MACHINE'))) + + pn = self.td.get('PN') + + ourenv = os.environ.copy() + origpath = self.td.get("ORIGPATH") + if origpath: + ourenv['PATH'] = ourenv['PATH'] + ":" + origpath + + for cmd in ["identify.im7", "convert.im7", "compare.im7"]: + try: + subprocess.check_output(["which", cmd], env=ourenv) + except subprocess.CalledProcessError: + self.skipTest("%s (from imagemagick) not available" % cmd) + + + # Store images so we can debug them if needed + saved_screenshots_dir = self.td.get('T') + "/saved-screenshots/" + + ### + # This is a really horrible way of doing this but I've not found the + # right event to determine "The system is loaded and screen is rendered" + # + # Using dbus-wait for matchbox is the wrong answer because while it + # ensures the system is up, it doesn't mean the screen is rendered. + # + # Checking the qmp socket doesn't work afaik either. + # + # One way to do this is to do compares of known good screendumps until + # we either get expected or close to expected or we time out. Part of the + # issue here with that is that there is a very fine difference in the + # diff between a screendump where the icons haven't loaded yet and + # one where they won't load. I'll look at that next, but, for now, this. + # + # Which is ugly and I hate it but it 'works' for various definitions of + # 'works'. + ### + # RP: if the signal is sent before we run this, it will never be seen and we'd timeout + #status, output = self.target.run('dbus-wait org.matchbox_project.desktop Loaded') + #if status != 0 or "Timeout" in output: + # self.fail('dbus-wait failed (%s, %s). This could mean that the image never loaded the matchbox desktop.' % (status, output)) + + # Start taking screenshots every 2 seconds until diff=0 or timeout is 60 seconds + timeout = time.time() + 60 + diff = True + with tempfile.NamedTemporaryFile(prefix="oeqa-screenshot-login", suffix=".png") as t: + while diff != 0 and time.time() < timeout: + time.sleep(2) + ret = self.target.runner.run_monitor("screendump", args={"filename": t.name, "format":"png"}) + + # Find out size of image so we can determine where to blank out clock. + # qemuarm and qemuppc are odd as it doesn't resize the window and returns + # incorrect widths + if self.td.get('MACHINE') == "qemuarm" or self.td.get('MACHINE') == "qemuppc": + width = "640" + else: + cmd = "identify.im7 -ping -format '%w' {0}".format(t.name) + width = subprocess.check_output(cmd, shell=True, env=ourenv).decode() + + rblank = int(float(width)) + lblank = rblank-80 + + # Use the meta-oe version of convert, along with it's suffix. This blanks out the clock. + cmd = "convert.im7 {0} -fill white -draw 'rectangle {1},4 {2},28' {3}".format(t.name, str(rblank), str(lblank), t.name) + convert_out=subprocess.check_output(cmd, shell=True, env=ourenv).decode() + + bb.utils.mkdirhier(saved_screenshots_dir) + savedfile = "{0}/saved-{1}-{2}-{3}.png".format(saved_screenshots_dir, \ + datetime.timestamp(datetime.now()), \ + pn, \ + self.td.get('MACHINE')) + shutil.copy2(t.name, savedfile) + + refimage = self.td.get('COREBASE') + "/meta/files/screenshot-tests/" + pn + "-" + self.td.get('MACHINE') +".png" + if not os.path.exists(refimage): + self.skipTest("No reference image for comparision (%s)" % refimage) + + cmd = "compare.im7 -metric MSE {0} {1} /dev/null".format(t.name, refimage) + compare_out = subprocess.run(cmd, shell=True, capture_output=True, text=True, env=ourenv) + diff=float(compare_out.stderr.replace("(", "").replace(")","").split()[1]) + if diff > 0: + # Keep a copy of the failed screenshot so we can see what happened. + self.fail("Screenshot diff is {0}. Failed image stored in {1}".format(str(diff), savedfile)) + else: + self.assertEqual(0, diff, "Screenshot diff is {0}.".format(str(diff))) diff --git a/meta/lib/oeqa/runtime/cases/logrotate.py b/meta/lib/oeqa/runtime/cases/logrotate.py index 2bff08f9da..6ad980cb6a 100644 --- a/meta/lib/oeqa/runtime/cases/logrotate.py +++ b/meta/lib/oeqa/runtime/cases/logrotate.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/runtime/cases/ltp.py b/meta/lib/oeqa/runtime/cases/ltp.py index a66d5d13d7..f588a93200 100644 --- a/meta/lib/oeqa/runtime/cases/ltp.py +++ b/meta/lib/oeqa/runtime/cases/ltp.py @@ -57,37 +57,47 @@ class LtpTestBase(OERuntimeTestCase): class LtpTest(LtpTestBase): - ltp_groups = ["math", "syscalls", "dio", "io", "mm", "ipc", "sched", "nptl", "pty", "containers", "controllers", "filecaps", "cap_bounds", "fcntl-locktests", "connectors", "commands", "net.ipv6_lib", "input","fs_perms_simple"] + ltp_groups = ["math", "syscalls", "dio", "io", "mm", "ipc", "sched", "nptl", "pty", "containers", "controllers", "filecaps", "cap_bounds", "fcntl-locktests", "commands", "net.ipv6_lib", "input","fs_perms_simple", "cve", "crypto", "ima", "net.nfs", "net_stress.ipsec_icmp", "net.ipv6", "numa", "uevent", "ltp-aiodio.part1", "ltp-aiodio.part2", "ltp-aiodio.part3", "ltp-aiodio.part4"] - ltp_fs = ["fs", "fsx", "fs_bind"] + ltp_fs = ["fs", "fs_bind"] # skip kernel cpuhotplug ltp_kernel = ["power_management_tests", "hyperthreading ", "kernel_misc", "hugetlb"] ltp_groups += ltp_fs def runltp(self, ltp_group): - cmd = '/opt/ltp/runltp -f %s -p -q -r /opt/ltp -l /opt/ltp/results/%s -I 1 -d /opt/ltp' % (ltp_group, ltp_group) + # LTP appends to log files, so ensure we start with a clean log + self.target.deleteFiles("/opt/ltp/results/", ltp_group) + + cmd = '/opt/ltp/runltp -f %s -q -r /opt/ltp -l /opt/ltp/results/%s -I 1 -d /opt/ltp' % (ltp_group, ltp_group) + starttime = time.time() - (status, output) = self.target.run(cmd) + (status, output) = self.target.run(cmd, timeout=1200) endtime = time.time() + # status of 1 is 'just' tests failing. 255 likely was a command output timeout + if status and status != 1: + msg = 'Command %s returned exit code %s' % (cmd, status) + self.target.logger.warning(msg) + + # Write the console log to disk for convenience with open(os.path.join(self.ltptest_log_dir, "%s-raw.log" % ltp_group), 'w') as f: f.write(output) + # Also put the console log into the test result JSON self.extras['ltpresult.rawlogs']['log'] = self.extras['ltpresult.rawlogs']['log'] + output - # copy nice log from DUT - dst = os.path.join(self.ltptest_log_dir, "%s" % ltp_group ) + # Copy the machine-readable test results locally so we can parse it + dst = os.path.join(self.ltptest_log_dir, ltp_group) remote_src = "/opt/ltp/results/%s" % ltp_group (status, output) = self.target.copyFrom(remote_src, dst, True) - msg = 'File could not be copied. Output: %s' % output if status: + msg = 'File could not be copied. Output: %s' % output self.target.logger.warning(msg) parser = LtpParser() results, sections = parser.parse(dst) - runtime = int(endtime-starttime) - sections['duration'] = runtime + sections['duration'] = int(endtime-starttime) self.sections[ltp_group] = sections failed_tests = {} diff --git a/meta/lib/oeqa/runtime/cases/maturin.py b/meta/lib/oeqa/runtime/cases/maturin.py new file mode 100644 index 0000000000..4e6384fe5e --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/maturin.py @@ -0,0 +1,58 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +import os + +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.runtime.decorator.package import OEHasPackage + + +class MaturinTest(OERuntimeTestCase): + @OETestDepends(['ssh.SSHTest.test_ssh', 'python.PythonTest.test_python3']) + @OEHasPackage(['python3-maturin']) + def test_maturin_list_python(self): + status, output = self.target.run("maturin list-python") + self.assertEqual(status, 0) + _, py_major = self.target.run("python3 -c 'import sys; print(sys.version_info.major)'") + _, py_minor = self.target.run("python3 -c 'import sys; print(sys.version_info.minor)'") + python_version = "%s.%s" % (py_major, py_minor) + self.assertEqual(output, "🐍 1 python interpreter found:\n" + " - CPython %s at /usr/bin/python%s" % (python_version, python_version)) + + +class MaturinDevelopTest(OERuntimeTestCase): + @classmethod + def setUp(cls): + dst = '/tmp' + src = os.path.join(cls.tc.files_dir, "maturin/guessing-game") + cls.tc.target.copyTo(src, dst) + + @classmethod + def tearDown(cls): + cls.tc.target.run('rm -rf %s' % '/tmp/guessing-game/target') + + @OETestDepends(['ssh.SSHTest.test_ssh', 'python.PythonTest.test_python3']) + @OEHasPackage(['python3-maturin']) + def test_maturin_develop(self): + """ + This test case requires: + (1) that a .venv can been created. + (2) DNS nameserver to resolve crate URIs for fetching + (3) a functional 'rustc' and 'cargo' + """ + targetdir = os.path.join("/tmp", "guessing-game") + self.target.run("cd %s; python3 -m venv .venv" % targetdir) + self.target.run("echo 'nameserver 8.8.8.8' > /etc/resolv.conf") + cmd = "cd %s; maturin develop" % targetdir + status, output = self.target.run(cmd) + self.assertRegex(output, r"🔗 Found pyo3 bindings with abi3 support for Python ≥ 3.8") + self.assertRegex(output, r"🐍 Not using a specific python interpreter") + self.assertRegex(output, r"📡 Using build options features from pyproject.toml") + self.assertRegex(output, r"Compiling guessing-game v0.1.0") + self.assertRegex(output, r"📦 Built wheel for abi3 Python ≥ 3.8") + self.assertRegex(output, r"🛠 Installed guessing-game-0.1.0") + self.assertEqual(status, 0) diff --git a/meta/lib/oeqa/runtime/cases/multilib.py b/meta/lib/oeqa/runtime/cases/multilib.py index 0d1b9ae2c9..68556e45c5 100644 --- a/meta/lib/oeqa/runtime/cases/multilib.py +++ b/meta/lib/oeqa/runtime/cases/multilib.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/runtime/cases/oe_syslog.py b/meta/lib/oeqa/runtime/cases/oe_syslog.py index 150b70d9f0..adb876160d 100644 --- a/meta/lib/oeqa/runtime/cases/oe_syslog.py +++ b/meta/lib/oeqa/runtime/cases/oe_syslog.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -114,7 +116,7 @@ class SyslogTestConfig(OERuntimeTestCase): @OETestDepends(['oe_syslog.SyslogTestConfig.test_syslog_logger']) @OEHasPackage(["busybox-syslog"]) @skipIfDataVar('VIRTUAL-RUNTIME_init_manager', 'systemd', - 'Not appropiate for systemd image') + 'Not appropriate for systemd image') def test_syslog_startup_config(self): cmd = 'echo "LOGFILE=/var/log/test" >> /etc/syslog-startup.conf' self.target.run(cmd) diff --git a/meta/lib/oeqa/runtime/cases/opkg.py b/meta/lib/oeqa/runtime/cases/opkg.py index 9cfee1cd88..a29c93e59a 100644 --- a/meta/lib/oeqa/runtime/cases/opkg.py +++ b/meta/lib/oeqa/runtime/cases/opkg.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/runtime/cases/pam.py b/meta/lib/oeqa/runtime/cases/pam.py index a482ded945..b3e8b56c3c 100644 --- a/meta/lib/oeqa/runtime/cases/pam.py +++ b/meta/lib/oeqa/runtime/cases/pam.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/runtime/cases/parselogs-ignores-common.txt b/meta/lib/oeqa/runtime/cases/parselogs-ignores-common.txt new file mode 100644 index 0000000000..f91abbc941 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/parselogs-ignores-common.txt @@ -0,0 +1,62 @@ +# Xserver explains what the short codes mean +(WW) warning, (EE) error, (NI) not implemented, (??) unknown. + +# Xserver warns if compiled with ACPI but no acpid running +Open ACPI failed (/var/run/acpid.socket) (No such file or directory) + +# Some machines (eg qemux86) don't enable PAE (they probably should though) +NX (Execute Disable) protection cannot be enabled: non-PAE kernel! + +# Connman's pacrunner warns if external connectivity isn't available +Failed to find URL:http://ipv4.connman.net/online/status.html +Failed to find URL:http://ipv6.connman.net/online/status.html + +# x86 on 6.6+ outputs this message, it is informational, not an error +ACPI: _OSC evaluation for CPUs failed, trying _PDC + +# These should be reviewed to see if they are still needed +dma timeout +can\'t add hid device: +usbhid: probe of +_OSC failed (AE_ERROR) +_OSC failed (AE_SUPPORT) +AE_ALREADY_EXISTS +ACPI _OSC request failed (AE_SUPPORT) +can\'t disable ASPM +Failed to load module "vesa" +Failed to load module "modesetting" +Failed to load module "glx" +Failed to load module "fbdev" +Failed to load module "ati" +[drm] Cannot find any crtc or sizes +_OSC failed (AE_NOT_FOUND); disabling ASPM +hd.: possibly failed opcode +NETLINK INITIALIZATION FAILED +kernel: Cannot find map file +omap_hwmod: debugss: _wait_target_disable failed +VGA arbiter: cannot open kernel arbiter, no multi-card support +Online check failed for +netlink init failed +Fast TSC calibration +controller can't do DEVSLP, turning off +stmmac_dvr_probe: warning: cannot get CSR clock +error: couldn\'t mount because of unsupported optional features +GPT: Use GNU Parted to correct GPT errors +Cannot set xattr user.Librepo.DownloadInProgress +Failed to read /var/lib/nfs/statd/state: Success +error retry time-out = +logind: cannot setup systemd-logind helper (-61), using legacy fallback +Failed to rename network interface +Failed to process device, ignoring: Device or resource busy +Cannot find a map file +[rdrand]: Initialization Failed +[rndr ]: Initialization Failed +[pulseaudio] authkey.c: Failed to open cookie file +[pulseaudio] authkey.c: Failed to load authentication key +was skipped because of a failed condition check +was skipped because all trigger condition checks failed +xf86OpenConsole: Switching VT failed +Failed to read LoaderConfigTimeoutOneShot variable, ignoring: Operation not supported +Failed to read LoaderEntryOneShot variable, ignoring: Operation not supported +Direct firmware load for regulatory.db +failed to load regulatory.db diff --git a/meta/lib/oeqa/runtime/cases/parselogs-ignores-mipsarch.txt b/meta/lib/oeqa/runtime/cases/parselogs-ignores-mipsarch.txt new file mode 100644 index 0000000000..2c0bd9a247 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/parselogs-ignores-mipsarch.txt @@ -0,0 +1,2 @@ +# These should be reviewed to see if they are still needed +cacheinfo: Failed to find cpu0 device node diff --git a/meta/lib/oeqa/runtime/cases/parselogs-ignores-qemuall.txt b/meta/lib/oeqa/runtime/cases/parselogs-ignores-qemuall.txt new file mode 100644 index 0000000000..b0c0fc9ddf --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/parselogs-ignores-qemuall.txt @@ -0,0 +1,27 @@ +# psplash +FBIOPUT_VSCREENINFO failed, double buffering disabled + +# PCI host bridge to bus 0000:00 +# pci_bus 0000:00: root bus resource [mem 0x10000000-0x17ffffff] +# pci_bus 0000:00: root bus resource [io 0x1000-0x1fffff] +# pci_bus 0000:00: No busn resource found for root bus, will use [bus 00-ff] +# pci 0000:00:00.0: [2046:ab11] type 00 class 0x100000 +# pci 0000:00:00.0: [Firmware Bug]: reg 0x10: invalid BAR (can't size) +# pci 0000:00:00.0: [Firmware Bug]: reg 0x14: invalid BAR (can't size) +# pci 0000:00:00.0: [Firmware Bug]: reg 0x18: invalid BAR (can't size) +# pci 0000:00:00.0: [Firmware Bug]: reg 0x1c: invalid BAR (can't size) +# pci 0000:00:00.0: [Firmware Bug]: reg 0x20: invalid BAR (can't size) +# pci 0000:00:00.0: [Firmware Bug]: reg 0x24: invalid BAR (can't size) +invalid BAR (can't size) + +# These should be reviewed to see if they are still needed +wrong ELF class +fail to add MMCONFIG information, can't access extended PCI configuration space under this bridge +can't claim BAR +amd_nb: Cannot enumerate AMD northbridges +tsc: HPET/PMTIMER calibration failed +modeset(0): Failed to initialize the DRI2 extension +glamor initialization failed +blk_update_request: I/O error, dev fd0, sector 0 op 0x0:(READ) +floppy: error +failed to IDENTIFY (I/O error, err_mask=0x4) diff --git a/meta/lib/oeqa/runtime/cases/parselogs-ignores-qemuarm64.txt b/meta/lib/oeqa/runtime/cases/parselogs-ignores-qemuarm64.txt new file mode 100644 index 0000000000..260cdde620 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/parselogs-ignores-qemuarm64.txt @@ -0,0 +1,6 @@ +# These should be reviewed to see if they are still needed +Fatal server error: +(EE) Server terminated with error (1). Closing log file. +dmi: Firmware registration failed. +irq: type mismatch, failed to map hwirq-27 for /intc +logind: failed to get session seat
\ No newline at end of file diff --git a/meta/lib/oeqa/runtime/cases/parselogs-ignores-qemuarmv5.txt b/meta/lib/oeqa/runtime/cases/parselogs-ignores-qemuarmv5.txt new file mode 100644 index 0000000000..ed91107b7d --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/parselogs-ignores-qemuarmv5.txt @@ -0,0 +1,19 @@ +# Code is 2 JENT_ECOARSETIME: Timer too coarse for RNG. +jitterentropy: Initialization failed with host not compliant with requirements: 2 + +# These should be reviewed to see if they are still needed +mmci-pl18x: probe of fpga:05 failed with error -22 +mmci-pl18x: probe of fpga:0b failed with error -22 + +OF: amba_device_add() failed (-19) for /amba/smc@10100000 +OF: amba_device_add() failed (-19) for /amba/mpmc@10110000 +OF: amba_device_add() failed (-19) for /amba/sctl@101e0000 +OF: amba_device_add() failed (-19) for /amba/watchdog@101e1000 +OF: amba_device_add() failed (-19) for /amba/sci@101f0000 +OF: amba_device_add() failed (-19) for /amba/spi@101f4000 +OF: amba_device_add() failed (-19) for /amba/ssp@101f4000 +OF: amba_device_add() failed (-19) for /amba/fpga/sci@a000 +Failed to initialize '/amba/timer@101e3000': -22 + +clcd-pl11x: probe of 10120000.display failed with error -2 +arm-charlcd 10008000.lcd: error -ENXIO: IRQ index 0 not found diff --git a/meta/lib/oeqa/runtime/cases/parselogs-ignores-qemuppc.txt b/meta/lib/oeqa/runtime/cases/parselogs-ignores-qemuppc.txt new file mode 100644 index 0000000000..d9b58b58f1 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/parselogs-ignores-qemuppc.txt @@ -0,0 +1,6 @@ +# These should be reviewed to see if they are still needed +PCI 0000:00 Cannot reserve Legacy IO [io 0x0000-0x0fff] +host side 80-wire cable detection failed, limiting max speed +mode "640x480" test failed +can't handle BAR above 4GB +Cannot reserve Legacy IO
\ No newline at end of file diff --git a/meta/lib/oeqa/runtime/cases/parselogs-ignores-qemuppc64.txt b/meta/lib/oeqa/runtime/cases/parselogs-ignores-qemuppc64.txt new file mode 100644 index 0000000000..b736a2aeb7 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/parselogs-ignores-qemuppc64.txt @@ -0,0 +1,4 @@ +# These should be reviewed to see if they are still needed +vio vio: uevent: failed to send synthetic uevent +synth uevent: /devices/vio: failed to send uevent +PCI 0000:00 Cannot reserve Legacy IO [io 0x10000-0x10fff]
\ No newline at end of file diff --git a/meta/lib/oeqa/runtime/cases/parselogs-ignores-qemux86.txt b/meta/lib/oeqa/runtime/cases/parselogs-ignores-qemux86.txt new file mode 100644 index 0000000000..ebb76f1221 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/parselogs-ignores-qemux86.txt @@ -0,0 +1,2 @@ +# These should be reviewed to see if they are still needed +Failed to access perfctr msr (MSR diff --git a/meta/lib/oeqa/runtime/cases/parselogs-ignores-x86.txt b/meta/lib/oeqa/runtime/cases/parselogs-ignores-x86.txt new file mode 100644 index 0000000000..5985247daf --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/parselogs-ignores-x86.txt @@ -0,0 +1,10 @@ +# These should be reviewed to see if they are still needed +[drm:psb_do_init] *ERROR* Debug is +wrong ELF class +Could not enable PowerButton event +probe of LNXPWRBN:00 failed with error -22 +pmd_set_huge: Cannot satisfy +failed to setup card detect gpio +amd_nb: Cannot enumerate AMD northbridges +failed to retrieve link info, disabling eDP +Direct firmware load for iwlwifi diff --git a/meta/lib/oeqa/runtime/cases/parselogs-ignores-x86_64.txt b/meta/lib/oeqa/runtime/cases/parselogs-ignores-x86_64.txt new file mode 120000 index 0000000000..404e384c32 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/parselogs-ignores-x86_64.txt @@ -0,0 +1 @@ +parselogs-ignores-x86.txt
\ No newline at end of file diff --git a/meta/lib/oeqa/runtime/cases/parselogs.py b/meta/lib/oeqa/runtime/cases/parselogs.py index 1f9365f3a8..6966923c94 100644 --- a/meta/lib/oeqa/runtime/cases/parselogs.py +++ b/meta/lib/oeqa/runtime/cases/parselogs.py @@ -1,216 +1,49 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # +import collections import os +import sys -from subprocess import check_output from shutil import rmtree from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends -from oeqa.core.decorator.data import skipIfDataVar -from oeqa.runtime.decorator.package import OEHasPackage - -#in the future these lists could be moved outside of module -errors = ["error", "cannot", "can\'t", "failed"] - -common_errors = [ - "(WW) warning, (EE) error, (NI) not implemented, (??) unknown.", - "dma timeout", - "can\'t add hid device:", - "usbhid: probe of ", - "_OSC failed (AE_ERROR)", - "_OSC failed (AE_SUPPORT)", - "AE_ALREADY_EXISTS", - "ACPI _OSC request failed (AE_SUPPORT)", - "can\'t disable ASPM", - "Failed to load module \"vesa\"", - "Failed to load module vesa", - "Failed to load module \"modesetting\"", - "Failed to load module modesetting", - "Failed to load module \"glx\"", - "Failed to load module \"fbdev\"", - "Failed to load module fbdev", - "Failed to load module glx", - "[drm] Cannot find any crtc or sizes", - "_OSC failed (AE_NOT_FOUND); disabling ASPM", - "Open ACPI failed (/var/run/acpid.socket) (No such file or directory)", - "NX (Execute Disable) protection cannot be enabled: non-PAE kernel!", - "hd.: possibly failed opcode", - 'NETLINK INITIALIZATION FAILED', - 'kernel: Cannot find map file', - 'omap_hwmod: debugss: _wait_target_disable failed', - 'VGA arbiter: cannot open kernel arbiter, no multi-card support', - 'Failed to find URL:http://ipv4.connman.net/online/status.html', - 'Online check failed for', - 'netlink init failed', - 'Fast TSC calibration', - "BAR 0-9", - "Failed to load module \"ati\"", - "controller can't do DEVSLP, turning off", - "stmmac_dvr_probe: warning: cannot get CSR clock", - "error: couldn\'t mount because of unsupported optional features", - "GPT: Use GNU Parted to correct GPT errors", - "Cannot set xattr user.Librepo.DownloadInProgress", - "Failed to read /var/lib/nfs/statd/state: Success", - "error retry time-out =", - "logind: cannot setup systemd-logind helper (-61), using legacy fallback", - "Failed to rename network interface", - "Failed to process device, ignoring: Device or resource busy", - "Cannot find a map file", - "[rdrand]: Initialization Failed", - "[rndr ]: Initialization Failed", - "[pulseaudio] authkey.c: Failed to open cookie file", - "[pulseaudio] authkey.c: Failed to load authentication key", - "was skipped because of a failed condition check", - "was skipped because all trigger condition checks failed", - ] -video_related = [ -] +# importlib.resources.open_text in Python <3.10 doesn't search all directories +# when a package is split across multiple directories. Until we can rely on +# 3.10+, reimplement the searching logic. +if sys.version_info < (3, 10): + def _open_text(package, resource): + import importlib, pathlib + module = importlib.import_module(package) + for path in module.__path__: + candidate = pathlib.Path(path) / resource + if candidate.exists(): + return candidate.open(encoding='utf-8') + raise FileNotFoundError +else: + from importlib.resources import open_text as _open_text -x86_common = [ - '[drm:psb_do_init] *ERROR* Debug is', - 'wrong ELF class', - 'Could not enable PowerButton event', - 'probe of LNXPWRBN:00 failed with error -22', - 'pmd_set_huge: Cannot satisfy', - 'failed to setup card detect gpio', - 'amd_nb: Cannot enumerate AMD northbridges', - 'failed to retrieve link info, disabling eDP', - 'Direct firmware load for iwlwifi', - 'Direct firmware load for regulatory.db', - 'failed to load regulatory.db', -] + common_errors -qemux86_common = [ - 'wrong ELF class', - "fail to add MMCONFIG information, can't access extended PCI configuration space under this bridge.", - "can't claim BAR ", - 'amd_nb: Cannot enumerate AMD northbridges', - 'tsc: HPET/PMTIMER calibration failed', - "modeset(0): Failed to initialize the DRI2 extension", - "glamor initialization failed", - "blk_update_request: I/O error, dev fd0, sector 0 op 0x0:(READ)", - "floppy: error", - 'failed to IDENTIFY (I/O error, err_mask=0x4)', -] + common_errors +class ParseLogsTest(OERuntimeTestCase): -ignore_errors = { - 'default' : common_errors, - 'qemux86' : [ - 'Failed to access perfctr msr (MSR', - 'pci 0000:00:00.0: [Firmware Bug]: reg 0x..: invalid BAR (can\'t size)', - ] + qemux86_common, - 'qemux86-64' : qemux86_common, - 'qemumips' : [ - 'Failed to load module "glx"', - 'pci 0000:00:00.0: [Firmware Bug]: reg 0x..: invalid BAR (can\'t size)', - 'cacheinfo: Failed to find cpu0 device node', - ] + common_errors, - 'qemumips64' : [ - 'pci 0000:00:00.0: [Firmware Bug]: reg 0x..: invalid BAR (can\'t size)', - 'cacheinfo: Failed to find cpu0 device node', - ] + common_errors, - 'qemuppc' : [ - 'PCI 0000:00 Cannot reserve Legacy IO [io 0x0000-0x0fff]', - 'host side 80-wire cable detection failed, limiting max speed', - 'mode "640x480" test failed', - 'Failed to load module "glx"', - 'can\'t handle BAR above 4GB', - 'Cannot reserve Legacy IO', - ] + common_errors, - 'qemuppc64' : [ - 'vio vio: uevent: failed to send synthetic uevent', - 'synth uevent: /devices/vio: failed to send uevent', - 'PCI 0000:00 Cannot reserve Legacy IO [io 0x10000-0x10fff]', - ] + common_errors, - 'qemuarmv5' : [ - 'mmci-pl18x: probe of fpga:05 failed with error -22', - 'mmci-pl18x: probe of fpga:0b failed with error -22', - 'Failed to load module "glx"', - 'OF: amba_device_add() failed (-19) for /amba/smc@10100000', - 'OF: amba_device_add() failed (-19) for /amba/mpmc@10110000', - 'OF: amba_device_add() failed (-19) for /amba/sctl@101e0000', - 'OF: amba_device_add() failed (-19) for /amba/watchdog@101e1000', - 'OF: amba_device_add() failed (-19) for /amba/sci@101f0000', - 'OF: amba_device_add() failed (-19) for /amba/spi@101f4000', - 'OF: amba_device_add() failed (-19) for /amba/ssp@101f4000', - 'OF: amba_device_add() failed (-19) for /amba/fpga/sci@a000', - 'Failed to initialize \'/amba/timer@101e3000\': -22', - 'jitterentropy: Initialization failed with host not compliant with requirements: 2', - 'clcd-pl11x: probe of 10120000.display failed with error -2', - ] + common_errors, - 'qemuarm64' : [ - 'Fatal server error:', - '(EE) Server terminated with error (1). Closing log file.', - 'dmi: Firmware registration failed.', - 'irq: type mismatch, failed to map hwirq-27 for /intc', - 'logind: failed to get session seat', - ] + common_errors, - 'intel-core2-32' : [ - 'ACPI: No _BQC method, cannot determine initial brightness', - '[Firmware Bug]: ACPI: No _BQC method, cannot determine initial brightness', - '(EE) Failed to load module "psb"', - '(EE) Failed to load module psb', - '(EE) Failed to load module "psbdrv"', - '(EE) Failed to load module psbdrv', - '(EE) open /dev/fb0: No such file or directory', - '(EE) AIGLX: reverting to software rendering', - 'dmi: Firmware registration failed.', - 'ioremap error for 0x78', - ] + x86_common, - 'intel-corei7-64' : [ - 'can\'t set Max Payload Size to 256', - 'intel_punit_ipc: can\'t request region for resource', - '[drm] parse error at position 4 in video mode \'efifb\'', - 'ACPI Error: Could not enable RealTimeClock event', - 'ACPI Warning: Could not enable fixed event - RealTimeClock', - 'hci_intel INT33E1:00: Unable to retrieve gpio', - 'hci_intel: probe of INT33E1:00 failed', - 'can\'t derive routing for PCI INT A', - 'failed to read out thermal zone', - 'Bluetooth: hci0: Setting Intel event mask failed', - 'ttyS2 - failed to request DMA', - 'Bluetooth: hci0: Failed to send firmware data (-38)', - 'atkbd serio0: Failed to enable keyboard on isa0060/serio0', - ] + x86_common, - 'genericx86' : x86_common, - 'genericx86-64' : [ - 'Direct firmware load for i915', - 'Failed to load firmware i915', - 'Failed to fetch GuC', - 'Failed to initialize GuC', - 'Failed to load DMC firmware', - 'The driver is built-in, so to load the firmware you need to', - ] + x86_common, - 'edgerouter' : [ - 'not creating \'/sys/firmware/fdt\'', - 'Failed to find cpu0 device node', - 'Fatal server error:', - 'Server terminated with error', - ] + common_errors, - 'beaglebone-yocto' : [ - 'Direct firmware load for regulatory.db', - 'failed to load regulatory.db', - 'l4_wkup_cm', - 'Failed to load module "glx"', - 'Failed to make EGL context current', - 'glamor initialization failed', - ] + common_errors, -} + # Which log files should be collected + log_locations = ["/var/log/", "/var/log/dmesg", "/tmp/dmesg_output.log"] -log_locations = ["/var/log/","/var/log/dmesg", "/tmp/dmesg_output.log"] + # The keywords that identify error messages in the log files + errors = ["error", "cannot", "can't", "failed"] -class ParseLogsTest(OERuntimeTestCase): + # A list of error messages that should be ignored + ignore_errors = [] @classmethod def setUpClass(cls): - cls.errors = errors - # When systemd is enabled we need to notice errors on # circular dependencies in units. - if 'systemd' in cls.td.get('DISTRO_FEATURES', ''): + if 'systemd' in cls.td.get('DISTRO_FEATURES'): cls.errors.extend([ 'Found ordering cycle on', 'Breaking ordering cycle by deleting job', @@ -218,48 +51,22 @@ class ParseLogsTest(OERuntimeTestCase): 'Ordering cycle found, skipping', ]) - cls.ignore_errors = ignore_errors - cls.log_locations = log_locations - cls.msg = '' - is_lsb, _ = cls.tc.target.run("which LSB_Test.sh") - if is_lsb == 0: - for machine in cls.ignore_errors: - cls.ignore_errors[machine] = cls.ignore_errors[machine] \ - + video_related - - def getMachine(self): - return self.td.get('MACHINE', '') - - def getWorkdir(self): - return self.td.get('WORKDIR', '') - - # Get some information on the CPU of the machine to display at the - # beginning of the output. This info might be useful in some cases. - def getHardwareInfo(self): - hwi = "" - cmd = ('cat /proc/cpuinfo | grep "model name" | head -n1 | ' - " awk 'BEGIN{FS=\":\"}{print $2}'") - _, cpu_name = self.target.run(cmd) - - cmd = ('cat /proc/cpuinfo | grep "cpu cores" | head -n1 | ' - "awk {'print $4'}") - _, cpu_physical_cores = self.target.run(cmd) - - cmd = 'cat /proc/cpuinfo | grep "processor" | wc -l' - _, cpu_logical_cores = self.target.run(cmd) - - _, cpu_arch = self.target.run('uname -m') + cls.errors = [s.casefold() for s in cls.errors] - hwi += 'Machine information: \n' - hwi += '*******************************\n' - hwi += 'Machine name: ' + self.getMachine() + '\n' - hwi += 'CPU: ' + str(cpu_name) + '\n' - hwi += 'Arch: ' + str(cpu_arch)+ '\n' - hwi += 'Physical cores: ' + str(cpu_physical_cores) + '\n' - hwi += 'Logical cores: ' + str(cpu_logical_cores) + '\n' - hwi += '*******************************\n' + cls.load_machine_ignores() - return hwi + @classmethod + def load_machine_ignores(cls): + # Add TARGET_ARCH explicitly as not every machine has that in MACHINEOVERRDES (eg qemux86-64) + for candidate in ["common", cls.td.get("TARGET_ARCH")] + cls.td.get("MACHINEOVERRIDES").split(":"): + try: + name = f"parselogs-ignores-{candidate}.txt" + for line in _open_text("oeqa.runtime.cases", name): + line = line.strip() + if line and not line.startswith("#"): + cls.ignore_errors.append(line.casefold()) + except FileNotFoundError: + pass # Go through the log locations provided and if it's a folder # create a list with all the .log files in it, if it's a file @@ -267,23 +74,23 @@ class ParseLogsTest(OERuntimeTestCase): def getLogList(self, log_locations): logs = [] for location in log_locations: - status, _ = self.target.run('test -f ' + str(location)) + status, _ = self.target.run('test -f %s' % location) if status == 0: - logs.append(str(location)) + logs.append(location) else: - status, _ = self.target.run('test -d ' + str(location)) + status, _ = self.target.run('test -d %s' % location) if status == 0: - cmd = 'find ' + str(location) + '/*.log -maxdepth 1 -type f' + cmd = 'find %s -name \\*.log -maxdepth 1 -type f' % location status, output = self.target.run(cmd) if status == 0: output = output.splitlines() for logfile in output: - logs.append(os.path.join(location, str(logfile))) + logs.append(os.path.join(location, logfile)) return logs # Copy the log files to be parsed locally def transfer_logs(self, log_list): - workdir = self.getWorkdir() + workdir = self.td.get('WORKDIR') self.target_logs = workdir + '/' + 'target_logs' target_logs = self.target_logs if os.path.exists(target_logs): @@ -300,65 +107,55 @@ class ParseLogsTest(OERuntimeTestCase): logs = [f for f in dir_files if os.path.isfile(f)] return logs - # Build the grep command to be used with filters and exclusions - def build_grepcmd(self, errors, ignore_errors, log): - grepcmd = 'grep ' - grepcmd += '-Ei "' - for error in errors: - grepcmd += r'\<' + error + r'\>' + '|' - grepcmd = grepcmd[:-1] - grepcmd += '" ' + str(log) + " | grep -Eiv \'" - - try: - errorlist = ignore_errors[self.getMachine()] - except KeyError: - self.msg += 'No ignore list found for this machine, using default\n' - errorlist = ignore_errors['default'] - - for ignore_error in errorlist: - ignore_error = ignore_error.replace('(', r'\(') - ignore_error = ignore_error.replace(')', r'\)') - ignore_error = ignore_error.replace("'", '.') - ignore_error = ignore_error.replace('?', r'\?') - ignore_error = ignore_error.replace('[', r'\[') - ignore_error = ignore_error.replace(']', r'\]') - ignore_error = ignore_error.replace('*', r'\*') - ignore_error = ignore_error.replace('0-9', '[0-9]') - grepcmd += ignore_error + '|' - grepcmd = grepcmd[:-1] - grepcmd += "\'" - - return grepcmd - - # Grep only the errors so that their context could be collected. - # Default context is 10 lines before and after the error itself - def parse_logs(self, errors, ignore_errors, logs, - lines_before = 10, lines_after = 10): - results = {} - rez = [] - grep_output = '' + def get_context(self, lines, index, before=6, after=3): + """ + Given a set of lines and the index of the line that is important, return + a number of lines surrounding that line. + """ + last = len(lines) + + start = index - before + end = index + after + 1 + + if start < 0: + end -= start + start = 0 + if end > last: + start -= end - last + end = last + + return lines[start:end] + + def test_get_context(self): + """ + A test case for the test case. + """ + lines = list(range(0,10)) + self.assertEqual(self.get_context(lines, 0, 2, 1), [0, 1, 2, 3]) + self.assertEqual(self.get_context(lines, 5, 2, 1), [3, 4, 5, 6]) + self.assertEqual(self.get_context(lines, 9, 2, 1), [6, 7, 8, 9]) + + def parse_logs(self, logs, lines_before=10, lines_after=10): + """ + Search the log files @logs looking for error lines (marked by + @self.errors), ignoring anything listed in @self.ignore_errors. + + Returns a dictionary of log filenames to a dictionary of error lines to + the error context (controlled by @lines_before and @lines_after). + """ + results = collections.defaultdict(dict) for log in logs: - result = None - thegrep = self.build_grepcmd(errors, ignore_errors, log) + with open(log) as f: + lines = f.readlines() - try: - result = check_output(thegrep, shell=True).decode('utf-8') - except: - pass + for i, line in enumerate(lines): + line = line.strip() + line_lower = line.casefold() - if result is not None: - results[log] = {} - rez = result.splitlines() - - for xrez in rez: - try: - cmd = ['grep', '-F', xrez, '-B', str(lines_before)] - cmd += ['-A', str(lines_after), log] - grep_output = check_output(cmd).decode('utf-8') - except: - pass - results[log][xrez]=grep_output + if any(keyword in line_lower for keyword in self.errors): + if not any(ignore in line_lower for ignore in self.ignore_errors): + results[log][line] = "".join(self.get_context(lines, i, lines_before, lines_after)) return results @@ -371,17 +168,18 @@ class ParseLogsTest(OERuntimeTestCase): def test_parselogs(self): self.write_dmesg() log_list = self.get_local_log_list(self.log_locations) - result = self.parse_logs(self.errors, self.ignore_errors, log_list) - print(self.getHardwareInfo()) + result = self.parse_logs(log_list) + errcount = 0 + self.msg = "" for log in result: self.msg += 'Log: ' + log + '\n' self.msg += '-----------------------\n' for error in result[log]: errcount += 1 - self.msg += 'Central error: ' + str(error) + '\n' + self.msg += 'Central error: ' + error + '\n' self.msg += '***********************\n' - self.msg += result[str(log)][str(error)] + '\n' + self.msg += result[log][error] + '\n' self.msg += '***********************\n' self.msg += '%s errors found in logs.' % errcount self.assertEqual(errcount, 0, msg=self.msg) diff --git a/meta/lib/oeqa/runtime/cases/perl.py b/meta/lib/oeqa/runtime/cases/perl.py index 2c6b3b7846..f11b300836 100644 --- a/meta/lib/oeqa/runtime/cases/perl.py +++ b/meta/lib/oeqa/runtime/cases/perl.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/runtime/cases/ping.py b/meta/lib/oeqa/runtime/cases/ping.py index 498f80d0a5..f72460e7f3 100644 --- a/meta/lib/oeqa/runtime/cases/ping.py +++ b/meta/lib/oeqa/runtime/cases/ping.py @@ -1,8 +1,11 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # from subprocess import Popen, PIPE +from time import sleep from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.oetimeout import OETimeout @@ -14,6 +17,7 @@ class PingTest(OERuntimeTestCase): def test_ping(self): output = '' count = 0 + self.assertNotEqual(len(self.target.ip), 0, msg="No target IP address set") try: while count < 5: cmd = 'ping -c 1 %s' % self.target.ip @@ -23,6 +27,7 @@ class PingTest(OERuntimeTestCase): count += 1 else: count = 0 + sleep(1) except OEQATimeoutError: self.fail("Ping timeout error for address %s, count %s, output: %s" % (self.target.ip, count, output)) msg = ('Expected 5 consecutive, got %d.\n' diff --git a/meta/lib/oeqa/runtime/cases/ptest.py b/meta/lib/oeqa/runtime/cases/ptest.py index 00742da2b5..fbaeb84d00 100644 --- a/meta/lib/oeqa/runtime/cases/ptest.py +++ b/meta/lib/oeqa/runtime/cases/ptest.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -81,17 +83,20 @@ class PtestRunnerTest(OERuntimeTestCase): extras['ptestresult.sections'] = sections + zerolength = [] trans = str.maketrans("()", "__") for section in results: for test in results[section]: result = results[section][test] testname = "ptestresult." + (section or "No-section") + "." + "_".join(test.translate(trans).split()) extras[testname] = {'status': result} + if not results[section]: + zerolength.append(section) failed_tests = {} for section in sections: - if 'exitcode' in sections[section].keys(): + if 'exitcode' in sections[section].keys() or 'timeout' in sections[section].keys(): failed_tests[section] = sections[section]["log"] for section in results: @@ -105,7 +110,10 @@ class PtestRunnerTest(OERuntimeTestCase): failmsg = "ERROR: Processes were killed by the OOM Killer:\n%s\n" % output if failed_tests: - failmsg = failmsg + "Failed ptests:\n%s" % pprint.pformat(failed_tests) + failmsg = failmsg + "\nFailed ptests:\n%s\n" % pprint.pformat(failed_tests) + + if zerolength: + failmsg = failmsg + "\nptests which had no test results:\n%s" % pprint.pformat(zerolength) if failmsg: self.logger.warning("There were failing ptests.") diff --git a/meta/lib/oeqa/runtime/cases/python.py b/meta/lib/oeqa/runtime/cases/python.py index ec54f1e1db..5d6d133480 100644 --- a/meta/lib/oeqa/runtime/cases/python.py +++ b/meta/lib/oeqa/runtime/cases/python.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/runtime/cases/rpm.py b/meta/lib/oeqa/runtime/cases/rpm.py index a4339116bf..ea5619ffea 100644 --- a/meta/lib/oeqa/runtime/cases/rpm.py +++ b/meta/lib/oeqa/runtime/cases/rpm.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -49,21 +51,20 @@ class RpmBasicTest(OERuntimeTestCase): msg = 'status: %s. Cannot run rpm -qa: %s' % (status, output) self.assertEqual(status, 0, msg=msg) - def check_no_process_for_user(u): - _, output = self.target.run(self.tc.target_cmds['ps']) - if u + ' ' in output: - return False - else: - return True + def wait_for_no_process_for_user(u, timeout = 120): + timeout_at = time.time() + timeout + while time.time() < timeout_at: + _, output = self.target.run(self.tc.target_cmds['ps']) + if u + ' ' not in output: + return + time.sleep(1) + user_pss = [ps for ps in output.split("\n") if u + ' ' in ps] + msg = "User %s has processes still running: %s" % (u, "\n".join(user_pss)) + self.fail(msg=msg) def unset_up_test_user(u): # ensure no test1 process in running - timeout = time.time() + 30 - while time.time() < timeout: - if check_no_process_for_user(u): - break - else: - time.sleep(1) + wait_for_no_process_for_user(u) status, output = self.target.run('userdel -r %s' % u) msg = 'Failed to erase user: %s' % output self.assertTrue(status == 0, msg=msg) @@ -79,21 +80,24 @@ class RpmBasicTest(OERuntimeTestCase): class RpmInstallRemoveTest(OERuntimeTestCase): - @classmethod - def setUpClass(cls): - pkgarch = cls.td['TUNE_PKGARCH'].replace('-', '_') - rpmdir = os.path.join(cls.tc.td['DEPLOY_DIR'], 'rpm', pkgarch) + def _find_test_file(self): + pkgarch = self.td['TUNE_PKGARCH'].replace('-', '_') + rpmdir = os.path.join(self.tc.td['DEPLOY_DIR'], 'rpm', pkgarch) # Pick base-passwd-doc as a test file to get installed, because it's small # and it will always be built for standard targets rpm_doc = 'base-passwd-doc-*.%s.rpm' % pkgarch if not os.path.exists(rpmdir): - return + self.fail("Rpm directory {} does not exist".format(rpmdir)) for f in fnmatch.filter(os.listdir(rpmdir), rpm_doc): - cls.test_file = os.path.join(rpmdir, f) - cls.dst = '/tmp/base-passwd-doc.rpm' + self.test_file = os.path.join(rpmdir, f) + break + else: + self.fail("Couldn't find the test rpm file {} in {}".format(rpm_doc, rpmdir)) + self.dst = '/tmp/base-passwd-doc.rpm' @OETestDepends(['rpm.RpmBasicTest.test_rpm_query']) def test_rpm_install(self): + self._find_test_file() self.tc.target.copyTo(self.test_file, self.dst) status, output = self.target.run('rpm -ivh /tmp/base-passwd-doc.rpm') msg = 'Failed to install base-passwd-doc package: %s' % output @@ -116,6 +120,7 @@ class RpmInstallRemoveTest(OERuntimeTestCase): Author: Alexander Kanavin <alex.kanavin@gmail.com> AutomatedBy: Daniel Istrate <daniel.alexandrux.istrate@intel.com> """ + self._find_test_file() db_files_cmd = 'ls /var/lib/rpm/rpmdb.sqlite*' check_log_cmd = "grep RPM /var/log/messages | wc -l" diff --git a/meta/lib/oeqa/runtime/cases/rt.py b/meta/lib/oeqa/runtime/cases/rt.py index 849ac1914e..15ab4dbbbb 100644 --- a/meta/lib/oeqa/runtime/cases/rt.py +++ b/meta/lib/oeqa/runtime/cases/rt.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/runtime/cases/rtc.py b/meta/lib/oeqa/runtime/cases/rtc.py index c4e6681324..6e45c5db4f 100644 --- a/meta/lib/oeqa/runtime/cases/rtc.py +++ b/meta/lib/oeqa/runtime/cases/rtc.py @@ -1,5 +1,11 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.data import skipIfFeature from oeqa.runtime.decorator.package import OEHasPackage import re @@ -16,12 +22,14 @@ class RTCTest(OERuntimeTestCase): self.logger.debug('Starting systemd-timesyncd daemon') self.target.run('systemctl enable --now --runtime systemd-timesyncd') + @skipIfFeature('read-only-rootfs', + 'Test does not work with read-only-rootfs in IMAGE_FEATURES') @OETestDepends(['ssh.SSHTest.test_ssh']) @OEHasPackage(['coreutils', 'busybox']) def test_rtc(self): (status, output) = self.target.run('hwclock -r') self.assertEqual(status, 0, msg='Failed to get RTC time, output: %s' % output) - + (status, current_datetime) = self.target.run('date +"%m%d%H%M%Y"') self.assertEqual(status, 0, msg='Failed to get system current date & time, output: %s' % current_datetime) @@ -32,7 +40,6 @@ class RTCTest(OERuntimeTestCase): (status, output) = self.target.run('date %s' % current_datetime) self.assertEqual(status, 0, msg='Failed to reset system date & time, output: %s' % output) - + (status, output) = self.target.run('hwclock -w') self.assertEqual(status, 0, msg='Failed to reset RTC time, output: %s' % output) - diff --git a/meta/lib/oeqa/runtime/cases/runlevel.py b/meta/lib/oeqa/runtime/cases/runlevel.py index 3a4df8ace1..6734b0f5ed 100644 --- a/meta/lib/oeqa/runtime/cases/runlevel.py +++ b/meta/lib/oeqa/runtime/cases/runlevel.py @@ -1,3 +1,8 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends diff --git a/meta/lib/oeqa/runtime/cases/rust.py b/meta/lib/oeqa/runtime/cases/rust.py index b3d6cf7f37..123c942012 100644 --- a/meta/lib/oeqa/runtime/cases/rust.py +++ b/meta/lib/oeqa/runtime/cases/rust.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -6,14 +8,57 @@ from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends from oeqa.runtime.decorator.package import OEHasPackage -class RustHelloworldTest(OERuntimeTestCase): +class RustCompileTest(OERuntimeTestCase): + + @classmethod + def setUp(cls): + dst = '/tmp/' + src = os.path.join(cls.tc.files_dir, 'test.rs') + cls.tc.target.copyTo(src, dst) + + @classmethod + def tearDown(cls): + files = '/tmp/test.rs /tmp/test' + cls.tc.target.run('rm %s' % files) + dirs = '/tmp/hello' + cls.tc.target.run('rm -r %s' % dirs) + + @OETestDepends(['ssh.SSHTest.test_ssh']) + @OEHasPackage('rust') + @OEHasPackage('openssh-scp') + def test_rust_compile(self): + status, output = self.target.run('rustc /tmp/test.rs -o /tmp/test') + msg = 'rust compile failed, output: %s' % output + self.assertEqual(status, 0, msg=msg) + + status, output = self.target.run('/tmp/test') + msg = 'running compiled file failed, output: %s' % output + self.assertEqual(status, 0, msg=msg) + + @OETestDepends(['ssh.SSHTest.test_ssh']) + @OEHasPackage('cargo') + @OEHasPackage('openssh-scp') + def test_cargo_compile(self): + status, output = self.target.run('cargo new /tmp/hello') + msg = 'cargo new failed, output: %s' % output + self.assertEqual(status, 0, msg=msg) + + status, output = self.target.run('cargo build --manifest-path=/tmp/hello/Cargo.toml') + msg = 'cargo build failed, output: %s' % output + self.assertEqual(status, 0, msg=msg) + + status, output = self.target.run('cargo run --manifest-path=/tmp/hello/Cargo.toml') + msg = 'running compiled file failed, output: %s' % output + self.assertEqual(status, 0, msg=msg) + +class RustCLibExampleTest(OERuntimeTestCase): @OETestDepends(['ssh.SSHTest.test_ssh']) - @OEHasPackage(['rust-hello-world']) - def test_rusthelloworld(self): - cmd = "rust-hello-world" + @OEHasPackage('rust-c-lib-example-bin') + def test_rust_c_lib_example(self): + cmd = "rust-c-lib-example-bin test" status, output = self.target.run(cmd) msg = 'Exit status was not 0. Output: %s' % output self.assertEqual(status, 0, msg=msg) msg = 'Incorrect output: %s' % output - self.assertEqual(output, "Hello, world!", msg=msg) + self.assertEqual(output, "Hello world in rust from C!", msg=msg) diff --git a/meta/lib/oeqa/runtime/cases/scons.py b/meta/lib/oeqa/runtime/cases/scons.py index 3c7c7f7270..4a8d4d40ba 100644 --- a/meta/lib/oeqa/runtime/cases/scons.py +++ b/meta/lib/oeqa/runtime/cases/scons.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/runtime/cases/scp.py b/meta/lib/oeqa/runtime/cases/scp.py index f2bbc947d6..ee97b8ef66 100644 --- a/meta/lib/oeqa/runtime/cases/scp.py +++ b/meta/lib/oeqa/runtime/cases/scp.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/runtime/cases/skeletoninit.py b/meta/lib/oeqa/runtime/cases/skeletoninit.py index a12f1e9aae..d0fdcbded9 100644 --- a/meta/lib/oeqa/runtime/cases/skeletoninit.py +++ b/meta/lib/oeqa/runtime/cases/skeletoninit.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -15,7 +17,7 @@ class SkeletonBasicTest(OERuntimeTestCase): @OETestDepends(['ssh.SSHTest.test_ssh']) @OEHasPackage(['service']) @skipIfDataVar('VIRTUAL-RUNTIME_init_manager', 'systemd', - 'Not appropiate for systemd image') + 'Not appropriate for systemd image') def test_skeleton_availability(self): status, output = self.target.run('ls /etc/init.d/skeleton') msg = 'skeleton init script not found. Output:\n%s' % output diff --git a/meta/lib/oeqa/runtime/cases/ssh.py b/meta/lib/oeqa/runtime/cases/ssh.py index e31224b3af..cdbef59500 100644 --- a/meta/lib/oeqa/runtime/cases/ssh.py +++ b/meta/lib/oeqa/runtime/cases/ssh.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -11,6 +13,9 @@ class SSHTest(OERuntimeTestCase): @OETestDepends(['ping.PingTest.test_ping']) @OEHasPackage(['dropbear', 'openssh-sshd']) def test_ssh(self): + (status, output) = self.target.run('sleep 20', timeout=2) + msg='run() timed out but return code was zero.' + self.assertNotEqual(status, 0, msg=msg) (status, output) = self.target.run('uname -a') self.assertEqual(status, 0, msg='SSH Test failed: %s' % output) (status, output) = self.target.run('cat /etc/controllerimage') diff --git a/meta/lib/oeqa/runtime/cases/stap.py b/meta/lib/oeqa/runtime/cases/stap.py index 480eaabf2d..3be4162108 100644 --- a/meta/lib/oeqa/runtime/cases/stap.py +++ b/meta/lib/oeqa/runtime/cases/stap.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/runtime/cases/storage.py b/meta/lib/oeqa/runtime/cases/storage.py index 972ef8210c..b05622fea8 100644 --- a/meta/lib/oeqa/runtime/cases/storage.py +++ b/meta/lib/oeqa/runtime/cases/storage.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/runtime/cases/suspend.py b/meta/lib/oeqa/runtime/cases/suspend.py index 0382d48f45..a625cc5901 100644 --- a/meta/lib/oeqa/runtime/cases/suspend.py +++ b/meta/lib/oeqa/runtime/cases/suspend.py @@ -1,3 +1,8 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends from oeqa.core.decorator.data import skipIfQemu diff --git a/meta/lib/oeqa/runtime/cases/systemd.py b/meta/lib/oeqa/runtime/cases/systemd.py index 7c44abe8ed..5481e1d840 100644 --- a/meta/lib/oeqa/runtime/cases/systemd.py +++ b/meta/lib/oeqa/runtime/cases/systemd.py @@ -1,8 +1,11 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # import re +import threading import time from oeqa.runtime.case import OERuntimeTestCase @@ -66,8 +69,8 @@ class SystemdBasicTests(SystemdTest): """ endtime = time.time() + (60 * 2) while True: - status, output = self.target.run('SYSTEMD_BUS_TIMEOUT=240s systemctl --state=activating') - if "0 loaded units listed" in output: + status, output = self.target.run('SYSTEMD_BUS_TIMEOUT=240s systemctl is-system-running') + if "running" in output or "degraded" in output: return (True, '') if time.time() >= endtime: return (False, output) @@ -134,6 +137,27 @@ class SystemdServiceTests(SystemdTest): status = self.target.run('mount -oro,remount /')[0] self.assertTrue(status == 0, msg='Remounting / as r/o failed') + @OETestDepends(['systemd.SystemdBasicTests.test_systemd_basic']) + @skipIfNotFeature('minidebuginfo', 'Test requires minidebuginfo to be in DISTRO_FEATURES') + @OEHasPackage(['busybox']) + def test_systemd_coredump_minidebuginfo(self): + """ + Verify that call-stacks generated by systemd-coredump contain symbolicated call-stacks, + extracted from the minidebuginfo metadata (.gnu_debugdata elf section). + """ + t_thread = threading.Thread(target=self.target.run, args=("ulimit -c unlimited && sleep 1000",)) + t_thread.start() + time.sleep(1) + + status, output = self.target.run('pidof sleep') + # cause segfault on purpose + self.target.run('kill -SEGV %s' % output) + self.assertEqual(status, 0, msg = 'Not able to find process that runs sleep, output : %s' % output) + + (status, output) = self.target.run('coredumpctl info') + self.assertEqual(status, 0, msg='MiniDebugInfo Test failed: %s' % output) + self.assertEqual('sleep_for_duration (busybox.nosuid' in output, True, msg='Call stack is missing minidebuginfo symbols (functions shown as "n/a"): %s' % output) + class SystemdJournalTests(SystemdTest): @OETestDepends(['systemd.SystemdBasicTests.test_systemd_basic']) @@ -152,7 +176,7 @@ class SystemdJournalTests(SystemdTest): """ # The expression chain that uniquely identifies the time boot message. - expr_items=['Startup finished', 'kernel', 'userspace','\.$'] + expr_items=['Startup finished', 'kernel', 'userspace', r'\.$'] try: output = self.journalctl(args='-o cat --reverse') except AssertionError: diff --git a/meta/lib/oeqa/runtime/cases/terminal.py b/meta/lib/oeqa/runtime/cases/terminal.py index 8fcca99f47..96ba3c3195 100644 --- a/meta/lib/oeqa/runtime/cases/terminal.py +++ b/meta/lib/oeqa/runtime/cases/terminal.py @@ -1,3 +1,8 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends from oeqa.runtime.decorator.package import OEHasPackage diff --git a/meta/lib/oeqa/runtime/cases/usb_hid.py b/meta/lib/oeqa/runtime/cases/usb_hid.py index 8743174370..6f23d2ff51 100644 --- a/meta/lib/oeqa/runtime/cases/usb_hid.py +++ b/meta/lib/oeqa/runtime/cases/usb_hid.py @@ -1,3 +1,8 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends from oeqa.core.decorator.data import skipIfQemu diff --git a/meta/lib/oeqa/runtime/cases/weston.py b/meta/lib/oeqa/runtime/cases/weston.py index 1fd471e611..ee4d336482 100644 --- a/meta/lib/oeqa/runtime/cases/weston.py +++ b/meta/lib/oeqa/runtime/cases/weston.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/runtime/cases/x32lib.py b/meta/lib/oeqa/runtime/cases/x32lib.py index f419c8f181..014da4b386 100644 --- a/meta/lib/oeqa/runtime/cases/x32lib.py +++ b/meta/lib/oeqa/runtime/cases/x32lib.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/runtime/cases/xorg.py b/meta/lib/oeqa/runtime/cases/xorg.py index d6845587c2..09afb1e3d1 100644 --- a/meta/lib/oeqa/runtime/cases/xorg.py +++ b/meta/lib/oeqa/runtime/cases/xorg.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/runtime/context.py b/meta/lib/oeqa/runtime/context.py index 8092dd0bae..cb7227a8df 100644 --- a/meta/lib/oeqa/runtime/context.py +++ b/meta/lib/oeqa/runtime/context.py @@ -10,7 +10,6 @@ import sys from oeqa.core.context import OETestContext, OETestContextExecutor from oeqa.core.target.ssh import OESSHTarget from oeqa.core.target.qemu import OEQemuTarget -from oeqa.utils.dump import HostDumper from oeqa.runtime.loader import OERuntimeTestLoader @@ -20,12 +19,11 @@ class OERuntimeTestContext(OETestContext): os.path.dirname(os.path.abspath(__file__)), "files") def __init__(self, td, logger, target, - host_dumper, image_packages, extract_dir): + image_packages, extract_dir): super(OERuntimeTestContext, self).__init__(td, logger) self.target = target self.image_packages = image_packages - self.host_dumper = host_dumper self.extract_dir = extract_dir self._set_target_cmds() @@ -67,11 +65,11 @@ class OERuntimeTestContextExecutor(OETestContextExecutor): % self.default_target_type) runtime_group.add_argument('--target-ip', action='store', default=self.default_target_ip, - help="IP address of device under test, default: %s" \ + help="IP address and optionally ssh port (default 22) of device under test, for example '192.168.0.7:22'. Default: %s" \ % self.default_target_ip) runtime_group.add_argument('--server-ip', action='store', default=self.default_target_ip, - help="IP address of device under test, default: %s" \ + help="IP address of the test host from test target machine, default: %s" \ % self.default_server_ip) runtime_group.add_argument('--host-dumper-dir', action='store', @@ -199,10 +197,6 @@ class OERuntimeTestContextExecutor(OETestContextExecutor): return image_packages - @staticmethod - def getHostDumper(cmds, directory): - return HostDumper(cmds, directory) - def _process_args(self, logger, args): if not args.packages_manifest: raise TypeError('Manifest file not provided') @@ -215,9 +209,6 @@ class OERuntimeTestContextExecutor(OETestContextExecutor): self.tc_kwargs['init']['target'] = \ OERuntimeTestContextExecutor.getTarget(args.target_type, None, args.target_ip, args.server_ip, **target_kwargs) - self.tc_kwargs['init']['host_dumper'] = \ - OERuntimeTestContextExecutor.getHostDumper(None, - args.host_dumper_dir) self.tc_kwargs['init']['image_packages'] = \ OERuntimeTestContextExecutor.readPackagesManifest( args.packages_manifest) diff --git a/meta/lib/oeqa/runtime/decorator/package.py b/meta/lib/oeqa/runtime/decorator/package.py index 8aba3f325b..b78ac9fc38 100644 --- a/meta/lib/oeqa/runtime/decorator/package.py +++ b/meta/lib/oeqa/runtime/decorator/package.py @@ -38,11 +38,12 @@ class OEHasPackage(OETestDecorator): if isinstance(self.need_pkgs, str): self.need_pkgs = [self.need_pkgs,] + mlprefix = self.case.td.get("MLPREFIX") for pkg in self.need_pkgs: if pkg.startswith('!'): - unneed_pkgs.add(pkg[1:]) + unneed_pkgs.add(mlprefix + pkg[1:]) else: - need_pkgs.add(pkg) + need_pkgs.add(mlprefix + pkg) if unneed_pkgs: msg = 'Checking if %s is not installed' % ', '.join(unneed_pkgs) diff --git a/meta/lib/oeqa/sdk/buildtools-cases/build.py b/meta/lib/oeqa/sdk/buildtools-cases/build.py index aee2e5a8c0..c85c32496b 100644 --- a/meta/lib/oeqa/sdk/buildtools-cases/build.py +++ b/meta/lib/oeqa/sdk/buildtools-cases/build.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/sdk/buildtools-cases/gcc.py b/meta/lib/oeqa/sdk/buildtools-cases/gcc.py index 36ba15b134..a62c4d0bc4 100644 --- a/meta/lib/oeqa/sdk/buildtools-cases/gcc.py +++ b/meta/lib/oeqa/sdk/buildtools-cases/gcc.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/sdk/buildtools-cases/https.py b/meta/lib/oeqa/sdk/buildtools-cases/https.py index 35e549eb40..4525e3d758 100644 --- a/meta/lib/oeqa/sdk/buildtools-cases/https.py +++ b/meta/lib/oeqa/sdk/buildtools-cases/https.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/sdk/buildtools-cases/sanity.py b/meta/lib/oeqa/sdk/buildtools-cases/sanity.py index 64baaa8f84..a55d456656 100644 --- a/meta/lib/oeqa/sdk/buildtools-cases/sanity.py +++ b/meta/lib/oeqa/sdk/buildtools-cases/sanity.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -19,4 +21,4 @@ class SanityTests(OESDKTestCase): # Canonicalise the location of this command tool_path = os.path.realpath(self._run("command -v %s" % command).strip()) # Assert that the tool was found inside the SDK root - self.assertEquals(os.path.commonprefix((sdk_base, tool_path)), sdk_base) + self.assertEqual(os.path.commonprefix((sdk_base, tool_path)), sdk_base) diff --git a/meta/lib/oeqa/sdk/buildtools-docs-cases/build.py b/meta/lib/oeqa/sdk/buildtools-docs-cases/build.py index 5b0eca046f..6e3ee94292 100644 --- a/meta/lib/oeqa/sdk/buildtools-docs-cases/build.py +++ b/meta/lib/oeqa/sdk/buildtools-docs-cases/build.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/sdk/cases/assimp.py b/meta/lib/oeqa/sdk/cases/assimp.py index f166758e49..e986838aea 100644 --- a/meta/lib/oeqa/sdk/cases/assimp.py +++ b/meta/lib/oeqa/sdk/cases/assimp.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -23,18 +25,21 @@ class BuildAssimp(OESDKTestCase): def test_assimp(self): with tempfile.TemporaryDirectory(prefix="assimp", dir=self.tc.sdk_dir) as testdir: - tarball = self.fetch(testdir, self.td["DL_DIR"], "https://github.com/assimp/assimp/archive/v4.1.0.tar.gz") + tarball = self.fetch(testdir, self.td["DL_DIR"], "https://github.com/assimp/assimp/archive/v5.3.1.tar.gz") dirs = {} - dirs["source"] = os.path.join(testdir, "assimp-4.1.0") + dirs["source"] = os.path.join(testdir, "assimp-5.3.1") dirs["build"] = os.path.join(testdir, "build") dirs["install"] = os.path.join(testdir, "install") subprocess.check_output(["tar", "xf", tarball, "-C", testdir], stderr=subprocess.STDOUT) self.assertTrue(os.path.isdir(dirs["source"])) + # Apply the zlib patch https://github.com/madler/zlib/commit/a566e156b3fa07b566ddbf6801b517a9dba04fa3 + # this sed wont be needed once assimp moves its zlib copy to v1.3.1+ + self._run("sed -i '/# ifdef _FILE_OFFSET_BITS/I,+2 d' {source}/contrib/zlib/gzguts.h".format(**dirs)) os.makedirs(dirs["build"]) - self._run("cd {build} && cmake -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON {source}".format(**dirs)) + self._run("cd {build} && cmake -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -DASSIMP_BUILD_ZLIB=ON {source}".format(**dirs)) self._run("cmake --build {build} -- -j".format(**dirs)) self._run("cmake --build {build} --target install -- DESTDIR={install}".format(**dirs)) - self.check_elf(os.path.join(dirs["install"], "usr", "local", "lib", "libassimp.so.4.1.0")) + self.check_elf(os.path.join(dirs["install"], "usr", "local", "lib", "libassimp.so.5.3.0")) diff --git a/meta/lib/oeqa/sdk/cases/buildcpio.py b/meta/lib/oeqa/sdk/cases/buildcpio.py index e7fc211a47..51003b19cd 100644 --- a/meta/lib/oeqa/sdk/cases/buildcpio.py +++ b/meta/lib/oeqa/sdk/cases/buildcpio.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -17,10 +19,10 @@ class BuildCpioTest(OESDKTestCase): """ def test_cpio(self): with tempfile.TemporaryDirectory(prefix="cpio-", dir=self.tc.sdk_dir) as testdir: - tarball = self.fetch(testdir, self.td["DL_DIR"], "https://ftp.gnu.org/gnu/cpio/cpio-2.13.tar.gz") + tarball = self.fetch(testdir, self.td["DL_DIR"], "https://ftp.gnu.org/gnu/cpio/cpio-2.15.tar.gz") dirs = {} - dirs["source"] = os.path.join(testdir, "cpio-2.13") + dirs["source"] = os.path.join(testdir, "cpio-2.15") dirs["build"] = os.path.join(testdir, "build") dirs["install"] = os.path.join(testdir, "install") @@ -28,8 +30,7 @@ class BuildCpioTest(OESDKTestCase): self.assertTrue(os.path.isdir(dirs["source"])) os.makedirs(dirs["build"]) - self._run("sed -i -e '/char.*program_name/d' {source}/src/global.c".format(**dirs)) - self._run("cd {build} && {source}/configure --disable-maintainer-mode $CONFIGURE_FLAGS".format(**dirs)) + self._run("cd {build} && {source}/configure $CONFIGURE_FLAGS".format(**dirs)) self._run("cd {build} && make -j".format(**dirs)) self._run("cd {build} && make install DESTDIR={install}".format(**dirs)) diff --git a/meta/lib/oeqa/sdk/cases/buildepoxy.py b/meta/lib/oeqa/sdk/cases/buildepoxy.py index ad08b777f2..147ee3e0ee 100644 --- a/meta/lib/oeqa/sdk/cases/buildepoxy.py +++ b/meta/lib/oeqa/sdk/cases/buildepoxy.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -33,7 +35,7 @@ class EpoxyTest(OESDKTestCase): self.assertTrue(os.path.isdir(dirs["source"])) os.makedirs(dirs["build"]) - log = self._run("meson -Degl=no -Dglx=no -Dx11=false {build} {source}".format(**dirs)) + log = self._run("meson --warnlevel 1 -Degl=no -Dglx=no -Dx11=false {build} {source}".format(**dirs)) # Check that Meson thinks we're doing a cross build and not a native self.assertIn("Build type: cross build", log) self._run("ninja -C {build} -v".format(**dirs)) diff --git a/meta/lib/oeqa/sdk/cases/buildgalculator.py b/meta/lib/oeqa/sdk/cases/buildgalculator.py index 58ade920c9..178f07472d 100644 --- a/meta/lib/oeqa/sdk/cases/buildgalculator.py +++ b/meta/lib/oeqa/sdk/cases/buildgalculator.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/sdk/cases/buildlzip.py b/meta/lib/oeqa/sdk/cases/buildlzip.py index 49ae756bf3..b4b7d85b88 100644 --- a/meta/lib/oeqa/sdk/cases/buildlzip.py +++ b/meta/lib/oeqa/sdk/cases/buildlzip.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/sdk/cases/gcc.py b/meta/lib/oeqa/sdk/cases/gcc.py index eb08eadd28..fc28b9c3d4 100644 --- a/meta/lib/oeqa/sdk/cases/gcc.py +++ b/meta/lib/oeqa/sdk/cases/gcc.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/sdk/cases/maturin.py b/meta/lib/oeqa/sdk/cases/maturin.py new file mode 100644 index 0000000000..ea10f568b2 --- /dev/null +++ b/meta/lib/oeqa/sdk/cases/maturin.py @@ -0,0 +1,79 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +import os +import shutil +import unittest + +from oeqa.core.utils.path import remove_safe +from oeqa.sdk.case import OESDKTestCase +from oeqa.utils.subprocesstweak import errors_have_output + +errors_have_output() + + +class MaturinTest(OESDKTestCase): + def setUp(self): + if not ( + self.tc.hasHostPackage("nativesdk-python3-maturin") + or self.tc.hasHostPackage("python3-maturin-native") + ): + raise unittest.SkipTest("No python3-maturin package in the SDK") + + def test_maturin_list_python(self): + py_major = self._run("python3 -c 'import sys; print(sys.version_info.major)'") + py_minor = self._run("python3 -c 'import sys; print(sys.version_info.minor)'") + python_version = "%s.%s" % (py_major.strip(), py_minor.strip()) + cmd = "maturin list-python" + output = self._run(cmd) + self.assertRegex(output, r"^🐍 1 python interpreter found:\n") + self.assertRegex( + output, + r" - CPython %s (.+)/usr/bin/python%s$" % (python_version, python_version), + ) + + +class MaturinDevelopTest(OESDKTestCase): + @classmethod + def setUpClass(self): + targetdir = os.path.join(self.tc.sdk_dir, "guessing-game") + try: + shutil.rmtree(targetdir) + except FileNotFoundError: + pass + shutil.copytree( + os.path.join(self.tc.files_dir, "maturin/guessing-game"), targetdir + ) + + def setUp(self): + machine = self.td.get("MACHINE") + if not ( + self.tc.hasHostPackage("nativesdk-python3-maturin") + or self.tc.hasHostPackage("python3-maturin-native") + ): + raise unittest.SkipTest("No python3-maturin package in the SDK") + if not ( + self.tc.hasHostPackage("packagegroup-rust-cross-canadian-%s" % machine) + ): + raise unittest.SkipTest( + "Testing 'maturin develop' requires Rust cross-canadian in the SDK" + ) + + def test_maturin_develop(self): + """ + This test case requires: + (1) that a .venv can been created. + (2) a functional 'rustc' and 'cargo' + """ + self._run("cd %s/guessing-game; python3 -m venv .venv" % self.tc.sdk_dir) + cmd = "cd %s/guessing-game; maturin develop" % self.tc.sdk_dir + output = self._run(cmd) + self.assertRegex(output, r"🔗 Found pyo3 bindings with abi3 support for Python ≥ 3.8") + self.assertRegex(output, r"🐍 Not using a specific python interpreter") + self.assertRegex(output, r"📡 Using build options features from pyproject.toml") + self.assertRegex(output, r"Compiling guessing-game v0.1.0") + self.assertRegex(output, r"📦 Built wheel for abi3 Python ≥ 3.8") + self.assertRegex(output, r"🛠 Installed guessing-game-0.1.0") diff --git a/meta/lib/oeqa/sdk/cases/perl.py b/meta/lib/oeqa/sdk/cases/perl.py index 14d76d820f..8eab4442e8 100644 --- a/meta/lib/oeqa/sdk/cases/perl.py +++ b/meta/lib/oeqa/sdk/cases/perl.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/sdk/cases/python.py b/meta/lib/oeqa/sdk/cases/python.py index d43354c32a..5ea992b9f3 100644 --- a/meta/lib/oeqa/sdk/cases/python.py +++ b/meta/lib/oeqa/sdk/cases/python.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/sdk/cases/rust.py b/meta/lib/oeqa/sdk/cases/rust.py index 1075d37b49..f5d437bb19 100644 --- a/meta/lib/oeqa/sdk/cases/rust.py +++ b/meta/lib/oeqa/sdk/cases/rust.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -31,3 +33,25 @@ class RustCompileTest(OESDKTestCase): def test_cargo_build(self): self._run('cd %s/hello; cargo build' % self.tc.sdk_dir) + +class RustHostCompileTest(OESDKTestCase): + td_vars = ['MACHINE', 'SDK_SYS'] + + @classmethod + def setUpClass(self): + targetdir = os.path.join(self.tc.sdk_dir, "hello") + try: + shutil.rmtree(targetdir) + except FileNotFoundError: + pass + shutil.copytree(os.path.join(self.tc.sdk_files_dir, "rust/hello"), targetdir) + + def setUp(self): + machine = self.td.get("MACHINE") + if not self.tc.hasHostPackage("packagegroup-rust-cross-canadian-%s" % machine): + raise unittest.SkipTest("RustCompileTest class: SDK doesn't contain a Rust cross-canadian toolchain") + + def test_cargo_build(self): + sdksys = self.td.get("SDK_SYS") + self._run('cd %s/hello; cargo build --target %s-gnu' % (self.tc.sdk_dir, sdksys)) + self._run('cd %s/hello; cargo run --target %s-gnu' % (self.tc.sdk_dir, sdksys)) diff --git a/meta/lib/oeqa/sdk/files/rust/hello/build.rs b/meta/lib/oeqa/sdk/files/rust/hello/build.rs new file mode 100644 index 0000000000..b1a533d5df --- /dev/null +++ b/meta/lib/oeqa/sdk/files/rust/hello/build.rs @@ -0,0 +1,3 @@ +/* This is the simplest build script just to invoke host compiler + in the build process. */ +fn main() {} diff --git a/meta/lib/oeqa/sdk/testmetaidesupport.py b/meta/lib/oeqa/sdk/testmetaidesupport.py index 2ff76fd8e0..00ef30e82e 100644 --- a/meta/lib/oeqa/sdk/testmetaidesupport.py +++ b/meta/lib/oeqa/sdk/testmetaidesupport.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/sdk/testsdk.py b/meta/lib/oeqa/sdk/testsdk.py index 35e40187bc..518b09febb 100644 --- a/meta/lib/oeqa/sdk/testsdk.py +++ b/meta/lib/oeqa/sdk/testsdk.py @@ -23,14 +23,6 @@ class TestSDKBase(object): return configuration @staticmethod - def get_sdk_json_result_dir(d): - json_result_dir = os.path.join(d.getVar("LOG_DIR"), 'oeqa') - custom_json_result_dir = d.getVar("OEQA_JSON_RESULT_DIR") - if custom_json_result_dir: - json_result_dir = custom_json_result_dir - return json_result_dir - - @staticmethod def get_sdk_result_id(configuration): return '%s_%s_%s_%s_%s' % (configuration['TEST_TYPE'], configuration['IMAGE_BASENAME'], configuration['SDKMACHINE'], configuration['MACHINE'], configuration['STARTTIME']) @@ -72,6 +64,7 @@ class TestSDK(TestSDKBase): from bb.utils import export_proxies from oeqa.utils import make_logger_bitbake_compatible + from oeqa.utils import get_json_result_dir pn = d.getVar("PN") logger = make_logger_bitbake_compatible(logging.getLogger("BitBake")) @@ -79,6 +72,9 @@ class TestSDK(TestSDKBase): # sdk use network for download projects for build export_proxies(d) + # We need the original PATH for testing the eSDK, not with our manipulations + os.environ['PATH'] = d.getVar("BB_ORIGENV", False).getVar("PATH") + tcname = self.get_tcname(d) if not os.path.exists(tcname): @@ -131,7 +127,7 @@ class TestSDK(TestSDKBase): component = "%s %s" % (pn, self.context_executor_class.name) context_msg = "%s:%s" % (os.path.basename(tcname), os.path.basename(sdk_env)) configuration = self.get_sdk_configuration(d, self.test_type) - result.logDetails(self.get_sdk_json_result_dir(d), + result.logDetails(get_json_result_dir(d), configuration, self.get_sdk_result_id(configuration)) result.logSummary(component, context_msg) diff --git a/meta/lib/oeqa/sdkext/cases/devtool.py b/meta/lib/oeqa/sdkext/cases/devtool.py index a5c6a76e02..5ffb732556 100644 --- a/meta/lib/oeqa/sdkext/cases/devtool.py +++ b/meta/lib/oeqa/sdkext/cases/devtool.py @@ -112,7 +112,7 @@ class SdkUpdateTest(OESDKExtTestCase): cmd = 'oe-publish-sdk %s %s' % (tcname_new, self.publish_dir) subprocess.check_output(cmd, shell=True) - self.http_service = HTTPService(self.publish_dir) + self.http_service = HTTPService(self.publish_dir, logger=self.logger) self.http_service.start() self.http_url = "http://127.0.0.1:%d" % self.http_service.port diff --git a/meta/lib/oeqa/sdkext/testsdk.py b/meta/lib/oeqa/sdkext/testsdk.py index 159f0d135b..9d5a99d900 100644 --- a/meta/lib/oeqa/sdkext/testsdk.py +++ b/meta/lib/oeqa/sdkext/testsdk.py @@ -16,6 +16,7 @@ class TestSDKExt(TestSDKBase): from bb.utils import export_proxies from oeqa.utils import avoid_paths_in_environ, make_logger_bitbake_compatible, subprocesstweak from oeqa.sdkext.context import OESDKExtTestContext, OESDKExtTestContextExecutor + from oeqa.utils import get_json_result_dir pn = d.getVar("PN") logger = make_logger_bitbake_compatible(logging.getLogger("BitBake")) @@ -91,7 +92,7 @@ class TestSDKExt(TestSDKBase): component = "%s %s" % (pn, OESDKExtTestContextExecutor.name) context_msg = "%s:%s" % (os.path.basename(tcname), os.path.basename(sdk_env)) configuration = self.get_sdk_configuration(d, 'sdkext') - result.logDetails(self.get_sdk_json_result_dir(d), + result.logDetails(get_json_result_dir(d), configuration, self.get_sdk_result_id(configuration)) result.logSummary(component, context_msg) diff --git a/meta/lib/oeqa/selftest/case.py b/meta/lib/oeqa/selftest/case.py index dcad4f76ec..da35b25f68 100644 --- a/meta/lib/oeqa/selftest/case.py +++ b/meta/lib/oeqa/selftest/case.py @@ -117,10 +117,6 @@ class OESelftestTestCase(OETestCase): if e.errno != errno.ENOENT: raise - if self.tc.custommachine: - machine_conf = 'MACHINE ??= "%s"\n' % self.tc.custommachine - self.set_machine_config(machine_conf) - # tests might need their own setup # but if they overwrite this one they have to call # super each time, so let's give them an alternative @@ -178,19 +174,11 @@ class OESelftestTestCase(OETestCase): self.logger.debug("Writing to: %s\n%s\n" % (dest_path, data)) ftools.write_file(dest_path, data) - if not multiconfig and self.tc.custommachine and 'MACHINE' in data: - machine = get_bb_var('MACHINE') - self.logger.warning('MACHINE overridden: %s' % machine) - def append_config(self, data): """Append to <builddir>/conf/selftest.inc""" self.logger.debug("Appending to: %s\n%s\n" % (self.testinc_path, data)) ftools.append_file(self.testinc_path, data) - if self.tc.custommachine and 'MACHINE' in data: - machine = get_bb_var('MACHINE') - self.logger.warning('MACHINE overridden: %s' % machine) - def remove_config(self, data): """Remove data from <builddir>/conf/selftest.inc""" self.logger.debug("Removing from: %s\n%s\n" % (self.testinc_path, data)) @@ -249,6 +237,13 @@ class OESelftestTestCase(OETestCase): self.logger.debug("Writing to: %s\n%s\n" % (self.machineinc_path, data)) ftools.write_file(self.machineinc_path, data) + def disable_class(self, classname): + destfile = "%s/classes/%s.bbclass" % (self.builddir, classname) + os.makedirs(os.path.dirname(destfile), exist_ok=True) + self.track_for_cleanup(destfile) + self.logger.debug("Creating empty class: %s\n" % (destfile)) + ftools.write_file(destfile, "") + # check does path exist def assertExists(self, expr, msg=None): if not os.path.exists(expr): diff --git a/meta/lib/oeqa/selftest/cases/_sstatetests_noauto.py b/meta/lib/oeqa/selftest/cases/_sstatetests_noauto.py index bff6e7740c..2c9584d329 100644 --- a/meta/lib/oeqa/selftest/cases/_sstatetests_noauto.py +++ b/meta/lib/oeqa/selftest/cases/_sstatetests_noauto.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -7,7 +9,7 @@ import shutil import oeqa.utils.ftools as ftools from oeqa.utils.commands import runCmd, bitbake, get_bb_var -from oeqa.selftest.cases.sstate import SStateBase +from oeqa.selftest.cases.sstatetests import SStateBase class RebuildFromSState(SStateBase): diff --git a/meta/lib/oeqa/selftest/cases/archiver.py b/meta/lib/oeqa/selftest/cases/archiver.py index 75195241b7..3cb888c506 100644 --- a/meta/lib/oeqa/selftest/cases/archiver.py +++ b/meta/lib/oeqa/selftest/cases/archiver.py @@ -1,9 +1,12 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # import os import glob +import re from oeqa.utils.commands import bitbake, get_bb_vars from oeqa.selftest.case import OESelftestTestCase @@ -117,7 +120,38 @@ class Archiver(OESelftestTestCase): excluded_present = len(glob.glob(src_path_target + '/%s-*/*' % target_recipes[1])) self.assertFalse(excluded_present, 'Recipe %s was not excluded.' % target_recipes[1]) + def test_archiver_multiconfig_shared_unpack_and_patch(self): + """ + Test that shared recipes in original mode with diff enabled works in multiconfig, + otherwise it will not build when using the same TMP dir. + """ + + features = 'BBMULTICONFIG = "mc1 mc2"\n' + features += 'INHERIT += "archiver"\n' + features += 'ARCHIVER_MODE[src] = "original"\n' + features += 'ARCHIVER_MODE[diff] = "1"\n' + self.write_config(features) + # We can use any machine in multiconfig as long as they are different + self.write_config('MACHINE = "qemuarm"\n', 'mc1') + self.write_config('MACHINE = "qemux86"\n', 'mc2') + + task = 'do_unpack_and_patch' + # Use gcc-source as it is a shared recipe (appends the pv to the pn) + pn = 'gcc-source-%s' % get_bb_vars(['PV'], 'gcc')['PV'] + + # Generate the tasks signatures + bitbake('mc:mc1:%s mc:mc2:%s -c %s -S lockedsigs' % (pn, pn, task)) + + # Check the tasks signatures + # To be machine agnostic the tasks needs to generate the same signature for each machine + locked_sigs_inc = "%s/locked-sigs.inc" % self.builddir + locked_sigs = open(locked_sigs_inc).read() + task_sigs = re.findall(r"%s:%s:.*" % (pn, task), locked_sigs) + uniq_sigs = set(task_sigs) + self.assertFalse(len(uniq_sigs) - 1, \ + 'The task "%s" of the recipe "%s" has different signatures in "%s" for each machine in multiconfig' \ + % (task, pn, locked_sigs_inc)) def test_archiver_srpm_mode(self): """ diff --git a/meta/lib/oeqa/selftest/cases/baremetal.py b/meta/lib/oeqa/selftest/cases/baremetal.py new file mode 100644 index 0000000000..cadaea2f1a --- /dev/null +++ b/meta/lib/oeqa/selftest/cases/baremetal.py @@ -0,0 +1,14 @@ + +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +from oeqa.selftest.case import OESelftestTestCase +from oeqa.utils.commands import bitbake + +class BaremetalTest(OESelftestTestCase): + def test_baremetal(self): + self.write_config('TCLIBC = "baremetal"') + bitbake('baremetal-helloworld') diff --git a/meta/lib/oeqa/selftest/cases/bblayers.py b/meta/lib/oeqa/selftest/cases/bblayers.py index 7d74833f61..695d17377d 100644 --- a/meta/lib/oeqa/selftest/cases/bblayers.py +++ b/meta/lib/oeqa/selftest/cases/bblayers.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -6,12 +8,18 @@ import os import re import oeqa.utils.ftools as ftools -from oeqa.utils.commands import runCmd, get_bb_var, get_bb_vars +from oeqa.utils.commands import runCmd, get_bb_var, get_bb_vars, bitbake from oeqa.selftest.case import OESelftestTestCase class BitbakeLayers(OESelftestTestCase): + @classmethod + def setUpClass(cls): + super(BitbakeLayers, cls).setUpClass() + bitbake("python3-jsonschema-native") + bitbake("-c addto_recipe_sysroot python3-jsonschema-native") + def test_bitbakelayers_layerindexshowdepends(self): result = runCmd('bitbake-layers layerindex-show-depends meta-poky') find_in_contents = re.search("openembedded-core", result.output) @@ -46,7 +54,7 @@ class BitbakeLayers(OESelftestTestCase): bb_file = os.path.join(testoutdir, recipe_path, recipe_file) self.assertTrue(os.path.isfile(bb_file), msg = "Cannot find xcursor-transparent-theme_0.1.1.bb in the test_bitbakelayers_flatten local dir.") contents = ftools.read_file(bb_file) - find_in_contents = re.search("##### bbappended from meta-selftest #####\n(.*\n)*include test_recipe.inc", contents) + find_in_contents = re.search(r"##### bbappended from meta-selftest #####\n(.*\n)*include test_recipe.inc", contents) self.assertTrue(find_in_contents, msg = "Flattening layers did not work. bitbake-layers flatten output: %s" % result.output) def test_bitbakelayers_add_remove(self): @@ -77,8 +85,9 @@ class BitbakeLayers(OESelftestTestCase): result = runCmd('bitbake-layers show-recipes -i image') self.assertIn('core-image-minimal', result.output) self.assertNotIn('mtd-utils:', result.output) - result = runCmd('bitbake-layers show-recipes -i cmake,pkgconfig') + result = runCmd('bitbake-layers show-recipes -i meson,pkgconfig') self.assertIn('libproxy:', result.output) + result = runCmd('bitbake-layers show-recipes -i cmake,pkgconfig') self.assertNotIn('mtd-utils:', result.output) # doesn't inherit either self.assertNotIn('wget:', result.output) # doesn't inherit cmake self.assertNotIn('waffle:', result.output) # doesn't inherit pkgconfig @@ -111,6 +120,11 @@ class BitbakeLayers(OESelftestTestCase): self.assertEqual(bb_vars['BBFILE_PRIORITY_%s' % layername], str(priority), 'BBFILE_PRIORITY_%s != %d' % (layername, priority)) + result = runCmd('bitbake-layers save-build-conf {} {}'.format(layerpath, "buildconf-1")) + for f in ('local.conf.sample', 'bblayers.conf.sample', 'conf-summary.txt', 'conf-notes.txt'): + fullpath = os.path.join(layerpath, "conf", "templates", "buildconf-1", f) + self.assertTrue(os.path.exists(fullpath), "Template configuration file {} not found".format(fullpath)) + def get_recipe_basename(self, recipe): recipe_file = "" result = runCmd("bitbake-layers show-recipes -f %s" % recipe) @@ -121,3 +135,108 @@ class BitbakeLayers(OESelftestTestCase): self.assertTrue(os.path.isfile(recipe_file), msg = "Can't find recipe file for %s" % recipe) return os.path.basename(recipe_file) + + def validate_layersjson(self, json): + python = os.path.join(get_bb_var('STAGING_BINDIR', 'python3-jsonschema-native'), 'nativepython3') + jsonvalidator = os.path.join(get_bb_var('STAGING_BINDIR', 'python3-jsonschema-native'), 'jsonschema') + jsonschema = os.path.join(get_bb_var('COREBASE'), 'meta/files/layers.schema.json') + result = runCmd("{} {} -i {} {}".format(python, jsonvalidator, json, jsonschema)) + + def test_validate_examplelayersjson(self): + json = os.path.join(get_bb_var('COREBASE'), "meta/files/layers.example.json") + self.validate_layersjson(json) + + def test_bitbakelayers_setup(self): + result = runCmd('bitbake-layers create-layers-setup {}'.format(self.testlayer_path)) + jsonfile = os.path.join(self.testlayer_path, "setup-layers.json") + self.validate_layersjson(jsonfile) + + # The revision-under-test may not necessarily be available on the remote server, + # so replace it with a revision that has a yocto-4.1 tag. + import json + with open(jsonfile) as f: + data = json.load(f) + for s in data['sources']: + data['sources'][s]['git-remote']['rev'] = '5200799866b92259e855051112520006e1aaaac0' + with open(jsonfile, 'w') as f: + json.dump(data, f) + + testcheckoutdir = os.path.join(self.builddir, 'test-layer-checkout') + result = runCmd('{}/setup-layers --destdir {}'.format(self.testlayer_path, testcheckoutdir)) + layers_json = os.path.join(testcheckoutdir, ".oe-layers.json") + self.assertTrue(os.path.exists(layers_json), "File {} not found in test layer checkout".format(layers_json)) + + # As setup-layers checkout out an old revision of poky, there is no setup-build symlink, + # and we need to run oe-setup-build directly from the current poky tree under test + oe_setup_build = os.path.join(get_bb_var('COREBASE'), 'scripts/oe-setup-build') + oe_setup_build_l = os.path.join(testcheckoutdir, 'setup-build') + os.symlink(oe_setup_build,oe_setup_build_l) + + cmd = '{} --layerlist {} list -v'.format(oe_setup_build_l, layers_json) + result = runCmd(cmd) + cond = "conf/templates/default" in result.output + self.assertTrue(cond, "Incorrect output from {}: {}".format(cmd, result.output)) + + # rather than hardcode the build setup cmdline here, let's actually run what the tool suggests to the user + conf = None + if 'poky-default' in result.output: + conf = 'poky-default' + elif 'meta-default' in result.output: + conf = 'meta-default' + self.assertIsNotNone(conf, "Could not find the configuration to set up a build in the output: {}".format(result.output)) + + cmd = '{} --layerlist {} setup -c {} --no-shell'.format(oe_setup_build_l, layers_json, conf) + result = runCmd(cmd) + + def test_bitbakelayers_updatelayer(self): + result = runCmd('bitbake-layers create-layers-setup {}'.format(self.testlayer_path)) + jsonfile = os.path.join(self.testlayer_path, "setup-layers.json") + self.validate_layersjson(jsonfile) + + import json + with open(jsonfile) as f: + data = json.load(f) + repos = [] + for s in data['sources']: + repos.append(s) + + self.assertTrue(len(repos) > 1, "Not enough repositories available") + self.validate_layersjson(jsonfile) + + test_ref_1 = 'ref_1' + test_ref_2 = 'ref_2' + + # Create a new layers setup using custom references + result = runCmd('bitbake-layers create-layers-setup --use-custom-reference {first_repo}:{test_ref} --use-custom-reference {second_repo}:{test_ref} {path}' + .format(first_repo=repos[0], second_repo=repos[1], test_ref=test_ref_1, path=self.testlayer_path)) + self.validate_layersjson(jsonfile) + + with open(jsonfile) as f: + data = json.load(f) + first_rev_1 = data['sources'][repos[0]]['git-remote']['rev'] + first_desc_1 = data['sources'][repos[0]]['git-remote']['describe'] + second_rev_1 = data['sources'][repos[1]]['git-remote']['rev'] + second_desc_1 = data['sources'][repos[1]]['git-remote']['describe'] + + self.assertEqual(first_rev_1, test_ref_1, "Revision not set correctly: '{}'".format(first_rev_1)) + self.assertEqual(first_desc_1, '', "Describe not cleared: '{}'".format(first_desc_1)) + self.assertEqual(second_rev_1, test_ref_1, "Revision not set correctly: '{}'".format(second_rev_1)) + self.assertEqual(second_desc_1, '', "Describe not cleared: '{}'".format(second_desc_1)) + + # Update one of the repositories in the layers setup using a different custom reference + # This should only update the selected repository, everything else should remain as is + result = runCmd('bitbake-layers create-layers-setup --update --use-custom-reference {first_repo}:{test_ref} {path}' + .format(first_repo=repos[0], test_ref=test_ref_2, path=self.testlayer_path)) + self.validate_layersjson(jsonfile) + + with open(jsonfile) as f: + data = json.load(f) + first_rev_2 = data['sources'][repos[0]]['git-remote']['rev'] + first_desc_2 = data['sources'][repos[0]]['git-remote']['describe'] + second_rev_2 = data['sources'][repos[1]]['git-remote']['rev'] + second_desc_2 = data['sources'][repos[1]]['git-remote']['describe'] + + self.assertEqual(first_rev_2, test_ref_2, "Revision not set correctly: '{}'".format(first_rev_2)) + self.assertEqual(first_desc_2, '', "Describe not cleared: '{}'".format(first_desc_2)) + self.assertEqual(second_rev_2, second_rev_1, "Revision should not be updated: '{}'".format(second_rev_2)) + self.assertEqual(second_desc_2, second_desc_1, "Describe should not be updated: '{}'".format(second_desc_2)) diff --git a/meta/lib/oeqa/selftest/cases/bblock.py b/meta/lib/oeqa/selftest/cases/bblock.py new file mode 100644 index 0000000000..2b62d2a0aa --- /dev/null +++ b/meta/lib/oeqa/selftest/cases/bblock.py @@ -0,0 +1,203 @@ +# +# Copyright (c) 2023 BayLibre, SAS +# Author: Julien Stepahn <jstephan@baylibre.com> +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import os +import re +import bb.tinfoil + +import oeqa.utils.ftools as ftools +from oeqa.utils.commands import runCmd, get_bb_var, get_bb_vars, bitbake + +from oeqa.selftest.case import OESelftestTestCase + + +class BBLock(OESelftestTestCase): + @classmethod + def setUpClass(cls): + super(BBLock, cls).setUpClass() + cls.lockfile = cls.builddir + "/conf/bblock.conf" + + def unlock_recipes(self, recipes=None, tasks=None): + cmd = "bblock -r " + if recipes: + cmd += " ".join(recipes) + if tasks: + cmd += " -t " + ",".join(tasks) + result = runCmd(cmd) + + if recipes: + # ensure all signatures are removed from lockfile + contents = ftools.read_file(self.lockfile) + for recipe in recipes: + for task in tasks: + find_in_contents = re.search( + 'SIGGEN_LOCKEDSIGS_.+\s\+=\s"%s:%s:.*"' % (recipe, task), + contents, + ) + self.assertFalse( + find_in_contents, + msg="%s:%s should not be present into bblock.conf anymore" + % (recipe, task), + ) + self.assertExists(self.lockfile) + else: + self.assertNotExists(self.lockfile) + + def lock_recipes(self, recipes, tasks=None): + cmd = "bblock " + " ".join(recipes) + if tasks: + cmd += " -t " + ",".join(tasks) + + result = runCmd(cmd) + + self.assertExists(self.lockfile) + + # ensure all signatures are added to lockfile + contents = ftools.read_file(self.lockfile) + for recipe in recipes: + if tasks: + for task in tasks: + find_in_contents = re.search( + 'SIGGEN_LOCKEDSIGS_.+\s\+=\s"%s:%s:.*"' % (recipe, task), + contents, + ) + self.assertTrue( + find_in_contents, + msg="%s:%s was not added into bblock.conf. bblock output: %s" + % (recipe, task, result.output), + ) + + def modify_tasks(self, recipes, tasks): + task_append = "" + for recipe in recipes: + bb_vars = get_bb_vars(["PV"], recipe) + recipe_pv = bb_vars["PV"] + recipe_append_file = recipe + "_" + recipe_pv + ".bbappend" + + os.mkdir(os.path.join(self.testlayer_path, "recipes-test", recipe)) + recipe_append_path = os.path.join( + self.testlayer_path, "recipes-test", recipe, recipe_append_file + ) + + for task in tasks: + task_append += "%s:append() {\n#modify task hash \n}\n" % task + ftools.write_file(recipe_append_path, task_append) + self.add_command_to_tearDown( + "rm -rf %s" % os.path.join(self.testlayer_path, "recipes-test", recipe) + ) + + def test_lock_single_recipe_single_task(self): + recipes = ["quilt"] + tasks = ["do_compile"] + self._run_test(recipes, tasks) + + def test_lock_single_recipe_multiple_tasks(self): + recipes = ["quilt"] + tasks = ["do_compile", "do_install"] + self._run_test(recipes, tasks) + + def test_lock_single_recipe_all_tasks(self): + recipes = ["quilt"] + self._run_test(recipes, None) + + def test_lock_multiple_recipe_single_task(self): + recipes = ["quilt", "bc"] + tasks = ["do_compile"] + self._run_test(recipes, tasks) + + def test_lock_architecture_specific(self): + # unlock all recipes and ensure no bblock.conf file exist + self.unlock_recipes() + + recipes = ["quilt"] + tasks = ["do_compile"] + + # lock quilt's do_compile task for another machine + if self.td["MACHINE"] == "qemux86-64": + machine = "qemuarm" + else: + machine = "qemux86-64" + + self.write_config('MACHINE = "%s"\n' % machine) + + self.lock_recipes(recipes, tasks) + + self.write_config('MACHINE = "%s"\n' % self.td["MACHINE"]) + # modify quilt's do_compile task + self.modify_tasks(recipes, tasks) + + # build quilt using the default machine + # No Note/Warning should be emitted since sig is locked for another machine + # (quilt package is architecture dependant) + info_message = "NOTE: The following recipes have locked tasks: " + recipes[0] + warn_message = "The %s:%s sig is computed to be" % (recipes[0], tasks[0]) + result = bitbake(recipes[0] + " -n") + self.assertNotIn(info_message, result.output) + self.assertNotIn(warn_message, result.output) + + # unlock all recipes + self.unlock_recipes() + + def _run_test(self, recipes, tasks=None): + # unlock all recipes and ensure no bblock.conf file exist + self.unlock_recipes() + + self.write_config('BB_SIGNATURE_HANDLER = "OEBasicHash"') + + # lock tasks for recipes + result = self.lock_recipes(recipes, tasks) + + if not tasks: + tasks = [] + result = bitbake("-c listtasks " + recipes[0]) + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(config_only=False, quiet=2) + d = tinfoil.parse_recipe(recipes[0]) + + for line in result.output.splitlines(): + if line.startswith("do_"): + task = line.split()[0] + if "setscene" in task: + continue + if d.getVarFlag(task, "nostamp"): + continue + tasks.append(task) + + # build recipes. At this stage we should have a Note about recipes + # having locked task's sig, but no warning since sig still match + info_message = "NOTE: The following recipes have locked tasks: " + " ".join( + recipes + ) + for recipe in recipes: + result = bitbake(recipe + " -n") + self.assertIn(info_message, result.output) + for task in tasks: + warn_message = "The %s:%s sig is computed to be" % (recipe, task) + self.assertNotIn(warn_message, result.output) + + # modify all tasks that are locked to trigger a sig change then build the recipes + # at this stage we should have a Note as before, but also a Warning for all + # locked tasks indicating the sig mismatch + self.modify_tasks(recipes, tasks) + for recipe in recipes: + result = bitbake(recipe + " -n") + self.assertIn(info_message, result.output) + for task in tasks: + warn_message = "The %s:%s sig is computed to be" % (recipe, task) + self.assertIn(warn_message, result.output) + + # unlock all tasks and rebuild, no more Note/Warning should remain + self.unlock_recipes(recipes, tasks) + for recipe in recipes: + result = bitbake(recipe + " -n") + self.assertNotIn(info_message, result.output) + for task in tasks: + warn_message = "The %s:%s sig is computed to be" % (recipe, task) + self.assertNotIn(warn_message, result.output) + + # unlock all recipes + self.unlock_recipes() diff --git a/meta/lib/oeqa/selftest/cases/bblogging.py b/meta/lib/oeqa/selftest/cases/bblogging.py index 317e68b82f..040c6db089 100644 --- a/meta/lib/oeqa/selftest/cases/bblogging.py +++ b/meta/lib/oeqa/selftest/cases/bblogging.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -103,16 +105,14 @@ class BitBakeLogging(OESelftestTestCase): self.write_config('BBINCLUDELOGS = ""') result = bitbake("logging-test -c pythontest_exit -f -v", ignore_status = True) self.assertIn("ERROR: Logfile of failure stored in:", result.output) - # python tasks don't log output with -v currently - #self.assertCount(result.output, "This is python stdout", 1) + self.assertCount(result.output, "This is python stdout", 1) def test_python_exit_loggingD(self): # logs, verbose self.write_config('BBINCLUDELOGS = "yes"') result = bitbake("logging-test -c pythontest_exit -f -v", ignore_status = True) self.assertIn("ERROR: Logfile of failure stored in:", result.output) - # python tasks don't log output with -v currently - #self.assertCount(result.output, "This is python stdout", 1) + self.assertCount(result.output, "This is python stdout", 1) def test_python_exec_func_python_loggingA(self): # no logs, no verbose @@ -137,8 +137,7 @@ class BitBakeLogging(OESelftestTestCase): result = bitbake("logging-test -c pythontest_exec_func_python -f -v", ignore_status = True) self.assertIn("ERROR: Logfile of failure stored in:", result.output) - # python tasks don't log output with -v currently - #self.assertCount(result.output, "This is python stdout", 1) + self.assertCount(result.output, "This is python stdout", 1) def test_python_exec_func_python_loggingD(self): # logs, verbose @@ -146,8 +145,7 @@ class BitBakeLogging(OESelftestTestCase): result = bitbake("logging-test -c pythontest_exec_func_python -f -v", ignore_status = True) self.assertIn("ERROR: Logfile of failure stored in:", result.output) - # python tasks don't log output with -v currently - #self.assertCount(result.output, "This is python stdout", 1) + self.assertCount(result.output, "This is python stdout", 1) def test_python_fatal_loggingA(self): # no logs, no verbose @@ -171,8 +169,7 @@ class BitBakeLogging(OESelftestTestCase): self.write_config('BBINCLUDELOGS = ""') result = bitbake("logging-test -c pythontest_fatal -f -v", ignore_status = True) self.assertIn("ERROR: Logfile of failure stored in:", result.output) - # python tasks don't log output with -v currently - #self.assertCount(result.output, "This is python fatal test stdout", 1) + self.assertCount(result.output, "This is python fatal test stdout", 1) self.assertCount(result.output, "This is a fatal error", 1) def test_python_fatal_loggingD(self): @@ -180,7 +177,6 @@ class BitBakeLogging(OESelftestTestCase): self.write_config('BBINCLUDELOGS = "yes"') result = bitbake("logging-test -c pythontest_fatal -f -v", ignore_status = True) self.assertIn("ERROR: Logfile of failure stored in:", result.output) - # python tasks don't log output with -v currently - #self.assertCount(result.output, "This is python fatal test stdout", 1) + self.assertCount(result.output, "This is python fatal test stdout", 1) self.assertCount(result.output, "This is a fatal error", 1) diff --git a/meta/lib/oeqa/selftest/cases/bbtests.py b/meta/lib/oeqa/selftest/cases/bbtests.py index 89267c7145..98e9f81661 100644 --- a/meta/lib/oeqa/selftest/cases/bbtests.py +++ b/meta/lib/oeqa/selftest/cases/bbtests.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -39,7 +41,7 @@ class BitbakeTests(OESelftestTestCase): def test_event_handler(self): self.write_config("INHERIT += \"test_events\"") - result = bitbake('m4-native') + result = bitbake('selftest-hello-native') find_build_started = re.search(r"NOTE: Test for bb\.event\.BuildStarted(\n.*)*NOTE: Executing.*Tasks", result.output) find_build_completed = re.search(r"Tasks Summary:.*(\n.*)*NOTE: Test for bb\.event\.BuildCompleted", result.output) self.assertTrue(find_build_started, msg = "Match failed in:\n%s" % result.output) @@ -47,11 +49,11 @@ class BitbakeTests(OESelftestTestCase): self.assertNotIn('Test for bb.event.InvalidEvent', result.output) def test_local_sstate(self): - bitbake('m4-native') - bitbake('m4-native -cclean') - result = bitbake('m4-native') - find_setscene = re.search("m4-native.*do_.*_setscene", result.output) - self.assertTrue(find_setscene, msg = "No \"m4-native.*do_.*_setscene\" message found during bitbake m4-native. bitbake output: %s" % result.output ) + bitbake('selftest-hello-native') + bitbake('selftest-hello-native -cclean') + result = bitbake('selftest-hello-native') + find_setscene = re.search("selftest-hello-native.*do_.*_setscene", result.output) + self.assertTrue(find_setscene, msg = "No \"selftest-hello-native.*do_.*_setscene\" message found during bitbake selftest-hello-native. bitbake output: %s" % result.output ) def test_bitbake_invalid_recipe(self): result = bitbake('-b asdf', ignore_status=True) @@ -173,7 +175,7 @@ SSTATE_DIR = \"${TOPDIR}/download-selftest\" self.assertIn('localconf', result.output) def test_dry_run(self): - result = runCmd('bitbake -n m4-native') + result = runCmd('bitbake -n selftest-hello-native') self.assertEqual(0, result.status, "bitbake dry run didn't run as expected. %s" % result.output) def test_just_parse(self): @@ -186,6 +188,10 @@ SSTATE_DIR = \"${TOPDIR}/download-selftest\" self.assertTrue(find, "No version returned for searched recipe. bitbake output: %s" % result.output) def test_prefile(self): + # Test when the prefile does not exist + result = runCmd('bitbake -r conf/prefile.conf', ignore_status=True) + self.assertEqual(1, result.status, "bitbake didn't error and should have when a specified prefile didn't exist: %s" % result.output) + # Test when the prefile exists preconf = os.path.join(self.builddir, 'conf/prefile.conf') self.track_for_cleanup(preconf) ftools.write_file(preconf ,"TEST_PREFILE=\"prefile\"") @@ -196,6 +202,10 @@ SSTATE_DIR = \"${TOPDIR}/download-selftest\" self.assertIn('localconf', result.output) def test_postfile(self): + # Test when the postfile does not exist + result = runCmd('bitbake -R conf/postfile.conf', ignore_status=True) + self.assertEqual(1, result.status, "bitbake didn't error and should have when a specified postfile didn't exist: %s" % result.output) + # Test when the postfile exists postconf = os.path.join(self.builddir, 'conf/postfile.conf') self.track_for_cleanup(postconf) ftools.write_file(postconf , "TEST_POSTFILE=\"postfile\"") @@ -222,16 +232,21 @@ INHERIT:remove = \"report-error\" self.assertLess(errorpos,continuepos, msg = "bitbake didn't pass do_fail_task. bitbake output: %s" % result.output) def test_non_gplv3(self): - self.write_config('INCOMPATIBLE_LICENSE = "GPL-3.0-or-later"') + self.write_config('''INCOMPATIBLE_LICENSE = "GPL-3.0-or-later" +require conf/distro/include/no-gplv3.inc +''') result = bitbake('selftest-ed', ignore_status=True) self.assertEqual(result.status, 0, "Bitbake failed, exit code %s, output %s" % (result.status, result.output)) lic_dir = get_bb_var('LICENSE_DIRECTORY') - self.assertFalse(os.path.isfile(os.path.join(lic_dir, 'selftest-ed/generic_GPL-3.0-or-later'))) - self.assertTrue(os.path.isfile(os.path.join(lic_dir, 'selftest-ed/generic_GPL-2.0-or-later'))) + arch = get_bb_var('SSTATE_PKGARCH') + filename = os.path.join(lic_dir, arch, 'selftest-ed', 'generic_GPL-3.0-or-later') + self.assertFalse(os.path.isfile(filename), msg="License file %s exists and shouldn't" % filename) + filename = os.path.join(lic_dir, arch, 'selftest-ed', 'generic_GPL-2.0-or-later') + self.assertTrue(os.path.isfile(filename), msg="License file %s doesn't exist" % filename) def test_setscene_only(self): """ Bitbake option to restore from sstate only within a build (i.e. execute no real tasks, only setscene)""" - test_recipe = 'ed' + test_recipe = 'selftest-hello-native' bitbake(test_recipe) bitbake('-c clean %s' % test_recipe) @@ -244,7 +259,7 @@ INHERIT:remove = \"report-error\" 'Executed tasks were: %s' % (task, str(tasks))) def test_skip_setscene(self): - test_recipe = 'ed' + test_recipe = 'selftest-hello-native' bitbake(test_recipe) bitbake('-c clean %s' % test_recipe) @@ -349,3 +364,14 @@ INHERIT:remove = \"report-error\" result = bitbake('gitunpackoffline-fail -c fetch', ignore_status=True) self.assertTrue(re.search("Recipe uses a floating tag/branch .* for repo .* without a fixed SRCREV yet doesn't call bb.fetch2.get_srcrev()", result.output), msg = "Recipe without PV set to SRCPV should have failed: %s" % result.output) + + def test_unexpanded_variable_in_path(self): + """ + Test that bitbake fails if directory contains unexpanded bitbake variable in the name + """ + recipe_name = "gitunpackoffline" + self.write_config('PV:pn-gitunpackoffline:append = "+${UNDEFVAL}"') + result = bitbake('{}'.format(recipe_name), ignore_status=True) + self.assertGreater(result.status, 0, "Build should have failed if ${ is in the path") + self.assertTrue(re.search("ERROR: Directory name /.* contains unexpanded bitbake variable. This may cause build failures and WORKDIR polution", + result.output), msg = "mkdirhier with unexpanded variable should have failed: %s" % result.output) diff --git a/meta/lib/oeqa/selftest/cases/binutils.py b/meta/lib/oeqa/selftest/cases/binutils.py index 3b0b44b390..1688eabe4e 100644 --- a/meta/lib/oeqa/selftest/cases/binutils.py +++ b/meta/lib/oeqa/selftest/cases/binutils.py @@ -1,5 +1,10 @@ +# +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT +# import os +import time from oeqa.core.decorator import OETestTag from oeqa.core.case import OEPTestResultTestCase from oeqa.selftest.case import OESelftestTestCase @@ -32,15 +37,19 @@ class BinutilsCrossSelfTest(OESelftestTestCase, OEPTestResultTestCase): bb_vars = get_bb_vars(["B", "TARGET_SYS", "T"], recipe) builddir, target_sys, tdir = bb_vars["B"], bb_vars["TARGET_SYS"], bb_vars["T"] + start_time = time.time() + bitbake("{0} -c check".format(recipe)) + end_time = time.time() + sumspath = os.path.join(builddir, suite, "{0}.sum".format(suite)) if not os.path.exists(sumspath): sumspath = os.path.join(builddir, suite, "testsuite", "{0}.sum".format(suite)) logpath = os.path.splitext(sumspath)[0] + ".log" ptestsuite = "binutils-{}".format(suite) if suite != "binutils" else suite - self.ptest_section(ptestsuite, logfile = logpath) + self.ptest_section(ptestsuite, duration = int(end_time - start_time), logfile = logpath) with open(sumspath, "r") as f: for test, result in parse_values(f): self.ptest_result(ptestsuite, test, result) diff --git a/meta/lib/oeqa/selftest/cases/buildhistory.py b/meta/lib/oeqa/selftest/cases/buildhistory.py index d865da6252..2d55994916 100644 --- a/meta/lib/oeqa/selftest/cases/buildhistory.py +++ b/meta/lib/oeqa/selftest/cases/buildhistory.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/selftest/cases/buildoptions.py b/meta/lib/oeqa/selftest/cases/buildoptions.py index ad604d6ae2..31dafaa9c5 100644 --- a/meta/lib/oeqa/selftest/cases/buildoptions.py +++ b/meta/lib/oeqa/selftest/cases/buildoptions.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -9,8 +11,10 @@ import shutil import tempfile from oeqa.selftest.case import OESelftestTestCase from oeqa.selftest.cases.buildhistory import BuildhistoryBase +from oeqa.core.decorator.data import skipIfMachine from oeqa.utils.commands import bitbake, get_bb_var, get_bb_vars import oeqa.utils.ftools as ftools +from oeqa.core.decorator import OETestTag class ImageOptionsTests(OESelftestTestCase): @@ -201,6 +205,7 @@ class ToolchainOptions(OESelftestTestCase): self.write_config(features) bitbake('fortran-helloworld') +@OETestTag("yocto-mirrors") class SourceMirroring(OESelftestTestCase): # Can we download everything from the Yocto Sources Mirror over http only def test_yocto_source_mirror(self): diff --git a/meta/lib/oeqa/selftest/cases/c_cpp.py b/meta/lib/oeqa/selftest/cases/c_cpp.py new file mode 100644 index 0000000000..9a70ce29f5 --- /dev/null +++ b/meta/lib/oeqa/selftest/cases/c_cpp.py @@ -0,0 +1,60 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +from oeqa.selftest.case import OESelftestTestCase +from oeqa.core.decorator.data import skipIfNotQemuUsermode +from oeqa.utils.commands import bitbake + + +class CCppTests(OESelftestTestCase): + + @skipIfNotQemuUsermode() + def _qemu_usermode(self, recipe_name): + self.add_command_to_tearDown("bitbake -c clean %s" % recipe_name) + bitbake("%s -c run_tests" % recipe_name) + + @skipIfNotQemuUsermode() + def _qemu_usermode_failing(self, recipe_name): + config = 'PACKAGECONFIG:pn-%s = "failing_test"' % recipe_name + self.write_config(config) + self.add_command_to_tearDown("bitbake -c clean %s" % recipe_name) + result = bitbake("%s -c run_tests" % recipe_name, ignore_status=True) + self.assertNotEqual(0, result.status, "command: %s is expected to fail but passed, status: %s, output: %s, error: %s" % ( + result.command, result.status, result.output, result.error)) + + +class CMakeTests(CCppTests): + def test_cmake_qemu(self): + """Test for cmake-qemu.bbclass good case + + compile the cmake-example and verify the CTests pass in qemu-user. + qemu-user is configured by CMAKE_CROSSCOMPILING_EMULATOR. + """ + self._qemu_usermode("cmake-example") + + def test_cmake_qemu_failing(self): + """Test for cmake-qemu.bbclass bad case + + Break the comparison in the test code and verify the CTests do not pass. + """ + self._qemu_usermode_failing("cmake-example") + + +class MesonTests(CCppTests): + def test_meson_qemu(self): + """Test the qemu-user feature of the meson.bbclass good case + + compile the meson-example and verify the Unit Test pass in qemu-user. + qemu-user is configured by meson's exe_wrapper option. + """ + self._qemu_usermode("meson-example") + + def test_meson_qemu_failing(self): + """Test the qemu-user feature of the meson.bbclass bad case + + Break the comparison in the test code and verify the Unit Test does not pass in qemu-user. + """ + self._qemu_usermode_failing("meson-example") diff --git a/meta/lib/oeqa/selftest/cases/containerimage.py b/meta/lib/oeqa/selftest/cases/containerimage.py index e0aea1a1ef..23c0a1408a 100644 --- a/meta/lib/oeqa/selftest/cases/containerimage.py +++ b/meta/lib/oeqa/selftest/cases/containerimage.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/selftest/cases/cve_check.py b/meta/lib/oeqa/selftest/cases/cve_check.py index d0b2213703..60cecd1328 100644 --- a/meta/lib/oeqa/selftest/cases/cve_check.py +++ b/meta/lib/oeqa/selftest/cases/cve_check.py @@ -1,3 +1,9 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + import json import os from oeqa.selftest.case import OESelftestTestCase @@ -48,6 +54,25 @@ class CVECheck(OESelftestTestCase): self.assertTrue( result ,msg="Failed to compare version with suffix '1.0_patch2' < '1.0_patch3'") + def test_convert_cve_version(self): + from oe.cve_check import convert_cve_version + + # Default format + self.assertEqual(convert_cve_version("8.3"), "8.3") + self.assertEqual(convert_cve_version(""), "") + + # OpenSSL format version + self.assertEqual(convert_cve_version("1.1.1t"), "1.1.1t") + + # OpenSSH format + self.assertEqual(convert_cve_version("8.3_p1"), "8.3p1") + self.assertEqual(convert_cve_version("8.3_p22"), "8.3p22") + + # Linux kernel format + self.assertEqual(convert_cve_version("6.2_rc8"), "6.2-rc8") + self.assertEqual(convert_cve_version("6.2_rc31"), "6.2-rc31") + + def test_recipe_report_json(self): config = """ INHERIT += "cve-check" @@ -182,18 +207,34 @@ CVE_CHECK_REPORT_PATCHED = "1" self.assertEqual(len(report["package"]), 1) package = report["package"][0] self.assertEqual(package["name"], "logrotate") - found_cves = { issue["id"]: issue["status"] for issue in package["issue"]} + found_cves = {} + for issue in package["issue"]: + found_cves[issue["id"]] = { + "status" : issue["status"], + "detail" : issue["detail"] if "detail" in issue else "", + "description" : issue["description"] if "description" in issue else "" + } # m4 CVE should not be in logrotate self.assertNotIn("CVE-2008-1687", found_cves) # logrotate has both Patched and Ignored CVEs self.assertIn("CVE-2011-1098", found_cves) - self.assertEqual(found_cves["CVE-2011-1098"], "Patched") + self.assertEqual(found_cves["CVE-2011-1098"]["status"], "Patched") + self.assertEqual(len(found_cves["CVE-2011-1098"]["detail"]), 0) + self.assertEqual(len(found_cves["CVE-2011-1098"]["description"]), 0) + detail = "not-applicable-platform" + description = "CVE is debian, gentoo or SUSE specific on the way logrotate was installed/used" self.assertIn("CVE-2011-1548", found_cves) - self.assertEqual(found_cves["CVE-2011-1548"], "Ignored") + self.assertEqual(found_cves["CVE-2011-1548"]["status"], "Ignored") + self.assertEqual(found_cves["CVE-2011-1548"]["detail"], detail) + self.assertEqual(found_cves["CVE-2011-1548"]["description"], description) self.assertIn("CVE-2011-1549", found_cves) - self.assertEqual(found_cves["CVE-2011-1549"], "Ignored") + self.assertEqual(found_cves["CVE-2011-1549"]["status"], "Ignored") + self.assertEqual(found_cves["CVE-2011-1549"]["detail"], detail) + self.assertEqual(found_cves["CVE-2011-1549"]["description"], description) self.assertIn("CVE-2011-1550", found_cves) - self.assertEqual(found_cves["CVE-2011-1550"], "Ignored") + self.assertEqual(found_cves["CVE-2011-1550"]["status"], "Ignored") + self.assertEqual(found_cves["CVE-2011-1550"]["detail"], detail) + self.assertEqual(found_cves["CVE-2011-1550"]["description"], description) self.assertExists(summary_json) check_m4_json(summary_json) diff --git a/meta/lib/oeqa/selftest/cases/debuginfod.py b/meta/lib/oeqa/selftest/cases/debuginfod.py new file mode 100644 index 0000000000..505b4be837 --- /dev/null +++ b/meta/lib/oeqa/selftest/cases/debuginfod.py @@ -0,0 +1,158 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# +import os +import socketserver +import subprocess +import time +import urllib +import pathlib + +from oeqa.core.decorator import OETestTag +from oeqa.selftest.case import OESelftestTestCase +from oeqa.utils.commands import bitbake, get_bb_var, runqemu + + +class Debuginfod(OESelftestTestCase): + + def wait_for_debuginfod(self, port): + """ + debuginfod takes time to scan the packages and requesting too early may + result in a test failure if the right packages haven't been scanned yet. + + Request the metrics endpoint periodically and wait for there to be no + busy scanning threads. + + Returns if debuginfod is ready, raises an exception if not within the + timeout. + """ + + # Wait two minutes + countdown = 24 + delay = 5 + latest = None + + while countdown: + self.logger.info("waiting...") + time.sleep(delay) + + self.logger.info("polling server") + if self.debuginfod.poll(): + self.logger.info("server dead") + self.debuginfod.communicate() + self.fail("debuginfod terminated unexpectedly") + self.logger.info("server alive") + + try: + with urllib.request.urlopen("http://localhost:%d/metrics" % port, timeout=10) as f: + for line in f.read().decode("ascii").splitlines(): + key, value = line.rsplit(" ", 1) + if key == "thread_busy{role=\"scan\"}": + latest = int(value) + self.logger.info("Waiting for %d scan jobs to finish" % latest) + if latest == 0: + return + except urllib.error.URLError as e: + # TODO: how to catch just timeouts? + self.logger.error(e) + + countdown -= 1 + + raise TimeoutError("Cannot connect debuginfod, still %d scan jobs running" % latest) + + def start_debuginfod(self): + # We assume that the caller has already bitbake'd elfutils-native:do_addto_recipe_sysroot + + # Save some useful paths for later + native_sysroot = pathlib.Path(get_bb_var("RECIPE_SYSROOT_NATIVE", "elfutils-native")) + native_bindir = native_sysroot / "usr" / "bin" + self.debuginfod = native_bindir / "debuginfod" + self.debuginfod_find = native_bindir / "debuginfod-find" + + cmd = [ + self.debuginfod, + "--verbose", + # In-memory database, this is a one-shot test + "--database=:memory:", + # Don't use all the host cores + "--concurrency=8", + "--connection-pool=8", + # Disable rescanning, this is a one-shot test + "--rescan-time=0", + "--groom-time=0", + get_bb_var("DEPLOY_DIR"), + ] + + format = get_bb_var("PACKAGE_CLASSES").split()[0] + if format == "package_deb": + cmd.append("--scan-deb-dir") + elif format == "package_ipk": + cmd.append("--scan-deb-dir") + elif format == "package_rpm": + cmd.append("--scan-rpm-dir") + else: + self.fail("Unknown package class %s" % format) + + # Find a free port. Racey but the window is small. + with socketserver.TCPServer(("localhost", 0), None) as s: + self.port = s.server_address[1] + cmd.append("--port=%d" % self.port) + + self.logger.info(f"Starting server {cmd}") + self.debuginfod = subprocess.Popen(cmd, env={}) + self.wait_for_debuginfod(self.port) + + + def test_debuginfod_native(self): + """ + Test debuginfod outside of qemu, by building a package and looking up a + binary's debuginfo using elfutils-native. + """ + + self.write_config(""" +TMPDIR = "${TOPDIR}/tmp-debuginfod" +DISTRO_FEATURES:append = " debuginfod" +""") + bitbake("elfutils-native:do_addto_recipe_sysroot xz xz:do_package") + + try: + self.start_debuginfod() + + env = os.environ.copy() + env["DEBUGINFOD_URLS"] = "http://localhost:%d/" % self.port + + pkgs = pathlib.Path(get_bb_var("PKGDEST", "xz")) + cmd = (self.debuginfod_find, "debuginfo", pkgs / "xz" / "usr" / "bin" / "xz.xz") + self.logger.info(f"Starting client {cmd}") + output = subprocess.check_output(cmd, env=env, text=True) + # This should be more comprehensive + self.assertIn("/.cache/debuginfod_client/", output) + finally: + self.debuginfod.kill() + + @OETestTag("runqemu") + def test_debuginfod_qemu(self): + """ + Test debuginfod-find inside a qemu, talking to a debuginfod on the host. + """ + + self.write_config(""" +TMPDIR = "${TOPDIR}/tmp-debuginfod" +DISTRO_FEATURES:append = " debuginfod" +CORE_IMAGE_EXTRA_INSTALL += "elfutils xz" + """) + bitbake("core-image-minimal elfutils-native:do_addto_recipe_sysroot") + + try: + self.start_debuginfod() + + with runqemu("core-image-minimal", runqemuparams="nographic") as qemu: + cmd = "DEBUGINFOD_URLS=http://%s:%d/ debuginfod-find debuginfo /usr/bin/xz" % (qemu.server_ip, self.port) + self.logger.info(f"Starting client {cmd}") + status, output = qemu.run_serial(cmd) + # This should be more comprehensive + self.assertIn("/.cache/debuginfod_client/", output) + finally: + self.debuginfod.kill() diff --git a/meta/lib/oeqa/selftest/cases/devtool.py b/meta/lib/oeqa/selftest/cases/devtool.py index 34fc791f3a..51949e3c93 100644 --- a/meta/lib/oeqa/selftest/cases/devtool.py +++ b/meta/lib/oeqa/selftest/cases/devtool.py @@ -1,13 +1,18 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # +import errno import os import re import shutil import tempfile import glob import fnmatch +import unittest +import json from oeqa.selftest.case import OESelftestTestCase from oeqa.utils.commands import runCmd, bitbake, get_bb_var, create_temp_layer @@ -24,6 +29,9 @@ def setUpModule(): corecopydir = os.path.join(templayerdir, 'core-copy') bblayers_conf = os.path.join(os.environ['BUILDDIR'], 'conf', 'bblayers.conf') edited_layers = [] + # make sure user doesn't have a local workspace + result = runCmd('bitbake-layers show-layers') + assert "workspacelayer" not in result.output, "Devtool test suite cannot be run with a local workspace directory" # We need to take a copy of the meta layer so we can modify it and not # have any races against other tests that might be running in parallel @@ -38,10 +46,17 @@ def setUpModule(): canonical_layerpath = os.path.realpath(canonical_layerpath) + '/' edited_layers.append(layerpath) oldmetapath = os.path.realpath(layerpath) + + # when downloading poky from tar.gz some tests will be skipped (BUG 12389) + try: + runCmd('git rev-parse --is-inside-work-tree', cwd=canonical_layerpath) + except: + raise unittest.SkipTest("devtool tests require folder to be a git repo") + result = runCmd('git rev-parse --show-toplevel', cwd=canonical_layerpath) oldreporoot = result.output.rstrip() newmetapath = os.path.join(corecopydir, os.path.relpath(oldmetapath, oldreporoot)) - runCmd('git clone %s %s' % (oldreporoot, corecopydir), cwd=templayerdir) + runCmd('git clone file://%s %s' % (oldreporoot, corecopydir), cwd=templayerdir) # Now we need to copy any modified files # You might ask "why not just copy the entire tree instead of # cloning and doing this?" - well, the problem with that is @@ -246,6 +261,47 @@ class DevtoolTestCase(OESelftestTestCase): if remaining_removelines: self.fail('Expected removed lines not found: %s' % remaining_removelines) + def _check_runqemu_prerequisites(self): + """Check runqemu is available + + Whilst some tests would seemingly be better placed as a runtime test, + unfortunately the runtime tests run under bitbake and you can't run + devtool within bitbake (since devtool needs to run bitbake itself). + Additionally we are testing build-time functionality as well, so + really this has to be done as an oe-selftest test. + """ + machine = get_bb_var('MACHINE') + if not machine.startswith('qemu'): + self.skipTest('This test only works with qemu machines') + if not os.path.exists('/etc/runqemu-nosudo'): + self.skipTest('You must set up tap devices with scripts/runqemu-gen-tapdevs before running this test') + result = runCmd('PATH="$PATH:/sbin:/usr/sbin" ip tuntap show', ignore_status=True) + if result.status != 0: + result = runCmd('PATH="$PATH:/sbin:/usr/sbin" ifconfig -a', ignore_status=True) + if result.status != 0: + self.skipTest('Failed to determine if tap devices exist with ifconfig or ip: %s' % result.output) + for line in result.output.splitlines(): + if line.startswith('tap'): + break + else: + self.skipTest('No tap devices found - you must set up tap devices with scripts/runqemu-gen-tapdevs before running this test') + + def _test_devtool_add_git_url(self, git_url, version, pn, resulting_src_uri): + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool add --version %s %s %s' % (version, pn, git_url)) + self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created') + # Check the recipe name is correct + recipefile = get_bb_var('FILE', pn) + self.assertIn('%s_git.bb' % pn, recipefile, 'Recipe file incorrectly named') + self.assertIn(recipefile, result.output) + # Test devtool status + result = runCmd('devtool status') + self.assertIn(pn, result.output) + self.assertIn(recipefile, result.output) + checkvars = {} + checkvars['SRC_URI'] = resulting_src_uri + self._test_recipe_contents(recipefile, checkvars, []) class DevtoolBase(DevtoolTestCase): @@ -258,6 +314,7 @@ class DevtoolBase(DevtoolTestCase): cls.sstate_conf = 'SSTATE_DIR = "%s"\n' % cls.devtool_sstate cls.sstate_conf += ('SSTATE_MIRRORS += "file://.* file:///%s/PATH"\n' % cls.original_sstate) + cls.sstate_conf += ('BB_HASHSERVE_UPSTREAM = "hashserv.yocto.io:8687"\n') @classmethod def tearDownClass(cls): @@ -339,6 +396,38 @@ class DevtoolAddTests(DevtoolBase): bindir = bindir[1:] self.assertTrue(os.path.isfile(os.path.join(installdir, bindir, 'pv')), 'pv binary not found in D') + def test_devtool_add_binary(self): + # Create a binary package containing a known test file + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + pn = 'tst-bin' + pv = '1.0' + test_file_dir = "var/lib/%s/" % pn + test_file_name = "test_file" + test_file_content = "TEST CONTENT" + test_file_package_root = os.path.join(tempdir, pn) + test_file_dir_full = os.path.join(test_file_package_root, test_file_dir) + bb.utils.mkdirhier(test_file_dir_full) + with open(os.path.join(test_file_dir_full, test_file_name), "w") as f: + f.write(test_file_content) + bin_package_path = os.path.join(tempdir, "%s.tar.gz" % pn) + runCmd("tar czf %s -C %s ." % (bin_package_path, test_file_package_root)) + + # Test devtool add -b on the binary package + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake -c cleansstate %s' % pn) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool add -b %s %s' % (pn, bin_package_path)) + self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created') + + # Build the resulting recipe + result = runCmd('devtool build %s' % pn) + installdir = get_bb_var('D', pn) + self.assertTrue(installdir, 'Could not query installdir variable') + + # Check that a known file from the binary package has indeed been installed + self.assertTrue(os.path.isfile(os.path.join(installdir, test_file_dir, test_file_name)), '%s not found in D' % test_file_name) + def test_devtool_add_git_local(self): # We need dbus built so that DEPENDS recognition works bitbake('dbus') @@ -374,12 +463,28 @@ class DevtoolAddTests(DevtoolBase): checkvars['LICENSE'] = 'GPL-2.0-only' checkvars['LIC_FILES_CHKSUM'] = 'file://COPYING;md5=b234ee4d69f5fce4486a80fdaf4a4263' checkvars['S'] = '${WORKDIR}/git' - checkvars['PV'] = '0.1+git${SRCPV}' + checkvars['PV'] = '0.1+git' checkvars['SRC_URI'] = 'git://git.yoctoproject.org/git/dbus-wait;protocol=https;branch=master' checkvars['SRCREV'] = srcrev checkvars['DEPENDS'] = set(['dbus']) self._test_recipe_contents(recipefile, checkvars, []) + def test_devtool_add_git_style1(self): + version = 'v3.1.0' + pn = 'mbedtls' + # this will trigger reformat_git_uri with branch parameter in url + git_url = "'git://git@github.com/ARMmbed/mbedtls.git;branch=mbedtls-2.28;protocol=https'" + resulting_src_uri = "git://git@github.com/ARMmbed/mbedtls.git;branch=mbedtls-2.28;protocol=https" + self._test_devtool_add_git_url(git_url, version, pn, resulting_src_uri) + + def test_devtool_add_git_style2(self): + version = 'v3.1.0' + pn = 'mbedtls' + # this will trigger reformat_git_uri with branch parameter in url + git_url = "'git://git@github.com/ARMmbed/mbedtls.git;protocol=https'" + resulting_src_uri = "gitsm://git@github.com/ARMmbed/mbedtls.git;protocol=https;branch=master" + self._test_devtool_add_git_url(git_url, version, pn, resulting_src_uri) + def test_devtool_add_library(self): # Fetch source tempdir = tempfile.mkdtemp(prefix='devtoolqa') @@ -440,7 +545,7 @@ class DevtoolAddTests(DevtoolBase): self.track_for_cleanup(self.workspacedir) self.add_command_to_tearDown('bitbake -c cleansstate %s' % testrecipe) self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') - result = runCmd('devtool add %s %s -f %s' % (testrecipe, srcdir, url)) + result = runCmd('devtool add --no-pypi %s %s -f %s' % (testrecipe, srcdir, url)) self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created. %s' % result.output) self.assertTrue(os.path.isfile(os.path.join(srcdir, 'setup.py')), 'Unable to find setup.py in source directory') self.assertTrue(os.path.isdir(os.path.join(srcdir, '.git')), 'git repository for external source tree was not created') @@ -459,7 +564,7 @@ class DevtoolAddTests(DevtoolBase): result = runCmd('devtool reset -n %s' % testrecipe) shutil.rmtree(srcdir) fakever = '1.9' - result = runCmd('devtool add %s %s -f %s -V %s' % (testrecipe, srcdir, url, fakever)) + result = runCmd('devtool add --no-pypi %s %s -f %s -V %s' % (testrecipe, srcdir, url, fakever)) self.assertTrue(os.path.isfile(os.path.join(srcdir, 'setup.py')), 'Unable to find setup.py in source directory') # Test devtool status result = runCmd('devtool status') @@ -472,7 +577,7 @@ class DevtoolAddTests(DevtoolBase): checkvars['S'] = '${WORKDIR}/MarkupSafe-%s' % testver checkvars['SRC_URI'] = url self._test_recipe_contents(recipefile, checkvars, []) - + def test_devtool_add_fetch_git(self): tempdir = tempfile.mkdtemp(prefix='devtoolqa') self.track_for_cleanup(tempdir) @@ -497,7 +602,7 @@ class DevtoolAddTests(DevtoolBase): self.assertIn('_git.bb', recipefile, 'Recipe file incorrectly named') checkvars = {} checkvars['S'] = '${WORKDIR}/git' - checkvars['PV'] = '1.0+git${SRCPV}' + checkvars['PV'] = '1.0+git' checkvars['SRC_URI'] = url_branch checkvars['SRCREV'] = '${AUTOREV}' self._test_recipe_contents(recipefile, checkvars, []) @@ -516,7 +621,7 @@ class DevtoolAddTests(DevtoolBase): self.assertIn('_git.bb', recipefile, 'Recipe file incorrectly named') checkvars = {} checkvars['S'] = '${WORKDIR}/git' - checkvars['PV'] = '1.5+git${SRCPV}' + checkvars['PV'] = '1.5+git' checkvars['SRC_URI'] = url_branch checkvars['SRCREV'] = checkrev self._test_recipe_contents(recipefile, checkvars, []) @@ -540,7 +645,7 @@ class DevtoolAddTests(DevtoolBase): result = runCmd('devtool status') self.assertIn(testrecipe, result.output) self.assertIn(srcdir, result.output) - # Check recipe + # Check recipedevtool add recipefile = get_bb_var('FILE', testrecipe) self.assertIn('%s_%s.bb' % (testrecipe, testver), recipefile, 'Recipe file incorrectly named') checkvars = {} @@ -813,6 +918,122 @@ class DevtoolModifyTests(DevtoolBase): # Try building bitbake(testrecipe) + def test_devtool_modify_git_no_extract(self): + # Check preconditions + testrecipe = 'psplash' + src_uri = get_bb_var('SRC_URI', testrecipe) + self.assertIn('git://', src_uri, 'This test expects the %s recipe to be a git recipe' % testrecipe) + # Clean up anything in the workdir/sysroot/sstate cache + bitbake('%s -c cleansstate' % testrecipe) + # Try modifying a recipe + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('git clone https://git.yoctoproject.org/psplash %s && devtool modify -n %s %s' % (tempdir, testrecipe, tempdir)) + self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created. devtool output: %s' % result.output) + matches = glob.glob(os.path.join(self.workspacedir, 'appends', 'psplash_*.bbappend')) + self.assertTrue(matches, 'bbappend not created') + # Test devtool status + result = runCmd('devtool status') + self.assertIn(testrecipe, result.output) + self.assertIn(tempdir, result.output) + + def test_devtool_modify_git_crates_subpath(self): + # This tests two things in devtool context: + # - that we support local git dependencies for cargo based recipe + # - that we support patches in SRC_URI when git url contains subpath parameter + + # Check preconditions: + # recipe inherits cargo + # git:// uri with a subpath as the main package + # some crate:// in SRC_URI + # others git:// in SRC_URI + # cointains a patch + testrecipe = 'hello-rs' + bb_vars = get_bb_vars(['SRC_URI', 'FILE', 'WORKDIR', 'CARGO_HOME'], testrecipe) + recipefile = bb_vars['FILE'] + workdir = bb_vars['WORKDIR'] + cargo_home = bb_vars['CARGO_HOME'] + src_uri = bb_vars['SRC_URI'].split() + self.assertTrue(src_uri[0].startswith('git://'), + 'This test expects the %s recipe to have a git repo has its main uri' % testrecipe) + self.assertIn(';subpath=', src_uri[0], + 'This test expects the %s recipe to have a git uri with subpath' % testrecipe) + self.assertTrue(any([uri.startswith('crate://') for uri in src_uri]), + 'This test expects the %s recipe to have some crates in its src uris' % testrecipe) + self.assertGreaterEqual(sum(map(lambda x:x.startswith('git://'), src_uri)), 2, + 'This test expects the %s recipe to have several git:// uris' % testrecipe) + self.assertTrue(any([uri.startswith('file://') and '.patch' in uri for uri in src_uri]), + 'This test expects the %s recipe to have a patch in its src uris' % testrecipe) + + self._test_recipe_contents(recipefile, {}, ['ptest-cargo']) + + # Clean up anything in the workdir/sysroot/sstate cache + bitbake('%s -c cleansstate' % testrecipe) + # Try modifying a recipe + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir)) + self.assertExists(os.path.join(tempdir, 'Cargo.toml'), 'Extracted source could not be found') + self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created. devtool output: %s' % result.output) + matches = glob.glob(os.path.join(self.workspacedir, 'appends', '%s_*.bbappend' % testrecipe)) + self.assertTrue(matches, 'bbappend not created') + # Test devtool status + result = runCmd('devtool status') + self.assertIn(testrecipe, result.output) + self.assertIn(tempdir, result.output) + # Check git repo + self._check_src_repo(tempdir) + # Check that the patch is correctly applied. + # The last commit message in the tree must contain the following note: + # Notes (devtool): + # original patch: <patchname> + # .. + patchname = None + for uri in src_uri: + if uri.startswith('file://') and '.patch' in uri: + patchname = uri.replace("file://", "").partition('.patch')[0] + '.patch' + self.assertIsNotNone(patchname) + result = runCmd('git -C %s log -1' % tempdir) + self.assertIn("Notes (devtool):\n original patch: %s" % patchname, result.output) + + # Configure the recipe to check that the git dependencies are correctly patched in cargo config + bitbake('-c configure %s' % testrecipe) + + cargo_config_path = os.path.join(cargo_home, 'config') + with open(cargo_config_path, "r") as f: + cargo_config_contents = [line.strip('\n') for line in f.readlines()] + + # Get back git dependencies of the recipe (ignoring the main one) + # and check that they are all correctly patched to be fetched locally + git_deps = [uri for uri in src_uri if uri.startswith("git://")][1:] + for git_dep in git_deps: + raw_url, _, raw_parms = git_dep.partition(";") + parms = {} + for parm in raw_parms.split(";"): + name_parm, _, value_parm = parm.partition('=') + parms[name_parm]=value_parm + self.assertIn('protocol', parms, 'git dependencies uri should contain the "protocol" parameter') + self.assertIn('name', parms, 'git dependencies uri should contain the "name" parameter') + self.assertIn('destsuffix', parms, 'git dependencies uri should contain the "destsuffix" parameter') + self.assertIn('type', parms, 'git dependencies uri should contain the "type" parameter') + self.assertEqual(parms['type'], 'git-dependency', 'git dependencies uri should have "type=git-dependency"') + raw_url = raw_url.replace("git://", '%s://' % parms['protocol']) + patch_line = '[patch."%s"]' % raw_url + path_patched = os.path.join(workdir, parms['destsuffix']) + path_override_line = '%s = { path = "%s" }' % (parms['name'], path_patched) + # Would have been better to use tomllib to read this file :/ + self.assertIn(patch_line, cargo_config_contents) + self.assertIn(path_override_line, cargo_config_contents) + + # Try to package the recipe + bitbake('-c package_qa %s' % testrecipe) + def test_devtool_modify_localfiles(self): # Check preconditions testrecipe = 'lighttpd' @@ -878,12 +1099,43 @@ class DevtoolModifyTests(DevtoolBase): runCmd('git -C %s checkout %s' % (tempdir, branch)) with open(source, "rt") as f: content = f.read() - self.assertEquals(content, expected) - check('devtool', 'This is a test for something\n') + self.assertEqual(content, expected) + if self.td["MACHINE"] == "qemux86": + check('devtool', 'This is a test for qemux86\n') + elif self.td["MACHINE"] == "qemuarm": + check('devtool', 'This is a test for qemuarm\n') + else: + check('devtool', 'This is a test for something\n') check('devtool-no-overrides', 'This is a test for something\n') check('devtool-override-qemuarm', 'This is a test for qemuarm\n') check('devtool-override-qemux86', 'This is a test for qemux86\n') + def test_devtool_modify_multiple_sources(self): + # This test check that recipes fetching several sources can be used with devtool modify/build + # Check preconditions + testrecipe = 'bzip2' + src_uri = get_bb_var('SRC_URI', testrecipe) + src1 = 'https://' in src_uri + src2 = 'git://' in src_uri + self.assertTrue(src1 and src2, 'This test expects the %s recipe to fetch both a git source and a tarball and it seems that it no longer does' % testrecipe) + # Clean up anything in the workdir/sysroot/sstate cache + bitbake('%s -c cleansstate' % testrecipe) + # Try modifying a recipe + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir)) + self.assertEqual(result.status, 0, "Could not modify recipe %s. Output: %s" % (testrecipe, result.output)) + # Test devtool status + result = runCmd('devtool status') + self.assertIn(testrecipe, result.output) + self.assertIn(tempdir, result.output) + # Try building + result = bitbake(testrecipe) + self.assertEqual(result.status, 0, "Bitbake failed, exit code %s, output %s" % (result.status, result.output)) + class DevtoolUpdateTests(DevtoolBase): def test_devtool_update_recipe(self): @@ -913,14 +1165,15 @@ class DevtoolUpdateTests(DevtoolBase): result = runCmd('git commit -m "Add a new file"', cwd=tempdir) self.add_command_to_tearDown('cd %s; rm %s/*.patch; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile))) result = runCmd('devtool update-recipe %s' % testrecipe) + result = runCmd('git add minicom', cwd=os.path.dirname(recipefile)) expected_status = [(' M', '.*/%s$' % os.path.basename(recipefile)), - ('??', '.*/0001-Change-the-README.patch$'), - ('??', '.*/0002-Add-a-new-file.patch$')] + ('A ', '.*/0001-Change-the-README.patch$'), + ('A ', '.*/0002-Add-a-new-file.patch$')] self._check_repo_status(os.path.dirname(recipefile), expected_status) def test_devtool_update_recipe_git(self): # Check preconditions - testrecipe = 'mtd-utils' + testrecipe = 'mtd-utils-selftest' bb_vars = get_bb_vars(['FILE', 'SRC_URI'], testrecipe) recipefile = bb_vars['FILE'] src_uri = bb_vars['SRC_URI'] @@ -1041,10 +1294,11 @@ class DevtoolUpdateTests(DevtoolBase): def test_devtool_update_recipe_append_git(self): # Check preconditions - testrecipe = 'mtd-utils' - bb_vars = get_bb_vars(['FILE', 'SRC_URI'], testrecipe) + testrecipe = 'mtd-utils-selftest' + bb_vars = get_bb_vars(['FILE', 'SRC_URI', 'LAYERSERIES_CORENAMES'], testrecipe) recipefile = bb_vars['FILE'] src_uri = bb_vars['SRC_URI'] + corenames = bb_vars['LAYERSERIES_CORENAMES'] self.assertIn('git://', src_uri, 'This test expects the %s recipe to be a git recipe' % testrecipe) for entry in src_uri.split(): if entry.startswith('git://'): @@ -1075,7 +1329,7 @@ class DevtoolUpdateTests(DevtoolBase): f.write('BBFILE_PATTERN_oeselftesttemplayer = "^${LAYERDIR}/"\n') f.write('BBFILE_PRIORITY_oeselftesttemplayer = "999"\n') f.write('BBFILE_PATTERN_IGNORE_EMPTY_oeselftesttemplayer = "1"\n') - f.write('LAYERSERIES_COMPAT_oeselftesttemplayer = "${LAYERSERIES_COMPAT_core}"\n') + f.write('LAYERSERIES_COMPAT_oeselftesttemplayer = "%s"\n' % corenames) self.add_command_to_tearDown('bitbake-layers remove-layer %s || true' % templayerdir) result = runCmd('bitbake-layers add-layer %s' % templayerdir, cwd=self.builddir) # Create the bbappend @@ -1151,14 +1405,30 @@ class DevtoolUpdateTests(DevtoolBase): runCmd('echo "Bar" > new-file', cwd=tempdir) runCmd('git add new-file', cwd=tempdir) runCmd('git commit -m "Add new file"', cwd=tempdir) - self.add_command_to_tearDown('cd %s; git clean -fd .; git checkout .' % - os.path.dirname(recipefile)) runCmd('devtool update-recipe %s' % testrecipe) expected_status = [(' M', '.*/%s$' % os.path.basename(recipefile)), (' M', '.*/makedevs/makedevs.c$'), ('??', '.*/makedevs/new-local$'), ('??', '.*/makedevs/0001-Add-new-file.patch$')] self._check_repo_status(os.path.dirname(recipefile), expected_status) + # Now try to update recipe in another layer, so first, clean it + runCmd('cd %s; git clean -fd .; git checkout .' % os.path.dirname(recipefile)) + # Create a temporary layer and add it to bblayers.conf + self._create_temp_layer(templayerdir, True, 'templayer') + # Update recipe in templayer + result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir)) + self.assertNotIn('WARNING:', result.output) + # Check recipe is still clean + self._check_repo_status(os.path.dirname(recipefile), []) + splitpath = os.path.dirname(recipefile).split(os.sep) + appenddir = os.path.join(templayerdir, splitpath[-2], splitpath[-1]) + bbappendfile = self._check_bbappend(testrecipe, recipefile, appenddir) + patchfile = os.path.join(appenddir, testrecipe, '0001-Add-new-file.patch') + new_local_file = os.path.join(appenddir, testrecipe, 'new_local') + local_file = os.path.join(appenddir, testrecipe, 'makedevs.c') + self.assertExists(patchfile, 'Patch file 0001-Add-new-file.patch not created') + self.assertExists(local_file, 'File makedevs.c not created') + self.assertExists(patchfile, 'File new_local not created') def test_devtool_update_recipe_local_files_2(self): """Check local source files support when oe-local-files is in Git""" @@ -1293,7 +1563,7 @@ class DevtoolUpdateTests(DevtoolBase): # Modify one file srctree = os.path.join(self.workspacedir, 'sources', testrecipe) runCmd('echo "Another line" >> README', cwd=srctree) - runCmd('git commit -a --amend --no-edit', cwd=srctree) + runCmd('git commit -a --amend --no-edit --no-verify', cwd=srctree) self.add_command_to_tearDown('cd %s; rm %s/*; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile))) result = runCmd('devtool update-recipe %s' % testrecipe) expected_status = [(' M', '.*/%s/readme.patch.gz$' % testrecipe)] @@ -1332,6 +1602,7 @@ class DevtoolUpdateTests(DevtoolBase): def test_devtool_finish_modify_git_subdir(self): # Check preconditions testrecipe = 'dos2unix' + self.append_config('ERROR_QA:remove:pn-dos2unix = "patch-status"\n') bb_vars = get_bb_vars(['SRC_URI', 'S', 'WORKDIR', 'FILE'], testrecipe) self.assertIn('git://', bb_vars['SRC_URI'], 'This test expects the %s recipe to be a git recipe' % testrecipe) workdir_git = '%s/git/' % bb_vars['WORKDIR'] @@ -1395,6 +1666,53 @@ class DevtoolUpdateTests(DevtoolBase): # Try building bitbake('%s -c patch' % testrecipe) + def test_devtool_git_submodules(self): + # This tests if we can add a patch in a git submodule and extract it properly using devtool finish + # Check preconditions + self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') + self.track_for_cleanup(self.workspacedir) + recipe = 'vulkan-samples' + src_uri = get_bb_var('SRC_URI', recipe) + self.assertIn('gitsm://', src_uri, 'This test expects the %s recipe to be a git recipe with submodules' % recipe) + oldrecipefile = get_bb_var('FILE', recipe) + recipedir = os.path.dirname(oldrecipefile) + result = runCmd('git status --porcelain .', cwd=recipedir) + if result.output.strip(): + self.fail('Recipe directory for %s contains uncommitted changes' % recipe) + self.assertIn('/meta/', recipedir) + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool modify %s %s' % (recipe, tempdir)) + self.assertExists(os.path.join(tempdir, 'CMakeLists.txt'), 'Extracted source could not be found') + # Test devtool status + result = runCmd('devtool status') + self.assertIn(recipe, result.output) + self.assertIn(tempdir, result.output) + # Modify a source file in a submodule, (grab the first one) + result = runCmd('git submodule --quiet foreach \'echo $sm_path\'', cwd=tempdir) + submodule = result.output.splitlines()[0] + submodule_path = os.path.join(tempdir, submodule) + runCmd('echo "#This is a first comment" >> testfile', cwd=submodule_path) + result = runCmd('git status --porcelain . ', cwd=submodule_path) + self.assertIn("testfile", result.output) + runCmd('git add testfile; git commit -m "Adding a new file"', cwd=submodule_path) + + # Try finish to the original layer + self.add_command_to_tearDown('rm -rf %s ; cd %s ; git checkout %s' % (recipedir, os.path.dirname(recipedir), recipedir)) + runCmd('devtool finish -f %s meta' % recipe) + result = runCmd('devtool status') + self.assertNotIn(recipe, result.output, 'Recipe should have been reset by finish but wasn\'t') + self.assertNotExists(os.path.join(self.workspacedir, 'recipes', recipe), 'Recipe directory should not exist after finish') + expected_status = [(' M', '.*/%s$' % os.path.basename(oldrecipefile)), + ('??', '.*/.*-Adding-a-new-file.patch$')] + self._check_repo_status(recipedir, expected_status) + # Make sure the patch is added to the recipe with the correct "patchdir" option + result = runCmd('git diff .', cwd=recipedir) + addlines = [ + 'file://0001-Adding-a-new-file.patch;patchdir=%s \\\\' % submodule + ] + self._check_diff(result.output, addlines, []) class DevtoolExtractTests(DevtoolBase): @@ -1446,28 +1764,7 @@ class DevtoolExtractTests(DevtoolBase): @OETestTag("runqemu") def test_devtool_deploy_target(self): - # NOTE: Whilst this test would seemingly be better placed as a runtime test, - # unfortunately the runtime tests run under bitbake and you can't run - # devtool within bitbake (since devtool needs to run bitbake itself). - # Additionally we are testing build-time functionality as well, so - # really this has to be done as an oe-selftest test. - # - # Check preconditions - machine = get_bb_var('MACHINE') - if not machine.startswith('qemu'): - self.skipTest('This test only works with qemu machines') - if not os.path.exists('/etc/runqemu-nosudo'): - self.skipTest('You must set up tap devices with scripts/runqemu-gen-tapdevs before running this test') - result = runCmd('PATH="$PATH:/sbin:/usr/sbin" ip tuntap show', ignore_status=True) - if result.status != 0: - result = runCmd('PATH="$PATH:/sbin:/usr/sbin" ifconfig -a', ignore_status=True) - if result.status != 0: - self.skipTest('Failed to determine if tap devices exist with ifconfig or ip: %s' % result.output) - for line in result.output.splitlines(): - if line.startswith('tap'): - break - else: - self.skipTest('No tap devices found - you must set up tap devices with scripts/runqemu-gen-tapdevs before running this test') + self._check_runqemu_prerequisites() self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') # Definitions testrecipe = 'mdadm' @@ -1653,6 +1950,54 @@ class DevtoolUpgradeTests(DevtoolBase): self.assertNotIn(recipe, result.output) self.assertNotExists(os.path.join(self.workspacedir, 'recipes', recipe), 'Recipe directory should not exist after resetting') + def test_devtool_upgrade_drop_md5sum(self): + # Check preconditions + self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + # For the moment, we are using a real recipe. + recipe = 'devtool-upgrade-test3' + version = '1.6.0' + oldrecipefile = get_bb_var('FILE', recipe) + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + # Check upgrade. Code does not check if new PV is older or newer that current PV, so, it may be that + # we are downgrading instead of upgrading. + result = runCmd('devtool upgrade %s %s -V %s' % (recipe, tempdir, version)) + # Check new recipe file is present + newrecipefile = os.path.join(self.workspacedir, 'recipes', recipe, '%s_%s.bb' % (recipe, version)) + self.assertExists(newrecipefile, 'Recipe file should exist after upgrade') + # Check recipe got changed as expected + with open(oldrecipefile + '.upgraded', 'r') as f: + desiredlines = f.readlines() + with open(newrecipefile, 'r') as f: + newlines = f.readlines() + self.assertEqual(desiredlines, newlines) + + def test_devtool_upgrade_all_checksums(self): + # Check preconditions + self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + # For the moment, we are using a real recipe. + recipe = 'devtool-upgrade-test4' + version = '1.6.0' + oldrecipefile = get_bb_var('FILE', recipe) + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + # Check upgrade. Code does not check if new PV is older or newer that current PV, so, it may be that + # we are downgrading instead of upgrading. + result = runCmd('devtool upgrade %s %s -V %s' % (recipe, tempdir, version)) + # Check new recipe file is present + newrecipefile = os.path.join(self.workspacedir, 'recipes', recipe, '%s_%s.bb' % (recipe, version)) + self.assertExists(newrecipefile, 'Recipe file should exist after upgrade') + # Check recipe got changed as expected + with open(oldrecipefile + '.upgraded', 'r') as f: + desiredlines = f.readlines() + with open(newrecipefile, 'r') as f: + newlines = f.readlines() + self.assertEqual(desiredlines, newlines) + def test_devtool_layer_plugins(self): """Test that devtool can use plugins from other layers. @@ -1671,7 +2016,15 @@ class DevtoolUpgradeTests(DevtoolBase): for p in paths: dstdir = os.path.join(dstdir, p) if not os.path.exists(dstdir): - os.makedirs(dstdir) + try: + os.makedirs(dstdir) + except PermissionError: + return False + except OSError as e: + if e.errno == errno.EROFS: + return False + else: + raise e if p == "lib": # Can race with other tests self.add_command_to_tearDown('rmdir --ignore-fail-on-non-empty %s' % dstdir) @@ -1679,8 +2032,12 @@ class DevtoolUpgradeTests(DevtoolBase): self.track_for_cleanup(dstdir) dstfile = os.path.join(dstdir, os.path.basename(srcfile)) if srcfile != dstfile: - shutil.copy(srcfile, dstfile) + try: + shutil.copy(srcfile, dstfile) + except PermissionError: + return False self.track_for_cleanup(dstfile) + return True def test_devtool_load_plugin(self): """Test that devtool loads only the first found plugin in BBPATH.""" @@ -1698,15 +2055,17 @@ class DevtoolUpgradeTests(DevtoolBase): plugincontent = fh.readlines() try: self.assertIn('meta-selftest', srcfile, 'wrong bbpath plugin found') - for path in searchpath: - self._copy_file_with_cleanup(srcfile, path, 'lib', 'devtool') + searchpath = [ + path for path in searchpath + if self._copy_file_with_cleanup(srcfile, path, 'lib', 'devtool') + ] result = runCmd("devtool --quiet count") self.assertEqual(result.output, '1') result = runCmd("devtool --quiet multiloaded") self.assertEqual(result.output, "no") for path in searchpath: result = runCmd("devtool --quiet bbdir") - self.assertEqual(result.output, path) + self.assertEqual(os.path.realpath(result.output), os.path.realpath(path)) os.unlink(os.path.join(result.output, 'lib', 'devtool', 'bbpath.py')) finally: with open(srcfile, 'w') as fh: @@ -1887,6 +2246,52 @@ class DevtoolUpgradeTests(DevtoolBase): if files: self.fail('Unexpected file(s) copied next to bbappend: %s' % ', '.join(files)) + def test_devtool_finish_update_patch(self): + # This test uses a modified version of the sysdig recipe from meta-oe. + # - The patches have been renamed. + # - The dependencies are commented out since the recipe is not being + # built. + # + # The sysdig recipe is interesting in that it fetches two different Git + # repositories, and there are patches for both. This leads to that + # devtool will create ignore commits as it uses Git submodules to keep + # track of the second repository. + # + # This test will verify that the ignored commits actually are ignored + # when a commit in between is modified. It will also verify that the + # updated patch keeps its original name. + + # Check preconditions + self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') + # Try modifying a recipe + self.track_for_cleanup(self.workspacedir) + recipe = 'sysdig-selftest' + recipefile = get_bb_var('FILE', recipe) + recipedir = os.path.dirname(recipefile) + result = runCmd('git status --porcelain .', cwd=recipedir) + if result.output.strip(): + self.fail('Recipe directory for %s contains uncommitted changes' % recipe) + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool modify %s %s' % (recipe, tempdir)) + self.add_command_to_tearDown('cd %s; rm %s/*; git checkout %s %s' % (recipedir, recipe, recipe, os.path.basename(recipefile))) + self.assertExists(os.path.join(tempdir, 'CMakeLists.txt'), 'Extracted source could not be found') + # Make a change to one of the existing commits + result = runCmd('echo "# A comment " >> CMakeLists.txt', cwd=tempdir) + result = runCmd('git status --porcelain', cwd=tempdir) + self.assertIn('M CMakeLists.txt', result.output) + result = runCmd('git commit --fixup HEAD^ CMakeLists.txt', cwd=tempdir) + result = runCmd('git show -s --format=%s', cwd=tempdir) + self.assertIn('fixup! cmake: Pass PROBE_NAME via CFLAGS', result.output) + result = runCmd('GIT_SEQUENCE_EDITOR=true git rebase -i --autosquash devtool-base', cwd=tempdir) + result = runCmd('devtool finish %s meta-selftest' % recipe) + result = runCmd('devtool status') + self.assertNotIn(recipe, result.output, 'Recipe should have been reset by finish but wasn\'t') + self.assertNotExists(os.path.join(self.workspacedir, 'recipes', recipe), 'Recipe directory should not exist after finish') + expected_status = [(' M', '.*/0099-cmake-Pass-PROBE_NAME-via-CFLAGS.patch$')] + self._check_repo_status(recipedir, expected_status) + def test_devtool_rename(self): # Check preconditions self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') @@ -1923,7 +2328,6 @@ class DevtoolUpgradeTests(DevtoolBase): self._test_recipe_contents(newrecipefile, checkvars, []) # Try again - change just name this time result = runCmd('devtool reset -n %s' % newrecipename) - shutil.rmtree(newsrctree) add_recipe() newrecipefile = os.path.join(self.workspacedir, 'recipes', newrecipename, '%s_%s.bb' % (newrecipename, recipever)) result = runCmd('devtool rename %s %s' % (recipename, newrecipename)) @@ -1936,7 +2340,6 @@ class DevtoolUpgradeTests(DevtoolBase): self._test_recipe_contents(newrecipefile, checkvars, []) # Try again - change just version this time result = runCmd('devtool reset -n %s' % newrecipename) - shutil.rmtree(newsrctree) add_recipe() newrecipefile = os.path.join(self.workspacedir, 'recipes', recipename, '%s_%s.bb' % (recipename, newrecipever)) result = runCmd('devtool rename %s -V %s' % (recipename, newrecipever)) @@ -2007,7 +2410,9 @@ class DevtoolUpgradeTests(DevtoolBase): #Modify the kernel source modfile = os.path.join(tempdir, 'init/version.c') - runCmd("sed -i 's/Linux/LiNuX/g' %s" % (modfile)) + # Moved to uts.h in 6.1 onwards + modfile2 = os.path.join(tempdir, 'include/linux/uts.h') + runCmd("sed -i 's/Linux/LiNuX/g' %s %s" % (modfile, modfile2)) #Modify the configuration codeconfigfile = os.path.join(tempdir, '.config.new') @@ -2022,3 +2427,518 @@ class DevtoolUpgradeTests(DevtoolBase): #Step 4.5 runCmd("grep %s %s" % (modconfopt, codeconfigfile)) + + +class DevtoolIdeSdkTests(DevtoolBase): + def _write_bb_config(self, recipe_names): + """Helper to write the bitbake local.conf file""" + conf_lines = [ + 'IMAGE_CLASSES += "image-combined-dbg"', + 'IMAGE_GEN_DEBUGFS = "1"', + 'IMAGE_INSTALL:append = " gdbserver %s"' % ' '.join( + [r + '-ptest' for r in recipe_names]) + ] + self.write_config("\n".join(conf_lines)) + + def _check_workspace(self): + """Check if a workspace directory is available and setup the cleanup""" + self.assertTrue(not os.path.exists(self.workspacedir), + 'This test cannot be run with a workspace directory under the build directory') + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + + def _workspace_scripts_dir(self, recipe_name): + return os.path.realpath(os.path.join(self.builddir, 'workspace', 'ide-sdk', recipe_name, 'scripts')) + + def _sources_scripts_dir(self, src_dir): + return os.path.realpath(os.path.join(src_dir, 'oe-scripts')) + + def _workspace_gdbinit_dir(self, recipe_name): + return os.path.realpath(os.path.join(self.builddir, 'workspace', 'ide-sdk', recipe_name, 'scripts', 'gdbinit')) + + def _sources_gdbinit_dir(self, src_dir): + return os.path.realpath(os.path.join(src_dir, 'oe-gdbinit')) + + def _devtool_ide_sdk_recipe(self, recipe_name, build_file, testimage): + """Setup a recipe for working with devtool ide-sdk + + Basically devtool modify -x followed by some tests + """ + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.add_command_to_tearDown('bitbake -c clean %s' % recipe_name) + + result = runCmd('devtool modify %s -x %s' % (recipe_name, tempdir)) + self.assertExists(os.path.join(tempdir, build_file), + 'Extracted source could not be found') + self.assertExists(os.path.join(self.workspacedir, 'conf', + 'layer.conf'), 'Workspace directory not created') + matches = glob.glob(os.path.join(self.workspacedir, + 'appends', recipe_name + '.bbappend')) + self.assertTrue(matches, 'bbappend not created %s' % result.output) + + # Test devtool status + result = runCmd('devtool status') + self.assertIn(recipe_name, result.output) + self.assertIn(tempdir, result.output) + self._check_src_repo(tempdir) + + # Usually devtool ide-sdk would initiate the build of the SDK. + # But there is a circular dependency with starting Qemu and passing the IP of runqemu to devtool ide-sdk. + if testimage: + bitbake("%s qemu-native qemu-helper-native" % testimage) + deploy_dir_image = get_bb_var('DEPLOY_DIR_IMAGE') + self.add_command_to_tearDown('bitbake -c clean %s' % testimage) + self.add_command_to_tearDown( + 'rm -f %s/%s*' % (deploy_dir_image, testimage)) + + return tempdir + + def _get_recipe_ids(self, recipe_name): + """IDs needed to write recipe specific config entries into IDE config files""" + package_arch = get_bb_var('PACKAGE_ARCH', recipe_name) + recipe_id = recipe_name + "-" + package_arch + recipe_id_pretty = recipe_name + ": " + package_arch + return (recipe_id, recipe_id_pretty) + + def _verify_install_script_code(self, tempdir, recipe_name): + """Verify the scripts referred by the tasks.json file are fine. + + This function does not depend on Qemu. Therefore it verifies the scripts + exists and the delete step works as expected. But it does not try to + deploy to Qemu. + """ + recipe_id, recipe_id_pretty = self._get_recipe_ids(recipe_name) + with open(os.path.join(tempdir, '.vscode', 'tasks.json')) as tasks_j: + tasks_d = json.load(tasks_j) + tasks = tasks_d["tasks"] + task_install = next( + (task for task in tasks if task["label"] == "install && deploy-target %s" % recipe_id_pretty), None) + self.assertIsNot(task_install, None) + # execute only the bb_run_do_install script since the deploy would require e.g. Qemu running. + i_and_d_script = "install_and_deploy_" + recipe_id + i_and_d_script_path = os.path.join( + self._workspace_scripts_dir(recipe_name), i_and_d_script) + self.assertExists(i_and_d_script_path) + del_script = "delete_package_dirs_" + recipe_id + del_script_path = os.path.join( + self._workspace_scripts_dir(recipe_name), del_script) + self.assertExists(del_script_path) + runCmd(del_script_path, cwd=tempdir) + + def _devtool_ide_sdk_qemu(self, tempdir, qemu, recipe_name, example_exe): + """Verify deployment and execution in Qemu system work for one recipe. + + This function checks the entire SDK workflow: changing the code, recompiling + it and deploying it back to Qemu, and checking that the changes have been + incorporated into the provided binaries. It also runs the tests of the recipe. + """ + recipe_id, _ = self._get_recipe_ids(recipe_name) + i_and_d_script = "install_and_deploy_" + recipe_id + install_deploy_cmd = os.path.join( + self._workspace_scripts_dir(recipe_name), i_and_d_script) + self.assertExists(install_deploy_cmd, + '%s script not found' % install_deploy_cmd) + runCmd(install_deploy_cmd) + + MAGIC_STRING_ORIG = "Magic: 123456789" + MAGIC_STRING_NEW = "Magic: 987654321" + ptest_cmd = "ptest-runner " + recipe_name + + # validate that SSH is working + status, _ = qemu.run("uname") + self.assertEqual( + status, 0, msg="Failed to connect to the SSH server on Qemu") + + # Verify the unmodified example prints the magic string + status, output = qemu.run(example_exe) + self.assertEqual(status, 0, msg="%s failed: %s" % + (example_exe, output)) + self.assertIn(MAGIC_STRING_ORIG, output) + + # Verify the unmodified ptests work + status, output = qemu.run(ptest_cmd) + self.assertEqual(status, 0, msg="%s failed: %s" % (ptest_cmd, output)) + self.assertIn("PASS: cpp-example-lib", output) + + # Verify remote debugging works + self._gdb_cross_debugging( + qemu, recipe_name, example_exe, MAGIC_STRING_ORIG) + + # Replace the Magic String in the code, compile and deploy to Qemu + cpp_example_lib_hpp = os.path.join(tempdir, 'cpp-example-lib.hpp') + with open(cpp_example_lib_hpp, 'r') as file: + cpp_code = file.read() + cpp_code = cpp_code.replace(MAGIC_STRING_ORIG, MAGIC_STRING_NEW) + with open(cpp_example_lib_hpp, 'w') as file: + file.write(cpp_code) + runCmd(install_deploy_cmd, cwd=tempdir) + + # Verify the modified example prints the modified magic string + status, output = qemu.run(example_exe) + self.assertEqual(status, 0, msg="%s failed: %s" % + (example_exe, output)) + self.assertNotIn(MAGIC_STRING_ORIG, output) + self.assertIn(MAGIC_STRING_NEW, output) + + # Verify the modified example ptests work + status, output = qemu.run(ptest_cmd) + self.assertEqual(status, 0, msg="%s failed: %s" % (ptest_cmd, output)) + self.assertIn("PASS: cpp-example-lib", output) + + # Verify remote debugging works wit the modified magic string + self._gdb_cross_debugging( + qemu, recipe_name, example_exe, MAGIC_STRING_NEW) + + def _gdb_cross(self): + """Verify gdb-cross is provided by devtool ide-sdk""" + target_arch = self.td["TARGET_ARCH"] + target_sys = self.td["TARGET_SYS"] + gdb_recipe = "gdb-cross-" + target_arch + gdb_binary = target_sys + "-gdb" + + native_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", gdb_recipe) + r = runCmd("%s --version" % gdb_binary, + native_sysroot=native_sysroot, target_sys=target_sys) + self.assertEqual(r.status, 0) + self.assertIn("GNU gdb", r.output) + + def _gdb_cross_debugging(self, qemu, recipe_name, example_exe, magic_string): + """Verify gdb-cross is working + + Test remote debugging: + break main + run + continue + break CppExample::print_json() + continue + print CppExample::test_string.compare("cpp-example-lib Magic: 123456789") + $1 = 0 + print CppExample::test_string.compare("cpp-example-lib Magic: 123456789aaa") + $2 = -3 + list cpp-example-lib.hpp:13,13 + 13 inline static const std::string test_string = "cpp-example-lib Magic: 123456789"; + continue + """ + sshargs = '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' + gdbserver_script = os.path.join(self._workspace_scripts_dir( + recipe_name), 'gdbserver_1234_usr-bin-' + example_exe + '_m') + gdb_script = os.path.join(self._workspace_scripts_dir( + recipe_name), 'gdb_1234_usr-bin-' + example_exe) + + # Start a gdbserver + r = runCmd(gdbserver_script) + self.assertEqual(r.status, 0) + + # Check there is a gdbserver running + r = runCmd('ssh %s root@%s %s' % (sshargs, qemu.ip, 'ps')) + self.assertEqual(r.status, 0) + self.assertIn("gdbserver ", r.output) + + # Check the pid file is correct + test_cmd = "cat /proc/$(cat /tmp/gdbserver_1234_usr-bin-" + \ + example_exe + "/pid)/cmdline" + r = runCmd('ssh %s root@%s %s' % (sshargs, qemu.ip, test_cmd)) + self.assertEqual(r.status, 0) + self.assertIn("gdbserver", r.output) + + # Test remote debugging works + gdb_batch_cmd = " --batch -ex 'break main' -ex 'run'" + gdb_batch_cmd += " -ex 'break CppExample::print_json()' -ex 'continue'" + gdb_batch_cmd += " -ex 'print CppExample::test_string.compare(\"cpp-example-lib %s\")'" % magic_string + gdb_batch_cmd += " -ex 'print CppExample::test_string.compare(\"cpp-example-lib %saaa\")'" % magic_string + gdb_batch_cmd += " -ex 'list cpp-example-lib.hpp:13,13'" + gdb_batch_cmd += " -ex 'continue'" + r = runCmd(gdb_script + gdb_batch_cmd) + self.logger.debug("%s %s returned: %s", gdb_script, + gdb_batch_cmd, r.output) + self.assertEqual(r.status, 0) + self.assertIn("Breakpoint 1, main", r.output) + self.assertIn("$1 = 0", r.output) # test.string.compare equal + self.assertIn("$2 = -3", r.output) # test.string.compare longer + self.assertIn( + 'inline static const std::string test_string = "cpp-example-lib %s";' % magic_string, r.output) + self.assertIn("exited normally", r.output) + + # Stop the gdbserver + r = runCmd(gdbserver_script + ' stop') + self.assertEqual(r.status, 0) + + # Check there is no gdbserver running + r = runCmd('ssh %s root@%s %s' % (sshargs, qemu.ip, 'ps')) + self.assertEqual(r.status, 0) + self.assertNotIn("gdbserver ", r.output) + + def _verify_cmake_preset(self, tempdir): + """Verify the generated cmake preset works as expected + + Check if compiling works + Check if unit tests can be executed in qemu (not qemu-system) + """ + with open(os.path.join(tempdir, 'CMakeUserPresets.json')) as cmake_preset_j: + cmake_preset_d = json.load(cmake_preset_j) + config_presets = cmake_preset_d["configurePresets"] + self.assertEqual(len(config_presets), 1) + cmake_exe = config_presets[0]["cmakeExecutable"] + preset_name = config_presets[0]["name"] + + # Verify the wrapper for cmake native is available + self.assertExists(cmake_exe) + + # Verify the cmake preset generated by devtool ide-sdk is available + result = runCmd('%s --list-presets' % cmake_exe, cwd=tempdir) + self.assertIn(preset_name, result.output) + + # Verify cmake re-uses the o files compiled by bitbake + result = runCmd('%s --build --preset %s' % + (cmake_exe, preset_name), cwd=tempdir) + self.assertIn("ninja: no work to do.", result.output) + + # Verify the unit tests work (in Qemu user mode) + result = runCmd('%s --build --preset %s --target test' % + (cmake_exe, preset_name), cwd=tempdir) + self.assertIn("100% tests passed", result.output) + + # Verify re-building and testing works again + result = runCmd('%s --build --preset %s --target clean' % + (cmake_exe, preset_name), cwd=tempdir) + self.assertIn("Cleaning", result.output) + result = runCmd('%s --build --preset %s' % + (cmake_exe, preset_name), cwd=tempdir) + self.assertIn("Building", result.output) + self.assertIn("Linking", result.output) + result = runCmd('%s --build --preset %s --target test' % + (cmake_exe, preset_name), cwd=tempdir) + self.assertIn("Running tests...", result.output) + self.assertIn("100% tests passed", result.output) + + @OETestTag("runqemu") + def test_devtool_ide_sdk_none_qemu(self): + """Start qemu-system and run tests for multiple recipes. ide=none is used.""" + recipe_names = ["cmake-example", "meson-example"] + testimage = "oe-selftest-image" + + self._check_workspace() + self._write_bb_config(recipe_names) + self._check_runqemu_prerequisites() + + # Verify deployment to Qemu (system mode) works + bitbake(testimage) + with runqemu(testimage, runqemuparams="nographic") as qemu: + # cmake-example recipe + recipe_name = "cmake-example" + example_exe = "cmake-example" + build_file = "CMakeLists.txt" + tempdir = self._devtool_ide_sdk_recipe( + recipe_name, build_file, testimage) + bitbake_sdk_cmd = 'devtool ide-sdk %s %s -t root@%s -c --ide=none' % ( + recipe_name, testimage, qemu.ip) + runCmd(bitbake_sdk_cmd) + self._gdb_cross() + self._verify_cmake_preset(tempdir) + self._devtool_ide_sdk_qemu(tempdir, qemu, recipe_name, example_exe) + # Verify the oe-scripts sym-link is valid + self.assertEqual(self._workspace_scripts_dir( + recipe_name), self._sources_scripts_dir(tempdir)) + + # meson-example recipe + recipe_name = "meson-example" + example_exe = "mesonex" + build_file = "meson.build" + tempdir = self._devtool_ide_sdk_recipe( + recipe_name, build_file, testimage) + bitbake_sdk_cmd = 'devtool ide-sdk %s %s -t root@%s -c --ide=none' % ( + recipe_name, testimage, qemu.ip) + runCmd(bitbake_sdk_cmd) + self._gdb_cross() + self._devtool_ide_sdk_qemu(tempdir, qemu, recipe_name, example_exe) + # Verify the oe-scripts sym-link is valid + self.assertEqual(self._workspace_scripts_dir( + recipe_name), self._sources_scripts_dir(tempdir)) + + def test_devtool_ide_sdk_code_cmake(self): + """Verify a cmake recipe works with ide=code mode""" + recipe_name = "cmake-example" + build_file = "CMakeLists.txt" + testimage = "oe-selftest-image" + + self._check_workspace() + self._write_bb_config([recipe_name]) + tempdir = self._devtool_ide_sdk_recipe( + recipe_name, build_file, testimage) + bitbake_sdk_cmd = 'devtool ide-sdk %s %s -t root@192.168.17.17 -c --ide=code' % ( + recipe_name, testimage) + runCmd(bitbake_sdk_cmd) + self._verify_cmake_preset(tempdir) + self._verify_install_script_code(tempdir, recipe_name) + self._gdb_cross() + + def test_devtool_ide_sdk_code_meson(self): + """Verify a meson recipe works with ide=code mode""" + recipe_name = "meson-example" + build_file = "meson.build" + testimage = "oe-selftest-image" + + self._check_workspace() + self._write_bb_config([recipe_name]) + tempdir = self._devtool_ide_sdk_recipe( + recipe_name, build_file, testimage) + bitbake_sdk_cmd = 'devtool ide-sdk %s %s -t root@192.168.17.17 -c --ide=code' % ( + recipe_name, testimage) + runCmd(bitbake_sdk_cmd) + + with open(os.path.join(tempdir, '.vscode', 'settings.json')) as settings_j: + settings_d = json.load(settings_j) + meson_exe = settings_d["mesonbuild.mesonPath"] + meson_build_folder = settings_d["mesonbuild.buildFolder"] + + # Verify the wrapper for meson native is available + self.assertExists(meson_exe) + + # Verify meson re-uses the o files compiled by bitbake + result = runCmd('%s compile -C %s' % + (meson_exe, meson_build_folder), cwd=tempdir) + self.assertIn("ninja: no work to do.", result.output) + + # Verify the unit tests work (in Qemu) + runCmd('%s test -C %s' % (meson_exe, meson_build_folder), cwd=tempdir) + + # Verify re-building and testing works again + result = runCmd('%s compile -C %s --clean' % + (meson_exe, meson_build_folder), cwd=tempdir) + self.assertIn("Cleaning...", result.output) + result = runCmd('%s compile -C %s' % + (meson_exe, meson_build_folder), cwd=tempdir) + self.assertIn("Linking target", result.output) + runCmd('%s test -C %s' % (meson_exe, meson_build_folder), cwd=tempdir) + + self._verify_install_script_code(tempdir, recipe_name) + self._gdb_cross() + + def test_devtool_ide_sdk_shared_sysroots(self): + """Verify the shared sysroot SDK""" + + # Handle the workspace (which is not needed by this test case) + self._check_workspace() + + result_init = runCmd( + 'devtool ide-sdk -m shared oe-selftest-image cmake-example meson-example --ide=code') + bb_vars = get_bb_vars( + ['REAL_MULTIMACH_TARGET_SYS', 'DEPLOY_DIR_IMAGE', 'COREBASE'], "meta-ide-support") + environment_script = 'environment-setup-%s' % bb_vars['REAL_MULTIMACH_TARGET_SYS'] + deploydir = bb_vars['DEPLOY_DIR_IMAGE'] + environment_script_path = os.path.join(deploydir, environment_script) + cpp_example_src = os.path.join( + bb_vars['COREBASE'], 'meta-selftest', 'recipes-test', 'cpp', 'files') + + # Verify the cross environment script is available + self.assertExists(environment_script_path) + + def runCmdEnv(cmd, cwd): + cmd = '/bin/sh -c ". %s > /dev/null && %s"' % ( + environment_script_path, cmd) + return runCmd(cmd, cwd) + + # Verify building the C++ example works with CMake + tempdir_cmake = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir_cmake) + + result_cmake = runCmdEnv("which cmake", cwd=tempdir_cmake) + cmake_native = os.path.normpath(result_cmake.output.strip()) + self.assertExists(cmake_native) + + runCmdEnv('cmake %s' % cpp_example_src, cwd=tempdir_cmake) + runCmdEnv('cmake --build %s' % tempdir_cmake, cwd=tempdir_cmake) + + # Verify the printed note really referres to a cmake executable + cmake_native_code = "" + for line in result_init.output.splitlines(): + m = re.search(r'"cmake.cmakePath": "(.*)"', line) + if m: + cmake_native_code = m.group(1) + break + self.assertExists(cmake_native_code) + self.assertEqual(cmake_native, cmake_native_code) + + # Verify building the C++ example works with Meson + tempdir_meson = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir_meson) + + result_cmake = runCmdEnv("which meson", cwd=tempdir_meson) + meson_native = os.path.normpath(result_cmake.output.strip()) + self.assertExists(meson_native) + + runCmdEnv('meson setup %s' % tempdir_meson, cwd=cpp_example_src) + runCmdEnv('meson compile', cwd=tempdir_meson) + + def test_devtool_ide_sdk_plugins(self): + """Test that devtool ide-sdk can use plugins from other layers.""" + + # We need a workspace layer and a modified recipe (but no image) + modified_recipe_name = "meson-example" + modified_build_file = "meson.build" + testimage = "oe-selftest-image" + shared_recipe_name = "cmake-example" + + self._check_workspace() + self._write_bb_config([modified_recipe_name]) + tempdir = self._devtool_ide_sdk_recipe( + modified_recipe_name, modified_build_file, None) + + IDE_RE = re.compile(r'.*--ide \{(.*)\}.*') + + def get_ides_from_help(help_str): + m = IDE_RE.search(help_str) + return m.group(1).split(',') + + # verify the default plugins are available but the foo plugin is not + result = runCmd('devtool ide-sdk -h') + found_ides = get_ides_from_help(result.output) + self.assertIn('code', found_ides) + self.assertIn('none', found_ides) + self.assertNotIn('foo', found_ides) + + shared_config_file = os.path.join(tempdir, 'shared-config.txt') + shared_config_str = 'Dummy shared IDE config' + modified_config_file = os.path.join(tempdir, 'modified-config.txt') + modified_config_str = 'Dummy modified IDE config' + + # Generate a foo plugin in the workspace layer + plugin_dir = os.path.join( + self.workspacedir, 'lib', 'devtool', 'ide_plugins') + os.makedirs(plugin_dir) + plugin_code = 'from devtool.ide_plugins import IdeBase\n\n' + plugin_code += 'class IdeFoo(IdeBase):\n' + plugin_code += ' def setup_shared_sysroots(self, shared_env):\n' + plugin_code += ' with open("%s", "w") as config_file:\n' % shared_config_file + plugin_code += ' config_file.write("%s")\n\n' % shared_config_str + plugin_code += ' def setup_modified_recipe(self, args, image_recipe, modified_recipe):\n' + plugin_code += ' with open("%s", "w") as config_file:\n' % modified_config_file + plugin_code += ' config_file.write("%s")\n\n' % modified_config_str + plugin_code += 'def register_ide_plugin(ide_plugins):\n' + plugin_code += ' ide_plugins["foo"] = IdeFoo\n' + + plugin_py = os.path.join(plugin_dir, 'ide_foo.py') + with open(plugin_py, 'w') as plugin_file: + plugin_file.write(plugin_code) + + # Verify the foo plugin is available as well + result = runCmd('devtool ide-sdk -h') + found_ides = get_ides_from_help(result.output) + self.assertIn('code', found_ides) + self.assertIn('none', found_ides) + self.assertIn('foo', found_ides) + + # Verify the foo plugin generates a shared config + result = runCmd( + 'devtool ide-sdk -m shared --skip-bitbake --ide foo %s' % shared_recipe_name) + with open(shared_config_file) as shared_config: + shared_config_new = shared_config.read() + self.assertEqual(shared_config_str, shared_config_new) + + # Verify the foo plugin generates a modified config + result = runCmd('devtool ide-sdk --skip-bitbake --ide foo %s %s' % + (modified_recipe_name, testimage)) + with open(modified_config_file) as modified_config: + modified_config_new = modified_config.read() + self.assertEqual(modified_config_str, modified_config_new) diff --git a/meta/lib/oeqa/selftest/cases/distrodata.py b/meta/lib/oeqa/selftest/cases/distrodata.py index b80d091c1c..ad952c004b 100644 --- a/meta/lib/oeqa/selftest/cases/distrodata.py +++ b/meta/lib/oeqa/selftest/cases/distrodata.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -37,42 +39,6 @@ but their recipes claim otherwise by setting UPSTREAM_VERSION_UNKNOWN. Please re """ + "\n".join(regressed_successes) self.assertTrue(len(regressed_failures) == 0 and len(regressed_successes) == 0, msg) - def test_missing_homepg(self): - """ - Summary: Test for oe-core recipes that don't have a HOMEPAGE or DESCRIPTION - Expected: All oe-core recipes should have a DESCRIPTION entry - Expected: All oe-core recipes should have a HOMEPAGE entry except for recipes that are not fetched from external sources. - Product: oe-core - """ - with bb.tinfoil.Tinfoil() as tinfoil: - tinfoil.prepare(config_only=False) - no_description = [] - no_homepage = [] - for fn in tinfoil.all_recipe_files(variants=False): - if not '/meta/recipes-' in fn: - # We are only interested in OE-Core - continue - rd = tinfoil.parse_recipe_file(fn, appends=False) - pn = rd.getVar('BPN') - srcfile = rd.getVar('SRC_URI').split() - #Since DESCRIPTION defaults to SUMMARY if not set, we are only interested in recipes without DESCRIPTION or SUMMARY - if not (rd.getVar('SUMMARY') or rd.getVar('DESCRIPTION')): - no_description.append((pn, fn)) - if not rd.getVar('HOMEPAGE'): - if srcfile and srcfile[0].startswith('file') or not rd.getVar('SRC_URI'): - # We are only interested in recipes SRC_URI fetched from external sources - continue - no_homepage.append((pn, fn)) - if no_homepage: - self.fail(""" -The following recipes do not have a HOMEPAGE. Please add an entry for HOMEPAGE in the recipe. -""" + "\n".join(['%s (%s)' % i for i in no_homepage])) - - if no_description: - self.fail(""" -The following recipes do not have a DESCRIPTION. Please add an entry for DESCRIPTION in the recipe. -""" + "\n".join(['%s (%s)' % i for i in no_description])) - def test_maintainers(self): """ Summary: Test that oe-core recipes have a maintainer and entries in maintainers list have a recipe @@ -82,15 +48,15 @@ The following recipes do not have a DESCRIPTION. Please add an entry for DESCRIP Author: Alexander Kanavin <alex.kanavin@gmail.com> """ def is_exception(pkg): - exceptions = ["packagegroup-", "initramfs-", "systemd-machine-units", "target-sdk-provides-dummy"] + exceptions = ["packagegroup-",] for i in exceptions: if i in pkg: return True return False def is_maintainer_exception(entry): - exceptions = ["musl", "newlib", "linux-yocto", "linux-dummy", "mesa-gl", "libgfortran", - "cve-update-db-native", "rust"] + exceptions = ["musl", "newlib", "linux-yocto", "linux-dummy", "mesa-gl", "libgfortran", "libx11-compose-data", + "cve-update-nvd2-native",] for i in exceptions: if i in entry: return True diff --git a/meta/lib/oeqa/selftest/cases/efibootpartition.py b/meta/lib/oeqa/selftest/cases/efibootpartition.py index 26de3a07c9..fa74103dec 100644 --- a/meta/lib/oeqa/selftest/cases/efibootpartition.py +++ b/meta/lib/oeqa/selftest/cases/efibootpartition.py @@ -5,42 +5,29 @@ # SPDX-License-Identifier: MIT # -import re - from oeqa.selftest.case import OESelftestTestCase -from oeqa.utils.commands import bitbake, runqemu, get_bb_var +from oeqa.utils.commands import bitbake, runqemu +from oeqa.core.decorator.data import skipIfNotMachine +import oe.types class GenericEFITest(OESelftestTestCase): """EFI booting test class""" + @skipIfNotMachine("qemux86-64", "test is qemux86-64 specific currently") + def test_boot_efi(self): + cmd = "runqemu nographic serial wic ovmf" + if oe.types.qemu_use_kvm(self.td.get('QEMU_USE_KVM', 0), self.td["TARGET_ARCH"]): + cmd += " kvm" + image = "core-image-minimal" - cmd_common = "runqemu nographic serial wic ovmf" - efi_provider = "systemd-boot" - image = "core-image-minimal" - machine = "qemux86-64" - recipes_built = False - - @classmethod - def setUpLocal(self): - super(GenericEFITest, self).setUpLocal(self) - - self.write_config(self, -""" -EFI_PROVIDER = "%s" + self.write_config(""" +EFI_PROVIDER = "systemd-boot" IMAGE_FSTYPES:pn-%s:append = " wic" -MACHINE = "%s" MACHINE_FEATURES:append = " efi" WKS_FILE = "efi-bootdisk.wks.in" IMAGE_INSTALL:append = " grub-efi systemd-boot kernel-image-bzimage" """ -% (self.efi_provider, self.image, self.machine)) - if not self.recipes_built: - bitbake("ovmf") - bitbake(self.image) - self.recipes_built = True +% (image)) - @classmethod - def test_boot_efi(self): - """Test generic boot partition with qemu""" - cmd = "%s %s" % (self.cmd_common, self.machine) - with runqemu(self.image, ssh=False, launch_cmd=cmd) as qemu: + bitbake(image + " ovmf") + with runqemu(image, ssh=False, launch_cmd=cmd) as qemu: self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd) diff --git a/meta/lib/oeqa/selftest/cases/eSDK.py b/meta/lib/oeqa/selftest/cases/esdk.py index 3ea0f66357..9f5de2cde7 100644 --- a/meta/lib/oeqa/selftest/cases/eSDK.py +++ b/meta/lib/oeqa/selftest/cases/esdk.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/selftest/cases/externalsrc.py b/meta/lib/oeqa/selftest/cases/externalsrc.py new file mode 100644 index 0000000000..1d800dc82c --- /dev/null +++ b/meta/lib/oeqa/selftest/cases/externalsrc.py @@ -0,0 +1,44 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +import os +import shutil +import tempfile + +from oeqa.selftest.case import OESelftestTestCase +from oeqa.utils.commands import get_bb_var, runCmd + +class ExternalSrc(OESelftestTestCase): + # test that srctree_hash_files does not crash + # we should be actually checking do_compile[file-checksums] but oeqa currently does not support it + # so we check only that a recipe with externalsrc can be parsed + def test_externalsrc_srctree_hash_files(self): + test_recipe = "git-submodule-test" + git_url = "git://git.yoctoproject.org/git-submodule-test" + externalsrc_dir = tempfile.TemporaryDirectory(prefix="externalsrc").name + + self.write_config( + """ +INHERIT += "externalsrc" +EXTERNALSRC:pn-%s = "%s" +""" % (test_recipe, externalsrc_dir) + ) + + # test with git without submodules + runCmd('git clone %s %s' % (git_url, externalsrc_dir)) + os.unlink(externalsrc_dir + "/.gitmodules") + open(".gitmodules", 'w').close() # local file .gitmodules in cwd should not affect externalsrc parsing + self.assertEqual(get_bb_var("S", test_recipe), externalsrc_dir, msg = "S does not equal to EXTERNALSRC") + os.unlink(".gitmodules") + + # test with git with submodules + runCmd('git checkout .gitmodules', cwd=externalsrc_dir) + runCmd('git submodule update --init --recursive', cwd=externalsrc_dir) + self.assertEqual(get_bb_var("S", test_recipe), externalsrc_dir, msg = "S does not equal to EXTERNALSRC") + + # test without git + shutil.rmtree(os.path.join(externalsrc_dir, ".git")) + self.assertEqual(get_bb_var("S", test_recipe), externalsrc_dir, msg = "S does not equal to EXTERNALSRC") diff --git a/meta/lib/oeqa/selftest/cases/fetch.py b/meta/lib/oeqa/selftest/cases/fetch.py index be14272e63..44099176fc 100644 --- a/meta/lib/oeqa/selftest/cases/fetch.py +++ b/meta/lib/oeqa/selftest/cases/fetch.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -34,6 +36,7 @@ PREMIRRORS:forcevariable = "" # No mirrors and broken git, should fail features = """ DL_DIR = "%s" +SRC_URI:pn-dbus-wait = "git://git.yoctoproject.org/dbus-wait;branch=master;protocol=git" GIT_PROXY_COMMAND = "false" MIRRORS:forcevariable = "" PREMIRRORS:forcevariable = "" @@ -46,6 +49,7 @@ PREMIRRORS:forcevariable = "" # Broken git but a specific mirror features = """ DL_DIR = "%s" +SRC_URI:pn-dbus-wait = "git://git.yoctoproject.org/dbus-wait;branch=master;protocol=git" GIT_PROXY_COMMAND = "false" MIRRORS:forcevariable = "git://.*/.* http://downloads.yoctoproject.org/mirror/sources/" """ % dldir @@ -99,7 +103,7 @@ class Dependencies(OESelftestTestCase): r = """ LICENSE="CLOSED" - SRC_URI="git://example.com/repo;branch=master" + SRC_URI="git://example.com/repo;branch=master;rev=ffffffffffffffffffffffffffffffffffffffff" """ f = self.write_recipe(textwrap.dedent(r), tempdir) d = tinfoil.parse_recipe_file(f) diff --git a/meta/lib/oeqa/selftest/cases/fitimage.py b/meta/lib/oeqa/selftest/cases/fitimage.py index d732a9020d..347c065377 100644 --- a/meta/lib/oeqa/selftest/cases/fitimage.py +++ b/meta/lib/oeqa/selftest/cases/fitimage.py @@ -1,9 +1,11 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # from oeqa.selftest.case import OESelftestTestCase -from oeqa.utils.commands import runCmd, bitbake, get_bb_var +from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars import os import re @@ -31,6 +33,8 @@ KERNEL_CLASSES = " kernel-fitimage " # RAM disk variables including load address and entrypoint for kernel and RAM disk IMAGE_FSTYPES += "cpio.gz" INITRAMFS_IMAGE = "core-image-minimal" +# core-image-minimal is used as initramfs here, drop the rootfs suffix +IMAGE_NAME_SUFFIX:pn-core-image-minimal = "" UBOOT_RD_LOADADDRESS = "0x88000000" UBOOT_RD_ENTRYPOINT = "0x88000000" UBOOT_LOADADDRESS = "0x80080000" @@ -40,15 +44,14 @@ FIT_DESC = "A model description" self.write_config(config) # fitImage is created as part of linux recipe - bitbake("virtual/kernel") + image = "virtual/kernel" + bitbake(image) + bb_vars = get_bb_vars(['DEPLOY_DIR_IMAGE', 'INITRAMFS_IMAGE_NAME', 'KERNEL_FIT_LINK_NAME'], image) - image_type = "core-image-minimal" - deploy_dir_image = get_bb_var('DEPLOY_DIR_IMAGE') - machine = get_bb_var('MACHINE') - fitimage_its_path = os.path.join(deploy_dir_image, - "fitImage-its-%s-%s-%s" % (image_type, machine, machine)) - fitimage_path = os.path.join(deploy_dir_image, - "fitImage-%s-%s-%s" % (image_type, machine, machine)) + fitimage_its_path = os.path.join(bb_vars['DEPLOY_DIR_IMAGE'], + "fitImage-its-%s-%s" % (bb_vars['INITRAMFS_IMAGE_NAME'], bb_vars['KERNEL_FIT_LINK_NAME'])) + fitimage_path = os.path.join(bb_vars['DEPLOY_DIR_IMAGE'], + "fitImage-%s-%s" % (bb_vars['INITRAMFS_IMAGE_NAME'], bb_vars['KERNEL_FIT_LINK_NAME'])) self.assertTrue(os.path.exists(fitimage_its_path), "%s image tree source doesn't exist" % (fitimage_its_path)) @@ -121,15 +124,14 @@ UBOOT_MKIMAGE_SIGN_ARGS = "-c 'a smart comment'" self.write_config(config) # fitImage is created as part of linux recipe - bitbake("virtual/kernel") + image = "virtual/kernel" + bitbake(image) + bb_vars = get_bb_vars(['DEPLOY_DIR_IMAGE', 'KERNEL_FIT_LINK_NAME'], image) - image_type = "core-image-minimal" - deploy_dir_image = get_bb_var('DEPLOY_DIR_IMAGE') - machine = get_bb_var('MACHINE') - fitimage_its_path = os.path.join(deploy_dir_image, - "fitImage-its-%s" % (machine,)) - fitimage_path = os.path.join(deploy_dir_image, - "fitImage-%s.bin" % (machine,)) + fitimage_its_path = os.path.join(bb_vars['DEPLOY_DIR_IMAGE'], + "fitImage-its-%s" % (bb_vars['KERNEL_FIT_LINK_NAME'])) + fitimage_path = os.path.join(bb_vars['DEPLOY_DIR_IMAGE'], + "fitImage-%s.bin" % (bb_vars['KERNEL_FIT_LINK_NAME'])) self.assertTrue(os.path.exists(fitimage_its_path), "%s image tree source doesn't exist" % (fitimage_its_path)) @@ -202,7 +204,7 @@ UBOOT_MKIMAGE_SIGN_ARGS = "-c 'a smart comment'" signed_sections = {} for line in result.output.splitlines(): if line.startswith((' Configuration', ' Image')): - in_signed = re.search('\((.*)\)', line).groups()[0] + in_signed = re.search(r'\((.*)\)', line).groups()[0] elif re.match('^ *', line) in (' ', ''): in_signed = None elif in_signed: @@ -275,8 +277,8 @@ FIT_SIGN_INDIVIDUAL = "1" """ self.write_config(config) - # The U-Boot fitImage is created as part of linux recipe - bitbake("virtual/kernel") + # The U-Boot fitImage is created as part of the U-Boot recipe + bitbake("virtual/bootloader") deploy_dir_image = get_bb_var('DEPLOY_DIR_IMAGE') machine = get_bb_var('MACHINE') @@ -348,7 +350,8 @@ UBOOT_LOADADDRESS = "0x80080000" UBOOT_ENTRYPOINT = "0x80080000" UBOOT_FIT_DESC = "A model description" KERNEL_IMAGETYPES += " fitImage " -KERNEL_CLASSES = " kernel-fitimage test-mkimage-wrapper " +KERNEL_CLASSES = " kernel-fitimage " +INHERIT += "test-mkimage-wrapper" UBOOT_SIGN_ENABLE = "1" FIT_GENERATE_KEYS = "1" UBOOT_SIGN_KEYDIR = "${TOPDIR}/signing-keys" @@ -359,8 +362,8 @@ UBOOT_MKIMAGE_SIGN_ARGS = "-c 'a smart U-Boot comment'" """ self.write_config(config) - # The U-Boot fitImage is created as part of linux recipe - bitbake("virtual/kernel") + # The U-Boot fitImage is created as part of the U-Boot recipe + bitbake("virtual/bootloader") deploy_dir_image = get_bb_var('DEPLOY_DIR_IMAGE') machine = get_bb_var('MACHINE') @@ -430,7 +433,8 @@ UBOOT_MACHINE = "am57xx_evm_defconfig" SPL_BINARY = "MLO" # The kernel-fitimage class is a dependency even if we're only # creating/signing the U-Boot fitImage -KERNEL_CLASSES = " kernel-fitimage test-mkimage-wrapper " +KERNEL_CLASSES = " kernel-fitimage" +INHERIT += "test-mkimage-wrapper" # Enable creation and signing of the U-Boot fitImage UBOOT_FITIMAGE_ENABLE = "1" SPL_SIGN_ENABLE = "1" @@ -449,8 +453,8 @@ UBOOT_FIT_HASH_ALG = "sha256" """ self.write_config(config) - # The U-Boot fitImage is created as part of linux recipe - bitbake("virtual/kernel") + # The U-Boot fitImage is created as part of the U-Boot recipe + bitbake("virtual/bootloader") image_type = "core-image-minimal" deploy_dir_image = get_bb_var('DEPLOY_DIR_IMAGE') @@ -521,7 +525,7 @@ UBOOT_FIT_HASH_ALG = "sha256" signed_sections = {} for line in result.output.splitlines(): if line.startswith((' Image')): - in_signed = re.search('\((.*)\)', line).groups()[0] + in_signed = re.search(r'\((.*)\)', line).groups()[0] elif re.match(' \w', line): in_signed = None elif in_signed: @@ -538,7 +542,7 @@ UBOOT_FIT_HASH_ALG = "sha256" self.assertEqual(len(value), 512, 'Signature value for section %s not expected length' % signed_section) # Check for SPL_MKIMAGE_SIGN_ARGS - result = runCmd('bitbake -e virtual/kernel | grep ^T=') + result = runCmd('bitbake -e virtual/bootloader | grep ^T=') tempdir = result.output.split('=', 1)[1].strip().strip('') result = runCmd('grep "a smart U-Boot comment" %s/run.do_uboot_assemble_fitimage' % tempdir, ignore_status=True) self.assertEqual(result.status, 0, 'SPL_MKIMAGE_SIGN_ARGS value did not get used') @@ -593,7 +597,8 @@ UBOOT_EXTLINUX = "0" UBOOT_FIT_GENERATE_KEYS = "1" UBOOT_FIT_HASH_ALG = "sha256" KERNEL_IMAGETYPES += " fitImage " -KERNEL_CLASSES = " kernel-fitimage test-mkimage-wrapper " +KERNEL_CLASSES = " kernel-fitimage " +INHERIT += "test-mkimage-wrapper" UBOOT_SIGN_ENABLE = "1" FIT_GENERATE_KEYS = "1" UBOOT_SIGN_KEYDIR = "${TOPDIR}/signing-keys" @@ -603,8 +608,8 @@ FIT_SIGN_INDIVIDUAL = "1" """ self.write_config(config) - # The U-Boot fitImage is created as part of linux recipe - bitbake("virtual/kernel") + # The U-Boot fitImage is created as part of the U-Boot recipe + bitbake("virtual/bootloader") image_type = "core-image-minimal" deploy_dir_image = get_bb_var('DEPLOY_DIR_IMAGE') @@ -675,7 +680,7 @@ FIT_SIGN_INDIVIDUAL = "1" signed_sections = {} for line in result.output.splitlines(): if line.startswith((' Image')): - in_signed = re.search('\((.*)\)', line).groups()[0] + in_signed = re.search(r'\((.*)\)', line).groups()[0] elif re.match(' \w', line): in_signed = None elif in_signed: @@ -692,7 +697,7 @@ FIT_SIGN_INDIVIDUAL = "1" self.assertEqual(len(value), 512, 'Signature value for section %s not expected length' % signed_section) # Check for SPL_MKIMAGE_SIGN_ARGS - result = runCmd('bitbake -e virtual/kernel | grep ^T=') + result = runCmd('bitbake -e virtual/bootloader | grep ^T=') tempdir = result.output.split('=', 1)[1].strip().strip('') result = runCmd('grep "a smart cascaded U-Boot comment" %s/run.do_uboot_assemble_fitimage' % tempdir, ignore_status=True) self.assertEqual(result.status, 0, 'SPL_MKIMAGE_SIGN_ARGS value did not get used') diff --git a/meta/lib/oeqa/selftest/cases/gcc.py b/meta/lib/oeqa/selftest/cases/gcc.py index b9ea03ae62..89360178fe 100644 --- a/meta/lib/oeqa/selftest/cases/gcc.py +++ b/meta/lib/oeqa/selftest/cases/gcc.py @@ -1,5 +1,10 @@ +# +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT +# import os +import time from oeqa.core.decorator import OETestTag from oeqa.core.case import OEPTestResultTestCase from oeqa.selftest.case import OESelftestTestCase @@ -39,8 +44,13 @@ class GccSelfTestBase(OESelftestTestCase, OEPTestResultTestCase): self.write_config("\n".join(features)) recipe = "gcc-runtime" + + start_time = time.time() + bitbake("{} -c check".format(recipe)) + end_time = time.time() + bb_vars = get_bb_vars(["B", "TARGET_SYS"], recipe) builddir, target_sys = bb_vars["B"], bb_vars["TARGET_SYS"] @@ -54,7 +64,7 @@ class GccSelfTestBase(OESelftestTestCase, OEPTestResultTestCase): ptestsuite = "gcc-{}".format(suite) if suite != "gcc" else suite ptestsuite = ptestsuite + "-user" if ssh is None else ptestsuite - self.ptest_section(ptestsuite, logfile = logpath) + self.ptest_section(ptestsuite, duration = int(end_time - start_time), logfile = logpath) with open(sumspath, "r") as f: for test, result in parse_values(f): self.ptest_result(ptestsuite, test, result) diff --git a/meta/lib/oeqa/selftest/cases/gdbserver.py b/meta/lib/oeqa/selftest/cases/gdbserver.py new file mode 100644 index 0000000000..9da97ae780 --- /dev/null +++ b/meta/lib/oeqa/selftest/cases/gdbserver.py @@ -0,0 +1,67 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# +import os +import time +import tempfile +import shutil +import concurrent.futures + +from oeqa.selftest.case import OESelftestTestCase +from oeqa.utils.commands import bitbake, get_bb_var, get_bb_vars , runqemu, runCmd + +class GdbServerTest(OESelftestTestCase): + def test_gdb_server(self): + target_arch = self.td["TARGET_ARCH"] + target_sys = self.td["TARGET_SYS"] + + features = """ +IMAGE_GEN_DEBUGFS = "1" +IMAGE_FSTYPES_DEBUGFS = "tar.bz2" +CORE_IMAGE_EXTRA_INSTALL = "gdbserver" + """ + self.write_config(features) + + gdb_recipe = "gdb-cross-" + target_arch + gdb_binary = target_sys + "-gdb" + + bitbake("core-image-minimal %s:do_addto_recipe_sysroot" % gdb_recipe) + + native_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", gdb_recipe) + r = runCmd("%s --version" % gdb_binary, native_sysroot=native_sysroot, target_sys=target_sys) + self.assertEqual(r.status, 0) + self.assertIn("GNU gdb", r.output) + image = 'core-image-minimal' + bb_vars = get_bb_vars(['DEPLOY_DIR_IMAGE', 'IMAGE_LINK_NAME'], image) + + with tempfile.TemporaryDirectory(prefix="debugfs-") as debugfs: + filename = os.path.join(bb_vars['DEPLOY_DIR_IMAGE'], "%s-dbg.tar.bz2" % bb_vars['IMAGE_LINK_NAME']) + shutil.unpack_archive(filename, debugfs) + filename = os.path.join(bb_vars['DEPLOY_DIR_IMAGE'], "%s.tar.bz2" % bb_vars['IMAGE_LINK_NAME']) + shutil.unpack_archive(filename, debugfs) + + with runqemu("core-image-minimal", runqemuparams="nographic") as qemu: + status, output = qemu.run_serial("kmod --help") + self.assertIn("modprobe", output) + + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor: + def run_gdb(): + for _ in range(5): + time.sleep(2) + cmd = "%s --batch -ex 'set sysroot %s' -ex \"target extended-remote %s:9999\" -ex \"info line kmod_help\"" % (gdb_binary, debugfs, qemu.ip) + self.logger.warning("starting gdb %s" % cmd) + r = runCmd(cmd, native_sysroot=native_sysroot, target_sys=target_sys) + self.assertEqual(0, r.status) + line_re = r"Line \d+ of \"/usr/src/debug/kmod/.*/tools/kmod.c\" starts at address 0x[0-9A-Fa-f]+ <kmod_help>" + self.assertRegex(r.output, line_re) + break + else: + self.fail("Timed out connecting to gdb") + future = executor.submit(run_gdb) + + status, output = qemu.run_serial("gdbserver --once :9999 kmod --help") + self.assertEqual(status, 1) + # The future either returns None, or raises an exception + future.result() diff --git a/meta/lib/oeqa/selftest/cases/gitarchivetests.py b/meta/lib/oeqa/selftest/cases/gitarchivetests.py new file mode 100644 index 0000000000..71382089c1 --- /dev/null +++ b/meta/lib/oeqa/selftest/cases/gitarchivetests.py @@ -0,0 +1,136 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +import os +import sys +basepath = os.path.abspath(os.path.dirname(__file__) + '/../../../../../') +lib_path = basepath + '/scripts/lib' +sys.path = sys.path + [lib_path] +import oeqa.utils.gitarchive as ga +from oeqa.utils.git import GitError +import tempfile +import shutil +import scriptutils +import logging +from oeqa.selftest.case import OESelftestTestCase + +logger = scriptutils.logger_create('resulttool') + +def create_fake_repository(commit, tag_list=[], add_remote=True): + """ Create a testing git directory + + Initialize a simple git repository with one initial commit, and as many + tags on this commit as listed in tag_list + Returns both git directory path and gitarchive git object + If commit is true, fake data will be commited, otherwise it will stay in staging area + If commit is true and tag_lsit is non empty, all tags in tag_list will be + created on the initial commit + Fake remote will also be added to make git ls-remote work + """ + fake_data_file = "fake_data.txt" + tempdir = tempfile.mkdtemp(prefix='fake_results.') + repo = ga.init_git_repo(tempdir, False, False, logger) + if add_remote: + repo.run_cmd(["remote", "add", "origin", "."]) + with open(os.path.join(tempdir, fake_data_file), "w") as fake_data: + fake_data.write("Fake data") + if commit: + repo.run_cmd(["add", fake_data_file]) + repo.run_cmd(["commit", "-m", "\"Add fake data\""]) + for tag in tag_list: + repo.run_cmd(["tag", tag]) + + return tempdir, repo + +def delete_fake_repository(path): + shutil.rmtree(path) + +def tag_exists(git_obj, target_tag): + for tag in git_obj.run_cmd(["tag"]).splitlines(): + if target_tag == tag: + return True + return False + +class GitArchiveTests(OESelftestTestCase): + TEST_BRANCH="main" + TEST_COMMIT="0f7d5df" + TEST_COMMIT_COUNT="42" + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.log = logging.getLogger('gitarchivetests') + cls.log.setLevel(logging.DEBUG) + + def test_create_first_test_tag(self): + path, git_obj = create_fake_repository(False) + keywords = {'commit': self.TEST_COMMIT, 'branch': self.TEST_BRANCH, "commit_count": self.TEST_COMMIT_COUNT} + target_tag = f"{self.TEST_BRANCH}/{self.TEST_COMMIT_COUNT}-g{self.TEST_COMMIT}/0" + + ga.gitarchive(path, path, True, False, + "Results of {branch}:{commit}", "branch: {branch}\ncommit: {commit}", "{branch}", + False, "{branch}/{commit_count}-g{commit}/{tag_number}", + 'Test run #{tag_number} of {branch}:{commit}', '', + [], [], False, keywords, logger) + self.assertTrue(tag_exists(git_obj, target_tag), msg=f"Tag {target_tag} has not been created") + delete_fake_repository(path) + + def test_create_second_test_tag(self): + first_tag = f"{self.TEST_BRANCH}/{self.TEST_COMMIT_COUNT}-g{self.TEST_COMMIT}/0" + second_tag = f"{self.TEST_BRANCH}/{self.TEST_COMMIT_COUNT}-g{self.TEST_COMMIT}/1" + keywords = {'commit': self.TEST_COMMIT, 'branch': self.TEST_BRANCH, "commit_count": self.TEST_COMMIT_COUNT} + + path, git_obj = create_fake_repository(True, [first_tag]) + ga.gitarchive(path, path, True, False, + "Results of {branch}:{commit}", "branch: {branch}\ncommit: {commit}", "{branch}", + False, "{branch}/{commit_count}-g{commit}/{tag_number}", + 'Test run #{tag_number} of {branch}:{commit}', '', + [], [], False, keywords, logger) + self.assertTrue(tag_exists(git_obj, second_tag), msg=f"Second tag {second_tag} has not been created") + delete_fake_repository(path) + + def test_get_revs_on_branch(self): + fake_tags_list=["main/10-g0f7d5df/0", "main/10-g0f7d5df/1", "foo/20-g2468f5d/0"] + tag_name = "{branch}/{commit_number}-g{commit}/{tag_number}" + + path, git_obj = create_fake_repository(True, fake_tags_list) + revs = ga.get_test_revs(logger, git_obj, tag_name, branch="main") + self.assertEqual(len(revs), 1) + self.assertEqual(revs[0].commit, "0f7d5df") + self.assertEqual(len(revs[0].tags), 2) + self.assertEqual(revs[0].tags, ['main/10-g0f7d5df/0', 'main/10-g0f7d5df/1']) + delete_fake_repository(path) + + def test_get_tags_without_valid_remote(self): + url = 'git://git.yoctoproject.org/poky' + path, git_obj = create_fake_repository(False, None, False) + + tags = ga.get_tags(git_obj, self.log, pattern="yocto-*", url=url) + """Test for some well established tags (released tags)""" + self.assertIn("yocto-4.0", tags) + self.assertIn("yocto-4.1", tags) + self.assertIn("yocto-4.2", tags) + delete_fake_repository(path) + + def test_get_tags_with_only_local_tag(self): + fake_tags_list=["main/10-g0f7d5df/0", "main/10-g0f7d5df/1", "foo/20-g2468f5d/0"] + path, git_obj = create_fake_repository(True, fake_tags_list, False) + + """No remote is configured and no url is passed: get_tags must fall + back to local tags + """ + tags = ga.get_tags(git_obj, self.log) + self.assertCountEqual(tags, fake_tags_list) + delete_fake_repository(path) + + def test_get_tags_without_valid_remote_and_wrong_url(self): + url = 'git://git.foo.org/bar' + path, git_obj = create_fake_repository(False, None, False) + + """Test for some well established tags (released tags)""" + with self.assertRaises(GitError): + tags = ga.get_tags(git_obj, self.log, pattern="yocto-*", url=url) + delete_fake_repository(path) diff --git a/meta/lib/oeqa/selftest/cases/glibc.py b/meta/lib/oeqa/selftest/cases/glibc.py index 6fc98e9cb4..bd56b2f6e7 100644 --- a/meta/lib/oeqa/selftest/cases/glibc.py +++ b/meta/lib/oeqa/selftest/cases/glibc.py @@ -1,5 +1,10 @@ +# +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT +# import os +import time import contextlib from oeqa.core.decorator import OETestTag from oeqa.core.case import OEPTestResultTestCase @@ -24,15 +29,19 @@ class GlibcSelfTestBase(OESelftestTestCase, OEPTestResultTestCase): features.append('TOOLCHAIN_TEST_HOST_USER = "root"') features.append('TOOLCHAIN_TEST_HOST_PORT = "22"') # force single threaded test execution - features.append('EGLIBCPARALLELISM_task-check:pn-glibc-testsuite = "PARALLELMFLAGS="-j1""') + features.append('EGLIBCPARALLELISM:task-check:pn-glibc-testsuite = "PARALLELMFLAGS="-j1""') self.write_config("\n".join(features)) + start_time = time.time() + bitbake("glibc-testsuite -c check") + end_time = time.time() + builddir = get_bb_var("B", "glibc-testsuite") ptestsuite = "glibc-user" if ssh is None else "glibc" - self.ptest_section(ptestsuite) + self.ptest_section(ptestsuite, duration = int(end_time - start_time)) with open(os.path.join(builddir, "tests.sum"), "r", errors='replace') as f: for test, result in parse_values(f): self.ptest_result(ptestsuite, test, result) @@ -41,7 +50,7 @@ class GlibcSelfTestBase(OESelftestTestCase, OEPTestResultTestCase): with contextlib.ExitStack() as s: # use the base work dir, as the nfs mount, since the recipe directory may not exist tmpdir = get_bb_var("BASE_WORKDIR") - nfsport, mountport = s.enter_context(unfs_server(tmpdir)) + nfsport, mountport = s.enter_context(unfs_server(tmpdir, udp = False)) # build core-image-minimal with required packages default_installed_packages = [ @@ -61,7 +70,7 @@ class GlibcSelfTestBase(OESelftestTestCase, OEPTestResultTestCase): bitbake("core-image-minimal") # start runqemu - qemu = s.enter_context(runqemu("core-image-minimal", runqemuparams = "nographic")) + qemu = s.enter_context(runqemu("core-image-minimal", runqemuparams = "nographic", qemuparams = "-m 1024")) # validate that SSH is working status, _ = qemu.run("uname") @@ -70,7 +79,7 @@ class GlibcSelfTestBase(OESelftestTestCase, OEPTestResultTestCase): # setup nfs mount if qemu.run("mkdir -p \"{0}\"".format(tmpdir))[0] != 0: raise Exception("Failed to setup NFS mount directory on target") - mountcmd = "mount -o noac,nfsvers=3,port={0},udp,mountport={1} \"{2}:{3}\" \"{3}\"".format(nfsport, mountport, qemu.server_ip, tmpdir) + mountcmd = "mount -o noac,nfsvers=3,port={0},mountport={1} \"{2}:{3}\" \"{3}\"".format(nfsport, mountport, qemu.server_ip, tmpdir) status, output = qemu.run(mountcmd) if status != 0: raise Exception("Failed to setup NFS mount on target ({})".format(repr(output))) diff --git a/meta/lib/oeqa/selftest/cases/gotoolchain.py b/meta/lib/oeqa/selftest/cases/gotoolchain.py index 978898b86f..ee2cf4b09a 100644 --- a/meta/lib/oeqa/selftest/cases/gotoolchain.py +++ b/meta/lib/oeqa/selftest/cases/gotoolchain.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -52,6 +54,7 @@ class oeGoToolchainSelfTest(OESelftestTestCase): cmd = cmd + "export GOPATH=%s; " % self.go_path cmd = cmd + "export GOFLAGS=-modcacherw; " cmd = cmd + "export CGO_ENABLED=1; " + cmd = cmd + "export GOPROXY=https://proxy.golang.org,direct; " cmd = cmd + "${CROSS_COMPILE}go %s" % gocmd return runCmd(cmd).status diff --git a/meta/lib/oeqa/selftest/cases/image_typedep.py b/meta/lib/oeqa/selftest/cases/image_typedep.py index 5b182a8f94..17c98baf14 100644 --- a/meta/lib/oeqa/selftest/cases/image_typedep.py +++ b/meta/lib/oeqa/selftest/cases/image_typedep.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/selftest/cases/imagefeatures.py b/meta/lib/oeqa/selftest/cases/imagefeatures.py index 7deafef6b2..dc88c222bd 100644 --- a/meta/lib/oeqa/selftest/cases/imagefeatures.py +++ b/meta/lib/oeqa/selftest/cases/imagefeatures.py @@ -1,10 +1,12 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # from oeqa.selftest.case import OESelftestTestCase from oeqa.core.decorator import OETestTag -from oeqa.utils.commands import runCmd, bitbake, get_bb_var, runqemu +from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars, runqemu from oeqa.utils.sshcontrol import SSHControl import glob import os @@ -100,12 +102,11 @@ class ImageFeatures(OESelftestTestCase): features = 'IMAGE_FSTYPES += " ext4 ext4.bmap ext4.bmap.gz"' self.write_config(features) - image_name = 'core-image-minimal' - bitbake(image_name) + image = 'core-image-minimal' + bitbake(image) + bb_vars = get_bb_vars(['DEPLOY_DIR_IMAGE', 'IMAGE_LINK_NAME'], image) - deploy_dir_image = get_bb_var('DEPLOY_DIR_IMAGE') - link_name = get_bb_var('IMAGE_LINK_NAME', image_name) - image_path = os.path.join(deploy_dir_image, "%s.ext4" % link_name) + image_path = os.path.join(bb_vars['DEPLOY_DIR_IMAGE'], "%s.ext4" % bb_vars['IMAGE_LINK_NAME']) bmap_path = "%s.bmap" % image_path gzip_path = "%s.gz" % bmap_path @@ -118,8 +119,8 @@ class ImageFeatures(OESelftestTestCase): image_stat = os.stat(image_path) self.assertGreater(image_stat.st_size, image_stat.st_blocks * 512) - # check if the resulting gzip is valid - self.assertTrue(runCmd('gzip -t %s' % gzip_path)) + # check if the resulting gzip is valid, --force is needed in case gzip_path is a symlink + self.assertTrue(runCmd('gzip --test --force %s' % gzip_path)) def test_hypervisor_fmts(self): """ @@ -134,17 +135,16 @@ class ImageFeatures(OESelftestTestCase): img_types = [ 'vmdk', 'vdi', 'qcow2' ] features = "" for itype in img_types: - features += 'IMAGE_FSTYPES += "wic.%s"\n' % itype + features += 'IMAGE_FSTYPES += "ext4.%s"\n' % itype self.write_config(features) - image_name = 'core-image-minimal' - bitbake(image_name) + image = 'core-image-minimal' + bitbake(image) + bb_vars = get_bb_vars(['DEPLOY_DIR_IMAGE', 'IMAGE_LINK_NAME'], image) - deploy_dir_image = get_bb_var('DEPLOY_DIR_IMAGE') - link_name = get_bb_var('IMAGE_LINK_NAME', image_name) for itype in img_types: - image_path = os.path.join(deploy_dir_image, "%s.wic.%s" % - (link_name, itype)) + image_path = os.path.join(bb_vars['DEPLOY_DIR_IMAGE'], "%s.ext4.%s" % + (bb_vars['IMAGE_LINK_NAME'], itype)) # check if result image file is in deploy directory self.assertTrue(os.path.exists(image_path)) @@ -175,13 +175,11 @@ class ImageFeatures(OESelftestTestCase): features = 'IMAGE_FSTYPES += "%s %s.sha256sum"' % (conv, conv) self.write_config(features) - image_name = 'core-image-minimal' - bitbake(image_name) - - deploy_dir_image = get_bb_var('DEPLOY_DIR_IMAGE') - link_name = get_bb_var('IMAGE_LINK_NAME', image_name) - image_path = os.path.join(deploy_dir_image, "%s.%s" % - (link_name, conv)) + image = 'core-image-minimal' + bitbake(image) + bb_vars = get_bb_vars(['DEPLOY_DIR_IMAGE', 'IMAGE_LINK_NAME'], image) + image_path = os.path.join(bb_vars['DEPLOY_DIR_IMAGE'], "%s.%s" % + (bb_vars['IMAGE_LINK_NAME'], conv)) # check if resulting image is in the deploy directory self.assertTrue(os.path.exists(image_path)) @@ -189,7 +187,7 @@ class ImageFeatures(OESelftestTestCase): # check if the resulting sha256sum agrees self.assertTrue(runCmd('cd %s;sha256sum -c %s.%s.sha256sum' % - (deploy_dir_image, link_name, conv))) + (bb_vars['DEPLOY_DIR_IMAGE'], bb_vars['IMAGE_LINK_NAME'], conv))) def test_image_fstypes(self): """ @@ -198,26 +196,43 @@ class ImageFeatures(OESelftestTestCase): Product: oe-core Author: Ed Bartosh <ed.bartosh@linux.intel.com> """ - image_name = 'core-image-minimal' + image = 'core-image-minimal' - all_image_types = set(get_bb_var("IMAGE_TYPES", image_name).split()) - skip_image_types = set(('container', 'elf', 'f2fs', 'multiubi', 'tar.zst', 'wic.zst', 'squashfs-lzo')) + all_image_types = set(get_bb_var("IMAGE_TYPES", image).split()) + skip_image_types = set(('container', 'elf', 'f2fs', 'tar.zst', 'wic.zst', 'squashfs-lzo', 'vfat')) img_types = all_image_types - skip_image_types - config = 'IMAGE_FSTYPES += "%s"\n'\ - 'MKUBIFS_ARGS ?= "-m 2048 -e 129024 -c 2047"\n'\ - 'UBINIZE_ARGS ?= "-m 2048 -p 128KiB -s 512"' % ' '.join(img_types) + config = """ +IMAGE_FSTYPES += "%s" +WKS_FILE = "wictestdisk.wks" +MKUBIFS_ARGS ?= "-m 2048 -e 129024 -c 2047" +UBINIZE_ARGS ?= "-m 2048 -p 128KiB -s 512" +MULTIUBI_BUILD += "mtd_2_128" +MKUBIFS_ARGS_mtd_2_128 ?= "-m 2048 -e 129024 -c 2047" +UBINIZE_ARGS_mtd_2_128 ?= "-m 2048 -p 128KiB -s 512" +MULTIUBI_BUILD += "mtd_4_256" +MKUBIFS_ARGS_mtd_4_256 ?= "-m 4096 -e 253952 -c 4096" +UBINIZE_ARGS_mtd_4_256 ?= "-m 4096 -p 256KiB" +""" % ' '.join(img_types) self.write_config(config) - bitbake(image_name) + bitbake(image) + bb_vars = get_bb_vars(['DEPLOY_DIR_IMAGE', 'IMAGE_LINK_NAME', 'MULTIUBI_BUILD'], image) - deploy_dir_image = get_bb_var('DEPLOY_DIR_IMAGE') - link_name = get_bb_var('IMAGE_LINK_NAME', image_name) for itype in img_types: - image_path = os.path.join(deploy_dir_image, "%s.%s" % (link_name, itype)) - # check if result image is in deploy directory - self.assertTrue(os.path.exists(image_path), - "%s image %s doesn't exist" % (itype, image_path)) + if itype == 'multiubi': + # For multiubi build we need to manage MULTIUBI_BUILD entry to append + # specific name to IMAGE_LINK_NAME + for vname in bb_vars['MULTIUBI_BUILD'].split(): + image_path = os.path.join(bb_vars['DEPLOY_DIR_IMAGE'], "%s_%s.ubifs" % (bb_vars['IMAGE_LINK_NAME'], vname)) + # check if result image is in deploy directory + self.assertTrue(os.path.exists(image_path), + "%s image %s doesn't exist" % (itype, image_path)) + else: + image_path = os.path.join(bb_vars['DEPLOY_DIR_IMAGE'], "%s.%s" % (bb_vars['IMAGE_LINK_NAME'], itype)) + # check if result image is in deploy directory + self.assertTrue(os.path.exists(image_path), + "%s image %s doesn't exist" % (itype, image_path)) def test_useradd_static(self): config = """ @@ -235,7 +250,7 @@ USERADD_GID_TABLES += "files/static-group" DISTRO_FEATURES:append = " pam opengl wayland" # Switch to systemd -DISTRO_FEATURES:append = " systemd" +DISTRO_FEATURES:append = " systemd usrmerge" VIRTUAL-RUNTIME_init_manager = "systemd" VIRTUAL-RUNTIME_initscripts = "" VIRTUAL-RUNTIME_syslog = "" @@ -266,20 +281,20 @@ SKIP_RECIPE[busybox] = "Don't build this" Yeoh Ee Peng <ee.peng.yeoh@intel.com> """ - image_name = 'core-image-minimal' + image = 'core-image-minimal' + image_fstypes_debugfs = 'tar.bz2' features = 'IMAGE_GEN_DEBUGFS = "1"\n' - features += 'IMAGE_FSTYPES_DEBUGFS = "tar.bz2"\n' - features += 'MACHINE = "genericx86-64"\n' + features += 'IMAGE_FSTYPES_DEBUGFS = "%s"\n' % image_fstypes_debugfs self.write_config(features) - bitbake(image_name) - deploy_dir_image = get_bb_var('DEPLOY_DIR_IMAGE') - dbg_tar_file = os.path.join(deploy_dir_image, "*-dbg.rootfs.tar.bz2") - debug_files = glob.glob(dbg_tar_file) - self.assertNotEqual(len(debug_files), 0, 'debug filesystem not generated at %s' % dbg_tar_file) - result = runCmd('cd %s; tar xvf %s' % (deploy_dir_image, dbg_tar_file)) + bitbake(image) + bb_vars = get_bb_vars(['DEPLOY_DIR_IMAGE', 'IMAGE_LINK_NAME'], image) + + dbg_tar_file = os.path.join(bb_vars['DEPLOY_DIR_IMAGE'], "%s-dbg.%s" % (bb_vars['IMAGE_LINK_NAME'], image_fstypes_debugfs)) + self.assertTrue(os.path.exists(dbg_tar_file), 'debug filesystem not generated at %s' % dbg_tar_file) + result = runCmd('cd %s; tar xvf %s' % (bb_vars['DEPLOY_DIR_IMAGE'], dbg_tar_file)) self.assertEqual(result.status, 0, msg='Failed to extract %s: %s' % (dbg_tar_file, result.output)) - result = runCmd('find %s -name %s' % (deploy_dir_image, "udevadm")) + result = runCmd('find %s -name %s' % (bb_vars['DEPLOY_DIR_IMAGE'], "udevadm")) self.assertTrue("udevadm" in result.output, msg='Failed to find udevadm: %s' % result.output) dbg_symbols_targets = result.output.splitlines() self.assertTrue(dbg_symbols_targets, msg='Failed to split udevadm: %s' % dbg_symbols_targets) @@ -289,9 +304,33 @@ SKIP_RECIPE[busybox] = "Don't build this" def test_empty_image(self): """Test creation of image with no packages""" - bitbake('test-empty-image') - res_dir = get_bb_var('DEPLOY_DIR_IMAGE') - images = os.path.join(res_dir, "test-empty-image-*.manifest") - result = glob.glob(images) - with open(result[1],"r") as f: + image = 'test-empty-image' + bitbake(image) + bb_vars = get_bb_vars(['DEPLOY_DIR_IMAGE', 'IMAGE_LINK_NAME'], image) + manifest = os.path.join(bb_vars['DEPLOY_DIR_IMAGE'], "%s.manifest" % bb_vars['IMAGE_LINK_NAME']) + self.assertTrue(os.path.exists(manifest)) + + with open(manifest, "r") as f: self.assertEqual(len(f.read().strip()),0) + + def test_mandb(self): + """ + Test that an image containing manpages has working man and apropos commands. + """ + config = """ +DISTRO_FEATURES:append = " api-documentation" +CORE_IMAGE_EXTRA_INSTALL = "man-pages kmod-doc" +""" + self.write_config(config) + bitbake("core-image-minimal") + + with runqemu('core-image-minimal', ssh=False, runqemuparams='nographic') as qemu: + # This manpage is provided by man-pages + status, output = qemu.run_serial("apropos 8859") + self.assertEqual(status, 1, 'Failed to run apropos: %s' % (output)) + self.assertIn("iso_8859_15", output) + + # This manpage is provided by kmod + status, output = qemu.run_serial("man --pager=cat modprobe") + self.assertEqual(status, 1, 'Failed to run man: %s' % (output)) + self.assertIn("force-modversion", output) diff --git a/meta/lib/oeqa/selftest/cases/incompatible_lic.py b/meta/lib/oeqa/selftest/cases/incompatible_lic.py index 6279d74245..f4af67a239 100644 --- a/meta/lib/oeqa/selftest/cases/incompatible_lic.py +++ b/meta/lib/oeqa/selftest/cases/incompatible_lic.py @@ -1,3 +1,8 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# from oeqa.selftest.case import OESelftestTestCase from oeqa.utils.commands import bitbake @@ -108,6 +113,7 @@ INCOMPATIBLE_LICENSE:pn-core-image-minimal = "GPL-3.0* LGPL-3.0*" raise AssertionError(result.output) def test_bash_and_license(self): + self.disable_class("create-spdx") self.write_config(self.default_config() + '\nLICENSE:append:pn-bash = " & SomeLicense"') error_msg = "ERROR: core-image-minimal-1.0-r0 do_rootfs: Package bash cannot be installed into the image because it has incompatible license(s): GPL-3.0-or-later" @@ -116,6 +122,7 @@ INCOMPATIBLE_LICENSE:pn-core-image-minimal = "GPL-3.0* LGPL-3.0*" raise AssertionError(result.output) def test_bash_or_license(self): + self.disable_class("create-spdx") self.write_config(self.default_config() + '\nLICENSE:append:pn-bash = " | SomeLicense"') bitbake('core-image-minimal') @@ -134,23 +141,11 @@ INCOMPATIBLE_LICENSE:pn-core-image-minimal = "GPL-3.0* LGPL-3.0*" def test_core_image_full_cmdline_weston(self): self.write_config(""" -INHERIT += "testimage" +IMAGE_CLASSES += "testimage" INCOMPATIBLE_LICENSE:pn-core-image-full-cmdline = "GPL-3.0* LGPL-3.0*" INCOMPATIBLE_LICENSE:pn-core-image-weston = "GPL-3.0* LGPL-3.0*" -# Settings for full-cmdline -RDEPENDS:packagegroup-core-full-cmdline-utils:remove = "bash bc coreutils cpio ed findutils gawk grep mc mc-fish mc-helpers mc-helpers-perl sed tar time" -RDEPENDS:packagegroup-core-full-cmdline-dev-utils:remove = "diffutils m4 make patch" -RDEPENDS:packagegroup-core-full-cmdline-multiuser:remove = "gzip" -# Settings for weston -# direct gpl3 dependencies -RRECOMMENDS:packagegroup-base-vfat:remove = "dosfstools" -PACKAGECONFIG:remove:pn-bluez5 = "readline" -# dnf pulls in gpg which is gpl3; it also pulls in python3-rpm which pulls in rpm-build which pulls in bash -# so install rpm but not dnf -IMAGE_FEATURES:remove:pn-core-image-weston = "package-management" -CORE_IMAGE_EXTRA_INSTALL:pn-core-image-weston += "rpm" -# matchbox-terminal depends on vte, which is gpl3 -CORE_IMAGE_BASE_INSTALL:remove:pn-core-image-weston = "matchbox-terminal" + +require conf/distro/include/no-gplv3.inc """) bitbake('core-image-full-cmdline core-image-weston') bitbake('-c testimage core-image-full-cmdline core-image-weston') diff --git a/meta/lib/oeqa/selftest/cases/intercept.py b/meta/lib/oeqa/selftest/cases/intercept.py index f12874dc7d..12583c3099 100644 --- a/meta/lib/oeqa/selftest/cases/intercept.py +++ b/meta/lib/oeqa/selftest/cases/intercept.py @@ -1,3 +1,9 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + from oeqa.selftest.case import OESelftestTestCase from oeqa.utils.commands import bitbake diff --git a/meta/lib/oeqa/selftest/cases/kerneldevelopment.py b/meta/lib/oeqa/selftest/cases/kerneldevelopment.py index b1623a1885..b1f78a0cd1 100644 --- a/meta/lib/oeqa/selftest/cases/kerneldevelopment.py +++ b/meta/lib/oeqa/selftest/cases/kerneldevelopment.py @@ -1,3 +1,9 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + import os from oeqa.selftest.case import OESelftestTestCase from oeqa.utils.commands import runCmd, get_bb_var @@ -58,6 +64,7 @@ class KernelDev(OESelftestTestCase): recipe_append = os.path.join(self.recipeskernel_dir, 'linux-yocto_%.bbappend') with open(recipe_append, 'w+') as fh: fh.write('SRC_URI += "file://%s"\n' % patch_name) + fh.write('ERROR_QA:remove:pn-linux-yocto = "patch-status"\n') fh.write('FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"') runCmd('bitbake virtual/kernel -c clean') diff --git a/meta/lib/oeqa/selftest/cases/layerappend.py b/meta/lib/oeqa/selftest/cases/layerappend.py index 8fb1e6c530..379ed589ad 100644 --- a/meta/lib/oeqa/selftest/cases/layerappend.py +++ b/meta/lib/oeqa/selftest/cases/layerappend.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/selftest/cases/liboe.py b/meta/lib/oeqa/selftest/cases/liboe.py index afe8f8809f..d5ffffdcb4 100644 --- a/meta/lib/oeqa/selftest/cases/liboe.py +++ b/meta/lib/oeqa/selftest/cases/liboe.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -97,6 +99,6 @@ class LibOE(OESelftestTestCase): dstcnt = len(os.listdir(dst)) srccnt = len(os.listdir(src)) - self.assertEquals(dstcnt, len(testfiles), "Number of files in dst (%s) differs from number of files in src(%s)." % (dstcnt, srccnt)) + self.assertEqual(dstcnt, len(testfiles), "Number of files in dst (%s) differs from number of files in src(%s)." % (dstcnt, srccnt)) oe.path.remove(testloc) diff --git a/meta/lib/oeqa/selftest/cases/lic_checksum.py b/meta/lib/oeqa/selftest/cases/lic_checksum.py index 8f1226e6a5..2d0b805b90 100644 --- a/meta/lib/oeqa/selftest/cases/lic_checksum.py +++ b/meta/lib/oeqa/selftest/cases/lic_checksum.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -26,6 +28,7 @@ LIC_FILES_CHKSUM = "file://%s;md5=d41d8cd98f00b204e9800998ecf8427e" SRC_URI = "file://%s;md5=d41d8cd98f00b204e9800998ecf8427e" """ % (urllib.parse.quote(lic_path), urllib.parse.quote(lic_path))) result = bitbake(bitbake_cmd) + self.delete_recipeinc('emptytest') # Verify that changing a license file that has an absolute path causes @@ -51,5 +54,6 @@ SRC_URI = "file://%s;md5=d41d8cd98f00b204e9800998ecf8427e" f.write("data") result = bitbake(bitbake_cmd, ignore_status=True) + self.delete_recipeinc('emptytest') if error_msg not in result.output: raise AssertionError(result.output) diff --git a/meta/lib/oeqa/selftest/cases/locales.py b/meta/lib/oeqa/selftest/cases/locales.py new file mode 100644 index 0000000000..4ca8ffb7aa --- /dev/null +++ b/meta/lib/oeqa/selftest/cases/locales.py @@ -0,0 +1,54 @@ +# +# SPDX-License-Identifier: MIT +# + +from oeqa.selftest.case import OESelftestTestCase +from oeqa.core.decorator import OETestTag +from oeqa.utils.commands import bitbake, runqemu + +class LocalesTest(OESelftestTestCase): + + @OETestTag("runqemu") + + def run_locales_test(self, binary_enabled): + features = [] + features.append('EXTRA_IMAGE_FEATURES = "empty-root-password allow-empty-password allow-root-login"') + features.append('IMAGE_INSTALL:append = " glibc-utils localedef"') + features.append('GLIBC_GENERATE_LOCALES = "en_US.UTF-8 fr_FR.UTF-8"') + features.append('IMAGE_LINGUAS:append = " en-us fr-fr"') + if binary_enabled: + features.append('ENABLE_BINARY_LOCALE_GENERATION = "1"') + else: + features.append('ENABLE_BINARY_LOCALE_GENERATION = "0"') + self.write_config("\n".join(features)) + + # Build a core-image-minimal + bitbake('core-image-minimal') + + with runqemu("core-image-minimal", ssh=False, runqemuparams='nographic') as qemu: + cmd = "locale -a" + status, output = qemu.run_serial(cmd) + # output must includes fr_FR or fr_FR.UTF-8 + self.assertEqual(status, 1, msg='locale test command failed: output: %s' % output) + self.assertIn("fr_FR", output, msg='locale -a test failed: output: %s' % output) + + cmd = "localedef --list-archive -v" + status, output = qemu.run_serial(cmd) + # output must includes fr_FR.utf8 + self.assertEqual(status, 1, msg='localedef test command failed: output: %s' % output) + self.assertIn("fr_FR.utf8", output, msg='localedef test failed: output: %s' % output) + + def test_locales_on(self): + """ + Summary: Test the locales are generated + Expected: 1. Check the locale exist in the locale-archive + 2. Check the locale exist for the glibc + 3. Check the locale can be generated + Product: oe-core + Author: Louis Rannou <lrannou@baylibre.com> + AutomatedBy: Louis Rannou <lrannou@baylibre.com> + """ + self.run_locales_test(True) + + def test_locales_off(self): + self.run_locales_test(False) diff --git a/meta/lib/oeqa/selftest/cases/manifest.py b/meta/lib/oeqa/selftest/cases/manifest.py index 0a04c13a85..07a6c80489 100644 --- a/meta/lib/oeqa/selftest/cases/manifest.py +++ b/meta/lib/oeqa/selftest/cases/manifest.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/selftest/cases/meta_ide.py b/meta/lib/oeqa/selftest/cases/meta_ide.py index ce7bba401d..ffe0d2604d 100644 --- a/meta/lib/oeqa/selftest/cases/meta_ide.py +++ b/meta/lib/oeqa/selftest/cases/meta_ide.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -16,7 +18,8 @@ class MetaIDE(OESelftestTestCase): def setUpClass(cls): super(MetaIDE, cls).setUpClass() bitbake('meta-ide-support') - bitbake('build-sysroots') + bitbake('build-sysroots -c build_native_sysroot') + bitbake('build-sysroots -c build_target_sysroot') bb_vars = get_bb_vars(['MULTIMACH_TARGET_SYS', 'DEPLOY_DIR_IMAGE', 'COREBASE']) cls.environment_script = 'environment-setup-%s' % bb_vars['MULTIMACH_TARGET_SYS'] cls.deploydir = bb_vars['DEPLOY_DIR_IMAGE'] @@ -41,10 +44,10 @@ class MetaIDE(OESelftestTestCase): def test_meta_ide_can_build_cpio_project(self): dl_dir = self.td.get('DL_DIR', None) self.project = SDKBuildProject(self.tmpdir_metaideQA + "/cpio/", self.environment_script_path, - "https://ftp.gnu.org/gnu/cpio/cpio-2.13.tar.gz", + "https://ftp.gnu.org/gnu/cpio/cpio-2.15.tar.gz", self.tmpdir_metaideQA, self.td['DATETIME'], dl_dir=dl_dir) self.project.download_archive() - self.assertEqual(self.project.run_configure('$CONFIGURE_FLAGS --disable-maintainer-mode','sed -i -e "/char \*program_name/d" src/global.c;'), 0, + self.assertEqual(self.project.run_configure('$CONFIGURE_FLAGS'), 0, msg="Running configure failed") self.assertEqual(self.project.run_make(), 0, msg="Running make failed") @@ -53,5 +56,5 @@ class MetaIDE(OESelftestTestCase): def test_meta_ide_can_run_sdk_tests(self): bitbake('-c populate_sysroot gtk+3') - bitbake('build-sysroots') + bitbake('build-sysroots -c build_target_sysroot') bitbake('-c testsdk meta-ide-support') diff --git a/meta/lib/oeqa/selftest/cases/minidebuginfo.py b/meta/lib/oeqa/selftest/cases/minidebuginfo.py new file mode 100644 index 0000000000..2919f07939 --- /dev/null +++ b/meta/lib/oeqa/selftest/cases/minidebuginfo.py @@ -0,0 +1,44 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# +import os +import subprocess +import tempfile +import shutil + +from oeqa.selftest.case import OESelftestTestCase +from oeqa.utils.commands import bitbake, get_bb_var, get_bb_vars, runCmd + + +class Minidebuginfo(OESelftestTestCase): + def test_minidebuginfo(self): + target_sys = get_bb_var("TARGET_SYS") + binutils = "binutils-cross-{}".format(get_bb_var("TARGET_ARCH")) + + image = 'core-image-minimal' + bb_vars = get_bb_vars(['DEPLOY_DIR_IMAGE', 'IMAGE_LINK_NAME', 'READELF'], image) + + self.write_config(""" +DISTRO_FEATURES:append = " minidebuginfo" +IMAGE_FSTYPES = "tar.bz2" +""") + bitbake("{} {}:do_addto_recipe_sysroot".format(image, binutils)) + + native_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", binutils) + + # confirm that executables and shared libraries contain an ELF section + # ".gnu_debugdata" which stores minidebuginfo. + with tempfile.TemporaryDirectory(prefix = "unpackfs-") as unpackedfs: + filename = os.path.join(bb_vars['DEPLOY_DIR_IMAGE'], "{}.tar.bz2".format(bb_vars['IMAGE_LINK_NAME'])) + shutil.unpack_archive(filename, unpackedfs) + + r = runCmd([bb_vars['READELF'], "-W", "-S", os.path.join(unpackedfs, "bin", "busybox")], + native_sysroot = native_sysroot, target_sys = target_sys) + self.assertIn(".gnu_debugdata", r.output) + + r = runCmd([bb_vars['READELF'], "-W", "-S", os.path.join(unpackedfs, "lib", "libc.so.6")], + native_sysroot = native_sysroot, target_sys = target_sys) + self.assertIn(".gnu_debugdata", r.output) + diff --git a/meta/lib/oeqa/selftest/cases/multiconfig.py b/meta/lib/oeqa/selftest/cases/multiconfig.py index 83cbd1345d..f509cbf607 100644 --- a/meta/lib/oeqa/selftest/cases/multiconfig.py +++ b/meta/lib/oeqa/selftest/cases/multiconfig.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/selftest/cases/newlib.py b/meta/lib/oeqa/selftest/cases/newlib.py index 999e3e78b0..fe57aa51f2 100644 --- a/meta/lib/oeqa/selftest/cases/newlib.py +++ b/meta/lib/oeqa/selftest/cases/newlib.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/selftest/cases/oelib/buildhistory.py b/meta/lib/oeqa/selftest/cases/oelib/buildhistory.py index 33bd6df2f3..042ccdd2b4 100644 --- a/meta/lib/oeqa/selftest/cases/oelib/buildhistory.py +++ b/meta/lib/oeqa/selftest/cases/oelib/buildhistory.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -28,6 +30,16 @@ class TestBlobParsing(OESelftestTestCase): import shutil shutil.rmtree(self.repo_path) + @property + def heads_default(self): + """ + Support repos defaulting to master or to main branch + """ + try: + return self.repo.heads.main + except AttributeError: + return self.repo.heads.master + def commit_vars(self, to_add={}, to_remove = [], msg="A commit message"): if len(to_add) == 0 and len(to_remove) == 0: return @@ -65,10 +77,10 @@ class TestBlobParsing(OESelftestTestCase): changesmap = { "foo-2" : ("2", "8"), "bar" : ("","4"), "bar-2" : ("","5")} self.commit_vars(to_add = { "foo" : "1", "foo-2" : "2", "foo-3" : "3" }) - blob1 = self.repo.heads.master.commit.tree.blobs[0] + blob1 = self.heads_default.commit.tree.blobs[0] self.commit_vars(to_add = { "foo-2" : "8", "bar" : "4", "bar-2" : "5" }) - blob2 = self.repo.heads.master.commit.tree.blobs[0] + blob2 = self.heads_default.commit.tree.blobs[0] change_records = compare_dict_blobs(os.path.join(self.repo_path, self.test_file), blob1, blob2, False, False) @@ -84,10 +96,10 @@ class TestBlobParsing(OESelftestTestCase): defaultmap = { x : ("default", "1") for x in ["PKG", "PKGE", "PKGV", "PKGR"]} self.commit_vars(to_add = { "foo" : "1" }) - blob1 = self.repo.heads.master.commit.tree.blobs[0] + blob1 = self.heads_default.commit.tree.blobs[0] self.commit_vars(to_add = { "PKG" : "1", "PKGE" : "1", "PKGV" : "1", "PKGR" : "1" }) - blob2 = self.repo.heads.master.commit.tree.blobs[0] + blob2 = self.heads_default.commit.tree.blobs[0] change_records = compare_dict_blobs(os.path.join(self.repo_path, self.test_file), blob1, blob2, False, False) diff --git a/meta/lib/oeqa/selftest/cases/oelib/elf.py b/meta/lib/oeqa/selftest/cases/oelib/elf.py index 5a5f9b4fdf..7bf550b6fd 100644 --- a/meta/lib/oeqa/selftest/cases/oelib/elf.py +++ b/meta/lib/oeqa/selftest/cases/oelib/elf.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/selftest/cases/oelib/license.py b/meta/lib/oeqa/selftest/cases/oelib/license.py index 3b359396b6..5eea12e761 100644 --- a/meta/lib/oeqa/selftest/cases/oelib/license.py +++ b/meta/lib/oeqa/selftest/cases/oelib/license.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/selftest/cases/oelib/path.py b/meta/lib/oeqa/selftest/cases/oelib/path.py index a1cfa08c09..b963e447e3 100644 --- a/meta/lib/oeqa/selftest/cases/oelib/path.py +++ b/meta/lib/oeqa/selftest/cases/oelib/path.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/selftest/cases/oelib/types.py b/meta/lib/oeqa/selftest/cases/oelib/types.py index 7eb49e6f95..58318b18b2 100644 --- a/meta/lib/oeqa/selftest/cases/oelib/types.py +++ b/meta/lib/oeqa/selftest/cases/oelib/types.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/selftest/cases/oelib/utils.py b/meta/lib/oeqa/selftest/cases/oelib/utils.py index bbf67bf9c9..0cb46425a0 100644 --- a/meta/lib/oeqa/selftest/cases/oelib/utils.py +++ b/meta/lib/oeqa/selftest/cases/oelib/utils.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/selftest/cases/oescripts.py b/meta/lib/oeqa/selftest/cases/oescripts.py index d3a789a6a7..f69efccfee 100644 --- a/meta/lib/oeqa/selftest/cases/oescripts.py +++ b/meta/lib/oeqa/selftest/cases/oescripts.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -34,18 +36,16 @@ class BuildhistoryDiffTests(BuildhistoryBase): if expected_endlines: self.fail('Missing expected line endings:\n %s' % '\n '.join(expected_endlines)) -class OEScriptTests(OESelftestTestCase): - scripts_dir = os.path.join(get_bb_var('COREBASE'), 'scripts') - @unittest.skipUnless(importlib.util.find_spec("cairo"), "Python cairo module is not present") -class OEPybootchartguyTests(OEScriptTests): +class OEPybootchartguyTests(OESelftestTestCase): @classmethod def setUpClass(cls): - super(OEScriptTests, cls).setUpClass() + super().setUpClass() bitbake("core-image-minimal -c rootfs -f") cls.tmpdir = get_bb_var('TMPDIR') cls.buildstats = cls.tmpdir + "/buildstats/" + sorted(os.listdir(cls.tmpdir + "/buildstats"))[-1] + cls.scripts_dir = os.path.join(get_bb_var('COREBASE'), 'scripts') def test_pybootchartguy_help(self): runCmd('%s/pybootchartgui/pybootchartgui.py --help' % self.scripts_dir) @@ -63,7 +63,12 @@ class OEPybootchartguyTests(OEScriptTests): self.assertTrue(os.path.exists(self.tmpdir + "/charts.pdf")) -class OEGitproxyTests(OEScriptTests): +class OEGitproxyTests(OESelftestTestCase): + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.scripts_dir = os.path.join(get_bb_var('COREBASE'), 'scripts') def test_oegitproxy_help(self): try: @@ -121,10 +126,16 @@ class OEGitproxyTests(OEScriptTests): class OeRunNativeTest(OESelftestTestCase): def test_oe_run_native(self): bitbake("qemu-helper-native -c addto_recipe_sysroot") - result = runCmd("oe-run-native qemu-helper-native tunctl -h") - self.assertIn("Delete: tunctl -d device-name [-f tun-clone-device]", result.output) + result = runCmd("oe-run-native qemu-helper-native qemu-oe-bridge-helper --help") + self.assertIn("Helper function to find and exec qemu-bridge-helper", result.output) + +class OEListPackageconfigTests(OESelftestTestCase): + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.scripts_dir = os.path.join(get_bb_var('COREBASE'), 'scripts') -class OEListPackageconfigTests(OEScriptTests): #oe-core.scripts.List_all_the_PACKAGECONFIG's_flags def check_endlines(self, results, expected_endlines): for line in results.output.splitlines(): @@ -146,7 +157,7 @@ class OEListPackageconfigTests(OEScriptTests): results = runCmd('%s/contrib/list-packageconfig-flags.py' % self.scripts_dir) expected_endlines = [] expected_endlines.append("RECIPE NAME PACKAGECONFIG FLAGS") - expected_endlines.append("pinentry gtk2 libcap ncurses qt secret") + expected_endlines.append("pinentry gtk2 ncurses qt secret") expected_endlines.append("tar acl selinux") self.check_endlines(results, expected_endlines) @@ -164,11 +175,10 @@ class OEListPackageconfigTests(OEScriptTests): def test_packageconfig_flags_option_all(self): results = runCmd('%s/contrib/list-packageconfig-flags.py -a' % self.scripts_dir) expected_endlines = [] - expected_endlines.append("pinentry-1.2.0") - expected_endlines.append("PACKAGECONFIG ncurses libcap") + expected_endlines.append("pinentry-1.2.1") + expected_endlines.append("PACKAGECONFIG ncurses") expected_endlines.append("PACKAGECONFIG[qt] --enable-pinentry-qt, --disable-pinentry-qt, qtbase-native qtbase") expected_endlines.append("PACKAGECONFIG[gtk2] --enable-pinentry-gtk2, --disable-pinentry-gtk2, gtk+ glib-2.0") - expected_endlines.append("PACKAGECONFIG[libcap] --with-libcap, --without-libcap, libcap") expected_endlines.append("PACKAGECONFIG[ncurses] --enable-ncurses --with-ncurses-include-dir=${STAGING_INCDIR}, --disable-ncurses, ncurses") expected_endlines.append("PACKAGECONFIG[secret] --enable-libsecret, --disable-libsecret, libsecret") @@ -178,7 +188,7 @@ class OEListPackageconfigTests(OEScriptTests): results = runCmd('%s/contrib/list-packageconfig-flags.py -p' % self.scripts_dir) expected_endlines = [] expected_endlines.append("RECIPE NAME PACKAGECONFIG FLAGS") - expected_endlines.append("pinentry gtk2 libcap ncurses qt secret") + expected_endlines.append("pinentry gtk2 ncurses qt secret") self.check_endlines(results, expected_endlines) diff --git a/meta/lib/oeqa/selftest/cases/overlayfs.py b/meta/lib/oeqa/selftest/cases/overlayfs.py index 96beb8b869..e31063567b 100644 --- a/meta/lib/oeqa/selftest/cases/overlayfs.py +++ b/meta/lib/oeqa/selftest/cases/overlayfs.py @@ -1,10 +1,13 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # from oeqa.selftest.case import OESelftestTestCase from oeqa.utils.commands import bitbake, runqemu from oeqa.core.decorator import OETestTag +from oeqa.core.decorator.data import skipIfNotMachine def getline_qemu(out, line): for l in out.split('\n'): @@ -55,7 +58,7 @@ inherit overlayfs config = """ IMAGE_INSTALL:append = " overlayfs-user" -DISTRO_FEATURES:append = " systemd overlayfs" +DISTRO_FEATURES:append = " systemd overlayfs usrmerge" """ self.write_config(config) @@ -76,7 +79,7 @@ DISTRO_FEATURES:append = " systemd overlayfs" config = """ IMAGE_INSTALL:append = " overlayfs-user" -DISTRO_FEATURES += "systemd overlayfs" +DISTRO_FEATURES:append = " systemd overlayfs usrmerge" OVERLAYFS_QA_SKIP[mnt-overlay] = "mount-configured" """ @@ -94,7 +97,7 @@ OVERLAYFS_QA_SKIP[mnt-overlay] = "mount-configured" config = """ IMAGE_INSTALL:append = " overlayfs-user" -DISTRO_FEATURES:append = " systemd overlayfs" +DISTRO_FEATURES:append = " systemd overlayfs usrmerge" """ self.write_config(config) @@ -112,7 +115,7 @@ DISTRO_FEATURES:append = " systemd overlayfs" config = """ IMAGE_INSTALL:append = " overlayfs-user" -DISTRO_FEATURES:append = " systemd overlayfs" +DISTRO_FEATURES:append = " systemd overlayfs usrmerge" """ wrong_machine_config = """ @@ -136,10 +139,10 @@ OVERLAYFS_MOUNT_POINT[usr-share-overlay] = "/usr/share/overlay" config = """ IMAGE_INSTALL:append = " overlayfs-user systemd-machine-units" -DISTRO_FEATURES:append = " systemd overlayfs" +DISTRO_FEATURES:append = " overlayfs" # Use systemd as init manager -VIRTUAL-RUNTIME_init_manager = "systemd" +INIT_MANAGER = "systemd" # enable overlayfs in the kernel KERNEL_EXTRA_FEATURES:append = " features/overlayfs/overlayfs.scc" @@ -271,10 +274,8 @@ class OverlayFSEtcRunTimeTests(OESelftestTestCase): """ configBase = """ -DISTRO_FEATURES:append = " systemd" - # Use systemd as init manager -VIRTUAL-RUNTIME_init_manager = "systemd" +INIT_MANAGER = "systemd" # enable overlayfs in the kernel KERNEL_EXTRA_FEATURES:append = " features/overlayfs/overlayfs.scc" @@ -313,10 +314,8 @@ OVERLAYFS_ETC_DEVICE = "/dev/mmcblk0p1" """ config = """ -DISTRO_FEATURES:append = " systemd" - # Use systemd as init manager -VIRTUAL-RUNTIME_init_manager = "systemd" +INIT_MANAGER = "systemd" # enable overlayfs in the kernel KERNEL_EXTRA_FEATURES:append = " features/overlayfs/overlayfs.scc" @@ -331,28 +330,18 @@ EXTRA_IMAGE_FEATURES += "package-management" self.assertTrue("overlayfs-etc" in res.output, msg=res.output) self.assertTrue("package-management" in res.output, msg=res.output) - def test_image_feature_is_missing_class_included(self): - configAppend = """ -INHERIT += "overlayfs-etc" -""" - self.run_check_image_feature(configAppend) - + # https://bugzilla.yoctoproject.org/show_bug.cgi?id=14963 + @skipIfNotMachine("qemux86-64", "tests are qemux86-64 specific currently") def test_image_feature_is_missing(self): - self.run_check_image_feature() - - def run_check_image_feature(self, appendToConfig=""): """ Summary: Overlayfs-etc class is not applied when image feature is not set - even if we inherit it directly, Expected: Image is created successfully but /etc is not an overlay Author: Vyacheslav Yurkov <uvv.mail@gmail.com> """ - config = f""" -DISTRO_FEATURES:append = " systemd" - + config = """ # Use systemd as init manager -VIRTUAL-RUNTIME_init_manager = "systemd" +INIT_MANAGER = "systemd" # enable overlayfs in the kernel KERNEL_EXTRA_FEATURES:append = " features/overlayfs/overlayfs.scc" @@ -364,7 +353,7 @@ EXTRA_IMAGE_FEATURES += "read-only-rootfs" # Image configuration for overlayfs-etc OVERLAYFS_ETC_MOUNT_POINT = "/data" OVERLAYFS_ETC_DEVICE = "/dev/sda3" -{appendToConfig} +OVERLAYFS_ROOTFS_TYPE = "ext4" """ self.write_config(config) @@ -377,45 +366,32 @@ OVERLAYFS_ETC_DEVICE = "/dev/sda3" line = getline_qemu(output, "upperdir=/data/overlay-etc/upper") self.assertFalse(line, msg=output) + @skipIfNotMachine("qemux86-64", "tests are qemux86-64 specific currently") def test_sbin_init_preinit(self): - self.run_sbin_init(False) + self.run_sbin_init(False, "ext4") + @skipIfNotMachine("qemux86-64", "tests are qemux86-64 specific currently") def test_sbin_init_original(self): - self.run_sbin_init(True) + self.run_sbin_init(True, "ext4") + + @skipIfNotMachine("qemux86-64", "tests are qemux86-64 specific currently") + def test_sbin_init_read_only(self): + self.run_sbin_init(True, "squashfs") - def run_sbin_init(self, origInit): + def run_sbin_init(self, origInit, rootfsType): """ Summary: Confirm we can replace original init and mount overlay on top of /etc Expected: Image is created successfully and /etc is mounted as an overlay Author: Vyacheslav Yurkov <uvv.mail@gmail.com> """ - config = """ -DISTRO_FEATURES:append = " systemd" - -# Use systemd as init manager -VIRTUAL-RUNTIME_init_manager = "systemd" - -# enable overlayfs in the kernel -KERNEL_EXTRA_FEATURES:append = " features/overlayfs/overlayfs.scc" - -IMAGE_FSTYPES += "wic" -OVERLAYFS_INIT_OPTION = "{OVERLAYFS_INIT_OPTION}" -WKS_FILE = "overlayfs_etc.wks.in" - -EXTRA_IMAGE_FEATURES += "read-only-rootfs" -# Image configuration for overlayfs-etc -EXTRA_IMAGE_FEATURES += "overlayfs-etc" -IMAGE_FEATURES:remove = "package-management" -OVERLAYFS_ETC_MOUNT_POINT = "/data" -OVERLAYFS_ETC_FSTYPE = "ext4" -OVERLAYFS_ETC_DEVICE = "/dev/sda3" -OVERLAYFS_ETC_USE_ORIG_INIT_NAME = "{OVERLAYFS_ETC_USE_ORIG_INIT_NAME}" -""" + config = self.get_working_config() args = { 'OVERLAYFS_INIT_OPTION': "" if origInit else "init=/sbin/preinit", - 'OVERLAYFS_ETC_USE_ORIG_INIT_NAME': int(origInit == True) + 'OVERLAYFS_ETC_USE_ORIG_INIT_NAME': int(origInit == True), + 'OVERLAYFS_ROOTFS_TYPE': rootfsType, + 'OVERLAYFS_ETC_CREATE_MOUNT_DIRS': int(rootfsType == "ext4") } self.write_config(config.format(**args)) @@ -432,6 +408,11 @@ OVERLAYFS_ETC_USE_ORIG_INIT_NAME = "{OVERLAYFS_ETC_USE_ORIG_INIT_NAME}" line = getline_qemu(output, "upperdir=/data/overlay-etc/upper") self.assertTrue(line and line.startswith("/data/overlay-etc/upper on /etc type overlay"), msg=output) + # check that lower layer is not available + status, output = qemu.run_serial("ls -1 /data/overlay-etc/lower") + line = getline_qemu(output, "No such file or directory") + self.assertTrue(line, msg=output) + status, output = qemu.run_serial("touch " + testFile) status, output = qemu.run_serial("sync") status, output = qemu.run_serial("ls -1 " + testFile) @@ -443,3 +424,79 @@ OVERLAYFS_ETC_USE_ORIG_INIT_NAME = "{OVERLAYFS_ETC_USE_ORIG_INIT_NAME}" status, output = qemu.run_serial("ls -1 " + testFile) line = getline_qemu(output, testFile) self.assertTrue(line and line.startswith(testFile), msg=output) + + @skipIfNotMachine("qemux86-64", "tests are qemux86-64 specific currently") + def test_lower_layer_access(self): + """ + Summary: Test that lower layer of /etc is available read-only when configured + Expected: Can't write to lower layer. The files on lower and upper different after + modification + Author: Vyacheslav Yurkov <uvv.mail@gmail.com> + """ + + config = self.get_working_config() + + configLower = """ +OVERLAYFS_ETC_EXPOSE_LOWER = "1" +IMAGE_INSTALL:append = " overlayfs-user" +""" + testFile = "lower-layer-test.txt" + + args = { + 'OVERLAYFS_INIT_OPTION': "", + 'OVERLAYFS_ETC_USE_ORIG_INIT_NAME': 1, + 'OVERLAYFS_ROOTFS_TYPE': "ext4", + 'OVERLAYFS_ETC_CREATE_MOUNT_DIRS': 1 + } + + self.write_config(config.format(**args)) + + self.append_config(configLower) + bitbake('core-image-minimal') + + with runqemu('core-image-minimal', image_fstype='wic') as qemu: + status, output = qemu.run_serial("echo \"Modified in upper\" > /etc/" + testFile) + status, output = qemu.run_serial("diff /etc/" + testFile + " /data/overlay-etc/lower/" + testFile) + line = getline_qemu(output, "Modified in upper") + self.assertTrue(line, msg=output) + line = getline_qemu(output, "Original file") + self.assertTrue(line, msg=output) + + status, output = qemu.run_serial("touch /data/overlay-etc/lower/ro-test.txt") + line = getline_qemu(output, "Read-only file system") + self.assertTrue(line, msg=output) + + def get_working_config(self): + return """ +# Use systemd as init manager +INIT_MANAGER = "systemd" + +# enable overlayfs in the kernel +KERNEL_EXTRA_FEATURES:append = " \ + features/overlayfs/overlayfs.scc \ + cfg/fs/squashfs.scc" + +IMAGE_FSTYPES += "wic" +OVERLAYFS_INIT_OPTION = "{OVERLAYFS_INIT_OPTION}" +OVERLAYFS_ROOTFS_TYPE = "{OVERLAYFS_ROOTFS_TYPE}" +OVERLAYFS_ETC_CREATE_MOUNT_DIRS = "{OVERLAYFS_ETC_CREATE_MOUNT_DIRS}" +WKS_FILE = "overlayfs_etc.wks.in" + +EXTRA_IMAGE_FEATURES += "read-only-rootfs" +# Image configuration for overlayfs-etc +EXTRA_IMAGE_FEATURES += "overlayfs-etc" +IMAGE_FEATURES:remove = "package-management" +OVERLAYFS_ETC_MOUNT_POINT = "/data" +OVERLAYFS_ETC_FSTYPE = "ext4" +OVERLAYFS_ETC_DEVICE = "/dev/sda3" +OVERLAYFS_ETC_USE_ORIG_INIT_NAME = "{OVERLAYFS_ETC_USE_ORIG_INIT_NAME}" + +ROOTFS_POSTPROCESS_COMMAND += "{OVERLAYFS_ROOTFS_TYPE}_rootfs" + +ext4_rootfs() {{ +}} + +squashfs_rootfs() {{ + mkdir -p ${{IMAGE_ROOTFS}}/data +}} +""" diff --git a/meta/lib/oeqa/selftest/cases/package.py b/meta/lib/oeqa/selftest/cases/package.py index 51d835259e..1aa6c03f8a 100644 --- a/meta/lib/oeqa/selftest/cases/package.py +++ b/meta/lib/oeqa/selftest/cases/package.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -87,6 +89,13 @@ class VersionOrdering(OESelftestTestCase): self.assertEqual(status - 100, sort, "%s %s (%d) failed" % (ver1, ver2, sort)) class PackageTests(OESelftestTestCase): + # Verify that a recipe cannot rename a package into an existing one + def test_package_name_conflict(self): + res = bitbake("packagenameconflict", ignore_status=True) + self.assertNotEqual(res.status, 0) + err = "package name already exists" + self.assertTrue(err in res.output) + # Verify that a recipe which sets up hardlink files has those preserved into split packages # Also test file sparseness is preserved def test_preserve_sparse_hardlinks(self): @@ -133,8 +142,10 @@ class PackageTests(OESelftestTestCase): self.logger.error("No debugging symbols found. GDB result:\n%s" % output) return False - # Check debugging symbols works correctly - elif re.match(r"Breakpoint 1.*hello\.c.*4", l): + # Check debugging symbols works correctly. Don't look for a + # source file as optimisation can put the breakpoint inside + # stdio.h. + elif "Breakpoint 1 at" in l: return True self.logger.error("GDB result:\n%d: %s", status, output) @@ -149,25 +160,25 @@ class PackageTests(OESelftestTestCase): self.fail('GDB %s failed' % binary) def test_preserve_ownership(self): - import os, stat, oe.cachedpath features = 'IMAGE_INSTALL:append = " selftest-chown"\n' self.write_config(features) bitbake("core-image-minimal") - sysconfdir = get_bb_var('sysconfdir', 'selftest-chown') - def check_ownership(qemu, gid, uid, path): + def check_ownership(qemu, expected_gid, expected_uid, path): self.logger.info("Check ownership of %s", path) - status, output = qemu.run_serial(r'/bin/stat -c "%U %G" ' + path, timeout=60) - output = output.split(" ") - if output[0] != uid or output[1] != gid : - self.logger.error("Incrrect ownership %s [%s:%s]", path, output[0], output[1]) - return False - return True + status, output = qemu.run_serial('stat -c "%U %G" ' + path) + self.assertEqual(status, 1, "stat failed: " + output) + try: + uid, gid = output.split() + self.assertEqual(uid, expected_uid) + self.assertEqual(gid, expected_gid) + except ValueError: + self.fail("Cannot parse output: " + output) + sysconfdir = get_bb_var('sysconfdir', 'selftest-chown') with runqemu('core-image-minimal') as qemu: for path in [ sysconfdir + "/selftest-chown/file", sysconfdir + "/selftest-chown/dir", sysconfdir + "/selftest-chown/symlink", sysconfdir + "/selftest-chown/fifotest/fifo"]: - if not check_ownership(qemu, "test", "test", path): - self.fail('Test ownership %s failed' % path) + check_ownership(qemu, "test", "test", path) diff --git a/meta/lib/oeqa/selftest/cases/pkgdata.py b/meta/lib/oeqa/selftest/cases/pkgdata.py index 87d069de65..d786c33018 100644 --- a/meta/lib/oeqa/selftest/cases/pkgdata.py +++ b/meta/lib/oeqa/selftest/cases/pkgdata.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/selftest/cases/prservice.py b/meta/lib/oeqa/selftest/cases/prservice.py index 10158ca7c2..8da3739c57 100644 --- a/meta/lib/oeqa/selftest/cases/prservice.py +++ b/meta/lib/oeqa/selftest/cases/prservice.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -12,6 +14,8 @@ from oeqa.selftest.case import OESelftestTestCase from oeqa.utils.commands import runCmd, bitbake, get_bb_var from oeqa.utils.network import get_free_port +import bb.utils + class BitbakePrTests(OESelftestTestCase): @classmethod @@ -19,6 +23,16 @@ class BitbakePrTests(OESelftestTestCase): super(BitbakePrTests, cls).setUpClass() cls.pkgdata_dir = get_bb_var('PKGDATA_DIR') + cls.exported_db_path = os.path.join(cls.builddir, 'export.inc') + cls.current_db_path = os.path.join(get_bb_var('PERSISTENT_DIR'), 'prserv.sqlite3') + + def cleanup(self): + # Ensure any memory resident bitbake is stopped + bitbake("-m") + # Remove any existing export file or prserv database + bb.utils.remove(self.exported_db_path) + bb.utils.remove(self.current_db_path + "*") + def get_pr_version(self, package_name): package_data_file = os.path.join(self.pkgdata_dir, 'runtime', package_name) package_data = ftools.read_file(package_data_file) @@ -47,6 +61,7 @@ class BitbakePrTests(OESelftestTestCase): self.assertEqual(res.status, 0, msg=res.output) def config_pr_tests(self, package_name, package_type='rpm', pr_socket='localhost:0'): + self.cleanup() config_package_data = 'PACKAGE_CLASSES = "package_%s"' % package_type self.write_config(config_package_data) config_server_data = 'PRSERV_HOST = "%s"' % pr_socket @@ -66,24 +81,24 @@ class BitbakePrTests(OESelftestTestCase): self.assertTrue(pr_2 - pr_1 == 1, "New PR %s did not increment as expected (from %s), difference should be 1" % (pr_2, pr_1)) self.assertTrue(stamp_1 != stamp_2, "Different pkg rev. but same stamp: %s" % stamp_1) + self.cleanup() + def run_test_pr_export_import(self, package_name, replace_current_db=True): self.config_pr_tests(package_name) self.increment_package_pr(package_name) pr_1 = self.get_pr_version(package_name) - exported_db_path = os.path.join(self.builddir, 'export.inc') - export_result = runCmd("bitbake-prserv-tool export %s" % exported_db_path, ignore_status=True) + export_result = runCmd("bitbake-prserv-tool export %s" % self.exported_db_path, ignore_status=True) self.assertEqual(export_result.status, 0, msg="PR Service database export failed: %s" % export_result.output) - self.assertTrue(os.path.exists(exported_db_path)) + self.assertTrue(os.path.exists(self.exported_db_path), msg="%s didn't exist, tool output %s" % (self.exported_db_path, export_result.output)) if replace_current_db: - current_db_path = os.path.join(get_bb_var('PERSISTENT_DIR'), 'prserv.sqlite3') - self.assertTrue(os.path.exists(current_db_path), msg="Path to current PR Service database is invalid: %s" % current_db_path) - os.remove(current_db_path) + self.assertTrue(os.path.exists(self.current_db_path), msg="Path to current PR Service database is invalid: %s" % self.current_db_path) + os.remove(self.current_db_path) - import_result = runCmd("bitbake-prserv-tool import %s" % exported_db_path, ignore_status=True) - os.remove(exported_db_path) + import_result = runCmd("bitbake-prserv-tool import %s" % self.exported_db_path, ignore_status=True) + #os.remove(self.exported_db_path) self.assertEqual(import_result.status, 0, msg="PR Service database import failed: %s" % import_result.output) self.increment_package_pr(package_name) @@ -91,6 +106,8 @@ class BitbakePrTests(OESelftestTestCase): self.assertTrue(pr_2 - pr_1 == 1, "New PR %s did not increment as expected (from %s), difference should be 1" % (pr_2, pr_1)) + self.cleanup() + def test_import_export_replace_db(self): self.run_test_pr_export_import('m4') diff --git a/meta/lib/oeqa/selftest/cases/pseudo.py b/meta/lib/oeqa/selftest/cases/pseudo.py index 33593d5ce9..3ef8786022 100644 --- a/meta/lib/oeqa/selftest/cases/pseudo.py +++ b/meta/lib/oeqa/selftest/cases/pseudo.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/selftest/cases/recipetool.py b/meta/lib/oeqa/selftest/cases/recipetool.py index 510dae6bad..aebea42502 100644 --- a/meta/lib/oeqa/selftest/cases/recipetool.py +++ b/meta/lib/oeqa/selftest/cases/recipetool.py @@ -1,7 +1,10 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # +import errno import os import shutil import tempfile @@ -25,6 +28,16 @@ def tearDownModule(): runCmd('rm -rf %s' % templayerdir) +def needTomllib(test): + # This test require python 3.11 or above for the tomllib module or tomli module to be installed + try: + import tomllib + except ImportError: + try: + import tomli + except ImportError: + test.skipTest('Test requires python 3.11 or above for tomllib module or tomli module') + class RecipetoolBase(devtool.DevtoolTestCase): def setUpLocal(self): @@ -35,6 +48,8 @@ class RecipetoolBase(devtool.DevtoolTestCase): self.testfile = os.path.join(self.tempdir, 'testfile') with open(self.testfile, 'w') as f: f.write('Test file\n') + config = 'BBMASK += "meta-poky/recipes-core/base-files/base-files_%.bbappend"\n' + self.append_config(config) def tearDownLocal(self): runCmd('rm -rf %s/recipes-*' % self.templayerdir) @@ -346,7 +361,6 @@ class RecipetoolCreateTests(RecipetoolBase): checkvars['LICENSE'] = 'GPL-2.0-only' checkvars['LIC_FILES_CHKSUM'] = 'file://COPYING;md5=b234ee4d69f5fce4486a80fdaf4a4263' checkvars['SRC_URI'] = 'https://github.com/logrotate/logrotate/releases/download/${PV}/logrotate-${PV}.tar.xz' - checkvars['SRC_URI[md5sum]'] = 'a560c57fac87c45b2fc17406cdf79288' checkvars['SRC_URI[sha256sum]'] = '2e6a401cac9024db2288297e3be1a8ab60e7401ba8e91225218aaf4a27e82a07' self._test_recipe_contents(recipefile, checkvars, []) @@ -359,14 +373,14 @@ class RecipetoolCreateTests(RecipetoolBase): tempsrc = os.path.join(self.tempdir, 'srctree') os.makedirs(tempsrc) recipefile = os.path.join(self.tempdir, 'libmatchbox.bb') - srcuri = 'git://git.yoctoproject.org/libmatchbox' + srcuri = 'git://git.yoctoproject.org/libmatchbox;protocol=https' result = runCmd(['recipetool', 'create', '-o', recipefile, srcuri + ";rev=9f7cf8895ae2d39c465c04cc78e918c157420269", '-x', tempsrc]) self.assertTrue(os.path.isfile(recipefile), 'recipetool did not create recipe file; output:\n%s' % result.output) checkvars = {} checkvars['LICENSE'] = 'LGPL-2.1-only' checkvars['LIC_FILES_CHKSUM'] = 'file://COPYING;md5=7fbc338309ac38fefcd64b04bb903e34' checkvars['S'] = '${WORKDIR}/git' - checkvars['PV'] = '1.11+git${SRCPV}' + checkvars['PV'] = '1.11+git' checkvars['SRC_URI'] = srcuri + ';branch=master' checkvars['DEPENDS'] = set(['libcheck', 'libjpeg-turbo', 'libpng', 'libx11', 'libxext', 'pango']) inherits = ['autotools', 'pkgconfig'] @@ -404,7 +418,6 @@ class RecipetoolCreateTests(RecipetoolBase): checkvars = {} checkvars['LICENSE'] = set(['LGPL-2.1-only', 'MPL-1.1-only']) checkvars['SRC_URI'] = 'http://taglib.github.io/releases/taglib-${PV}.tar.gz' - checkvars['SRC_URI[md5sum]'] = 'cee7be0ccfc892fa433d6c837df9522a' checkvars['SRC_URI[sha256sum]'] = 'b6d1a5a610aae6ff39d93de5efd0fdc787aa9e9dc1e7026fa4c961b26563526b' checkvars['DEPENDS'] = set(['boost', 'zlib']) inherits = ['cmake'] @@ -440,44 +453,257 @@ class RecipetoolCreateTests(RecipetoolBase): self._test_recipe_contents(recipefile, checkvars, inherits) def test_recipetool_create_github(self): - # Basic test to see if github URL mangling works + # Basic test to see if github URL mangling works. Deliberately use an + # older release of Meson at present so we don't need a toml parser. temprecipe = os.path.join(self.tempdir, 'recipe') os.makedirs(temprecipe) - recipefile = os.path.join(temprecipe, 'meson_git.bb') - srcuri = 'https://github.com/mesonbuild/meson;rev=0.32.0' - result = runCmd(['recipetool', 'create', '-o', temprecipe, srcuri]) - self.assertTrue(os.path.isfile(recipefile)) + recipefile = os.path.join(temprecipe, 'python3-meson_git.bb') + srcuri = 'https://github.com/mesonbuild/meson;rev=0.52.1' + cmd = ['recipetool', 'create', '-o', temprecipe, srcuri] + result = runCmd(cmd) + self.assertTrue(os.path.isfile(recipefile), msg="recipe %s not created for command %s, output %s" % (recipefile, " ".join(cmd), result.output)) checkvars = {} - checkvars['LICENSE'] = set(['Apache-2.0']) - checkvars['SRC_URI'] = 'git://github.com/mesonbuild/meson;protocol=https;branch=master' + checkvars['LICENSE'] = set(['Apache-2.0', "Unknown"]) + checkvars['SRC_URI'] = 'git://github.com/mesonbuild/meson;protocol=https;branch=0.52' inherits = ['setuptools3'] self._test_recipe_contents(recipefile, checkvars, inherits) def test_recipetool_create_python3_setuptools(self): # Test creating python3 package from tarball (using setuptools3 class) + # Use the --no-pypi switch to avoid creating a pypi enabled recipe and + # and check the created recipe as if it was a more general tarball temprecipe = os.path.join(self.tempdir, 'recipe') os.makedirs(temprecipe) pn = 'python-magic' pv = '0.4.15' recipefile = os.path.join(temprecipe, '%s_%s.bb' % (pn, pv)) srcuri = 'https://files.pythonhosted.org/packages/84/30/80932401906eaf787f2e9bd86dc458f1d2e75b064b4c187341f29516945c/python-magic-%s.tar.gz' % pv - result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) + result = runCmd('recipetool create --no-pypi -o %s %s' % (temprecipe, srcuri)) self.assertTrue(os.path.isfile(recipefile)) checkvars = {} checkvars['LICENSE'] = set(['MIT']) checkvars['LIC_FILES_CHKSUM'] = 'file://LICENSE;md5=16a934f165e8c3245f241e77d401bb88' checkvars['SRC_URI'] = 'https://files.pythonhosted.org/packages/84/30/80932401906eaf787f2e9bd86dc458f1d2e75b064b4c187341f29516945c/python-magic-${PV}.tar.gz' - checkvars['SRC_URI[md5sum]'] = 'e384c95a47218f66c6501cd6dd45ff59' checkvars['SRC_URI[sha256sum]'] = 'f3765c0f582d2dfc72c15f3b5a82aecfae9498bd29ca840d72f37d7bd38bfcd5' inherits = ['setuptools3'] self._test_recipe_contents(recipefile, checkvars, inherits) + def test_recipetool_create_python3_setuptools_pypi_tarball(self): + # Test creating python3 package from tarball (using setuptools3 and pypi classes) + temprecipe = os.path.join(self.tempdir, 'recipe') + os.makedirs(temprecipe) + pn = 'python-magic' + pv = '0.4.15' + recipefile = os.path.join(temprecipe, '%s_%s.bb' % (pn, pv)) + srcuri = 'https://files.pythonhosted.org/packages/84/30/80932401906eaf787f2e9bd86dc458f1d2e75b064b4c187341f29516945c/python-magic-%s.tar.gz' % pv + result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) + self.assertTrue(os.path.isfile(recipefile)) + checkvars = {} + checkvars['LICENSE'] = set(['MIT']) + checkvars['LIC_FILES_CHKSUM'] = 'file://LICENSE;md5=16a934f165e8c3245f241e77d401bb88' + checkvars['SRC_URI[sha256sum]'] = 'f3765c0f582d2dfc72c15f3b5a82aecfae9498bd29ca840d72f37d7bd38bfcd5' + checkvars['PYPI_PACKAGE'] = pn + inherits = ['setuptools3', 'pypi'] + self._test_recipe_contents(recipefile, checkvars, inherits) + + def test_recipetool_create_python3_setuptools_pypi(self): + # Test creating python3 package from pypi url (using setuptools3 and pypi classes) + # Intentionnaly using setuptools3 class here instead of any of the pep517 class + # to avoid the toml dependency and allows this test to run on host autobuilders + # with older version of python + temprecipe = os.path.join(self.tempdir, 'recipe') + os.makedirs(temprecipe) + pn = 'python-magic' + pv = '0.4.15' + recipefile = os.path.join(temprecipe, '%s_%s.bb' % (pn, pv)) + # First specify the required version in the url + srcuri = 'https://pypi.org/project/%s/%s' % (pn, pv) + runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) + self.assertTrue(os.path.isfile(recipefile)) + checkvars = {} + checkvars['LICENSE'] = set(['MIT']) + checkvars['LIC_FILES_CHKSUM'] = 'file://LICENSE;md5=16a934f165e8c3245f241e77d401bb88' + checkvars['SRC_URI[sha256sum]'] = 'f3765c0f582d2dfc72c15f3b5a82aecfae9498bd29ca840d72f37d7bd38bfcd5' + checkvars['PYPI_PACKAGE'] = pn + inherits = ['setuptools3', "pypi"] + self._test_recipe_contents(recipefile, checkvars, inherits) + + # Now specify the version as a recipetool parameter + runCmd('rm -rf %s' % recipefile) + self.assertFalse(os.path.isfile(recipefile)) + srcuri = 'https://pypi.org/project/%s' % pn + runCmd('recipetool create -o %s %s --version %s' % (temprecipe, srcuri, pv)) + self.assertTrue(os.path.isfile(recipefile)) + checkvars = {} + checkvars['LICENSE'] = set(['MIT']) + checkvars['LIC_FILES_CHKSUM'] = 'file://LICENSE;md5=16a934f165e8c3245f241e77d401bb88' + checkvars['SRC_URI[sha256sum]'] = 'f3765c0f582d2dfc72c15f3b5a82aecfae9498bd29ca840d72f37d7bd38bfcd5' + checkvars['PYPI_PACKAGE'] = pn + inherits = ['setuptools3', "pypi"] + self._test_recipe_contents(recipefile, checkvars, inherits) + + # Now, try to grab latest version of the package, so we cannot guess the name of the recipe, + # unless hardcoding the latest version but it means we will need to update the test for each release, + # so use a regexp + runCmd('rm -rf %s' % recipefile) + self.assertFalse(os.path.isfile(recipefile)) + recipefile_re = r'%s_(.*)\.bb' % pn + result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) + dirlist = os.listdir(temprecipe) + if len(dirlist) > 1: + self.fail('recipetool created more than just one file; output:\n%s\ndirlist:\n%s' % (result.output, str(dirlist))) + if len(dirlist) < 1 or not os.path.isfile(os.path.join(temprecipe, dirlist[0])): + self.fail('recipetool did not create recipe file; output:\n%s\ndirlist:\n%s' % (result.output, str(dirlist))) + import re + match = re.match(recipefile_re, dirlist[0]) + self.assertTrue(match) + latest_pv = match.group(1) + self.assertTrue(latest_pv != pv) + recipefile = os.path.join(temprecipe, '%s_%s.bb' % (pn, latest_pv)) + # Do not check LIC_FILES_CHKSUM and SRC_URI checksum here to avoid having updating the test on each release + checkvars = {} + checkvars['LICENSE'] = set(['MIT']) + checkvars['PYPI_PACKAGE'] = pn + inherits = ['setuptools3', "pypi"] + self._test_recipe_contents(recipefile, checkvars, inherits) + + def test_recipetool_create_python3_pep517_setuptools_build_meta(self): + # This test require python 3.11 or above for the tomllib module or tomli module to be installed + needTomllib(self) + + # Test creating python3 package from tarball (using setuptools.build_meta class) + temprecipe = os.path.join(self.tempdir, 'recipe') + os.makedirs(temprecipe) + pn = 'webcolors' + pv = '1.13' + recipefile = os.path.join(temprecipe, 'python3-%s_%s.bb' % (pn, pv)) + srcuri = 'https://files.pythonhosted.org/packages/a1/fb/f95560c6a5d4469d9c49e24cf1b5d4d21ffab5608251c6020a965fb7791c/%s-%s.tar.gz' % (pn, pv) + result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) + self.assertTrue(os.path.isfile(recipefile)) + checkvars = {} + checkvars['SUMMARY'] = 'A library for working with the color formats defined by HTML and CSS.' + checkvars['LICENSE'] = set(['BSD-3-Clause']) + checkvars['LIC_FILES_CHKSUM'] = 'file://LICENSE;md5=702b1ef12cf66832a88f24c8f2ee9c19' + checkvars['SRC_URI[sha256sum]'] = 'c225b674c83fa923be93d235330ce0300373d02885cef23238813b0d5668304a' + inherits = ['python_setuptools_build_meta', 'pypi'] + + self._test_recipe_contents(recipefile, checkvars, inherits) + + def test_recipetool_create_python3_pep517_poetry_core_masonry_api(self): + # This test require python 3.11 or above for the tomllib module or tomli module to be installed + needTomllib(self) + + # Test creating python3 package from tarball (using poetry.core.masonry.api class) + temprecipe = os.path.join(self.tempdir, 'recipe') + os.makedirs(temprecipe) + pn = 'iso8601' + pv = '2.1.0' + recipefile = os.path.join(temprecipe, 'python3-%s_%s.bb' % (pn, pv)) + srcuri = 'https://files.pythonhosted.org/packages/b9/f3/ef59cee614d5e0accf6fd0cbba025b93b272e626ca89fb70a3e9187c5d15/%s-%s.tar.gz' % (pn, pv) + result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) + self.assertTrue(os.path.isfile(recipefile)) + checkvars = {} + checkvars['SUMMARY'] = 'Simple module to parse ISO 8601 dates' + checkvars['LICENSE'] = set(['MIT']) + checkvars['LIC_FILES_CHKSUM'] = 'file://LICENSE;md5=aab31f2ef7ba214a5a341eaa47a7f367' + checkvars['SRC_URI[sha256sum]'] = '6b1d3829ee8921c4301998c909f7829fa9ed3cbdac0d3b16af2d743aed1ba8df' + inherits = ['python_poetry_core', 'pypi'] + + self._test_recipe_contents(recipefile, checkvars, inherits) + + def test_recipetool_create_python3_pep517_flit_core_buildapi(self): + # This test require python 3.11 or above for the tomllib module or tomli module to be installed + needTomllib(self) + + # Test creating python3 package from tarball (using flit_core.buildapi class) + temprecipe = os.path.join(self.tempdir, 'recipe') + os.makedirs(temprecipe) + pn = 'typing-extensions' + pv = '4.8.0' + recipefile = os.path.join(temprecipe, 'python3-%s_%s.bb' % (pn, pv)) + srcuri = 'https://files.pythonhosted.org/packages/1f/7a/8b94bb016069caa12fc9f587b28080ac33b4fbb8ca369b98bc0a4828543e/typing_extensions-%s.tar.gz' % pv + result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) + self.assertTrue(os.path.isfile(recipefile)) + checkvars = {} + checkvars['SUMMARY'] = 'Backported and Experimental Type Hints for Python 3.8+' + checkvars['LICENSE'] = set(['PSF-2.0']) + checkvars['LIC_FILES_CHKSUM'] = 'file://LICENSE;md5=fcf6b249c2641540219a727f35d8d2c2' + checkvars['SRC_URI[sha256sum]'] = 'df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef' + inherits = ['python_flit_core', 'pypi'] + + self._test_recipe_contents(recipefile, checkvars, inherits) + + def test_recipetool_create_python3_pep517_hatchling(self): + # This test require python 3.11 or above for the tomllib module or tomli module to be installed + needTomllib(self) + + # Test creating python3 package from tarball (using hatchling class) + temprecipe = os.path.join(self.tempdir, 'recipe') + os.makedirs(temprecipe) + pn = 'jsonschema' + pv = '4.19.1' + recipefile = os.path.join(temprecipe, 'python3-%s_%s.bb' % (pn, pv)) + srcuri = 'https://files.pythonhosted.org/packages/e4/43/087b24516db11722c8687e0caf0f66c7785c0b1c51b0ab951dfde924e3f5/jsonschema-%s.tar.gz' % pv + result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) + self.assertTrue(os.path.isfile(recipefile)) + checkvars = {} + checkvars['SUMMARY'] = 'An implementation of JSON Schema validation for Python' + checkvars['HOMEPAGE'] = 'https://github.com/python-jsonschema/jsonschema' + checkvars['LICENSE'] = set(['MIT']) + checkvars['LIC_FILES_CHKSUM'] = 'file://COPYING;md5=7a60a81c146ec25599a3e1dabb8610a8 file://json/LICENSE;md5=9d4de43111d33570c8fe49b4cb0e01af' + checkvars['SRC_URI[sha256sum]'] = 'ec84cc37cfa703ef7cd4928db24f9cb31428a5d0fa77747b8b51a847458e0bbf' + inherits = ['python_hatchling', 'pypi'] + + self._test_recipe_contents(recipefile, checkvars, inherits) + + def test_recipetool_create_python3_pep517_maturin(self): + # This test require python 3.11 or above for the tomllib module or tomli module to be installed + needTomllib(self) + + # Test creating python3 package from tarball (using maturin class) + temprecipe = os.path.join(self.tempdir, 'recipe') + os.makedirs(temprecipe) + pn = 'pydantic-core' + pv = '2.14.5' + recipefile = os.path.join(temprecipe, 'python3-%s_%s.bb' % (pn, pv)) + srcuri = 'https://files.pythonhosted.org/packages/64/26/cffb93fe9c6b5a91c497f37fae14a4b073ecbc47fc36a9979c7aa888b245/pydantic_core-%s.tar.gz' % pv + result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) + self.assertTrue(os.path.isfile(recipefile)) + checkvars = {} + checkvars['HOMEPAGE'] = 'https://github.com/pydantic/pydantic-core' + checkvars['LICENSE'] = set(['MIT']) + checkvars['LIC_FILES_CHKSUM'] = 'file://LICENSE;md5=ab599c188b4a314d2856b3a55030c75c' + checkvars['SRC_URI[sha256sum]'] = '6d30226dfc816dd0fdf120cae611dd2215117e4f9b124af8c60ab9093b6e8e71' + inherits = ['python_maturin', 'pypi'] + + self._test_recipe_contents(recipefile, checkvars, inherits) + + def test_recipetool_create_python3_pep517_mesonpy(self): + # This test require python 3.11 or above for the tomllib module or tomli module to be installed + needTomllib(self) + + # Test creating python3 package from tarball (using mesonpy class) + temprecipe = os.path.join(self.tempdir, 'recipe') + os.makedirs(temprecipe) + pn = 'siphash24' + pv = '1.4' + recipefile = os.path.join(temprecipe, 'python3-%s_%s.bb' % (pn, pv)) + srcuri = 'https://files.pythonhosted.org/packages/c2/32/b934a70592f314afcfa86c7f7e388804a8061be65b822e2aa07e573b6477/%s-%s.tar.gz' % (pn, pv) + result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) + self.assertTrue(os.path.isfile(recipefile)) + checkvars = {} + checkvars['SRC_URI[sha256sum]'] = '7fd65e39b2a7c8c4ddc3a168a687f4610751b0ac2ebb518783c0cdfc30bec4a0' + inherits = ['python_mesonpy', 'pypi'] + + self._test_recipe_contents(recipefile, checkvars, inherits) + def test_recipetool_create_github_tarball(self): - # Basic test to ensure github URL mangling doesn't apply to release tarballs + # Basic test to ensure github URL mangling doesn't apply to release tarballs. + # Deliberately use an older release of Meson at present so we don't need a toml parser. temprecipe = os.path.join(self.tempdir, 'recipe') os.makedirs(temprecipe) - pv = '0.32.0' - recipefile = os.path.join(temprecipe, 'meson_%s.bb' % pv) + pv = '0.52.1' + recipefile = os.path.join(temprecipe, 'python3-meson_%s.bb' % pv) srcuri = 'https://github.com/mesonbuild/meson/releases/download/%s/meson-%s.tar.gz' % (pv, pv) result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) self.assertTrue(os.path.isfile(recipefile)) @@ -509,14 +735,252 @@ class RecipetoolCreateTests(RecipetoolBase): self._test_recipetool_create_git('http://git.yoctoproject.org/git/matchbox-keyboard') def test_recipetool_create_git_srcuri_master(self): - self._test_recipetool_create_git('git://git.yoctoproject.org/matchbox-keyboard;branch=master') + self._test_recipetool_create_git('git://git.yoctoproject.org/matchbox-keyboard;branch=master;protocol=https') def test_recipetool_create_git_srcuri_branch(self): - self._test_recipetool_create_git('git://git.yoctoproject.org/matchbox-keyboard;branch=matchbox-keyboard-0-1') + self._test_recipetool_create_git('git://git.yoctoproject.org/matchbox-keyboard;branch=matchbox-keyboard-0-1;protocol=https') def test_recipetool_create_git_srcbranch(self): - self._test_recipetool_create_git('git://git.yoctoproject.org/matchbox-keyboard', 'matchbox-keyboard-0-1') + self._test_recipetool_create_git('git://git.yoctoproject.org/matchbox-keyboard;protocol=https', 'matchbox-keyboard-0-1') + + def _go_urifiy(self, url, version, modulepath = None, pathmajor = None, subdir = None): + modulepath = ",path='%s'" % modulepath if len(modulepath) else '' + pathmajor = ",pathmajor='%s'" % pathmajor if len(pathmajor) else '' + subdir = ",subdir='%s'" % subdir if len(subdir) else '' + return "${@go_src_uri('%s','%s'%s%s%s)}" % (url, version, modulepath, pathmajor, subdir) + + def test_recipetool_create_go(self): + # Basic test to check go recipe generation + temprecipe = os.path.join(self.tempdir, 'recipe') + os.makedirs(temprecipe) + + recipefile = os.path.join(temprecipe, 'edgex-go_git.bb') + deps_require_file = os.path.join(temprecipe, 'edgex-go', 'edgex-go-modules.inc') + lics_require_file = os.path.join(temprecipe, 'edgex-go', 'edgex-go-licenses.inc') + modules_txt_file = os.path.join(temprecipe, 'edgex-go', 'modules.txt') + + srcuri = 'https://github.com/edgexfoundry/edgex-go.git' + srcrev = "v3.0.0" + srcbranch = "main" + + result = runCmd('recipetool create -o %s %s -S %s -B %s' % (temprecipe, srcuri, srcrev, srcbranch)) + + self.maxDiff = None + inherits = ['go-vendor'] + + checkvars = {} + checkvars['GO_IMPORT'] = "github.com/edgexfoundry/edgex-go" + checkvars['SRC_URI'] = {'git://${GO_IMPORT};destsuffix=git/src/${GO_IMPORT};nobranch=1;name=${BPN};protocol=https', + 'file://modules.txt'} + checkvars['LIC_FILES_CHKSUM'] = {'file://src/${GO_IMPORT}/LICENSE;md5=8f8bc924cf73f6a32381e5fd4c58d603'} + + self.assertTrue(os.path.isfile(recipefile)) + self._test_recipe_contents(recipefile, checkvars, inherits) + + checkvars = {} + checkvars['VENDORED_LIC_FILES_CHKSUM'] = set( + ['file://src/${GO_IMPORT}/vendor/github.com/Microsoft/go-winio/LICENSE;md5=69205ff73858f2c22b2ca135b557e8ef', + 'file://src/${GO_IMPORT}/vendor/github.com/armon/go-metrics/LICENSE;md5=d2d77030c0183e3d1e66d26dc1f243be', + 'file://src/${GO_IMPORT}/vendor/github.com/cenkalti/backoff/LICENSE;md5=1571d94433e3f3aa05267efd4dbea68b', + 'file://src/${GO_IMPORT}/vendor/github.com/davecgh/go-spew/LICENSE;md5=c06795ed54b2a35ebeeb543cd3a73e56', + 'file://src/${GO_IMPORT}/vendor/github.com/eclipse/paho.mqtt.golang/LICENSE;md5=dcdb33474b60c38efd27356d8f2edec7', + 'file://src/${GO_IMPORT}/vendor/github.com/eclipse/paho.mqtt.golang/edl-v10;md5=3adfcc70f5aeb7a44f3f9b495aa1fbf3', + 'file://src/${GO_IMPORT}/vendor/github.com/edgexfoundry/go-mod-bootstrap/v3/LICENSE;md5=0d6dae39976133b2851fba4c1e1275ff', + 'file://src/${GO_IMPORT}/vendor/github.com/edgexfoundry/go-mod-configuration/v3/LICENSE;md5=0d6dae39976133b2851fba4c1e1275ff', + 'file://src/${GO_IMPORT}/vendor/github.com/edgexfoundry/go-mod-core-contracts/v3/LICENSE;md5=0d6dae39976133b2851fba4c1e1275ff', + 'file://src/${GO_IMPORT}/vendor/github.com/edgexfoundry/go-mod-messaging/v3/LICENSE;md5=0d6dae39976133b2851fba4c1e1275ff', + 'file://src/${GO_IMPORT}/vendor/github.com/edgexfoundry/go-mod-registry/v3/LICENSE;md5=0d6dae39976133b2851fba4c1e1275ff', + 'file://src/${GO_IMPORT}/vendor/github.com/edgexfoundry/go-mod-secrets/v3/LICENSE;md5=f9fa2f4f8e0ef8cc7b5dd150963eb457', + 'file://src/${GO_IMPORT}/vendor/github.com/fatih/color/LICENSE.md;md5=316e6d590bdcde7993fb175662c0dd5a', + 'file://src/${GO_IMPORT}/vendor/github.com/fxamacker/cbor/v2/LICENSE;md5=827f5a2fa861382d35a3943adf9ebb86', + 'file://src/${GO_IMPORT}/vendor/github.com/go-jose/go-jose/v3/LICENSE;md5=3b83ef96387f14655fc854ddc3c6bd57', + 'file://src/${GO_IMPORT}/vendor/github.com/go-jose/go-jose/v3/json/LICENSE;md5=591778525c869cdde0ab5a1bf283cd81', + 'file://src/${GO_IMPORT}/vendor/github.com/go-kit/log/LICENSE;md5=5b7c15ad5fffe2ff6e9d58a6c161f082', + 'file://src/${GO_IMPORT}/vendor/github.com/go-logfmt/logfmt/LICENSE;md5=98e39517c38127f969de33057067091e', + 'file://src/${GO_IMPORT}/vendor/github.com/go-playground/locales/LICENSE;md5=3ccbda375ee345400ad1da85ba522301', + 'file://src/${GO_IMPORT}/vendor/github.com/go-playground/universal-translator/LICENSE;md5=2e2b21ef8f61057977d27c727c84bef1', + 'file://src/${GO_IMPORT}/vendor/github.com/go-playground/validator/v10/LICENSE;md5=a718a0f318d76f7c5d510cbae84f0b60', + 'file://src/${GO_IMPORT}/vendor/github.com/go-redis/redis/v7/LICENSE;md5=58103aa5ea1ee9b7a369c9c4a95ef9b5', + 'file://src/${GO_IMPORT}/vendor/github.com/golang/protobuf/LICENSE;md5=939cce1ec101726fa754e698ac871622', + 'file://src/${GO_IMPORT}/vendor/github.com/gomodule/redigo/LICENSE;md5=2ee41112a44fe7014dce33e26468ba93', + 'file://src/${GO_IMPORT}/vendor/github.com/google/uuid/LICENSE;md5=88073b6dd8ec00fe09da59e0b6dfded1', + 'file://src/${GO_IMPORT}/vendor/github.com/gorilla/mux/LICENSE;md5=33fa1116c45f9e8de714033f99edde13', + 'file://src/${GO_IMPORT}/vendor/github.com/gorilla/websocket/LICENSE;md5=c007b54a1743d596f46b2748d9f8c044', + 'file://src/${GO_IMPORT}/vendor/github.com/hashicorp/consul/api/LICENSE;md5=b8a277a612171b7526e9be072f405ef4', + 'file://src/${GO_IMPORT}/vendor/github.com/hashicorp/errwrap/LICENSE;md5=b278a92d2c1509760384428817710378', + 'file://src/${GO_IMPORT}/vendor/github.com/hashicorp/go-cleanhttp/LICENSE;md5=65d26fcc2f35ea6a181ac777e42db1ea', + 'file://src/${GO_IMPORT}/vendor/github.com/hashicorp/go-hclog/LICENSE;md5=ec7f605b74b9ad03347d0a93a5cc7eb8', + 'file://src/${GO_IMPORT}/vendor/github.com/hashicorp/go-immutable-radix/LICENSE;md5=65d26fcc2f35ea6a181ac777e42db1ea', + 'file://src/${GO_IMPORT}/vendor/github.com/hashicorp/go-multierror/LICENSE;md5=d44fdeb607e2d2614db9464dbedd4094', + 'file://src/${GO_IMPORT}/vendor/github.com/hashicorp/go-rootcerts/LICENSE;md5=65d26fcc2f35ea6a181ac777e42db1ea', + 'file://src/${GO_IMPORT}/vendor/github.com/hashicorp/golang-lru/LICENSE;md5=f27a50d2e878867827842f2c60e30bfc', + 'file://src/${GO_IMPORT}/vendor/github.com/hashicorp/serf/LICENSE;md5=b278a92d2c1509760384428817710378', + 'file://src/${GO_IMPORT}/vendor/github.com/leodido/go-urn/LICENSE;md5=8f50db5538ec1148a9b3d14ed96c3418', + 'file://src/${GO_IMPORT}/vendor/github.com/mattn/go-colorable/LICENSE;md5=24ce168f90aec2456a73de1839037245', + 'file://src/${GO_IMPORT}/vendor/github.com/mattn/go-isatty/LICENSE;md5=f509beadd5a11227c27b5d2ad6c9f2c6', + 'file://src/${GO_IMPORT}/vendor/github.com/mitchellh/consulstructure/LICENSE;md5=96ada10a9e51c98c4656f2cede08c673', + 'file://src/${GO_IMPORT}/vendor/github.com/mitchellh/copystructure/LICENSE;md5=56da355a12d4821cda57b8f23ec34bc4', + 'file://src/${GO_IMPORT}/vendor/github.com/mitchellh/go-homedir/LICENSE;md5=3f7765c3d4f58e1f84c4313cecf0f5bd', + 'file://src/${GO_IMPORT}/vendor/github.com/mitchellh/mapstructure/LICENSE;md5=3f7765c3d4f58e1f84c4313cecf0f5bd', + 'file://src/${GO_IMPORT}/vendor/github.com/mitchellh/reflectwalk/LICENSE;md5=3f7765c3d4f58e1f84c4313cecf0f5bd', + 'file://src/${GO_IMPORT}/vendor/github.com/nats-io/nats.go/LICENSE;md5=86d3f3a95c324c9479bd8986968f4327', + 'file://src/${GO_IMPORT}/vendor/github.com/nats-io/nkeys/LICENSE;md5=86d3f3a95c324c9479bd8986968f4327', + 'file://src/${GO_IMPORT}/vendor/github.com/nats-io/nuid/LICENSE;md5=86d3f3a95c324c9479bd8986968f4327', + 'file://src/${GO_IMPORT}/vendor/github.com/pmezard/go-difflib/LICENSE;md5=e9a2ebb8de779a07500ddecca806145e', + 'file://src/${GO_IMPORT}/vendor/github.com/rcrowley/go-metrics/LICENSE;md5=1bdf5d819f50f141366dabce3be1460f', + 'file://src/${GO_IMPORT}/vendor/github.com/spiffe/go-spiffe/v2/LICENSE;md5=86d3f3a95c324c9479bd8986968f4327', + 'file://src/${GO_IMPORT}/vendor/github.com/stretchr/objx/LICENSE;md5=d023fd31d3ca39ec61eec65a91732735', + 'file://src/${GO_IMPORT}/vendor/github.com/stretchr/testify/LICENSE;md5=188f01994659f3c0d310612333d2a26f', + 'file://src/${GO_IMPORT}/vendor/github.com/x448/float16/LICENSE;md5=de8f8e025d57fe7ee0b67f30d571323b', + 'file://src/${GO_IMPORT}/vendor/github.com/zeebo/errs/LICENSE;md5=84914ab36fc0eb48edbaa53e66e8d326', + 'file://src/${GO_IMPORT}/vendor/golang.org/x/crypto/LICENSE;md5=5d4950ecb7b26d2c5e4e7b4e0dd74707', + 'file://src/${GO_IMPORT}/vendor/golang.org/x/mod/LICENSE;md5=5d4950ecb7b26d2c5e4e7b4e0dd74707', + 'file://src/${GO_IMPORT}/vendor/golang.org/x/net/LICENSE;md5=5d4950ecb7b26d2c5e4e7b4e0dd74707', + 'file://src/${GO_IMPORT}/vendor/golang.org/x/sync/LICENSE;md5=5d4950ecb7b26d2c5e4e7b4e0dd74707', + 'file://src/${GO_IMPORT}/vendor/golang.org/x/sys/LICENSE;md5=5d4950ecb7b26d2c5e4e7b4e0dd74707', + 'file://src/${GO_IMPORT}/vendor/golang.org/x/text/LICENSE;md5=5d4950ecb7b26d2c5e4e7b4e0dd74707', + 'file://src/${GO_IMPORT}/vendor/golang.org/x/tools/LICENSE;md5=5d4950ecb7b26d2c5e4e7b4e0dd74707', + 'file://src/${GO_IMPORT}/vendor/google.golang.org/genproto/LICENSE;md5=3b83ef96387f14655fc854ddc3c6bd57', + 'file://src/${GO_IMPORT}/vendor/google.golang.org/grpc/LICENSE;md5=3b83ef96387f14655fc854ddc3c6bd57', + 'file://src/${GO_IMPORT}/vendor/google.golang.org/protobuf/LICENSE;md5=02d4002e9171d41a8fad93aa7faf3956', + 'file://src/${GO_IMPORT}/vendor/gopkg.in/eapache/queue.v1/LICENSE;md5=1bfd4408d3de090ef6b908b0cc45a316', + 'file://src/${GO_IMPORT}/vendor/gopkg.in/yaml.v3/LICENSE;md5=3c91c17266710e16afdbb2b6d15c761c']) + + self.assertTrue(os.path.isfile(lics_require_file)) + self._test_recipe_contents(lics_require_file, checkvars, []) + + dependencies = \ + [ ('github.com/eclipse/paho.mqtt.golang','v1.4.2', '', '', ''), + ('github.com/edgexfoundry/go-mod-bootstrap','v3.0.1','github.com/edgexfoundry/go-mod-bootstrap/v3','/v3', ''), + ('github.com/edgexfoundry/go-mod-configuration','v3.0.0','github.com/edgexfoundry/go-mod-configuration/v3','/v3', ''), + ('github.com/edgexfoundry/go-mod-core-contracts','v3.0.0','github.com/edgexfoundry/go-mod-core-contracts/v3','/v3', ''), + ('github.com/edgexfoundry/go-mod-messaging','v3.0.0','github.com/edgexfoundry/go-mod-messaging/v3','/v3', ''), + ('github.com/edgexfoundry/go-mod-secrets','v3.0.1','github.com/edgexfoundry/go-mod-secrets/v3','/v3', ''), + ('github.com/fxamacker/cbor','v2.4.0','github.com/fxamacker/cbor/v2','/v2', ''), + ('github.com/gomodule/redigo','v1.8.9', '', '', ''), + ('github.com/google/uuid','v1.3.0', '', '', ''), + ('github.com/gorilla/mux','v1.8.0', '', '', ''), + ('github.com/rcrowley/go-metrics','v0.0.0-20201227073835-cf1acfcdf475', '', '', ''), + ('github.com/spiffe/go-spiffe','v2.1.4','github.com/spiffe/go-spiffe/v2','/v2', ''), + ('github.com/stretchr/testify','v1.8.2', '', '', ''), + ('go.googlesource.com/crypto','v0.8.0','golang.org/x/crypto', '', ''), + ('gopkg.in/eapache/queue.v1','v1.1.0', '', '', ''), + ('gopkg.in/yaml.v3','v3.0.1', '', '', ''), + ('github.com/microsoft/go-winio','v0.6.0','github.com/Microsoft/go-winio', '', ''), + ('github.com/hashicorp/go-metrics','v0.3.10','github.com/armon/go-metrics', '', ''), + ('github.com/cenkalti/backoff','v2.2.1+incompatible', '', '', ''), + ('github.com/davecgh/go-spew','v1.1.1', '', '', ''), + ('github.com/edgexfoundry/go-mod-registry','v3.0.0','github.com/edgexfoundry/go-mod-registry/v3','/v3', ''), + ('github.com/fatih/color','v1.9.0', '', '', ''), + ('github.com/go-jose/go-jose','v3.0.0','github.com/go-jose/go-jose/v3','/v3', ''), + ('github.com/go-kit/log','v0.2.1', '', '', ''), + ('github.com/go-logfmt/logfmt','v0.5.1', '', '', ''), + ('github.com/go-playground/locales','v0.14.1', '', '', ''), + ('github.com/go-playground/universal-translator','v0.18.1', '', '', ''), + ('github.com/go-playground/validator','v10.13.0','github.com/go-playground/validator/v10','/v10', ''), + ('github.com/go-redis/redis','v7.3.0','github.com/go-redis/redis/v7','/v7', ''), + ('github.com/golang/protobuf','v1.5.2', '', '', ''), + ('github.com/gorilla/websocket','v1.4.2', '', '', ''), + ('github.com/hashicorp/consul','v1.20.0','github.com/hashicorp/consul/api', '', 'api'), + ('github.com/hashicorp/errwrap','v1.0.0', '', '', ''), + ('github.com/hashicorp/go-cleanhttp','v0.5.1', '', '', ''), + ('github.com/hashicorp/go-hclog','v0.14.1', '', '', ''), + ('github.com/hashicorp/go-immutable-radix','v1.3.0', '', '', ''), + ('github.com/hashicorp/go-multierror','v1.1.1', '', '', ''), + ('github.com/hashicorp/go-rootcerts','v1.0.2', '', '', ''), + ('github.com/hashicorp/golang-lru','v0.5.4', '', '', ''), + ('github.com/hashicorp/serf','v0.10.1', '', '', ''), + ('github.com/leodido/go-urn','v1.2.3', '', '', ''), + ('github.com/mattn/go-colorable','v0.1.12', '', '', ''), + ('github.com/mattn/go-isatty','v0.0.14', '', '', ''), + ('github.com/mitchellh/consulstructure','v0.0.0-20190329231841-56fdc4d2da54', '', '', ''), + ('github.com/mitchellh/copystructure','v1.2.0', '', '', ''), + ('github.com/mitchellh/go-homedir','v1.1.0', '', '', ''), + ('github.com/mitchellh/mapstructure','v1.5.0', '', '', ''), + ('github.com/mitchellh/reflectwalk','v1.0.2', '', '', ''), + ('github.com/nats-io/nats.go','v1.25.0', '', '', ''), + ('github.com/nats-io/nkeys','v0.4.4', '', '', ''), + ('github.com/nats-io/nuid','v1.0.1', '', '', ''), + ('github.com/pmezard/go-difflib','v1.0.0', '', '', ''), + ('github.com/stretchr/objx','v0.5.0', '', '', ''), + ('github.com/x448/float16','v0.8.4', '', '', ''), + ('github.com/zeebo/errs','v1.3.0', '', '', ''), + ('go.googlesource.com/mod','v0.8.0','golang.org/x/mod', '', ''), + ('go.googlesource.com/net','v0.9.0','golang.org/x/net', '', ''), + ('go.googlesource.com/sync','v0.1.0','golang.org/x/sync', '', ''), + ('go.googlesource.com/sys','v0.7.0','golang.org/x/sys', '', ''), + ('go.googlesource.com/text','v0.9.0','golang.org/x/text', '', ''), + ('go.googlesource.com/tools','v0.6.0','golang.org/x/tools', '', ''), + ('github.com/googleapis/go-genproto','v0.0.0-20230223222841-637eb2293923','google.golang.org/genproto', '', ''), + ('github.com/grpc/grpc-go','v1.53.0','google.golang.org/grpc', '', ''), + ('go.googlesource.com/protobuf','v1.28.1','google.golang.org/protobuf', '', ''), + ] + src_uri = set() + for d in dependencies: + src_uri.add(self._go_urifiy(*d)) + + checkvars = {} + checkvars['GO_DEPENDENCIES_SRC_URI'] = src_uri + + self.assertTrue(os.path.isfile(deps_require_file)) + self._test_recipe_contents(deps_require_file, checkvars, []) + + def test_recipetool_create_go_replace_modules(self): + # Check handling of replaced modules + temprecipe = os.path.join(self.tempdir, 'recipe') + os.makedirs(temprecipe) + + recipefile = os.path.join(temprecipe, 'openapi-generator_git.bb') + deps_require_file = os.path.join(temprecipe, 'openapi-generator', 'go-modules.inc') + lics_require_file = os.path.join(temprecipe, 'openapi-generator', 'go-licenses.inc') + modules_txt_file = os.path.join(temprecipe, 'openapi-generator', 'modules.txt') + + srcuri = 'https://github.com/OpenAPITools/openapi-generator.git' + srcrev = "v7.2.0" + srcbranch = "master" + srcsubdir = "samples/openapi3/client/petstore/go" + + result = runCmd('recipetool create -o %s %s -S %s -B %s --src-subdir %s' % (temprecipe, srcuri, srcrev, srcbranch, srcsubdir)) + + self.maxDiff = None + inherits = ['go-vendor'] + + checkvars = {} + checkvars['GO_IMPORT'] = "github.com/OpenAPITools/openapi-generator/samples/openapi3/client/petstore/go" + checkvars['SRC_URI'] = {'git://${GO_IMPORT};destsuffix=git/src/${GO_IMPORT};nobranch=1;name=${BPN};protocol=https', + 'file://modules.txt'} + + self.assertNotIn('Traceback', result.output) + self.assertIn('No license file was detected for the main module', result.output) + self.assertTrue(os.path.isfile(recipefile)) + self._test_recipe_contents(recipefile, checkvars, inherits) + + # make sure that dependencies don't mention local directory ./go-petstore + dependencies = \ + [ ('github.com/stretchr/testify','v1.8.4', '', '', ''), + ('go.googlesource.com/oauth2','v0.10.0','golang.org/x/oauth2', '', ''), + ('github.com/davecgh/go-spew','v1.1.1', '', '', ''), + ('github.com/golang/protobuf','v1.5.3', '', '', ''), + ('github.com/kr/pretty','v0.3.0', '', '', ''), + ('github.com/pmezard/go-difflib','v1.0.0', '', '', ''), + ('github.com/rogpeppe/go-internal','v1.9.0', '', '', ''), + ('go.googlesource.com/net','v0.12.0','golang.org/x/net', '', ''), + ('github.com/golang/appengine','v1.6.7','google.golang.org/appengine', '', ''), + ('go.googlesource.com/protobuf','v1.31.0','google.golang.org/protobuf', '', ''), + ('gopkg.in/check.v1','v1.0.0-20201130134442-10cb98267c6c', '', '', ''), + ('gopkg.in/yaml.v3','v3.0.1', '', '', ''), + ] + + src_uri = set() + for d in dependencies: + src_uri.add(self._go_urifiy(*d)) + + checkvars = {} + checkvars['GO_DEPENDENCIES_SRC_URI'] = src_uri + + self.assertTrue(os.path.isfile(deps_require_file)) + self._test_recipe_contents(deps_require_file, checkvars, []) class RecipetoolTests(RecipetoolBase): @@ -536,7 +1000,15 @@ class RecipetoolTests(RecipetoolBase): for p in paths: dstdir = os.path.join(dstdir, p) if not os.path.exists(dstdir): - os.makedirs(dstdir) + try: + os.makedirs(dstdir) + except PermissionError: + return False + except OSError as e: + if e.errno == errno.EROFS: + return False + else: + raise e if p == "lib": # Can race with other tests self.add_command_to_tearDown('rmdir --ignore-fail-on-non-empty %s' % dstdir) @@ -544,8 +1016,12 @@ class RecipetoolTests(RecipetoolBase): self.track_for_cleanup(dstdir) dstfile = os.path.join(dstdir, os.path.basename(srcfile)) if srcfile != dstfile: - shutil.copy(srcfile, dstfile) + try: + shutil.copy(srcfile, dstfile) + except PermissionError: + return False self.track_for_cleanup(dstfile) + return True def test_recipetool_load_plugin(self): """Test that recipetool loads only the first found plugin in BBPATH.""" @@ -559,15 +1035,17 @@ class RecipetoolTests(RecipetoolBase): plugincontent = fh.readlines() try: self.assertIn('meta-selftest', srcfile, 'wrong bbpath plugin found') - for path in searchpath: - self._copy_file_with_cleanup(srcfile, path, 'lib', 'recipetool') + searchpath = [ + path for path in searchpath + if self._copy_file_with_cleanup(srcfile, path, 'lib', 'recipetool') + ] result = runCmd("recipetool --quiet count") self.assertEqual(result.output, '1') result = runCmd("recipetool --quiet multiloaded") self.assertEqual(result.output, "no") for path in searchpath: result = runCmd("recipetool --quiet bbdir") - self.assertEqual(result.output, path) + self.assertEqual(os.path.realpath(result.output), os.path.realpath(path)) os.unlink(os.path.join(result.output, 'lib', 'recipetool', 'bbpath.py')) finally: with open(srcfile, 'w') as fh: @@ -579,7 +1057,10 @@ class RecipetoolTests(RecipetoolBase): commonlicdir = get_bb_var('COMMON_LICENSE_DIR') - d = bb.tinfoil.TinfoilDataStoreConnector + class DataConnectorCopy(bb.tinfoil.TinfoilDataStoreConnector): + pass + + d = DataConnectorCopy d.getVar = Mock(return_value=commonlicdir) srctree = tempfile.mkdtemp(prefix='recipetoolqa') @@ -727,9 +1208,9 @@ class RecipetoolAppendsrcBase(RecipetoolBase): for uri in src_uri: p = urllib.parse.urlparse(uri) if p.scheme == 'file': - return p.netloc + p.path + return p.netloc + p.path, uri - def _test_appendsrcfile(self, testrecipe, filename=None, destdir=None, has_src_uri=True, srcdir=None, newfile=None, options=''): + def _test_appendsrcfile(self, testrecipe, filename=None, destdir=None, has_src_uri=True, srcdir=None, newfile=None, remove=None, machine=None , options=''): if newfile is None: newfile = self.testfile @@ -756,12 +1237,40 @@ class RecipetoolAppendsrcBase(RecipetoolBase): expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', '\n'] + + override = "" + if machine: + options += ' -m %s' % machine + override = ':append:%s' % machine + expectedlines.extend(['PACKAGE_ARCH = "${MACHINE_ARCH}"\n', + '\n']) + + if remove: + for entry in remove: + if machine: + entry_remove_line = 'SRC_URI:remove:%s = " %s"\n' % (machine, entry) + else: + entry_remove_line = 'SRC_URI:remove = "%s"\n' % entry + + expectedlines.extend([entry_remove_line, + '\n']) + if has_src_uri: uri = 'file://%s' % filename if expected_subdir: uri += ';subdir=%s' % expected_subdir - expectedlines[0:0] = ['SRC_URI += "%s"\n' % uri, - '\n'] + if machine: + src_uri_line = 'SRC_URI%s = " %s"\n' % (override, uri) + else: + src_uri_line = 'SRC_URI += "%s"\n' % uri + + expectedlines.extend([src_uri_line, '\n']) + + with open("/tmp/tmp.txt", "w") as file: + print(expectedlines, file=file) + + if machine: + filename = '%s/%s' % (machine, filename) return self._try_recipetool_appendsrcfile(testrecipe, newfile, destpath, options, expectedlines, [filename]) @@ -816,18 +1325,46 @@ class RecipetoolAppendsrcTests(RecipetoolAppendsrcBase): def test_recipetool_appendsrcfile_existing_in_src_uri(self): testrecipe = 'base-files' - filepath = self._get_first_file_uri(testrecipe) + filepath,_ = self._get_first_file_uri(testrecipe) self.assertTrue(filepath, 'Unable to test, no file:// uri found in SRC_URI for %s' % testrecipe) self._test_appendsrcfile(testrecipe, filepath, has_src_uri=False) - def test_recipetool_appendsrcfile_existing_in_src_uri_diff_params(self): + def test_recipetool_appendsrcfile_existing_in_src_uri_diff_params(self, machine=None): testrecipe = 'base-files' subdir = 'tmp' - filepath = self._get_first_file_uri(testrecipe) + filepath, srcuri_entry = self._get_first_file_uri(testrecipe) self.assertTrue(filepath, 'Unable to test, no file:// uri found in SRC_URI for %s' % testrecipe) - output = self._test_appendsrcfile(testrecipe, filepath, subdir, has_src_uri=False) - self.assertTrue(any('with different parameters' in l for l in output)) + self._test_appendsrcfile(testrecipe, filepath, subdir, machine=machine, remove=[srcuri_entry]) + + def test_recipetool_appendsrcfile_machine(self): + # A very basic test + self._test_appendsrcfile('base-files', 'a-file', machine='mymachine') + + # Force cleaning the output of previous test + self.tearDownLocal() + + # A more complex test: existing entry in src_uri with different param + self.test_recipetool_appendsrcfile_existing_in_src_uri_diff_params(machine='mymachine') + + def test_recipetool_appendsrcfile_update_recipe_basic(self): + testrecipe = "mtd-utils-selftest" + recipefile = get_bb_var('FILE', testrecipe) + self.assertIn('meta-selftest', recipefile, 'This test expect %s recipe to be in meta-selftest') + cmd = 'recipetool appendsrcfile -W -u meta-selftest %s %s' % (testrecipe, self.testfile) + result = runCmd(cmd) + self.assertNotIn('Traceback', result.output) + self.add_command_to_tearDown('cd %s; rm -f %s/%s; git checkout .' % (os.path.dirname(recipefile), testrecipe, os.path.basename(self.testfile))) + + expected_status = [(' M', '.*/%s$' % os.path.basename(recipefile)), + ('??', '.*/%s/%s$' % (testrecipe, os.path.basename(self.testfile)))] + self._check_repo_status(os.path.dirname(recipefile), expected_status) + result = runCmd('git diff %s' % os.path.basename(recipefile), cwd=os.path.dirname(recipefile)) + removelines = [] + addlines = [ + 'file://%s \\\\' % os.path.basename(self.testfile), + ] + self._check_diff(result.output, addlines, removelines) def test_recipetool_appendsrcfile_replace_file_srcdir(self): testrecipe = 'bash' diff --git a/meta/lib/oeqa/selftest/cases/recipeutils.py b/meta/lib/oeqa/selftest/cases/recipeutils.py index 74b2098ae8..2cb4445f81 100644 --- a/meta/lib/oeqa/selftest/cases/recipeutils.py +++ b/meta/lib/oeqa/selftest/cases/recipeutils.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -48,7 +50,7 @@ class RecipeUtilsTests(OESelftestTestCase): +SRC_URI[md5sum] = "aaaaaa" SRC_URI[sha256sum] = "ac6894d876e45878faae493b0cf61d0e28ec417334448ac0a6ea2229d8343051" - RDEPENDS:${PN} += "${PYTHON_PN}-threading" + RDEPENDS:${PN} += "python3-threading" """ patchlines = [] for f in patches: diff --git a/meta/lib/oeqa/selftest/cases/reproducible.py b/meta/lib/oeqa/selftest/cases/reproducible.py index 5042c11d8e..80e830136f 100644 --- a/meta/lib/oeqa/selftest/cases/reproducible.py +++ b/meta/lib/oeqa/selftest/cases/reproducible.py @@ -43,13 +43,14 @@ class CompareResult(object): return (self.status, self.test) < (other.status, other.test) class PackageCompareResults(object): - def __init__(self): + def __init__(self, exclusions): self.total = [] self.missing = [] self.different = [] self.different_excluded = [] self.same = [] self.active_exclusions = set() + exclude_packages.extend((exclusions or "").split()) def add_result(self, r): self.total.append(r) @@ -133,8 +134,10 @@ class ReproducibleTests(OESelftestTestCase): # targets are the things we want to test the reproducibility of targets = ['core-image-minimal', 'core-image-sato', 'core-image-full-cmdline', 'core-image-weston', 'world'] + # sstate targets are things to pull from sstate to potentially cut build/debugging time sstate_targets = [] + save_results = False if 'OEQA_DEBUGGING_SAVED_OUTPUT' in os.environ: save_results = os.environ['OEQA_DEBUGGING_SAVED_OUTPUT'] @@ -149,11 +152,29 @@ class ReproducibleTests(OESelftestTestCase): def setUpLocal(self): super().setUpLocal() - needed_vars = ['TOPDIR', 'TARGET_PREFIX', 'BB_NUMBER_THREADS'] + needed_vars = [ + 'TOPDIR', + 'TARGET_PREFIX', + 'BB_NUMBER_THREADS', + 'BB_HASHSERVE', + 'OEQA_REPRODUCIBLE_TEST_PACKAGE', + 'OEQA_REPRODUCIBLE_TEST_TARGET', + 'OEQA_REPRODUCIBLE_TEST_SSTATE_TARGETS', + 'OEQA_REPRODUCIBLE_EXCLUDED_PACKAGES', + ] bb_vars = get_bb_vars(needed_vars) for v in needed_vars: setattr(self, v.lower(), bb_vars[v]) + if bb_vars['OEQA_REPRODUCIBLE_TEST_PACKAGE']: + self.package_classes = bb_vars['OEQA_REPRODUCIBLE_TEST_PACKAGE'].split() + + if bb_vars['OEQA_REPRODUCIBLE_TEST_TARGET']: + self.targets = bb_vars['OEQA_REPRODUCIBLE_TEST_TARGET'].split() + + if bb_vars['OEQA_REPRODUCIBLE_TEST_SSTATE_TARGETS']: + self.sstate_targets = bb_vars['OEQA_REPRODUCIBLE_TEST_SSTATE_TARGETS'].split() + self.extraresults = {} self.extraresults.setdefault('reproducible.rawlogs', {})['log'] = '' self.extraresults.setdefault('reproducible', {}).setdefault('files', {}) @@ -162,7 +183,7 @@ class ReproducibleTests(OESelftestTestCase): self.extraresults['reproducible.rawlogs']['log'] += msg def compare_packages(self, reference_dir, test_dir, diffutils_sysroot): - result = PackageCompareResults() + result = PackageCompareResults(self.oeqa_reproducible_excluded_packages) old_cwd = os.getcwd() try: @@ -202,10 +223,9 @@ class ReproducibleTests(OESelftestTestCase): config = textwrap.dedent('''\ PACKAGE_CLASSES = "{package_classes}" - INHIBIT_PACKAGE_STRIP = "1" TMPDIR = "{tmpdir}" LICENSE_FLAGS_ACCEPTED = "commercial" - DISTRO_FEATURES:append = ' systemd pam' + DISTRO_FEATURES:append = ' pam' USERADDEXTENSION = "useradd-staticids" USERADD_ERROR_DYNAMIC = "skip" USERADD_UID_TABLES += "files/static-passwd" @@ -223,7 +243,7 @@ class ReproducibleTests(OESelftestTestCase): # mirror, forcing a complete build from scratch config += textwrap.dedent('''\ SSTATE_DIR = "${TMPDIR}/sstate" - SSTATE_MIRRORS = "" + SSTATE_MIRRORS = "file://.*/.*-native.* http://sstate.yoctoproject.org/all/PATH;downloadfilename=PATH file://.*/.*-cross.* http://sstate.yoctoproject.org/all/PATH;downloadfilename=PATH" ''') self.logger.info("Building %s (sstate%s allowed)..." % (name, '' if use_sstate else ' NOT')) @@ -290,9 +310,13 @@ class ReproducibleTests(OESelftestTestCase): self.copy_file(d.reference, '/'.join([save_dir, 'packages-excluded', strip_topdir(d.reference)])) self.copy_file(d.test, '/'.join([save_dir, 'packages-excluded', strip_topdir(d.test)])) - if result.missing or result.different: - fails.append("The following %s packages are missing or different and not in exclusion list: %s" % - (c, '\n'.join(r.test for r in (result.missing + result.different)))) + if result.different: + fails.append("The following %s packages are different and not in exclusion list:\n%s" % + (c, '\n'.join(r.test for r in (result.different)))) + + if result.missing and len(self.sstate_targets) == 0: + fails.append("The following %s packages are missing and not in exclusion list:\n%s" % + (c, '\n'.join(r.test for r in (result.missing)))) # Clean up empty directories if self.save_results: diff --git a/meta/lib/oeqa/selftest/cases/resulttooltests.py b/meta/lib/oeqa/selftest/cases/resulttooltests.py index dac5c46801..c3303f3fbb 100644 --- a/meta/lib/oeqa/selftest/cases/resulttooltests.py +++ b/meta/lib/oeqa/selftest/cases/resulttooltests.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -69,7 +71,7 @@ class ResultToolTests(OESelftestTestCase): self.assertTrue('target_result1' in results['runtime/mydistro/qemux86/image'], msg="Pair not correct:%s" % results) self.assertTrue('target_result3' in results['runtime/mydistro/qemux86-64/image'], msg="Pair not correct:%s" % results) - def test_regrresion_can_get_regression_result(self): + def test_regression_can_get_regression_result(self): base_result_data = {'result': {'test1': {'status': 'PASSED'}, 'test2': {'status': 'PASSED'}, 'test3': {'status': 'FAILED'}, @@ -96,3 +98,278 @@ class ResultToolTests(OESelftestTestCase): resultutils.append_resultsdata(results, ResultToolTests.target_results_data, configmap=resultutils.flatten_map) self.assertEqual(len(results[''].keys()), 5, msg="Flattened results not correct %s" % str(results)) + def test_results_without_metadata_can_be_compared(self): + base_configuration = {"configuration": { + "TEST_TYPE": "oeselftest", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86", + "STARTTIME": 1672527600 + }, "result": {}} + target_configuration = {"configuration": { + "TEST_TYPE": "oeselftest", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86", + "STARTTIME": 1672527600 + }, "result": {}} + self.assertTrue(regression.can_be_compared(self.logger, base_configuration, target_configuration), + msg="incorrect metadata filtering, tests without metadata should be compared") + + def test_target_result_with_missing_metadata_can_not_be_compared(self): + base_configuration = {"configuration": { + "TEST_TYPE": "oeselftest", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86", + "OESELFTEST_METADATA": { + "run_all_tests": True, + "run_tests": None, + "skips": None, + "machine": None, + "select_tags": ["toolchain-user", "toolchain-system"], + "exclude_tags": None + }}, "result": {}} + target_configuration = {"configuration": {"TEST_TYPE": "oeselftest", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86", + "STARTTIME": 1672527600 + }, "result": {}} + self.assertFalse(regression.can_be_compared(self.logger, base_configuration, target_configuration), + msg="incorrect metadata filtering, tests should not be compared") + + def test_results_with_matching_metadata_can_be_compared(self): + base_configuration = {"configuration": { + "TEST_TYPE": "oeselftest", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86", + "STARTTIME": 1672527600, + "OESELFTEST_METADATA": {"run_all_tests": True, + "run_tests": None, + "skips": None, + "machine": None, + "select_tags": ["toolchain-user", "toolchain-system"], + "exclude_tags": None} + }, "result": {}} + target_configuration = {"configuration": { + "TEST_TYPE": "oeselftest", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86", + "STARTTIME": 1672527600, + "OESELFTEST_METADATA": {"run_all_tests": True, + "run_tests": None, + "skips": None, + "machine": None, + "select_tags": ["toolchain-user", "toolchain-system"], + "exclude_tags": None} + }, "result": {}} + self.assertTrue(regression.can_be_compared(self.logger, base_configuration, target_configuration), + msg="incorrect metadata filtering, tests with matching metadata should be compared") + + def test_results_with_mismatching_metadata_can_not_be_compared(self): + base_configuration = {"configuration": { + "TEST_TYPE": "oeselftest", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86", + "STARTTIME": 1672527600, + "OESELFTEST_METADATA": {"run_all_tests": True, + "run_tests": None, + "skips": None, + "machine": None, + "select_tags": ["toolchain-user", "toolchain-system"], + "exclude_tags": None} + }, "result": {}} + target_configuration = {"configuration": { + "TEST_TYPE": "oeselftest", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86", + "STARTTIME": 1672527600, + "OESELFTEST_METADATA": {"run_all_tests": True, + "run_tests": None, + "skips": None, + "machine": None, + "select_tags": ["machine"], + "exclude_tags": None} + }, "result": {}} + self.assertFalse(regression.can_be_compared(self.logger, base_configuration, target_configuration), + msg="incorrect metadata filtering, tests with mismatching metadata should not be compared") + + def test_metadata_matching_is_only_checked_for_relevant_test_type(self): + base_configuration = {"configuration": {"TEST_TYPE": "runtime", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86", + "STARTTIME": 1672527600, + "OESELFTEST_METADATA": {"run_all_tests": True, + "run_tests": None, + "skips": None, + "machine": None, + "select_tags": ["toolchain-user", "toolchain-system"], + "exclude_tags": None}}, "result": {}} + target_configuration = {"configuration": {"TEST_TYPE": "runtime", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86", + "STARTTIME": 1672527600, + "OESELFTEST_METADATA": {"run_all_tests": True, + "run_tests": None, + "skips": None, + "machine": None, + "select_tags": ["machine"], + "exclude_tags": None}}, "result": {}} + self.assertTrue(regression.can_be_compared(self.logger, base_configuration, target_configuration), + msg="incorrect metadata filtering, %s tests should be compared" % base_configuration['configuration']['TEST_TYPE']) + + def test_machine_matches(self): + base_configuration = {"configuration": { + "TEST_TYPE": "runtime", + "MACHINE": "qemux86"}, "result": {}} + target_configuration = {"configuration": { + "TEST_TYPE": "runtime", + "MACHINE": "qemux86" + }, "result": {}} + self.assertTrue(regression.can_be_compared(self.logger, base_configuration, target_configuration), + msg="incorrect machine filtering, identical machine tests should be compared") + + def test_machine_mismatches(self): + base_configuration = {"configuration": { + "TEST_TYPE": "runtime", + "MACHINE": "qemux86" + }, "result": {}} + target_configuration = {"configuration": { + "TEST_TYPE": "runtime", + "MACHINE": "qemux86_64" + }, "result": {}} + self.assertFalse(regression.can_be_compared(self.logger, base_configuration, target_configuration), + msg="incorrect machine filtering, mismatching machine tests should not be compared") + + def test_can_not_compare_non_ltp_tests(self): + base_configuration = {"configuration": { + "TEST_TYPE": "runtime", + "MACHINE": "qemux86" + }, "result": { + "ltpresult_foo": { + "status": "PASSED" + }}} + target_configuration = {"configuration": { + "TEST_TYPE": "runtime", + "MACHINE": "qemux86_64" + }, "result": { + "bar": { + "status": "PASSED" + }}} + self.assertFalse(regression.can_be_compared(self.logger, base_configuration, target_configuration), + msg="incorrect ltpresult filtering, mismatching ltpresult content should not be compared") + + def test_can_compare_ltp_tests(self): + base_configuration = {"configuration": { + "TEST_TYPE": "runtime", + "MACHINE": "qemux86" + }, "result": { + "ltpresult_foo": { + "status": "PASSED" + }}} + target_configuration = {"configuration": { + "TEST_TYPE": "runtime", + "MACHINE": "qemux86" + }, "result": { + "ltpresult_foo": { + "status": "PASSED" + }}} + self.assertTrue(regression.can_be_compared(self.logger, base_configuration, target_configuration), + msg="incorrect ltpresult filtering, matching ltpresult content should be compared") + + def test_can_match_non_static_ptest_names(self): + base_configuration = {"a": { + "conf_X": { + "configuration": { + "TEST_TYPE": "runtime", + "MACHINE": "qemux86" + }, "result": { + "ptestresult.lttng-tools.foo_-_bar_-_moo": { + "status": "PASSED" + }, + "ptestresult.babeltrace.bar_-_moo_-_foo": { + "status": "PASSED" + }, + "ptestresult.babeltrace2.moo_-_foo_-_bar": { + "status": "PASSED" + }, + "ptestresult.curl.test_0000__foo_out_of_bar": { + "status": "PASSED" + }, + "ptestresult.dbus.test_0000__foo_out_of_bar,_remaining:_00:02,_took_0.032s,_duration:_03:32_": { + "status": "PASSED" + }, + "ptestresult.binutils-ld.in testcase /foo/build-st-bar/moo/ctf.exp": { + "status": "PASSED" + }, + "ptestresult.gcc-libstdc++-v3.Couldn't create remote directory /tmp/runtest.30975 on target": { + "status": "PASSED" + }, + "ptestresult.gcc-libgomp.Couldn't create remote directory /tmp/runtest.3657621 on": { + "status": "PASSED" + } + }}}} + target_configuration = {"a": { + "conf_Y": { + "configuration": { + "TEST_TYPE": "runtime", + "MACHINE": "qemux86" + }, "result": { + "ptestresult.lttng-tools.foo_-_yyy_-_zzz": { + "status": "PASSED" + }, + "ptestresult.babeltrace.bar_-_zzz_-_xxx": { + "status": "PASSED" + }, + "ptestresult.babeltrace2.moo_-_xxx_-_yyy": { + "status": "PASSED" + }, + "ptestresult.curl.test_0000__xxx_out_of_yyy": { + "status": "PASSED" + }, + "ptestresult.dbus.test_0000__yyy_out_of_zzz,_remaining:_00:03,_took_0.034s,_duration:_03:30_": { + "status": "PASSED" + }, + "ptestresult.binutils-ld.in testcase /xxx/build-st-yyy/zzz/ctf.exp": { + "status": "PASSED" + }, + "ptestresult.gcc-libstdc++-v3.Couldn't create remote directory /tmp/runtest.45678 on target": { + "status": "PASSED" + }, + "ptestresult.gcc-libgomp.Couldn't create remote directory /tmp/runtest.3657621 on": { + "status": "PASSED" + } + }}}} + regression.fixup_ptest_names(base_configuration, self.logger) + regression.fixup_ptest_names(target_configuration, self.logger) + result, resultstring = regression.compare_result( + self.logger, "A", "B", base_configuration["a"]["conf_X"], target_configuration["a"]["conf_Y"]) + self.assertDictEqual( + result, {}, msg=f"ptests should be compared: {resultstring}") diff --git a/meta/lib/oeqa/selftest/cases/rootfspostcommandstests.py b/meta/lib/oeqa/selftest/cases/rootfspostcommandstests.py new file mode 100644 index 0000000000..44e2c09a6f --- /dev/null +++ b/meta/lib/oeqa/selftest/cases/rootfspostcommandstests.py @@ -0,0 +1,97 @@ +# SPDX-FileCopyrightText: Huawei Inc. +# +# SPDX-License-Identifier: MIT + +import os +import oe +import unittest +from oeqa.selftest.case import OESelftestTestCase +from oeqa.utils.commands import bitbake, get_bb_vars + +class ShadowUtilsTidyFiles(OESelftestTestCase): + """ + Check if shadow image rootfs files are tidy. + + The tests are focused on testing the functionality provided by the + 'tidy_shadowutils_files' rootfs postprocess command (via + SORT_PASSWD_POSTPROCESS_COMMAND). + """ + + def sysconf_build(self): + """ + Verify if shadow tidy files tests are to be run and if yes, build a + test image and return its sysconf rootfs path. + """ + + test_image = "core-image-minimal" + + config = 'IMAGE_CLASSES += "extrausers"\n' + config += 'EXTRA_USERS_PARAMS = "groupadd -g 1000 oeqatester; "\n' + config += 'EXTRA_USERS_PARAMS += "useradd -p \'\' -u 1000 -N -g 1000 oeqatester; "\n' + self.write_config(config) + + vars = get_bb_vars(("IMAGE_ROOTFS", "SORT_PASSWD_POSTPROCESS_COMMAND", "sysconfdir"), + test_image) + passwd_postprocess_cmd = vars["SORT_PASSWD_POSTPROCESS_COMMAND"] + self.assertIsNotNone(passwd_postprocess_cmd) + if (passwd_postprocess_cmd.strip() != 'tidy_shadowutils_files;'): + raise unittest.SkipTest("Testcase skipped as 'tidy_shadowutils_files' " + "rootfs post process command is not the set SORT_PASSWD_POSTPROCESS_COMMAND.") + + rootfs = vars["IMAGE_ROOTFS"] + self.assertIsNotNone(rootfs) + sysconfdir = vars["sysconfdir"] + bitbake(test_image) + self.assertIsNotNone(sysconfdir) + + return oe.path.join(rootfs, sysconfdir) + + def test_shadowutils_backup_files(self): + """ + Test that the rootfs doesn't include any known shadow backup files. + """ + + backup_files = ( + 'group-', + 'gshadow-', + 'passwd-', + 'shadow-', + 'subgid-', + 'subuid-', + ) + + rootfs_sysconfdir = self.sysconf_build() + found = [] + for backup_file in backup_files: + backup_filepath = oe.path.join(rootfs_sysconfdir, backup_file) + if os.path.exists(backup_filepath): + found.append(backup_file) + if (found): + raise Exception('The following shadow backup files were found in ' + 'the rootfs: %s' % found) + + def test_shadowutils_sorted_files(self): + """ + Test that the 'passwd' and the 'group' shadow utils files are ordered + by ID. + """ + + files = ( + 'passwd', + 'group', + ) + + rootfs_sysconfdir = self.sysconf_build() + unsorted = [] + for file in files: + filepath = oe.path.join(rootfs_sysconfdir, file) + with open(filepath, 'rb') as f: + ids = [] + lines = f.readlines() + for line in lines: + entries = line.split(b':') + ids.append(int(entries[2])) + if (ids != sorted(ids)): + unsorted.append(file) + if (unsorted): + raise Exception("The following files were not sorted by ID as expected: %s" % unsorted) diff --git a/meta/lib/oeqa/selftest/cases/rpmtests.py b/meta/lib/oeqa/selftest/cases/rpmtests.py new file mode 100644 index 0000000000..902d7dca3d --- /dev/null +++ b/meta/lib/oeqa/selftest/cases/rpmtests.py @@ -0,0 +1,14 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +from oeqa.selftest.case import OESelftestTestCase +from oeqa.utils.commands import bitbake + +class BitbakeTests(OESelftestTestCase): + + def test_rpm_filenames(self): + test_recipe = "testrpm" + bitbake(test_recipe) diff --git a/meta/lib/oeqa/selftest/cases/runcmd.py b/meta/lib/oeqa/selftest/cases/runcmd.py index e9612389fe..70047ca0ca 100644 --- a/meta/lib/oeqa/selftest/cases/runcmd.py +++ b/meta/lib/oeqa/selftest/cases/runcmd.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -56,11 +58,11 @@ class RunCmdTests(OESelftestTestCase): self.assertEqual(result.status, 0) def test_result_assertion(self): - self.assertRaisesRegexp(AssertionError, "Command 'echo .* false' returned non-zero exit status 1:\nfoobar", + self.assertRaisesRegex(AssertionError, "Command 'echo .* false' returned non-zero exit status 1:\nfoobar", runCmd, "echo foobar >&2; false", shell=True) def test_result_exception(self): - self.assertRaisesRegexp(CommandError, "Command 'echo .* false' returned non-zero exit status 1 with output: foobar", + self.assertRaisesRegex(CommandError, "Command 'echo .* false' returned non-zero exit status 1 with output: foobar", runCmd, "echo foobar >&2; false", shell=True, assert_error=False) def test_output(self): diff --git a/meta/lib/oeqa/selftest/cases/runqemu.py b/meta/lib/oeqa/selftest/cases/runqemu.py index c1d277a095..f01e1eec66 100644 --- a/meta/lib/oeqa/selftest/cases/runqemu.py +++ b/meta/lib/oeqa/selftest/cases/runqemu.py @@ -4,13 +4,15 @@ # SPDX-License-Identifier: MIT # +import os import re -import tempfile import time import oe.types from oeqa.core.decorator import OETestTag +from oeqa.core.decorator.data import skipIfNotArch, skipIfNotMachine from oeqa.selftest.case import OESelftestTestCase -from oeqa.utils.commands import bitbake, runqemu, get_bb_var, runCmd +from oeqa.utils.commands import bitbake, runqemu, get_bb_var + @OETestTag("runqemu") class RunqemuTests(OESelftestTestCase): @@ -22,23 +24,26 @@ class RunqemuTests(OESelftestTestCase): def setUpLocal(self): super(RunqemuTests, self).setUpLocal() self.recipe = 'core-image-minimal' - self.machine = 'qemux86-64' - self.fstypes = "ext4 iso hddimg wic.vmdk wic.qcow2 wic.vdi" - self.cmd_common = "runqemu nographic" + self.machine = self.td['MACHINE'] + self.image_link_name = get_bb_var('IMAGE_LINK_NAME', self.recipe) - kvm = oe.types.qemu_use_kvm(get_bb_var('QEMU_USE_KVM'), 'x86_64') + self.fstypes = "ext4" + if self.td["HOST_ARCH"] in ('i586', 'i686', 'x86_64'): + self.fstypes += " iso hddimg" + if self.machine == "qemux86-64": + self.fstypes += " wic.vmdk wic.qcow2 wic.vdi" + + self.cmd_common = "runqemu nographic" + kvm = oe.types.qemu_use_kvm(get_bb_var('QEMU_USE_KVM'), self.td["TARGET_ARCH"]) if kvm: self.cmd_common += " kvm" self.write_config( """ -MACHINE = "%s" IMAGE_FSTYPES = "%s" # 10 means 1 second SYSLINUX_TIMEOUT = "10" -""" -% (self.machine, self.fstypes) - ) +""" % self.fstypes) if not RunqemuTests.image_is_ready: RunqemuTests.deploy_dir_image = get_bb_var('DEPLOY_DIR_IMAGE') @@ -57,14 +62,17 @@ SYSLINUX_TIMEOUT = "10" cmd = "%s %s ext4" % (self.cmd_common, self.machine) with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: with open(qemu.qemurunnerlog) as f: - self.assertIn('rootfs.ext4', f.read(), "Failed: %s" % cmd) + regexp = r'\nROOTFS: .*\.ext4]\n' + self.assertRegex(f.read(), regexp, "Failed to find '%s' in '%s' after running '%s'" % (regexp, qemu.qemurunnerlog, cmd)) + @skipIfNotArch(['i586', 'i686', 'x86_64']) def test_boot_machine_iso(self): """Test runqemu machine iso""" cmd = "%s %s iso" % (self.cmd_common, self.machine) with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: with open(qemu.qemurunnerlog) as f: - self.assertIn('media=cdrom', f.read(), "Failed: %s" % cmd) + text_in = 'media=cdrom' + self.assertIn(text_in, f.read(), "Failed to find '%s' in '%s' after running '%s'" % (text_in, qemu.qemurunnerlog, cmd)) def test_boot_recipe_image(self): """Test runqemu recipe-image""" @@ -73,20 +81,24 @@ SYSLINUX_TIMEOUT = "10" with open(qemu.qemurunnerlog) as f: self.assertTrue(qemu.runner.logged, "Failed: %s, %s" % (cmd, f.read())) - + # https://bugzilla.yoctoproject.org/show_bug.cgi?id=14963 + @skipIfNotMachine("qemux86-64", "tests are qemux86-64 specific currently") def test_boot_recipe_image_vmdk(self): """Test runqemu recipe-image vmdk""" cmd = "%s %s wic.vmdk" % (self.cmd_common, self.recipe) with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: with open(qemu.qemurunnerlog) as f: - self.assertIn('format=vmdk', f.read(), "Failed: %s" % cmd) + text_in = 'format=vmdk' + self.assertIn(text_in, f.read(), "Failed to find '%s' in '%s' after running '%s'" % (text_in, qemu.qemurunnerlog, cmd)) + @skipIfNotMachine("qemux86-64", "tests are qemux86-64 specific currently") def test_boot_recipe_image_vdi(self): """Test runqemu recipe-image vdi""" cmd = "%s %s wic.vdi" % (self.cmd_common, self.recipe) with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: with open(qemu.qemurunnerlog) as f: - self.assertIn('format=vdi', f.read(), "Failed: %s" % cmd) + text_in = 'format=vdi' + self.assertIn(text_in, f.read(), "Failed to find '%s' in '%s' after running '%s'" % (text_in, qemu.qemurunnerlog, cmd)) def test_boot_deploy(self): """Test runqemu deploy_dir_image""" @@ -95,7 +107,7 @@ SYSLINUX_TIMEOUT = "10" with open(qemu.qemurunnerlog) as f: self.assertTrue(qemu.runner.logged, "Failed: %s, %s" % (cmd, f.read())) - + @skipIfNotArch(['i586', 'i686', 'x86_64']) def test_boot_deploy_hddimg(self): """Test runqemu deploy_dir_image hddimg""" cmd = "%s %s hddimg" % (self.cmd_common, self.deploy_dir_image) @@ -110,6 +122,7 @@ SYSLINUX_TIMEOUT = "10" with open(qemu.qemurunnerlog) as f: self.assertIn(' -netdev user', f.read(), "Failed: %s" % cmd) + @skipIfNotMachine("qemux86-64", "tests are qemux86-64 specific currently") def test_boot_machine_slirp_qcow2(self): """Test runqemu machine slirp qcow2""" cmd = "%s slirp wic.qcow2 %s" % (self.cmd_common, self.machine) @@ -119,7 +132,7 @@ SYSLINUX_TIMEOUT = "10" def test_boot_qemu_boot(self): """Test runqemu /path/to/image.qemuboot.conf""" - qemuboot_conf = "%s-%s.qemuboot.conf" % (self.recipe, self.machine) + qemuboot_conf = "%s.qemuboot.conf" % (self.image_link_name) qemuboot_conf = os.path.join(self.deploy_dir_image, qemuboot_conf) if not os.path.exists(qemuboot_conf): self.skipTest("%s not found" % qemuboot_conf) @@ -130,7 +143,7 @@ SYSLINUX_TIMEOUT = "10" def test_boot_rootfs(self): """Test runqemu /path/to/rootfs.ext4""" - rootfs = "%s-%s.ext4" % (self.recipe, self.machine) + rootfs = "%s.ext4" % (self.image_link_name) rootfs = os.path.join(self.deploy_dir_image, rootfs) if not os.path.exists(rootfs): self.skipTest("%s not found" % rootfs) @@ -157,10 +170,11 @@ class QemuTest(OESelftestTestCase): def setUpClass(cls): super(QemuTest, cls).setUpClass() cls.recipe = 'core-image-minimal' - cls.machine = get_bb_var('MACHINE') - cls.deploy_dir_image = get_bb_var('DEPLOY_DIR_IMAGE') + cls.machine = get_bb_var('MACHINE') + cls.deploy_dir_image = get_bb_var('DEPLOY_DIR_IMAGE') + cls.image_link_name = get_bb_var('IMAGE_LINK_NAME', cls.recipe) cls.cmd_common = "runqemu nographic" - cls.qemuboot_conf = "%s-%s.qemuboot.conf" % (cls.recipe, cls.machine) + cls.qemuboot_conf = "%s.qemuboot.conf" % (cls.image_link_name) cls.qemuboot_conf = os.path.join(cls.deploy_dir_image, cls.qemuboot_conf) bitbake(cls.recipe) @@ -191,22 +205,12 @@ class QemuTest(OESelftestTestCase): qemu_shutdown_succeeded = self._start_qemu_shutdown_check_if_shutdown_succeeded(qemu, shutdown_timeout) self.assertTrue(qemu_shutdown_succeeded, 'Failed: %s does not shutdown within timeout(%s)' % (self.machine, shutdown_timeout)) - # Need to have portmap/rpcbind running to allow this test to work and - # current autobuilder setup does not have this. - def disabled_test_qemu_can_boot_nfs_and_shutdown(self): - self.assertExists(self.qemuboot_conf) - bitbake('meta-ide-support') - rootfs_tar = "%s-%s.tar.bz2" % (self.recipe, self.machine) + def test_qemu_can_boot_nfs_and_shutdown(self): + rootfs_tar = "%s.tar.bz2" % (self.image_link_name) rootfs_tar = os.path.join(self.deploy_dir_image, rootfs_tar) self.assertExists(rootfs_tar) - tmpdir = tempfile.mkdtemp(prefix='qemu_nfs') - tmpdir_nfs = os.path.join(tmpdir, 'nfs') - cmd_extract_nfs = 'runqemu-extract-sdk %s %s' % (rootfs_tar, tmpdir_nfs) - result = runCmd(cmd_extract_nfs) - self.assertEqual(0, result.status, "runqemu-extract-sdk didn't run as expected. %s" % result.output) - cmd = "%s nfs %s %s" % (self.cmd_common, self.qemuboot_conf, tmpdir_nfs) + cmd = "%s %s" % (self.cmd_common, rootfs_tar) shutdown_timeout = 120 with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: qemu_shutdown_succeeded = self._start_qemu_shutdown_check_if_shutdown_succeeded(qemu, shutdown_timeout) self.assertTrue(qemu_shutdown_succeeded, 'Failed: %s does not shutdown within timeout(%s)' % (self.machine, shutdown_timeout)) - runCmd('rm -rf %s' % tmpdir) diff --git a/meta/lib/oeqa/selftest/cases/runtime_test.py b/meta/lib/oeqa/selftest/cases/runtime_test.py index 857737f730..12000aac16 100644 --- a/meta/lib/oeqa/selftest/cases/runtime_test.py +++ b/meta/lib/oeqa/selftest/cases/runtime_test.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -8,7 +10,7 @@ from oeqa.core.decorator import OETestTag import os import tempfile import oe.lsb -from oeqa.core.decorator.data import skipIfNotQemu +from oeqa.core.decorator.data import skipIfNotQemu, skipIfNotMachine class TestExport(OESelftestTestCase): @@ -23,7 +25,7 @@ class TestExport(OESelftestTestCase): Author: Mariano Lopez <mariano.lopez@intel.com> """ - features = 'INHERIT += "testexport"\n' + features = 'IMAGE_CLASSES += "testexport"\n' # These aren't the actual IP addresses but testexport class needs something defined features += 'TEST_SERVER_IP = "192.168.7.1"\n' features += 'TEST_TARGET_IP = "192.168.7.1"\n' @@ -64,7 +66,7 @@ class TestExport(OESelftestTestCase): Author: Mariano Lopez <mariano.lopez@intel.com> """ - features = 'INHERIT += "testexport"\n' + features = 'IMAGE_CLASSES += "testexport"\n' # These aren't the actual IP addresses but testexport class needs something defined features += 'TEST_SERVER_IP = "192.168.7.1"\n' features += 'TEST_TARGET_IP = "192.168.7.1"\n' @@ -119,7 +121,7 @@ class TestImage(OESelftestTestCase): if get_bb_var('DISTRO') == 'poky-tiny': self.skipTest('core-image-full-cmdline not buildable for poky-tiny') - features = 'INHERIT += "testimage"\n' + features = 'IMAGE_CLASSES += "testimage"\n' features += 'IMAGE_INSTALL:append = " libssl"\n' features += 'TEST_SUITES = "ping ssh selftest"\n' self.write_config(features) @@ -127,6 +129,22 @@ class TestImage(OESelftestTestCase): bitbake('core-image-full-cmdline socat') bitbake('-c testimage core-image-full-cmdline') + def test_testimage_slirp(self): + """ + Summary: Check basic testimage functionality with qemu and slirp networking. + """ + + features = ''' +IMAGE_CLASSES:append = " testimage" +IMAGE_FEATURES:append = " ssh-server-dropbear" +IMAGE_ROOTFS_EXTRA_SPACE:append = "${@bb.utils.contains("IMAGE_CLASSES", "testimage", " + 5120", "", d)}" +TEST_RUNQEMUPARAMS += " slirp" +''' + self.write_config(features) + + bitbake('core-image-minimal') + bitbake('-c testimage core-image-minimal') + def test_testimage_dnf(self): """ Summary: Check package feeds functionality for dnf @@ -137,7 +155,7 @@ class TestImage(OESelftestTestCase): if get_bb_var('DISTRO') == 'poky-tiny': self.skipTest('core-image-full-cmdline not buildable for poky-tiny') - features = 'INHERIT += "testimage"\n' + features = 'IMAGE_CLASSES += "testimage"\n' features += 'TEST_SUITES = "ping ssh dnf_runtime dnf.DnfBasicTest.test_dnf_help"\n' # We don't yet know what the server ip and port will be - they will be patched # in at the start of the on-image test @@ -172,7 +190,7 @@ class TestImage(OESelftestTestCase): if get_bb_var('DISTRO') == 'poky-tiny': self.skipTest('core-image-full-cmdline not buildable for poky-tiny') - features = 'INHERIT += "testimage"\n' + features = 'IMAGE_CLASSES += "testimage"\n' features += 'TEST_SUITES = "ping ssh apt.AptRepoTest.test_apt_install_from_repo"\n' # We don't yet know what the server ip and port will be - they will be patched # in at the start of the on-image test @@ -200,6 +218,8 @@ class TestImage(OESelftestTestCase): bitbake('core-image-full-cmdline socat') bitbake('-c testimage core-image-full-cmdline') + # https://bugzilla.yoctoproject.org/show_bug.cgi?id=14966 + @skipIfNotMachine("qemux86-64", "test needs qemux86-64") def test_testimage_virgl_gtk_sdl(self): """ Summary: Check host-assisted accelerate OpenGL functionality in qemu with gtk and SDL frontends @@ -222,7 +242,7 @@ class TestImage(OESelftestTestCase): qemu_packageconfig = get_bb_var('PACKAGECONFIG', 'qemu-system-native') qemu_distrofeatures = get_bb_var('DISTRO_FEATURES', 'qemu-system-native') - features = 'INHERIT += "testimage"\n' + features = 'IMAGE_CLASSES += "testimage"\n' if 'gtk+' not in qemu_packageconfig: features += 'PACKAGECONFIG:append:pn-qemu-system-native = " gtk+"\n' if 'sdl' not in qemu_packageconfig: @@ -232,15 +252,16 @@ class TestImage(OESelftestTestCase): features += 'TEST_SUITES = "ping ssh virgl"\n' features += 'IMAGE_FEATURES:append = " ssh-server-dropbear"\n' features += 'IMAGE_INSTALL:append = " kmscube"\n' - features_gtk = features + 'TEST_RUNQEMUPARAMS = "gtk gl"\n' + features_gtk = features + 'TEST_RUNQEMUPARAMS += " gtk gl"\n' self.write_config(features_gtk) bitbake('core-image-minimal') bitbake('-c testimage core-image-minimal') - features_sdl = features + 'TEST_RUNQEMUPARAMS = "sdl gl"\n' + features_sdl = features + 'TEST_RUNQEMUPARAMS += " sdl gl"\n' self.write_config(features_sdl) bitbake('core-image-minimal') bitbake('-c testimage core-image-minimal') + @skipIfNotMachine("qemux86-64", "test needs qemux86-64") def test_testimage_virgl_headless(self): """ Summary: Check host-assisted accelerate OpenGL functionality in qemu with egl-headless frontend @@ -252,28 +273,18 @@ class TestImage(OESelftestTestCase): import subprocess, os distro = oe.lsb.distro_identifier() - if distro and (distro in ['debian-9', 'debian-10', 'centos-7', 'centos-8', 'ubuntu-16.04', 'ubuntu-18.04'] or distro.startswith('almalinux')): + if distro and (distro in ['debian-9', 'debian-10', 'centos-7', 'centos-8', 'ubuntu-16.04', 'ubuntu-18.04'] or + distro.startswith('almalinux') or distro.startswith('rocky')): self.skipTest('virgl headless cannot be tested with %s' %(distro)) - render_hint = """If /dev/dri/renderD* is absent due to lack of suitable GPU, 'modprobe vgem' will create one suitable for mesa llvmpipe software renderer.""" - try: - content = os.listdir("/dev/dri") - if len([i for i in content if i.startswith('render')]) == 0: - self.fail("No render nodes found in /dev/dri: %s. %s" %(content, render_hint)) - except FileNotFoundError: - self.fail("/dev/dri directory does not exist; no render nodes available on this machine. %s" %(render_hint)) - try: - dripath = subprocess.check_output("pkg-config --variable=dridriverdir dri", shell=True) - except subprocess.CalledProcessError as e: - self.fail("Could not determine the path to dri drivers on the host via pkg-config.\nPlease install Mesa development files (particularly, dri.pc) on the host machine.") qemu_distrofeatures = get_bb_var('DISTRO_FEATURES', 'qemu-system-native') - features = 'INHERIT += "testimage"\n' + features = 'IMAGE_CLASSES += "testimage"\n' if 'opengl' not in qemu_distrofeatures: features += 'DISTRO_FEATURES:append = " opengl"\n' features += 'TEST_SUITES = "ping ssh virgl"\n' features += 'IMAGE_FEATURES:append = " ssh-server-dropbear"\n' features += 'IMAGE_INSTALL:append = " kmscube"\n' - features += 'TEST_RUNQEMUPARAMS = "egl-headless"\n' + features += 'TEST_RUNQEMUPARAMS += " egl-headless"\n' self.write_config(features) bitbake('core-image-minimal') bitbake('-c testimage core-image-minimal') @@ -299,7 +310,7 @@ class Postinst(OESelftestTestCase): features += 'IMAGE_FEATURES += "package-management empty-root-password"\n' features += 'PACKAGE_CLASSES = "%s"\n' % classes if init_manager == "systemd": - features += 'DISTRO_FEATURES:append = " systemd"\n' + features += 'DISTRO_FEATURES:append = " systemd usrmerge"\n' features += 'VIRTUAL-RUNTIME_init_manager = "systemd"\n' features += 'DISTRO_FEATURES_BACKFILL_CONSIDERED = "sysvinit"\n' features += 'VIRTUAL-RUNTIME_initscripts = ""\n' diff --git a/meta/lib/oeqa/selftest/cases/rust.py b/meta/lib/oeqa/selftest/cases/rust.py new file mode 100644 index 0000000000..ad14189c6d --- /dev/null +++ b/meta/lib/oeqa/selftest/cases/rust.py @@ -0,0 +1,231 @@ +# SPDX-License-Identifier: MIT +import os +import subprocess +import time +from oeqa.core.decorator import OETestTag +from oeqa.core.case import OEPTestResultTestCase +from oeqa.selftest.case import OESelftestTestCase +from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars, runqemu, Command +from oeqa.utils.sshcontrol import SSHControl + +def parse_results(filename): + tests = {} + with open(filename, "r") as f: + lines = f.readlines() + for line in lines: + if "..." in line and "test [" in line: + test = line.split("test ")[1].split(" ... ")[0] + if "] " in test: + test = test.split("] ", 1)[1] + result = line.split(" ... ")[1].strip() + if result == "ok": + result = "PASS" + elif result == "failed": + result = "FAIL" + elif "ignored" in result: + result = "SKIPPED" + if test in tests: + if tests[test] != result: + print("Duplicate and mismatching result %s for %s" % (result, test)) + else: + print("Duplicate result %s for %s" % (result, test)) + else: + tests[test] = result + return tests + +# Total time taken for testing is of about 2hr 20min, with PARALLEL_MAKE set to 40 number of jobs. +@OETestTag("toolchain-system") +@OETestTag("toolchain-user") +@OETestTag("runqemu") +class RustSelfTestSystemEmulated(OESelftestTestCase, OEPTestResultTestCase): + def test_rust(self, *args, **kwargs): + # Disable Rust Oe-selftest + #self.skipTest("The Rust Oe-selftest is disabled.") + + # Skip mips32 target since it is unstable with rust tests + machine = get_bb_var('MACHINE') + if machine == "qemumips": + self.skipTest("The mips32 target is skipped for Rust Oe-selftest.") + + # build remote-test-server before image build + recipe = "rust" + start_time = time.time() + bitbake("{} -c test_compile".format(recipe)) + builddir = get_bb_var("RUSTSRC", "rust") + # build core-image-minimal with required packages + default_installed_packages = ["libgcc", "libstdc++", "libatomic", "libgomp"] + features = [] + features.append('IMAGE_FEATURES += "ssh-server-dropbear"') + features.append('CORE_IMAGE_EXTRA_INSTALL += "{0}"'.format(" ".join(default_installed_packages))) + self.write_config("\n".join(features)) + bitbake("core-image-minimal") + + # Exclude the test folders that error out while building + # TODO: Fix the errors and include them for testing + # no-fail-fast: Run all tests regardless of failure. + # bless: First runs rustfmt to format the codebase, + # then runs tidy checks. + exclude_list = [ + 'compiler/rustc', + 'compiler/rustc_interface/src/tests.rs', + 'library/panic_abort', + 'library/panic_unwind', + 'library/test/src/stats/tests.rs', + 'src/bootstrap/builder/tests.rs', + 'src/doc/rustc', + 'src/doc/rustdoc', + 'src/doc/unstable-book', + 'src/librustdoc', + 'src/rustdoc-json-types', + 'src/tools/compiletest/src/common.rs', + 'src/tools/lint-docs', + 'src/tools/rust-analyzer', + 'src/tools/rustdoc-themes', + 'src/tools/tidy', + 'tests/assembly/asm/aarch64-outline-atomics.rs', + 'tests/codegen/abi-main-signature-32bit-c-int.rs', + 'tests/codegen/abi-repr-ext.rs', + 'tests/codegen/abi-x86-interrupt.rs', + 'tests/codegen/branch-protection.rs', + 'tests/codegen/catch-unwind.rs', + 'tests/codegen/cf-protection.rs', + 'tests/codegen/enum-bounds-check-derived-idx.rs', + 'tests/codegen/force-unwind-tables.rs', + 'tests/codegen/intrinsic-no-unnamed-attr.rs', + 'tests/codegen/issues/issue-103840.rs', + 'tests/codegen/issues/issue-47278.rs', + 'tests/codegen/issues/issue-73827-bounds-check-index-in-subexpr.rs', + 'tests/codegen/lifetime_start_end.rs', + 'tests/codegen/local-generics-in-exe-internalized.rs', + 'tests/codegen/match-unoptimized.rs', + 'tests/codegen/noalias-rwlockreadguard.rs', + 'tests/codegen/non-terminate/nonempty-infinite-loop.rs', + 'tests/codegen/noreturn-uninhabited.rs', + 'tests/codegen/repr-transparent-aggregates-3.rs', + 'tests/codegen/riscv-abi/call-llvm-intrinsics.rs', + 'tests/codegen/riscv-abi/riscv64-lp64f-lp64d-abi.rs', + 'tests/codegen/riscv-abi/riscv64-lp64d-abi.rs', + 'tests/codegen/sse42-implies-crc32.rs', + 'tests/codegen/thread-local.rs', + 'tests/codegen/uninit-consts.rs', + 'tests/pretty/raw-str-nonexpr.rs', + 'tests/run-make', + 'tests/run-make-fulldeps', + 'tests/rustdoc', + 'tests/rustdoc-json', + 'tests/rustdoc-js-std', + 'tests/rustdoc-ui/cfg-test.rs', + 'tests/rustdoc-ui/check-cfg-test.rs', + 'tests/rustdoc-ui/display-output.rs', + 'tests/rustdoc-ui/doc-comment-multi-line-attr.rs', + 'tests/rustdoc-ui/doc-comment-multi-line-cfg-attr.rs', + 'tests/rustdoc-ui/doc-test-doctest-feature.rs', + 'tests/rustdoc-ui/doctest-multiline-crate-attribute.rs', + 'tests/rustdoc-ui/doctest-output.rs', + 'tests/rustdoc-ui/doc-test-rustdoc-feature.rs', + 'tests/rustdoc-ui/failed-doctest-compile-fail.rs', + 'tests/rustdoc-ui/issue-80992.rs', + 'tests/rustdoc-ui/issue-91134.rs', + 'tests/rustdoc-ui/nocapture-fail.rs', + 'tests/rustdoc-ui/nocapture.rs', + 'tests/rustdoc-ui/no-run-flag.rs', + 'tests/rustdoc-ui/run-directory.rs', + 'tests/rustdoc-ui/test-no_std.rs', + 'tests/rustdoc-ui/test-type.rs', + 'tests/rustdoc/unit-return.rs', + 'tests/ui/abi/stack-probes-lto.rs', + 'tests/ui/abi/stack-probes.rs', + 'tests/ui/array-slice-vec/subslice-patterns-const-eval-match.rs', + 'tests/ui/asm/x86_64/sym.rs', + 'tests/ui/associated-type-bounds/fn-apit.rs', + 'tests/ui/associated-type-bounds/fn-dyn-apit.rs', + 'tests/ui/associated-type-bounds/fn-wrap-apit.rs', + 'tests/ui/debuginfo/debuginfo-emit-llvm-ir-and-split-debuginfo.rs', + 'tests/ui/drop/dynamic-drop.rs', + 'tests/ui/empty_global_asm.rs', + 'tests/ui/functions-closures/fn-help-with-err.rs', + 'tests/ui/linkage-attr/issue-10755.rs', + 'tests/ui/macros/restricted-shadowing-legacy.rs', + 'tests/ui/process/nofile-limit.rs', + 'tests/ui/process/process-panic-after-fork.rs', + 'tests/ui/process/process-sigpipe.rs', + 'tests/ui/simd/target-feature-mixup.rs', + 'tests/ui/structs-enums/multiple-reprs.rs', + 'src/tools/jsondoclint', + 'src/tools/replace-version-placeholder', + 'tests/codegen/abi-efiapi.rs', + 'tests/codegen/abi-sysv64.rs', + 'tests/codegen/align-byval.rs', + 'tests/codegen/align-fn.rs', + 'tests/codegen/asm-powerpc-clobbers.rs', + 'tests/codegen/async-fn-debug-awaitee-field.rs', + 'tests/codegen/binary-search-index-no-bound-check.rs', + 'tests/codegen/call-metadata.rs', + 'tests/codegen/debug-column.rs', + 'tests/codegen/debug-limited.rs', + 'tests/codegen/debuginfo-generic-closure-env-names.rs', + 'tests/codegen/drop.rs', + 'tests/codegen/dst-vtable-align-nonzero.rs', + 'tests/codegen/enable-lto-unit-splitting.rs', + 'tests/codegen/enum/enum-u128.rs', + 'tests/codegen/fn-impl-trait-self.rs', + 'tests/codegen/inherit_overflow.rs', + 'tests/codegen/inline-function-args-debug-info.rs', + 'tests/codegen/intrinsics/mask.rs', + 'tests/codegen/intrinsics/transmute-niched.rs', + 'tests/codegen/issues/issue-73258.rs', + 'tests/codegen/issues/issue-75546.rs', + 'tests/codegen/issues/issue-77812.rs', + 'tests/codegen/issues/issue-98156-const-arg-temp-lifetime.rs', + 'tests/codegen/llvm-ident.rs', + 'tests/codegen/mainsubprogram.rs', + 'tests/codegen/move-operands.rs', + 'tests/codegen/repr/transparent-mips64.rs', + 'tests/mir-opt/', + 'tests/rustdoc-json', + 'tests/rustdoc-ui/doc-test-rustdoc-feature.rs', + 'tests/rustdoc-ui/no-run-flag.rs', + 'tests/ui-fulldeps/', + 'tests/ui/numbers-arithmetic/u128.rs' + ] + + exclude_fail_tests = " ".join([" --exclude " + item for item in exclude_list]) + # Add exclude_fail_tests with other test arguments + testargs = exclude_fail_tests + " --doc --no-fail-fast --bless" + + # wrap the execution with a qemu instance. + # Tests are run with 512 tasks in parallel to execute all tests very quickly + with runqemu("core-image-minimal", runqemuparams = "nographic", qemuparams = "-m 512") as qemu: + # Copy remote-test-server to image through scp + host_sys = get_bb_var("RUST_BUILD_SYS", "rust") + ssh = SSHControl(ip=qemu.ip, logfile=qemu.sshlog, user="root") + ssh.copy_to(builddir + "/build/" + host_sys + "/stage1-tools-bin/remote-test-server","~/") + # Execute remote-test-server on image through background ssh + command = '~/remote-test-server --bind 0.0.0.0:12345 -v' + sshrun=subprocess.Popen(("ssh", '-o', 'UserKnownHostsFile=/dev/null', '-o', 'StrictHostKeyChecking=no', '-f', "root@%s" % qemu.ip, command), shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # Get the values of variables. + tcpath = get_bb_var("TARGET_SYS", "rust") + targetsys = get_bb_var("RUST_TARGET_SYS", "rust") + rustlibpath = get_bb_var("WORKDIR", "rust") + tmpdir = get_bb_var("TMPDIR", "rust") + + # Set path for target-poky-linux-gcc, RUST_TARGET_PATH and hosttools. + cmd = " export PATH=%s/recipe-sysroot-native/usr/bin:$PATH;" % rustlibpath + cmd = cmd + " export TARGET_VENDOR=\"-poky\";" + cmd = cmd + " export PATH=%s/recipe-sysroot-native/usr/bin/%s:%s/hosttools:$PATH;" % (rustlibpath, tcpath, tmpdir) + cmd = cmd + " export RUST_TARGET_PATH=%s/rust-targets;" % rustlibpath + # Trigger testing. + cmd = cmd + " export TEST_DEVICE_ADDR=\"%s:12345\";" % qemu.ip + cmd = cmd + " cd %s; python3 src/bootstrap/bootstrap.py test %s --target %s" % (builddir, testargs, targetsys) + retval = runCmd(cmd) + end_time = time.time() + + resultlog = rustlibpath + "/results-log.txt" + with open(resultlog, "w") as f: + f.write(retval.output) + + ptestsuite = "rust" + self.ptest_section(ptestsuite, duration = int(end_time - start_time), logfile=resultlog) + test_results = parse_results(resultlog) + for test in test_results: + self.ptest_result(ptestsuite, test, test_results[test]) diff --git a/meta/lib/oeqa/selftest/cases/selftest.py b/meta/lib/oeqa/selftest/cases/selftest.py index 7268e25939..a80a8651a5 100644 --- a/meta/lib/oeqa/selftest/cases/selftest.py +++ b/meta/lib/oeqa/selftest/cases/selftest.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/selftest/cases/signing.py b/meta/lib/oeqa/selftest/cases/signing.py index 6f3d4aeae9..18cce0ba25 100644 --- a/meta/lib/oeqa/selftest/cases/signing.py +++ b/meta/lib/oeqa/selftest/cases/signing.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -189,7 +191,7 @@ class LockedSignatures(OESelftestTestCase): bitbake(test_recipe) # Generate locked sigs include file - bitbake('-S none %s' % test_recipe) + bitbake('-S lockedsigs %s' % test_recipe) feature = 'require %s\n' % locked_sigs_file feature += 'SIGGEN_LOCKEDSIGS_TASKSIG_CHECK = "warn"\n' diff --git a/meta/lib/oeqa/selftest/cases/spdx.py b/meta/lib/oeqa/selftest/cases/spdx.py new file mode 100644 index 0000000000..05fc4e390b --- /dev/null +++ b/meta/lib/oeqa/selftest/cases/spdx.py @@ -0,0 +1,54 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +import json +import os +from oeqa.selftest.case import OESelftestTestCase +from oeqa.utils.commands import bitbake, get_bb_var, runCmd + +class SPDXCheck(OESelftestTestCase): + + @classmethod + def setUpClass(cls): + super(SPDXCheck, cls).setUpClass() + bitbake("python3-spdx-tools-native") + bitbake("-c addto_recipe_sysroot python3-spdx-tools-native") + + def check_recipe_spdx(self, high_level_dir, spdx_file, target_name): + config = """ +INHERIT += "create-spdx" +""" + self.write_config(config) + + deploy_dir = get_bb_var("DEPLOY_DIR") + machine_var = get_bb_var("MACHINE") + # qemux86-64 creates the directory qemux86_64 + machine_dir = machine_var.replace("-", "_") + + full_file_path = os.path.join(deploy_dir, "spdx", machine_dir, high_level_dir, spdx_file) + + try: + os.remove(full_file_path) + except FileNotFoundError: + pass + + bitbake("%s -c create_spdx" % target_name) + + def check_spdx_json(filename): + with open(filename) as f: + report = json.load(f) + self.assertNotEqual(report, None) + self.assertNotEqual(report["SPDXID"], None) + + python = os.path.join(get_bb_var('STAGING_BINDIR', 'python3-spdx-tools-native'), 'nativepython3') + validator = os.path.join(get_bb_var('STAGING_BINDIR', 'python3-spdx-tools-native'), 'pyspdxtools') + result = runCmd("{} {} -i {}".format(python, validator, filename)) + + self.assertExists(full_file_path) + result = check_spdx_json(full_file_path) + + def test_spdx_base_files(self): + self.check_recipe_spdx("packages", "base-files.spdx.json", "base-files") diff --git a/meta/lib/oeqa/selftest/cases/sstate.py b/meta/lib/oeqa/selftest/cases/sstate.py deleted file mode 100644 index 176766331a..0000000000 --- a/meta/lib/oeqa/selftest/cases/sstate.py +++ /dev/null @@ -1,64 +0,0 @@ -# -# SPDX-License-Identifier: MIT -# - -import datetime -import os -import re - -from oeqa.selftest.case import OESelftestTestCase -from oeqa.utils.commands import get_bb_vars - - -class SStateBase(OESelftestTestCase): - - def setUpLocal(self): - super(SStateBase, self).setUpLocal() - self.temp_sstate_location = None - needed_vars = ['SSTATE_DIR', 'NATIVELSBSTRING', 'TCLIBC', 'TUNE_ARCH', - 'TOPDIR', 'TARGET_VENDOR', 'TARGET_OS'] - bb_vars = get_bb_vars(needed_vars) - self.sstate_path = bb_vars['SSTATE_DIR'] - self.hostdistro = bb_vars['NATIVELSBSTRING'] - self.tclibc = bb_vars['TCLIBC'] - self.tune_arch = bb_vars['TUNE_ARCH'] - self.topdir = bb_vars['TOPDIR'] - self.target_vendor = bb_vars['TARGET_VENDOR'] - self.target_os = bb_vars['TARGET_OS'] - self.distro_specific_sstate = os.path.join(self.sstate_path, self.hostdistro) - - # Creates a special sstate configuration with the option to add sstate mirrors - def config_sstate(self, temp_sstate_location=False, add_local_mirrors=[]): - self.temp_sstate_location = temp_sstate_location - - if self.temp_sstate_location: - temp_sstate_path = os.path.join(self.builddir, "temp_sstate_%s" % datetime.datetime.now().strftime('%Y%m%d%H%M%S')) - config_temp_sstate = "SSTATE_DIR = \"%s\"" % temp_sstate_path - self.append_config(config_temp_sstate) - self.track_for_cleanup(temp_sstate_path) - bb_vars = get_bb_vars(['SSTATE_DIR', 'NATIVELSBSTRING']) - self.sstate_path = bb_vars['SSTATE_DIR'] - self.hostdistro = bb_vars['NATIVELSBSTRING'] - self.distro_specific_sstate = os.path.join(self.sstate_path, self.hostdistro) - - if add_local_mirrors: - config_set_sstate_if_not_set = 'SSTATE_MIRRORS ?= ""' - self.append_config(config_set_sstate_if_not_set) - for local_mirror in add_local_mirrors: - self.assertFalse(os.path.join(local_mirror) == os.path.join(self.sstate_path), msg='Cannot add the current sstate path as a sstate mirror') - config_sstate_mirror = "SSTATE_MIRRORS += \"file://.* file:///%s/PATH\"" % local_mirror - self.append_config(config_sstate_mirror) - - # Returns a list containing sstate files - def search_sstate(self, filename_regex, distro_specific=True, distro_nonspecific=True): - result = [] - for root, dirs, files in os.walk(self.sstate_path): - if distro_specific and re.search(r"%s/%s/[a-z0-9]{2}/[a-z0-9]{2}$" % (self.sstate_path, self.hostdistro), root): - for f in files: - if re.search(filename_regex, f): - result.append(f) - if distro_nonspecific and re.search(r"%s/[a-z0-9]{2}/[a-z0-9]{2}$" % self.sstate_path, root): - for f in files: - if re.search(filename_regex, f): - result.append(f) - return result diff --git a/meta/lib/oeqa/selftest/cases/sstatetests.py b/meta/lib/oeqa/selftest/cases/sstatetests.py index 63827f3068..86d6cd7464 100644 --- a/meta/lib/oeqa/selftest/cases/sstatetests.py +++ b/meta/lib/oeqa/selftest/cases/sstatetests.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -7,54 +9,77 @@ import shutil import glob import subprocess import tempfile +import datetime +import re -from oeqa.utils.commands import runCmd, bitbake, get_bb_var, create_temp_layer -from oeqa.selftest.cases.sstate import SStateBase -import oe +from oeqa.utils.commands import runCmd, bitbake, get_bb_var, create_temp_layer, get_bb_vars +from oeqa.selftest.case import OESelftestTestCase +from oeqa.core.decorator import OETestTag +import oe import bb.siggen -class SStateTests(SStateBase): - def test_autorev_sstate_works(self): - # Test that a git repository which changes is correctly handled by SRCREV = ${AUTOREV} - # when PV does not contain SRCPV - - tempdir = tempfile.mkdtemp(prefix='sstate_autorev') - tempdldir = tempfile.mkdtemp(prefix='sstate_autorev_dldir') - self.track_for_cleanup(tempdir) - self.track_for_cleanup(tempdldir) - create_temp_layer(tempdir, 'selftestrecipetool') - self.add_command_to_tearDown('bitbake-layers remove-layer %s' % tempdir) - self.append_config("DL_DIR = \"%s\"" % tempdldir) - runCmd('bitbake-layers add-layer %s' % tempdir) - - # Use dbus-wait as a local git repo we can add a commit between two builds in - pn = 'dbus-wait' - srcrev = '6cc6077a36fe2648a5f993fe7c16c9632f946517' - url = 'git://git.yoctoproject.org/dbus-wait' - result = runCmd('git clone %s noname' % url, cwd=tempdir) - srcdir = os.path.join(tempdir, 'noname') - result = runCmd('git reset --hard %s' % srcrev, cwd=srcdir) - self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure.ac')), 'Unable to find configure script in source directory') - - recipefile = os.path.join(tempdir, "recipes-test", "dbus-wait-test", 'dbus-wait-test_git.bb') - os.makedirs(os.path.dirname(recipefile)) - srcuri = 'git://' + srcdir + ';protocol=file;branch=master' - result = runCmd(['recipetool', 'create', '-o', recipefile, srcuri]) - self.assertTrue(os.path.isfile(recipefile), 'recipetool did not create recipe file; output:\n%s' % result.output) - - with open(recipefile, 'a') as f: - f.write('SRCREV = "${AUTOREV}"\n') - f.write('PV = "1.0"\n') - - bitbake("dbus-wait-test -c fetch") - with open(os.path.join(srcdir, "bar.txt"), "w") as f: - f.write("foo") - result = runCmd('git add bar.txt; git commit -asm "add bar"', cwd=srcdir) - bitbake("dbus-wait-test -c unpack") - - - # Test sstate files creation and their location +# Set to True to preserve stamp files after test execution for debugging failures +keep_temp_files = False + +class SStateBase(OESelftestTestCase): + + def setUpLocal(self): + super(SStateBase, self).setUpLocal() + self.temp_sstate_location = None + needed_vars = ['SSTATE_DIR', 'NATIVELSBSTRING', 'TCLIBC', 'TUNE_ARCH', + 'TOPDIR', 'TARGET_VENDOR', 'TARGET_OS'] + bb_vars = get_bb_vars(needed_vars) + self.sstate_path = bb_vars['SSTATE_DIR'] + self.hostdistro = bb_vars['NATIVELSBSTRING'] + self.tclibc = bb_vars['TCLIBC'] + self.tune_arch = bb_vars['TUNE_ARCH'] + self.topdir = bb_vars['TOPDIR'] + self.target_vendor = bb_vars['TARGET_VENDOR'] + self.target_os = bb_vars['TARGET_OS'] + self.distro_specific_sstate = os.path.join(self.sstate_path, self.hostdistro) + + def track_for_cleanup(self, path): + if not keep_temp_files: + super().track_for_cleanup(path) + + # Creates a special sstate configuration with the option to add sstate mirrors + def config_sstate(self, temp_sstate_location=False, add_local_mirrors=[]): + self.temp_sstate_location = temp_sstate_location + + if self.temp_sstate_location: + temp_sstate_path = os.path.join(self.builddir, "temp_sstate_%s" % datetime.datetime.now().strftime('%Y%m%d%H%M%S')) + config_temp_sstate = "SSTATE_DIR = \"%s\"" % temp_sstate_path + self.append_config(config_temp_sstate) + self.track_for_cleanup(temp_sstate_path) + bb_vars = get_bb_vars(['SSTATE_DIR', 'NATIVELSBSTRING']) + self.sstate_path = bb_vars['SSTATE_DIR'] + self.hostdistro = bb_vars['NATIVELSBSTRING'] + self.distro_specific_sstate = os.path.join(self.sstate_path, self.hostdistro) + + if add_local_mirrors: + config_set_sstate_if_not_set = 'SSTATE_MIRRORS ?= ""' + self.append_config(config_set_sstate_if_not_set) + for local_mirror in add_local_mirrors: + self.assertFalse(os.path.join(local_mirror) == os.path.join(self.sstate_path), msg='Cannot add the current sstate path as a sstate mirror') + config_sstate_mirror = "SSTATE_MIRRORS += \"file://.* file:///%s/PATH\"" % local_mirror + self.append_config(config_sstate_mirror) + + # Returns a list containing sstate files + def search_sstate(self, filename_regex, distro_specific=True, distro_nonspecific=True): + result = [] + for root, dirs, files in os.walk(self.sstate_path): + if distro_specific and re.search(r"%s/%s/[a-z0-9]{2}/[a-z0-9]{2}$" % (self.sstate_path, self.hostdistro), root): + for f in files: + if re.search(filename_regex, f): + result.append(f) + if distro_nonspecific and re.search(r"%s/[a-z0-9]{2}/[a-z0-9]{2}$" % self.sstate_path, root): + for f in files: + if re.search(filename_regex, f): + result.append(f) + return result + + # Test sstate files creation and their location and directory perms def run_test_sstate_creation(self, targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True, should_pass=True): self.config_sstate(temp_sstate_location, [self.sstate_path]) @@ -63,6 +88,19 @@ class SStateTests(SStateBase): else: bitbake(['-ccleansstate'] + targets) + # We need to test that the env umask have does not effect sstate directory creation + # So, first, we'll get the current umask and set it to something we know incorrect + # See: sstate_task_postfunc for correct umask of os.umask(0o002) + import os + def current_umask(): + current_umask = os.umask(0) + os.umask(current_umask) + return current_umask + + orig_umask = current_umask() + # Set it to a umask we know will be 'wrong' + os.umask(0o022) + bitbake(targets) file_tracker = [] results = self.search_sstate('|'.join(map(str, targets)), distro_specific, distro_nonspecific) @@ -79,17 +117,18 @@ class SStateTests(SStateBase): else: self.assertTrue(not file_tracker , msg="Found sstate files in the wrong place for: %s (found %s)" % (', '.join(map(str, targets)), str(file_tracker))) - def test_sstate_creation_distro_specific_pass(self): - self.run_test_sstate_creation(['binutils-cross-'+ self.tune_arch, 'binutils-native'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True) - - def test_sstate_creation_distro_specific_fail(self): - self.run_test_sstate_creation(['binutils-cross-'+ self.tune_arch, 'binutils-native'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True, should_pass=False) + # Now we'll walk the tree to check the mode and see if things are incorrect. + badperms = [] + for root, dirs, files in os.walk(self.sstate_path): + for directory in dirs: + if (os.stat(os.path.join(root, directory)).st_mode & 0o777) != 0o775: + badperms.append(os.path.join(root, directory)) - def test_sstate_creation_distro_nonspecific_pass(self): - self.run_test_sstate_creation(['linux-libc-headers'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True) + # Return to original umask + os.umask(orig_umask) - def test_sstate_creation_distro_nonspecific_fail(self): - self.run_test_sstate_creation(['linux-libc-headers'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True, should_pass=False) + if should_pass: + self.assertTrue(badperms , msg="Found sstate directories with the wrong permissions: %s (found %s)" % (', '.join(map(str, targets)), str(badperms))) # Test the sstate files deletion part of the do_cleansstate task def run_test_cleansstate_task(self, targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True): @@ -108,20 +147,6 @@ class SStateTests(SStateBase): archives_removed = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific, distro_nonspecific) self.assertTrue(not archives_removed, msg="do_cleansstate didn't remove .tar.zst sstate files for: %s (%s)" % (', '.join(map(str, targets)), str(archives_removed))) - def test_cleansstate_task_distro_specific_nonspecific(self): - targets = ['binutils-cross-'+ self.tune_arch, 'binutils-native'] - targets.append('linux-libc-headers') - self.run_test_cleansstate_task(targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True) - - def test_cleansstate_task_distro_nonspecific(self): - self.run_test_cleansstate_task(['linux-libc-headers'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True) - - def test_cleansstate_task_distro_specific(self): - targets = ['binutils-cross-'+ self.tune_arch, 'binutils-native'] - targets.append('linux-libc-headers') - self.run_test_cleansstate_task(targets, distro_specific=True, distro_nonspecific=False, temp_sstate_location=True) - - # Test rebuilding of distro-specific sstate files def run_test_rebuild_distro_specific_sstate(self, targets, temp_sstate_location=True): self.config_sstate(temp_sstate_location, [self.sstate_path]) @@ -154,6 +179,105 @@ class SStateTests(SStateBase): created_once = [x for x in file_tracker_2 if x not in file_tracker_1] self.assertTrue(created_once == [], msg="The following sstate files were created only in the second run: %s" % ', '.join(map(str, created_once))) + def sstate_common_samesigs(self, configA, configB, allarch=False): + + self.write_config(configA) + self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") + bitbake("world meta-toolchain -S none") + self.write_config(configB) + self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") + bitbake("world meta-toolchain -S none") + + def get_files(d, result): + for root, dirs, files in os.walk(d): + for name in files: + if "meta-environment" in root or "cross-canadian" in root: + continue + if "do_build" not in name: + # 1.4.1+gitAUTOINC+302fca9f4c-r0.do_package_write_ipk.sigdata.f3a2a38697da743f0dbed8b56aafcf79 + (_, task, _, shash) = name.rsplit(".", 3) + result[os.path.join(os.path.basename(root), task)] = shash + + files1 = {} + files2 = {} + subdirs = sorted(glob.glob(self.topdir + "/tmp-sstatesamehash/stamps/*-nativesdk*-linux")) + if allarch: + subdirs.extend(sorted(glob.glob(self.topdir + "/tmp-sstatesamehash/stamps/all-*-linux"))) + + for subdir in subdirs: + nativesdkdir = os.path.basename(subdir) + get_files(self.topdir + "/tmp-sstatesamehash/stamps/" + nativesdkdir, files1) + get_files(self.topdir + "/tmp-sstatesamehash2/stamps/" + nativesdkdir, files2) + + self.maxDiff = None + self.assertEqual(files1, files2) + +class SStateTests(SStateBase): + def test_autorev_sstate_works(self): + # Test that a git repository which changes is correctly handled by SRCREV = ${AUTOREV} + + tempdir = tempfile.mkdtemp(prefix='sstate_autorev') + tempdldir = tempfile.mkdtemp(prefix='sstate_autorev_dldir') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(tempdldir) + create_temp_layer(tempdir, 'selftestrecipetool') + self.add_command_to_tearDown('bitbake-layers remove-layer %s' % tempdir) + self.append_config("DL_DIR = \"%s\"" % tempdldir) + runCmd('bitbake-layers add-layer %s' % tempdir) + + # Use dbus-wait as a local git repo we can add a commit between two builds in + pn = 'dbus-wait' + srcrev = '6cc6077a36fe2648a5f993fe7c16c9632f946517' + url = 'git://git.yoctoproject.org/dbus-wait' + result = runCmd('git clone %s noname' % url, cwd=tempdir) + srcdir = os.path.join(tempdir, 'noname') + result = runCmd('git reset --hard %s' % srcrev, cwd=srcdir) + self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure.ac')), 'Unable to find configure script in source directory') + + recipefile = os.path.join(tempdir, "recipes-test", "dbus-wait-test", 'dbus-wait-test_git.bb') + os.makedirs(os.path.dirname(recipefile)) + srcuri = 'git://' + srcdir + ';protocol=file;branch=master' + result = runCmd(['recipetool', 'create', '-o', recipefile, srcuri]) + self.assertTrue(os.path.isfile(recipefile), 'recipetool did not create recipe file; output:\n%s' % result.output) + + with open(recipefile, 'a') as f: + f.write('SRCREV = "${AUTOREV}"\n') + f.write('PV = "1.0"\n') + + bitbake("dbus-wait-test -c fetch") + with open(os.path.join(srcdir, "bar.txt"), "w") as f: + f.write("foo") + result = runCmd('git add bar.txt; git commit -asm "add bar"', cwd=srcdir) + bitbake("dbus-wait-test -c unpack") + +class SStateCreation(SStateBase): + def test_sstate_creation_distro_specific_pass(self): + self.run_test_sstate_creation(['binutils-cross-'+ self.tune_arch, 'binutils-native'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True) + + def test_sstate_creation_distro_specific_fail(self): + self.run_test_sstate_creation(['binutils-cross-'+ self.tune_arch, 'binutils-native'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True, should_pass=False) + + def test_sstate_creation_distro_nonspecific_pass(self): + self.run_test_sstate_creation(['linux-libc-headers'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True) + + def test_sstate_creation_distro_nonspecific_fail(self): + self.run_test_sstate_creation(['linux-libc-headers'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True, should_pass=False) + +class SStateCleanup(SStateBase): + def test_cleansstate_task_distro_specific_nonspecific(self): + targets = ['binutils-cross-'+ self.tune_arch, 'binutils-native'] + targets.append('linux-libc-headers') + self.run_test_cleansstate_task(targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True) + + def test_cleansstate_task_distro_nonspecific(self): + self.run_test_cleansstate_task(['linux-libc-headers'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True) + + def test_cleansstate_task_distro_specific(self): + targets = ['binutils-cross-'+ self.tune_arch, 'binutils-native'] + targets.append('linux-libc-headers') + self.run_test_cleansstate_task(targets, distro_specific=True, distro_nonspecific=False, temp_sstate_location=True) + +class SStateDistroTests(SStateBase): def test_rebuild_distro_specific_sstate_cross_native_targets(self): self.run_test_rebuild_distro_specific_sstate(['binutils-cross-' + self.tune_arch, 'binutils-native'], temp_sstate_location=True) @@ -163,30 +287,30 @@ class SStateTests(SStateBase): def test_rebuild_distro_specific_sstate_native_target(self): self.run_test_rebuild_distro_specific_sstate(['binutils-native'], temp_sstate_location=True) - +class SStateCacheManagement(SStateBase): # Test the sstate-cache-management script. Each element in the global_config list is used with the corresponding element in the target_config list - # global_config elements are expected to not generate any sstate files that would be removed by sstate-cache-management.sh (such as changing the value of MACHINE) + # global_config elements are expected to not generate any sstate files that would be removed by sstate-cache-management.py (such as changing the value of MACHINE) def run_test_sstate_cache_management_script(self, target, global_config=[''], target_config=[''], ignore_patterns=[]): self.assertTrue(global_config) self.assertTrue(target_config) self.assertTrue(len(global_config) == len(target_config), msg='Lists global_config and target_config should have the same number of elements') - self.config_sstate(temp_sstate_location=True, add_local_mirrors=[self.sstate_path]) - # If buildhistory is enabled, we need to disable version-going-backwards - # QA checks for this test. It may report errors otherwise. - self.append_config('ERROR_QA:remove = "version-going-backwards"') + for idx in range(len(target_config)): + self.append_config(global_config[idx]) + self.append_recipeinc(target, target_config[idx]) + bitbake(target) + self.remove_config(global_config[idx]) + self.remove_recipeinc(target, target_config[idx]) + + self.config_sstate(temp_sstate_location=True, add_local_mirrors=[self.sstate_path]) # For now this only checks if random sstate tasks are handled correctly as a group. # In the future we should add control over what tasks we check for. - sstate_archs_list = [] expected_remaining_sstate = [] for idx in range(len(target_config)): self.append_config(global_config[idx]) self.append_recipeinc(target, target_config[idx]) - sstate_arch = get_bb_var('SSTATE_PKGARCH', target) - if not sstate_arch in sstate_archs_list: - sstate_archs_list.append(sstate_arch) if target_config[idx] == target_config[-1]: target_sstate_before_build = self.search_sstate(target + r'.*?\.tar.zst$') bitbake("-cclean %s" % target) @@ -198,7 +322,7 @@ class SStateTests(SStateBase): self.remove_recipeinc(target, target_config[idx]) self.assertEqual(result.status, 0, msg = "build of %s failed with %s" % (target, result.output)) - runCmd("sstate-cache-management.sh -y --cache-dir=%s --remove-duplicated --extra-archs=%s" % (self.sstate_path, ','.join(map(str, sstate_archs_list)))) + runCmd("sstate-cache-management.py -y --cache-dir=%s --remove-duplicated" % (self.sstate_path)) actual_remaining_sstate = [x for x in self.search_sstate(target + r'.*?\.tar.zst$') if not any(pattern in x for pattern in ignore_patterns)] actual_not_expected = [x for x in actual_remaining_sstate if x not in expected_remaining_sstate] @@ -242,6 +366,7 @@ class SStateTests(SStateBase): target_config.append('') self.run_test_sstate_cache_management_script('m4', global_config, target_config, ignore_patterns=['populate_lic']) +class SStateHashSameSigs(SStateBase): def test_sstate_32_64_same_hash(self): """ The sstate checksums for both native and target should not vary whether @@ -327,6 +452,7 @@ BB_SIGNATURE_HANDLER = "OEBasicHash" self.maxDiff = None self.assertCountEqual(files1, files2) +class SStateHashSameSigs2(SStateBase): def test_sstate_allarch_samesigs(self): """ The sstate checksums of allarch packages should be independent of whichever @@ -375,41 +501,7 @@ BB_SIGNATURE_HANDLER = "OEBasicHash" """ self.sstate_common_samesigs(configA, configB) - def sstate_common_samesigs(self, configA, configB, allarch=False): - - self.write_config(configA) - self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") - bitbake("world meta-toolchain -S none") - self.write_config(configB) - self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") - bitbake("world meta-toolchain -S none") - - def get_files(d): - f = {} - for root, dirs, files in os.walk(d): - for name in files: - if "meta-environment" in root or "cross-canadian" in root: - continue - if "do_build" not in name: - # 1.4.1+gitAUTOINC+302fca9f4c-r0.do_package_write_ipk.sigdata.f3a2a38697da743f0dbed8b56aafcf79 - (_, task, _, shash) = name.rsplit(".", 3) - f[os.path.join(os.path.basename(root), task)] = shash - return f - - nativesdkdir = os.path.basename(glob.glob(self.topdir + "/tmp-sstatesamehash/stamps/*-nativesdk*-linux")[0]) - - files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps/" + nativesdkdir) - files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps/" + nativesdkdir) - self.maxDiff = None - self.assertEqual(files1, files2) - - if allarch: - allarchdir = os.path.basename(glob.glob(self.topdir + "/tmp-sstatesamehash/stamps/all-*-linux")[0]) - - files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps/" + allarchdir) - files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps/" + allarchdir) - self.assertEqual(files1, files2) - +class SStateHashSameSigs3(SStateBase): def test_sstate_sametune_samesigs(self): """ The sstate checksums of two identical machines (using the same tune) should be the @@ -497,7 +589,7 @@ BB_SIGNATURE_HANDLER = "OEBasicHash" self.maxDiff = None self.assertCountEqual(files1, files2) - +class SStateHashSameSigs4(SStateBase): def test_sstate_noop_samesigs(self): """ The sstate checksums of two builds with these variables changed or @@ -625,3 +717,292 @@ TMPDIR = "${TOPDIR}/tmp-sstatesamehash2" self.maxDiff = None self.assertCountEqual(files1, files2) +class SStateFindSiginfo(SStateBase): + def test_sstate_compare_sigfiles_and_find_siginfo(self): + """ + Test the functionality of the find_siginfo: basic function and callback in compare_sigfiles + """ + self.write_config(""" +TMPDIR = \"${TOPDIR}/tmp-sstates-findsiginfo\" +TCLIBCAPPEND = \"\" +MACHINE = \"qemux86-64\" +require conf/multilib.conf +MULTILIBS = "multilib:lib32" +DEFAULTTUNE:virtclass-multilib-lib32 = "x86" +BB_SIGNATURE_HANDLER = "OEBasicHash" +""") + self.track_for_cleanup(self.topdir + "/tmp-sstates-findsiginfo") + + pns = ["binutils", "binutils-native", "lib32-binutils"] + target_configs = [ +""" +TMPVAL1 = "tmpval1" +TMPVAL2 = "tmpval2" +do_tmptask1() { + echo ${TMPVAL1} +} +do_tmptask2() { + echo ${TMPVAL2} +} +addtask do_tmptask1 +addtask tmptask2 before do_tmptask1 +""", +""" +TMPVAL3 = "tmpval3" +TMPVAL4 = "tmpval4" +do_tmptask1() { + echo ${TMPVAL3} +} +do_tmptask2() { + echo ${TMPVAL4} +} +addtask do_tmptask1 +addtask tmptask2 before do_tmptask1 +""" + ] + + for target_config in target_configs: + self.write_recipeinc("binutils", target_config) + for pn in pns: + bitbake("%s -c do_tmptask1 -S none" % pn) + self.delete_recipeinc("binutils") + + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(config_only=True) + + def find_siginfo(pn, taskname, sigs=None): + result = None + command_complete = False + tinfoil.set_event_mask(["bb.event.FindSigInfoResult", + "bb.command.CommandCompleted"]) + ret = tinfoil.run_command("findSigInfo", pn, taskname, sigs) + if ret: + while result is None or not command_complete: + event = tinfoil.wait_event(1) + if event: + if isinstance(event, bb.command.CommandCompleted): + command_complete = True + elif isinstance(event, bb.event.FindSigInfoResult): + result = event.result + return result + + def recursecb(key, hash1, hash2): + nonlocal recursecb_count + recursecb_count += 1 + hashes = [hash1, hash2] + hashfiles = find_siginfo(key, None, hashes) + self.assertCountEqual(hashes, hashfiles) + bb.siggen.compare_sigfiles(hashfiles[hash1]['path'], hashfiles[hash2]['path'], recursecb) + + for pn in pns: + recursecb_count = 0 + matches = find_siginfo(pn, "do_tmptask1") + self.assertGreaterEqual(len(matches), 2) + latesthashes = sorted(matches.keys(), key=lambda h: matches[h]['time'])[-2:] + bb.siggen.compare_sigfiles(matches[latesthashes[-2]]['path'], matches[latesthashes[-1]]['path'], recursecb) + self.assertEqual(recursecb_count,1) + +class SStatePrintdiff(SStateBase): + def run_test_printdiff_changerecipe(self, target, change_recipe, change_bbtask, change_content, expected_sametmp_output, expected_difftmp_output): + import time + self.write_config(""" +TMPDIR = "${{TOPDIR}}/tmp-sstateprintdiff-sametmp-{}" +""".format(time.time())) + # Use runall do_build to ensure any indirect sstate is created, e.g. tzcode-native on both x86 and + # aarch64 hosts since only allarch target recipes depend upon it and it may not be built otherwise. + # A bitbake -c cleansstate tzcode-native would cause some of these tests to error for example. + bitbake("--runall build --runall deploy_source_date_epoch {}".format(target)) + bitbake("-S none {}".format(target)) + bitbake(change_bbtask) + self.write_recipeinc(change_recipe, change_content) + result_sametmp = bitbake("-S printdiff {}".format(target)) + + self.write_config(""" +TMPDIR = "${{TOPDIR}}/tmp-sstateprintdiff-difftmp-{}" +""".format(time.time())) + result_difftmp = bitbake("-S printdiff {}".format(target)) + + self.delete_recipeinc(change_recipe) + for item in expected_sametmp_output: + self.assertIn(item, result_sametmp.output, msg = "Item {} not found in output:\n{}".format(item, result_sametmp.output)) + for item in expected_difftmp_output: + self.assertIn(item, result_difftmp.output, msg = "Item {} not found in output:\n{}".format(item, result_difftmp.output)) + + def run_test_printdiff_changeconfig(self, target, change_bbtasks, change_content, expected_sametmp_output, expected_difftmp_output): + import time + self.write_config(""" +TMPDIR = "${{TOPDIR}}/tmp-sstateprintdiff-sametmp-{}" +""".format(time.time())) + bitbake("--runall build --runall deploy_source_date_epoch {}".format(target)) + bitbake("-S none {}".format(target)) + bitbake(" ".join(change_bbtasks)) + self.append_config(change_content) + result_sametmp = bitbake("-S printdiff {}".format(target)) + + self.write_config(""" +TMPDIR = "${{TOPDIR}}/tmp-sstateprintdiff-difftmp-{}" +""".format(time.time())) + self.append_config(change_content) + result_difftmp = bitbake("-S printdiff {}".format(target)) + + for item in expected_sametmp_output: + self.assertIn(item, result_sametmp.output, msg = "Item {} not found in output:\n{}".format(item, result_sametmp.output)) + for item in expected_difftmp_output: + self.assertIn(item, result_difftmp.output, msg = "Item {} not found in output:\n{}".format(item, result_difftmp.output)) + + + # Check if printdiff walks the full dependency chain from the image target to where the change is in a specific recipe + def test_image_minimal_vs_perlcross(self): + expected_output = ("Task perlcross-native:do_install couldn't be used from the cache because:", +"We need hash", +"most recent matching task was") + expected_sametmp_output = expected_output + ( +"Variable do_install value changed", +'+ echo "this changes the task signature"') + expected_difftmp_output = expected_output + + self.run_test_printdiff_changerecipe("core-image-minimal", "perlcross", "-c do_install perlcross-native", +""" +do_install:append() { + echo "this changes the task signature" +} +""", +expected_sametmp_output, expected_difftmp_output) + + # Check if changes to gcc-source (which uses tmp/work-shared) are correctly discovered + def test_gcc_runtime_vs_gcc_source(self): + gcc_source_pn = 'gcc-source-%s' % get_bb_vars(['PV'], 'gcc')['PV'] + + expected_output = ("Task {}:do_preconfigure couldn't be used from the cache because:".format(gcc_source_pn), +"We need hash", +"most recent matching task was") + expected_sametmp_output = expected_output + ( +"Variable do_preconfigure value changed", +'+ print("this changes the task signature")') + expected_difftmp_output = expected_output + + self.run_test_printdiff_changerecipe("gcc-runtime", "gcc-source", "-c do_preconfigure {}".format(gcc_source_pn), +""" +python do_preconfigure:append() { + print("this changes the task signature") +} +""", +expected_sametmp_output, expected_difftmp_output) + + # Check if changing a really base task definiton is reported against multiple core recipes using it + def test_image_minimal_vs_base_do_configure(self): + change_bbtasks = ('zstd-native:do_configure', +'texinfo-dummy-native:do_configure', +'ldconfig-native:do_configure', +'gettext-minimal-native:do_configure', +'tzcode-native:do_configure', +'makedevs-native:do_configure', +'pigz-native:do_configure', +'update-rc.d-native:do_configure', +'unzip-native:do_configure', +'gnu-config-native:do_configure') + + expected_output = ["Task {} couldn't be used from the cache because:".format(t) for t in change_bbtasks] + [ +"We need hash", +"most recent matching task was"] + + expected_sametmp_output = expected_output + [ +"Variable base_do_configure value changed", +'+ echo "this changes base_do_configure() definiton "'] + expected_difftmp_output = expected_output + + self.run_test_printdiff_changeconfig("core-image-minimal",change_bbtasks, +""" +INHERIT += "base-do-configure-modified" +""", +expected_sametmp_output, expected_difftmp_output) + +@OETestTag("yocto-mirrors") +class SStateMirrors(SStateBase): + def check_bb_output(self, output, exceptions, check_cdn): + def is_exception(object, exceptions): + for e in exceptions: + if re.search(e, object): + return True + return False + + output_l = output.splitlines() + for l in output_l: + if l.startswith("Sstate summary"): + for idx, item in enumerate(l.split()): + if item == 'Missed': + missing_objects = int(l.split()[idx+1]) + break + else: + self.fail("Did not find missing objects amount in sstate summary: {}".format(l)) + break + else: + self.fail("Did not find 'Sstate summary' line in bitbake output") + + failed_urls = [] + failed_urls_extrainfo = [] + for l in output_l: + if "SState: Unsuccessful fetch test for" in l and check_cdn: + missing_object = l.split()[6] + elif "SState: Looked for but didn't find file" in l and not check_cdn: + missing_object = l.split()[8] + else: + missing_object = None + if missing_object: + if not is_exception(missing_object, exceptions): + failed_urls.append(missing_object) + else: + missing_objects -= 1 + + if "urlopen failed for" in l and not is_exception(l, exceptions): + failed_urls_extrainfo.append(l) + + self.assertEqual(len(failed_urls), missing_objects, "Amount of reported missing objects does not match failed URLs: {}\nFailed URLs:\n{}\nFetcher diagnostics:\n{}".format(missing_objects, "\n".join(failed_urls), "\n".join(failed_urls_extrainfo))) + self.assertEqual(len(failed_urls), 0, "Missing objects in the cache:\n{}\nFetcher diagnostics:\n{}".format("\n".join(failed_urls), "\n".join(failed_urls_extrainfo))) + + def run_test(self, machine, targets, exceptions, check_cdn = True, ignore_errors = False): + # sstate is checked for existence of these, but they never get written out to begin with + exceptions += ["{}.*image_qa".format(t) for t in targets.split()] + exceptions += ["{}.*deploy_source_date_epoch".format(t) for t in targets.split()] + exceptions += ["{}.*image_complete".format(t) for t in targets.split()] + exceptions += ["linux-yocto.*shared_workdir"] + # these get influnced by IMAGE_FSTYPES tweaks in yocto-autobuilder-helper's config.json (on x86-64) + # additionally, they depend on noexec (thus, absent stamps) package, install, etc. image tasks, + # which makes tracing other changes difficult + exceptions += ["{}.*create_spdx".format(t) for t in targets.split()] + exceptions += ["{}.*create_runtime_spdx".format(t) for t in targets.split()] + + if check_cdn: + self.config_sstate(True) + self.append_config(""" +MACHINE = "{}" +BB_HASHSERVE_UPSTREAM = "hashserv.yocto.io:8687" +SSTATE_MIRRORS ?= "file://.* http://cdn.jsdelivr.net/yocto/sstate/all/PATH;downloadfilename=PATH" +""".format(machine)) + else: + self.append_config(""" +MACHINE = "{}" +""".format(machine)) + result = bitbake("-DD -n {}".format(targets)) + bitbake("-S none {}".format(targets)) + if ignore_errors: + return + self.check_bb_output(result.output, exceptions, check_cdn) + + def test_cdn_mirror_qemux86_64(self): + exceptions = [] + self.run_test("qemux86-64", "core-image-minimal core-image-full-cmdline core-image-sato-sdk", exceptions, ignore_errors = True) + self.run_test("qemux86-64", "core-image-minimal core-image-full-cmdline core-image-sato-sdk", exceptions) + + def test_cdn_mirror_qemuarm64(self): + exceptions = [] + self.run_test("qemuarm64", "core-image-minimal core-image-full-cmdline core-image-sato-sdk", exceptions, ignore_errors = True) + self.run_test("qemuarm64", "core-image-minimal core-image-full-cmdline core-image-sato-sdk", exceptions) + + def test_local_cache_qemux86_64(self): + exceptions = [] + self.run_test("qemux86-64", "core-image-minimal core-image-full-cmdline core-image-sato-sdk", exceptions, check_cdn = False) + + def test_local_cache_qemuarm64(self): + exceptions = [] + self.run_test("qemuarm64", "core-image-minimal core-image-full-cmdline core-image-sato-sdk", exceptions, check_cdn = False) diff --git a/meta/lib/oeqa/selftest/cases/sysroot.py b/meta/lib/oeqa/selftest/cases/sysroot.py index 294ba4a4a0..ef854f6fee 100644 --- a/meta/lib/oeqa/selftest/cases/sysroot.py +++ b/meta/lib/oeqa/selftest/cases/sysroot.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/selftest/cases/tinfoil.py b/meta/lib/oeqa/selftest/cases/tinfoil.py index c81d56d82b..21c8686b2a 100644 --- a/meta/lib/oeqa/selftest/cases/tinfoil.py +++ b/meta/lib/oeqa/selftest/cases/tinfoil.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -46,6 +48,17 @@ class TinfoilTests(OESelftestTestCase): rd = tinfoil.parse_recipe_file(best[3]) self.assertEqual(testrecipe, rd.getVar('PN')) + def test_parse_virtual_recipe(self): + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(config_only=False, quiet=2) + testrecipe = 'nativesdk-gcc' + best = tinfoil.find_best_provider(testrecipe) + if not best: + self.fail('Unable to find recipe providing %s' % testrecipe) + rd = tinfoil.parse_recipe_file(best[3]) + self.assertEqual(testrecipe, rd.getVar('PN')) + self.assertIsNotNone(rd.getVar('FILE_LAYERNAME')) + def test_parse_recipe_copy_expand(self): with bb.tinfoil.Tinfoil() as tinfoil: tinfoil.prepare(config_only=False, quiet=2) @@ -64,6 +77,32 @@ class TinfoilTests(OESelftestTestCase): localdata.setVar('PN', 'hello') self.assertEqual('hello', localdata.getVar('BPN')) + # The config_data API to parse_recipe_file is used by: + # layerindex-web layerindex/update_layer.py + def test_parse_recipe_custom_data(self): + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(config_only=False, quiet=2) + localdata = bb.data.createCopy(tinfoil.config_data) + localdata.setVar("TESTVAR", "testval") + testrecipe = 'mdadm' + best = tinfoil.find_best_provider(testrecipe) + if not best: + self.fail('Unable to find recipe providing %s' % testrecipe) + rd = tinfoil.parse_recipe_file(best[3], config_data=localdata) + self.assertEqual("testval", rd.getVar('TESTVAR')) + + def test_parse_virtual_recipe_custom_data(self): + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(config_only=False, quiet=2) + localdata = bb.data.createCopy(tinfoil.config_data) + localdata.setVar("TESTVAR", "testval") + testrecipe = 'nativesdk-gcc' + best = tinfoil.find_best_provider(testrecipe) + if not best: + self.fail('Unable to find recipe providing %s' % testrecipe) + rd = tinfoil.parse_recipe_file(best[3], config_data=localdata) + self.assertEqual("testval", rd.getVar('TESTVAR')) + def test_list_recipes(self): with bb.tinfoil.Tinfoil() as tinfoil: tinfoil.prepare(config_only=False, quiet=2) diff --git a/meta/lib/oeqa/selftest/cases/usergrouptests.py b/meta/lib/oeqa/selftest/cases/usergrouptests.py new file mode 100644 index 0000000000..3c59b0f290 --- /dev/null +++ b/meta/lib/oeqa/selftest/cases/usergrouptests.py @@ -0,0 +1,57 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +import os +import shutil +from oeqa.selftest.case import OESelftestTestCase +from oeqa.utils.commands import bitbake +from oeqa.utils.commands import bitbake, get_bb_var, get_test_layer + +class UserGroupTests(OESelftestTestCase): + def test_group_from_dep_package(self): + self.logger.info("Building creategroup2") + bitbake(' creategroup2 creategroup1') + bitbake(' creategroup2 creategroup1 -c clean') + self.logger.info("Packaging creategroup2") + self.assertTrue(bitbake(' creategroup2 -c package')) + + def test_add_task_between_p_sysroot_and_package(self): + # Test for YOCTO #14961 + self.assertTrue(bitbake('useraddbadtask -C fetch')) + + def test_postinst_order(self): + self.logger.info("Building dcreategroup") + self.assertTrue(bitbake(' dcreategroup')) + + def test_static_useradd_from_dynamic(self): + metaselftestpath = get_test_layer() + self.logger.info("Building core-image-minimal to generate passwd/group file") + bitbake(' core-image-minimal') + self.logger.info("Setting up useradd-staticids") + repropassdir = os.path.join(metaselftestpath, "conf/include") + os.makedirs(repropassdir) + etcdir=os.path.join(os.path.join(os.path.join(get_bb_var("TMPDIR"), "work"), \ + os.path.join(get_bb_var("MACHINE").replace("-","_")+"-poky-linux", "core-image-minimal/1.0/rootfs/etc"))) + shutil.copy(os.path.join(etcdir, "passwd"), os.path.join(repropassdir, "reproducable-passwd")) + shutil.copy(os.path.join(etcdir, "group"), os.path.join(repropassdir, "reproducable-group")) + # Copy the original local.conf + shutil.copyfile(os.path.join(os.environ.get('BUILDDIR'), 'conf/local.conf'), os.path.join(os.environ.get('BUILDDIR'), 'conf/local.conf.orig')) + + self.write_config("USERADDEXTENSION = \"useradd-staticids\"") + self.write_config("USERADD_ERROR_DYNAMIC ??= \"error\"") + self.write_config("USERADD_UID_TABLES += \"conf/include/reproducible-passwd\"") + self.write_config("USERADD_GID_TABLES += \"conf/include/reproducible-group\"") + self.logger.info("Rebuild with staticids") + bitbake(' core-image-minimal') + shutil.copyfile(os.path.join(os.environ.get('BUILDDIR'), 'conf/local.conf.orig'), os.path.join(os.environ.get('BUILDDIR'), 'conf/local.conf')) + self.logger.info("Rebuild without staticids") + bitbake(' core-image-minimal') + self.write_config("USERADDEXTENSION = \"useradd-staticids\"") + self.write_config("USERADD_ERROR_DYNAMIC ??= \"error\"") + self.write_config("USERADD_UID_TABLES += \"files/static-passwd\"") + self.write_config("USERADD_GID_TABLES += \"files/static-group\"") + self.logger.info("Rebuild with other staticids") + self.assertTrue(bitbake(' core-image-minimal')) diff --git a/meta/lib/oeqa/selftest/cases/wic.py b/meta/lib/oeqa/selftest/cases/wic.py index 53058df226..b616759209 100644 --- a/meta/lib/oeqa/selftest/cases/wic.py +++ b/meta/lib/oeqa/selftest/cases/wic.py @@ -15,33 +15,15 @@ import hashlib from glob import glob from shutil import rmtree, copy -from functools import wraps, lru_cache from tempfile import NamedTemporaryFile +from tempfile import TemporaryDirectory from oeqa.selftest.case import OESelftestTestCase from oeqa.core.decorator import OETestTag +from oeqa.core.decorator.data import skipIfNotArch from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars, runqemu -@lru_cache() -def get_host_arch(): - return get_bb_var('HOST_ARCH') - - -def only_for_arch(archs): - """Decorator for wrapping test cases that can be run only for specific target - architectures. A list of compatible architectures is passed in `archs`. - """ - def wrapper(func): - @wraps(func) - def wrapped_f(*args, **kwargs): - arch = get_host_arch() - if archs and arch not in archs: - raise unittest.SkipTest("Testcase arch dependency not met: %s" % arch) - return func(*args, **kwargs) - return wrapped_f - return wrapper - def extract_files(debugfs_output): """ extract file names from the output of debugfs -R 'ls -p', @@ -165,20 +147,87 @@ class CLITests(OESelftestTestCase): self.assertEqual(1, runCmd('wic', ignore_status=True).status) class Wic(WicTestCase): + def test_skip_kernel_install(self): + """Test the functionality of not installing the kernel in the boot directory using the wic plugin""" + # create a temporary file for the WKS content + with NamedTemporaryFile("w", suffix=".wks") as wks: + wks.write( + 'part --source bootimg-efi ' + '--sourceparams="loader=grub-efi,install-kernel-into-boot-dir=false" ' + '--label boot --active\n' + ) + wks.flush() + # create a temporary directory to extract the disk image to + with TemporaryDirectory() as tmpdir: + img = 'core-image-minimal' + # build the image using the WKS file + cmd = "wic create %s -e %s -o %s" % ( + wks.name, img, self.resultdir) + runCmd(cmd) + wksname = os.path.splitext(os.path.basename(wks.name))[0] + out = glob(os.path.join( + self.resultdir, "%s-*.direct" % wksname)) + self.assertEqual(1, len(out)) + sysroot = get_bb_var('RECIPE_SYSROOT_NATIVE', 'wic-tools') + # extract the content of the disk image to the temporary directory + cmd = "wic cp %s:1 %s -n %s" % (out[0], tmpdir, sysroot) + runCmd(cmd) + # check if the kernel is installed or not + kimgtype = get_bb_var('KERNEL_IMAGETYPE', img) + for file in os.listdir(tmpdir): + if file == kimgtype: + raise AssertionError( + "The kernel image '{}' was found in the partition".format(kimgtype) + ) + + def test_kernel_install(self): + """Test the installation of the kernel to the boot directory in the wic plugin""" + # create a temporary file for the WKS content + with NamedTemporaryFile("w", suffix=".wks") as wks: + wks.write( + 'part --source bootimg-efi ' + '--sourceparams="loader=grub-efi,install-kernel-into-boot-dir=true" ' + '--label boot --active\n' + ) + wks.flush() + # create a temporary directory to extract the disk image to + with TemporaryDirectory() as tmpdir: + img = 'core-image-minimal' + # build the image using the WKS file + cmd = "wic create %s -e %s -o %s" % (wks.name, img, self.resultdir) + runCmd(cmd) + wksname = os.path.splitext(os.path.basename(wks.name))[0] + out = glob(os.path.join(self.resultdir, "%s-*.direct" % wksname)) + self.assertEqual(1, len(out)) + sysroot = get_bb_var('RECIPE_SYSROOT_NATIVE', 'wic-tools') + # extract the content of the disk image to the temporary directory + cmd = "wic cp %s:1 %s -n %s" % (out[0], tmpdir, sysroot) + runCmd(cmd) + # check if the kernel is installed or not + kimgtype = get_bb_var('KERNEL_IMAGETYPE', img) + found = False + for file in os.listdir(tmpdir): + if file == kimgtype: + found = True + break + self.assertTrue( + found, "The kernel image '{}' was not found in the boot partition".format(kimgtype) + ) + def test_build_image_name(self): """Test wic create wictestdisk --image-name=core-image-minimal""" cmd = "wic create wictestdisk --image-name=core-image-minimal -o %s" % self.resultdir runCmd(cmd) self.assertEqual(1, len(glob(os.path.join (self.resultdir, "wictestdisk-*.direct")))) - @only_for_arch(['i586', 'i686', 'x86_64']) + @skipIfNotArch(['i586', 'i686', 'x86_64']) def test_gpt_image(self): """Test creation of core-image-minimal with gpt table and UUID boot""" cmd = "wic create directdisk-gpt --image-name core-image-minimal -o %s" % self.resultdir runCmd(cmd) self.assertEqual(1, len(glob(os.path.join(self.resultdir, "directdisk-*.direct")))) - @only_for_arch(['i586', 'i686', 'x86_64']) + @skipIfNotArch(['i586', 'i686', 'x86_64']) def test_iso_image(self): """Test creation of hybrid iso image with legacy and EFI boot""" config = 'INITRAMFS_IMAGE = "core-image-minimal-initramfs"\n'\ @@ -192,21 +241,21 @@ class Wic(WicTestCase): self.assertEqual(1, len(glob(os.path.join(self.resultdir, "HYBRID_ISO_IMG-*.direct")))) self.assertEqual(1, len(glob(os.path.join (self.resultdir, "HYBRID_ISO_IMG-*.iso")))) - @only_for_arch(['i586', 'i686', 'x86_64']) + @skipIfNotArch(['i586', 'i686', 'x86_64']) def test_qemux86_directdisk(self): """Test creation of qemux-86-directdisk image""" cmd = "wic create qemux86-directdisk -e core-image-minimal -o %s" % self.resultdir runCmd(cmd) self.assertEqual(1, len(glob(os.path.join(self.resultdir, "qemux86-directdisk-*direct")))) - @only_for_arch(['i586', 'i686', 'x86_64', 'aarch64']) + @skipIfNotArch(['i586', 'i686', 'x86_64', 'aarch64']) def test_mkefidisk(self): """Test creation of mkefidisk image""" cmd = "wic create mkefidisk -e core-image-minimal -o %s" % self.resultdir runCmd(cmd) self.assertEqual(1, len(glob(os.path.join(self.resultdir, "mkefidisk-*direct")))) - @only_for_arch(['i586', 'i686', 'x86_64']) + @skipIfNotArch(['i586', 'i686', 'x86_64']) def test_bootloader_config(self): """Test creation of directdisk-bootloader-config image""" config = 'DEPENDS:pn-core-image-minimal += "syslinux"\n' @@ -217,7 +266,7 @@ class Wic(WicTestCase): runCmd(cmd) self.assertEqual(1, len(glob(os.path.join(self.resultdir, "directdisk-bootloader-config-*direct")))) - @only_for_arch(['i586', 'i686', 'x86_64', 'aarch64']) + @skipIfNotArch(['i586', 'i686', 'x86_64', 'aarch64']) def test_systemd_bootdisk(self): """Test creation of systemd-bootdisk image""" config = 'MACHINE_FEATURES:append = " efi"\n' @@ -248,7 +297,7 @@ class Wic(WicTestCase): self.assertEqual(1, len(glob(os.path.join(self.resultdir, "sdimage-bootpart-*direct")))) # TODO this doesn't have to be x86-specific - @only_for_arch(['i586', 'i686', 'x86_64']) + @skipIfNotArch(['i586', 'i686', 'x86_64']) def test_default_output_dir(self): """Test default output location""" for fname in glob("directdisk-*.direct"): @@ -261,7 +310,7 @@ class Wic(WicTestCase): runCmd(cmd) self.assertEqual(1, len(glob("directdisk-*.direct"))) - @only_for_arch(['i586', 'i686', 'x86_64']) + @skipIfNotArch(['i586', 'i686', 'x86_64']) def test_build_artifacts(self): """Test wic create directdisk providing all artifacts.""" bb_vars = get_bb_vars(['STAGING_DATADIR', 'RECIPE_SYSROOT_NATIVE'], @@ -353,7 +402,7 @@ class Wic(WicTestCase): self.assertEqual(1, len(glob(os.path.join(self.resultdir, "wictestdisk-*.direct")))) # TODO this doesn't have to be x86-specific - @only_for_arch(['i586', 'i686', 'x86_64']) + @skipIfNotArch(['i586', 'i686', 'x86_64']) def test_rootfs_indirect_recipes(self): """Test usage of rootfs plugin with rootfs recipes""" runCmd("wic create directdisk-multi-rootfs " @@ -364,7 +413,7 @@ class Wic(WicTestCase): self.assertEqual(1, len(glob(os.path.join(self.resultdir, "directdisk-multi-rootfs*.direct")))) # TODO this doesn't have to be x86-specific - @only_for_arch(['i586', 'i686', 'x86_64']) + @skipIfNotArch(['i586', 'i686', 'x86_64']) def test_rootfs_artifacts(self): """Test usage of rootfs plugin with rootfs paths""" bb_vars = get_bb_vars(['STAGING_DATADIR', 'RECIPE_SYSROOT_NATIVE'], @@ -748,7 +797,48 @@ part /etc --source rootfs --fstype=ext4 --change-directory=etc wicout = glob(os.path.join(self.resultdir, "wictestdisk-*.direct")) self.assertEqual(1, len(wicout)) size = os.path.getsize(wicout[0]) - self.assertTrue(size > extraspace) + self.assertTrue(size > extraspace, msg="Extra space not present (%s vs %s)" % (size, extraspace)) + + def test_no_table(self): + """Test --no-table wks option.""" + wks_file = 'temp.wks' + + # Absolute argument. + with open(wks_file, 'w') as wks: + wks.write("part testspace --no-table --fixed-size 16k --offset 4080k") + runCmd("wic create %s --image-name core-image-minimal -o %s" % (wks_file, self.resultdir)) + + wicout = glob(os.path.join(self.resultdir, "*.*")) + + self.assertEqual(1, len(wicout)) + size = os.path.getsize(wicout[0]) + self.assertEqual(size, 4 * 1024 * 1024) + + os.remove(wks_file) + + def test_partition_hidden_attributes(self): + """Test --hidden wks option.""" + wks_file = 'temp.wks' + sysroot = get_bb_var('RECIPE_SYSROOT_NATIVE', 'wic-tools') + try: + with open(wks_file, 'w') as wks: + wks.write(""" +part / --source rootfs --fstype=ext4 +part / --source rootfs --fstype=ext4 --hidden +bootloader --ptable gpt""") + + runCmd("wic create %s -e core-image-minimal -o %s" \ + % (wks_file, self.resultdir)) + wicout = os.path.join(self.resultdir, "*.direct") + + result = runCmd("%s/usr/sbin/sfdisk --part-attrs %s 1" % (sysroot, wicout)) + self.assertEqual('', result.output) + result = runCmd("%s/usr/sbin/sfdisk --part-attrs %s 2" % (sysroot, wicout)) + self.assertEqual('RequiredPartition', result.output) + + finally: + os.remove(wks_file) + class Wic2(WicTestCase): @@ -775,7 +865,7 @@ class Wic2(WicTestCase): basename = bb_vars['IMAGE_BASENAME'] self.assertEqual(basename, image) path = os.path.join(imgdatadir, basename) + '.env' - self.assertTrue(os.path.isfile(path)) + self.assertTrue(os.path.isfile(path), msg="File %s wasn't generated as expected" % path) wicvars = set(bb_vars['WICVARS'].split()) # filter out optional variables @@ -788,7 +878,7 @@ class Wic2(WicTestCase): # test if variables used by wic present in the .env file for var in wicvars: self.assertTrue(var in content, "%s is not in .env file" % var) - self.assertTrue(content[var]) + self.assertTrue(content[var], "%s doesn't have a value (%s)" % (var, content[var])) def test_image_vars_dir_short(self): """Test image vars directory selection -v option""" @@ -817,27 +907,29 @@ class Wic2(WicTestCase): self.resultdir)) self.assertEqual(1, len(glob(os.path.join(self.resultdir, "wictestdisk-*direct")))) - @only_for_arch(['i586', 'i686', 'x86_64', 'aarch64']) + # TODO this test could also work on aarch64 + @skipIfNotArch(['i586', 'i686', 'x86_64']) def test_wic_image_type(self): """Test building wic images by bitbake""" config = 'IMAGE_FSTYPES += "wic"\nWKS_FILE = "wic-image-minimal"\n'\ 'MACHINE_FEATURES:append = " efi"\n' self.append_config(config) - bitbake('wic-image-minimal') + image = 'wic-image-minimal' + bitbake(image) self.remove_config(config) - deploy_dir = get_bb_var('DEPLOY_DIR_IMAGE') - machine = self.td['MACHINE'] - prefix = os.path.join(deploy_dir, 'wic-image-minimal-%s.' % machine) + bb_vars = get_bb_vars(['DEPLOY_DIR_IMAGE', 'IMAGE_LINK_NAME'], image) + prefix = os.path.join(bb_vars['DEPLOY_DIR_IMAGE'], '%s.' % bb_vars['IMAGE_LINK_NAME']) + # check if we have result image and manifests symlinks # pointing to existing files for suffix in ('wic', 'manifest'): path = prefix + suffix - self.assertTrue(os.path.islink(path)) - self.assertTrue(os.path.isfile(os.path.realpath(path))) + self.assertTrue(os.path.islink(path), msg="Link %s wasn't generated as expected" % path) + self.assertTrue(os.path.isfile(os.path.realpath(path)), msg="File linked to by %s wasn't generated as expected" % path) # TODO this should work on aarch64 - @only_for_arch(['i586', 'i686', 'x86_64']) + @skipIfNotArch(['i586', 'i686', 'x86_64']) @OETestTag("runqemu") def test_qemu(self): """Test wic-image-minimal under qemu""" @@ -858,7 +950,7 @@ class Wic2(WicTestCase): self.assertEqual(1, status, 'Failed to run command "%s": %s' % (cmd, output)) self.assertEqual(output, 'UUID=2c71ef06-a81d-4735-9d3a-379b69c6bdba\t/media\text4\tdefaults\t0\t0') - @only_for_arch(['i586', 'i686', 'x86_64']) + @skipIfNotArch(['i586', 'i686', 'x86_64']) @OETestTag("runqemu") def test_qemu_efi(self): """Test core-image-minimal efi image under qemu""" @@ -1042,7 +1134,8 @@ class Wic2(WicTestCase): size = int(size[:-3]) self.assertGreaterEqual(size, 204800) - @only_for_arch(['i586', 'i686', 'x86_64', 'aarch64']) + # TODO this test could also work on aarch64 + @skipIfNotArch(['i586', 'i686', 'x86_64']) @OETestTag("runqemu") def test_rawcopy_plugin_qemu(self): """Test rawcopy plugin in qemu""" @@ -1050,9 +1143,13 @@ class Wic2(WicTestCase): config = 'IMAGE_FSTYPES = "ext4"\n' self.append_config(config) bitbake('core-image-minimal') + image_link_name = get_bb_var('IMAGE_LINK_NAME', 'core-image-minimal') self.remove_config(config) - config = 'IMAGE_FSTYPES = "wic"\nWKS_FILE = "test_rawcopy_plugin.wks.in"\n' + config = 'IMAGE_FSTYPES = "wic"\n' \ + 'IMAGE_LINK_NAME_CORE_IMAGE_MINIMAL = "%s"\n'\ + 'WKS_FILE = "test_rawcopy_plugin.wks.in"\n'\ + % image_link_name self.append_config(config) bitbake('core-image-minimal-mtdutils') self.remove_config(config) @@ -1066,14 +1163,14 @@ class Wic2(WicTestCase): def _rawcopy_plugin(self, fstype): """Test rawcopy plugin""" - img = 'core-image-minimal' - machine = self.td["MACHINE"] + image = 'core-image-minimal' + bb_vars = get_bb_vars(['DEPLOY_DIR_IMAGE', 'IMAGE_LINK_NAME'], image) params = ',unpack' if fstype.endswith('.gz') else '' with NamedTemporaryFile("w", suffix=".wks") as wks: - wks.write('part / --source rawcopy --sourceparams="file=%s-%s.%s%s"\n'\ - % (img, machine, fstype, params)) + wks.write('part / --source rawcopy --sourceparams="file=%s.%s%s"\n'\ + % (bb_vars['IMAGE_LINK_NAME'], fstype, params)) wks.flush() - cmd = "wic create %s -e %s -o %s" % (wks.name, img, self.resultdir) + cmd = "wic create %s -e %s -o %s" % (wks.name, image, self.resultdir) runCmd(cmd) wksname = os.path.splitext(os.path.basename(wks.name))[0] out = glob(os.path.join(self.resultdir, "%s-*direct" % wksname)) @@ -1094,13 +1191,12 @@ class Wic2(WicTestCase): """Test empty plugin""" config = 'IMAGE_FSTYPES = "wic"\nWKS_FILE = "test_empty_plugin.wks"\n' self.append_config(config) - bitbake('core-image-minimal') + image = 'core-image-minimal' + bitbake(image) self.remove_config(config) - deploy_dir = get_bb_var('DEPLOY_DIR_IMAGE') - machine = self.td['MACHINE'] - - image_path = os.path.join(deploy_dir, 'core-image-minimal-%s.wic' % machine) - self.assertTrue(os.path.exists(image_path)) + bb_vars = get_bb_vars(['DEPLOY_DIR_IMAGE', 'IMAGE_LINK_NAME'], image) + image_path = os.path.join(bb_vars['DEPLOY_DIR_IMAGE'], '%s.wic' % bb_vars['IMAGE_LINK_NAME']) + self.assertTrue(os.path.exists(image_path), msg="Image file %s wasn't generated as expected" % image_path) sysroot = get_bb_var('RECIPE_SYSROOT_NATIVE', 'wic-tools') @@ -1109,7 +1205,7 @@ class Wic2(WicTestCase): result = runCmd("wic ls %s -n %s | awk -F ' ' '{print $1 \" \" $5}' | grep '^2' | wc -w" % (image_path, sysroot)) self.assertEqual('1', result.output) - @only_for_arch(['i586', 'i686', 'x86_64']) + @skipIfNotArch(['i586', 'i686', 'x86_64']) @OETestTag("runqemu") def test_biosplusefi_plugin_qemu(self): """Test biosplusefi plugin in qemu""" @@ -1142,7 +1238,7 @@ class Wic2(WicTestCase): self.assertEqual(1, status, 'Failed to run command "%s": %s' % (cmd, output)) self.assertEqual(output, '*') - @only_for_arch(['i586', 'i686', 'x86_64']) + @skipIfNotArch(['i586', 'i686', 'x86_64']) def test_biosplusefi_plugin(self): """Test biosplusefi plugin""" # Wic generation below may fail depending on the order of the unittests @@ -1168,8 +1264,28 @@ class Wic2(WicTestCase): out = glob(os.path.join(self.resultdir, "%s-*.direct" % wksname)) self.assertEqual(1, len(out)) + @skipIfNotArch(['i586', 'i686', 'x86_64', 'aarch64']) + def test_uefi_kernel(self): + """ Test uefi-kernel in wic """ + config = 'IMAGE_EFI_BOOT_FILES="/etc/fstab;testfile"\nIMAGE_FSTYPES = "wic"\nWKS_FILE = "test_uefikernel.wks"\nMACHINE_FEATURES:append = " efi"\n' + self.append_config(config) + bitbake('core-image-minimal') + self.remove_config(config) + + img = 'core-image-minimal' + with NamedTemporaryFile("w", suffix=".wks") as wks: + wks.writelines(['part /boot --source bootimg-efi --sourceparams="loader=uefi-kernel"\n' + 'part / --source rootfs --fstype=ext4 --align 1024 --use-uuid\n'\ + 'bootloader --timeout=0 --append="console=ttyS0,115200n8"\n']) + wks.flush() + cmd = "wic create %s -e %s -o %s" % (wks.name, img, self.resultdir) + runCmd(cmd) + wksname = os.path.splitext(os.path.basename(wks.name))[0] + out = glob(os.path.join(self.resultdir, "%s-*.direct" % wksname)) + self.assertEqual(1, len(out)) + # TODO this test could also work on aarch64 - @only_for_arch(['i586', 'i686', 'x86_64']) + @skipIfNotArch(['i586', 'i686', 'x86_64']) @OETestTag("runqemu") def test_efi_plugin_unified_kernel_image_qemu(self): """Test efi plugin's Unified Kernel Image feature in qemu""" @@ -1287,19 +1403,19 @@ class Wic2(WicTestCase): out = glob(os.path.join(self.resultdir, "%s-*direct" % wksname)) self.assertEqual(1, len(out)) - @only_for_arch(['i586', 'i686', 'x86_64']) + @skipIfNotArch(['i586', 'i686', 'x86_64']) @OETestTag("runqemu") def test_expand_mbr_image(self): """Test wic write --expand command for mbr image""" # build an image config = 'IMAGE_FSTYPES = "wic"\nWKS_FILE = "directdisk.wks"\n' self.append_config(config) - bitbake('core-image-minimal') + image = 'core-image-minimal' + bitbake(image) # get path to the image - deploy_dir = get_bb_var('DEPLOY_DIR_IMAGE') - machine = self.td['MACHINE'] - image_path = os.path.join(deploy_dir, 'core-image-minimal-%s.wic' % machine) + bb_vars = get_bb_vars(['DEPLOY_DIR_IMAGE', 'IMAGE_LINK_NAME'], image) + image_path = os.path.join(bb_vars['DEPLOY_DIR_IMAGE'], '%s.wic' % bb_vars['IMAGE_LINK_NAME']) self.remove_config(config) @@ -1307,7 +1423,7 @@ class Wic2(WicTestCase): # expand image to 1G new_image_path = None with NamedTemporaryFile(mode='wb', suffix='.wic.exp', - dir=deploy_dir, delete=False) as sparse: + dir=bb_vars['DEPLOY_DIR_IMAGE'], delete=False) as sparse: sparse.truncate(1024 ** 3) new_image_path = sparse.name @@ -1321,11 +1437,11 @@ class Wic2(WicTestCase): orig_sizes = [int(line.split()[3]) for line in orig.output.split('\n')[1:]] exp_sizes = [int(line.split()[3]) for line in exp.output.split('\n')[1:]] self.assertEqual(orig_sizes[0], exp_sizes[0]) # first partition is not resized - self.assertTrue(orig_sizes[1] < exp_sizes[1]) + self.assertTrue(orig_sizes[1] < exp_sizes[1], msg="Parition size wasn't enlarged (%s vs %s)" % (orig_sizes[1], exp_sizes[1])) # Check if all free space is partitioned result = runCmd("%s/usr/sbin/sfdisk -F %s" % (sysroot, new_image_path)) - self.assertTrue("0 B, 0 bytes, 0 sectors" in result.output) + self.assertIn("0 B, 0 bytes, 0 sectors", result.output) os.rename(image_path, image_path + '.bak') os.rename(new_image_path, image_path) @@ -1341,6 +1457,68 @@ class Wic2(WicTestCase): if os.path.exists(image_path + '.bak'): os.rename(image_path + '.bak', image_path) + def test_gpt_partition_name(self): + """Test --part-name argument to set partition name in GPT table""" + config = 'IMAGE_FSTYPES += "wic"\nWKS_FILE = "test_gpt_partition_name.wks"\n' + self.append_config(config) + image = 'core-image-minimal' + bitbake(image) + self.remove_config(config) + deploy_dir = get_bb_var('DEPLOY_DIR_IMAGE') + bb_vars = get_bb_vars(['DEPLOY_DIR_IMAGE', 'IMAGE_LINK_NAME'], image) + image_path = os.path.join(bb_vars['DEPLOY_DIR_IMAGE'], '%s.wic' % bb_vars['IMAGE_LINK_NAME']) + + sysroot = get_bb_var('RECIPE_SYSROOT_NATIVE', 'wic-tools') + + # Image is created + self.assertTrue(os.path.exists(image_path), "image file %s doesn't exist" % image_path) + + # Check the names of the three partitions + # as listed in test_gpt_partition_name.wks + result = runCmd("%s/usr/sbin/sfdisk --part-label %s 1" % (sysroot, image_path)) + self.assertEqual('boot-A', result.output) + result = runCmd("%s/usr/sbin/sfdisk --part-label %s 2" % (sysroot, image_path)) + self.assertEqual('root-A', result.output) + # When the --part-name is not defined, the partition name is equal to the --label + result = runCmd("%s/usr/sbin/sfdisk --part-label %s 3" % (sysroot, image_path)) + self.assertEqual('ext-space', result.output) + + def test_empty_zeroize_plugin(self): + img = 'core-image-minimal' + expected_size = [ 1024*1024, # 1M + 512*1024, # 512K + 2*1024*1024] # 2M + # Check combination of sourceparams + with NamedTemporaryFile("w", suffix=".wks") as wks: + wks.writelines( + ['part empty --source empty --sourceparams="fill" --ondisk sda --fixed-size 1M\n', + 'part empty --source empty --sourceparams="size=512K" --ondisk sda --size 1M --align 1024\n', + 'part empty --source empty --sourceparams="size=2048k,bs=512K" --ondisk sda --size 4M --align 1024\n' + ]) + wks.flush() + cmd = "wic create %s -e %s -o %s" % (wks.name, img, self.resultdir) + runCmd(cmd) + wksname = os.path.splitext(os.path.basename(wks.name))[0] + wicout = glob(os.path.join(self.resultdir, "%s-*direct" % wksname)) + # Skip the complete image and just look at the single partitions + for idx, value in enumerate(wicout[1:]): + self.logger.info(wicout[idx]) + # Check if partitions are actually zeroized + with open(wicout[idx], mode="rb") as fd: + ba = bytearray(fd.read()) + for b in ba: + self.assertEqual(b, 0) + self.assertEqual(expected_size[idx], os.path.getsize(wicout[idx])) + + # Check inconsistancy check between "fill" and "--size" parameter + with NamedTemporaryFile("w", suffix=".wks") as wks: + wks.writelines(['part empty --source empty --sourceparams="fill" --ondisk sda --size 1M\n']) + wks.flush() + cmd = "wic create %s -e %s -o %s" % (wks.name, img, self.resultdir) + result = runCmd(cmd, ignore_status=True) + self.assertIn("Source parameter 'fill' only works with the '--fixed-size' option, exiting.", result.output) + self.assertNotEqual(0, result.status) + class ModifyTests(WicTestCase): def test_wic_ls(self): """Test listing image content using 'wic ls'""" @@ -1383,7 +1561,7 @@ class ModifyTests(WicTestCase): # check if file is there result = runCmd("wic ls %s:1/ -n %s" % (images[0], sysroot)) self.assertEqual(7, len(result.output.split('\n'))) - self.assertTrue(os.path.basename(testfile.name) in result.output) + self.assertIn(os.path.basename(testfile.name), result.output) # prepare directory testdir = os.path.join(self.resultdir, 'wic-test-cp-dir') @@ -1397,13 +1575,13 @@ class ModifyTests(WicTestCase): # check if directory is there result = runCmd("wic ls %s:1/ -n %s" % (images[0], sysroot)) self.assertEqual(8, len(result.output.split('\n'))) - self.assertTrue(os.path.basename(testdir) in result.output) + self.assertIn(os.path.basename(testdir), result.output) # copy the file from the partition and check if it success dest = '%s-cp' % testfile.name runCmd("wic cp %s:1/%s %s -n %s" % (images[0], os.path.basename(testfile.name), dest, sysroot)) - self.assertTrue(os.path.exists(dest)) + self.assertTrue(os.path.exists(dest), msg="File %s wasn't generated as expected" % dest) def test_wic_rm(self): @@ -1420,7 +1598,7 @@ class ModifyTests(WicTestCase): # list directory content of the first partition result = runCmd("wic ls %s:1 -n %s" % (images[0], sysroot)) - self.assertIn('\n%s ' % kerneltype.upper(), result.output) + self.assertIn('\n%s ' % kerneltype.upper(), result.output) self.assertIn('\nEFI <DIR> ', result.output) # remove file. EFI partitions are case-insensitive so exercise that too @@ -1447,7 +1625,7 @@ class ModifyTests(WicTestCase): # list directory content of the second ext4 partition result = runCmd("wic ls %s:2/ -n %s" % (images[0], sysroot)) self.assertTrue(set(['bin', 'home', 'proc', 'usr', 'var', 'dev', 'lib', 'sbin']).issubset( - set(line.split()[-1] for line in result.output.split('\n') if line))) + set(line.split()[-1] for line in result.output.split('\n') if line)), msg="Expected directories not present %s" % result.output) def test_wic_cp_ext(self): """Test copy files and directories to the ext partition.""" @@ -1462,7 +1640,7 @@ class ModifyTests(WicTestCase): # list directory content of the ext4 partition result = runCmd("wic ls %s:2/ -n %s" % (images[0], sysroot)) dirs = set(line.split()[-1] for line in result.output.split('\n') if line) - self.assertTrue(set(['bin', 'home', 'proc', 'usr', 'var', 'dev', 'lib', 'sbin']).issubset(dirs)) + self.assertTrue(set(['bin', 'home', 'proc', 'usr', 'var', 'dev', 'lib', 'sbin']).issubset(dirs), msg="Expected directories not present %s" % dirs) with NamedTemporaryFile("w", suffix=".wic-cp") as testfile: testfile.write("test") @@ -1477,12 +1655,12 @@ class ModifyTests(WicTestCase): # check if the file to copy is in the partition result = runCmd("wic ls %s:2/etc/ -n %s" % (images[0], sysroot)) - self.assertTrue('fstab' in [line.split()[-1] for line in result.output.split('\n') if line]) + self.assertIn('fstab', [line.split()[-1] for line in result.output.split('\n') if line]) # copy file from the partition, replace the temporary file content with it and # check for the file size to validate the copy runCmd("wic cp %s:2/etc/fstab %s -n %s" % (images[0], testfile.name, sysroot)) - self.assertTrue(os.stat(testfile.name).st_size > 0) + self.assertTrue(os.stat(testfile.name).st_size > 0, msg="Filesize not as expected %s" % os.stat(testfile.name).st_size) def test_wic_rm_ext(self): @@ -1497,18 +1675,18 @@ class ModifyTests(WicTestCase): # list directory content of the /etc directory on ext4 partition result = runCmd("wic ls %s:2/etc/ -n %s" % (images[0], sysroot)) - self.assertTrue('fstab' in [line.split()[-1] for line in result.output.split('\n') if line]) + self.assertIn('fstab', [line.split()[-1] for line in result.output.split('\n') if line]) # remove file runCmd("wic rm %s:2/etc/fstab -n %s" % (images[0], sysroot)) # check if it's removed result = runCmd("wic ls %s:2/etc/ -n %s" % (images[0], sysroot)) - self.assertTrue('fstab' not in [line.split()[-1] for line in result.output.split('\n') if line]) + self.assertNotIn('fstab', [line.split()[-1] for line in result.output.split('\n') if line]) # remove non-empty directory runCmd("wic rm -r %s:2/etc/ -n %s" % (images[0], sysroot)) # check if it's removed result = runCmd("wic ls %s:2/ -n %s" % (images[0], sysroot)) - self.assertTrue('etc' not in [line.split()[-1] for line in result.output.split('\n') if line]) + self.assertNotIn('etc', [line.split()[-1] for line in result.output.split('\n') if line]) diff --git a/meta/lib/oeqa/selftest/cases/wrapper.py b/meta/lib/oeqa/selftest/cases/wrapper.py index 6de63310c0..f2be44262c 100644 --- a/meta/lib/oeqa/selftest/cases/wrapper.py +++ b/meta/lib/oeqa/selftest/cases/wrapper.py @@ -1,3 +1,8 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# from oeqa.selftest.case import OESelftestTestCase from oeqa.utils.commands import bitbake diff --git a/meta/lib/oeqa/selftest/cases/yoctotestresultsquerytests.py b/meta/lib/oeqa/selftest/cases/yoctotestresultsquerytests.py new file mode 100644 index 0000000000..312edb6431 --- /dev/null +++ b/meta/lib/oeqa/selftest/cases/yoctotestresultsquerytests.py @@ -0,0 +1,39 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +import os +import sys +import subprocess +import shutil +from oeqa.selftest.case import OESelftestTestCase +from yocto_testresults_query import get_sha1, create_workdir +basepath = os.path.abspath(os.path.dirname(__file__) + '/../../../../../') +lib_path = basepath + '/scripts/lib' +sys.path = sys.path + [lib_path] + + +class TestResultsQueryTests(OESelftestTestCase): + def test_get_sha1(self): + test_data_get_sha1 = [ + {"input": "yocto-4.0", "expected": "00cfdde791a0176c134f31e5a09eff725e75b905"}, + {"input": "4.1_M1", "expected": "95066dde6861ee08fdb505ab3e0422156cc24fae"}, + ] + for data in test_data_get_sha1: + test_name = data["input"] + with self.subTest(f"Test SHA1 from {test_name}"): + self.assertEqual( + get_sha1(basepath, data["input"]), data["expected"]) + + def test_create_workdir(self): + workdir = create_workdir() + try: + url = subprocess.check_output( + ["git", "-C", workdir, "remote", "get-url", "origin"]).strip().decode("utf-8") + except: + shutil.rmtree(workdir, ignore_errors=True) + self.fail(f"Can not execute git commands in {workdir}") + shutil.rmtree(workdir) + self.assertEqual(url, "git://git.yoctoproject.org/yocto-testresults") diff --git a/meta/lib/oeqa/selftest/context.py b/meta/lib/oeqa/selftest/context.py index 78c7a467e2..99186175e5 100644 --- a/meta/lib/oeqa/selftest/context.py +++ b/meta/lib/oeqa/selftest/context.py @@ -16,19 +16,32 @@ from random import choice import oeqa import oe import bb.utils +import bb.tinfoil from oeqa.core.context import OETestContext, OETestContextExecutor from oeqa.core.exception import OEQAPreRun, OEQATestNotFound from oeqa.utils.commands import runCmd, get_bb_vars, get_test_layer +OESELFTEST_METADATA=["run_all_tests", "run_tests", "skips", "machine", "select_tags", "exclude_tags"] + +def get_oeselftest_metadata(args): + result = {} + raw_args = vars(args) + for metadata in OESELFTEST_METADATA: + if metadata in raw_args: + result[metadata] = raw_args[metadata] + + return result + class NonConcurrentTestSuite(unittest.TestSuite): - def __init__(self, suite, processes, setupfunc, removefunc): + def __init__(self, suite, processes, setupfunc, removefunc, bb_vars): super().__init__([suite]) self.processes = processes self.suite = suite self.setupfunc = setupfunc self.removefunc = removefunc + self.bb_vars = bb_vars def run(self, result): (builddir, newbuilddir) = self.setupfunc("-st", None, self.suite) @@ -57,8 +70,6 @@ class OESelftestTestContext(OETestContext): def __init__(self, td=None, logger=None, machines=None, config_paths=None, newbuilddir=None, keep_builddir=None): super(OESelftestTestContext, self).__init__(td, logger) - self.machines = machines - self.custommachine = None self.config_paths = config_paths self.newbuilddir = newbuilddir @@ -67,10 +78,15 @@ class OESelftestTestContext(OETestContext): else: self.removebuilddir = removebuilddir + def set_variables(self, vars): + self.bb_vars = vars + def setup_builddir(self, suffix, selftestdir, suite): + sstatedir = self.bb_vars['SSTATE_DIR'] + builddir = os.environ['BUILDDIR'] if not selftestdir: - selftestdir = get_test_layer() + selftestdir = get_test_layer(self.bb_vars['BBLAYERS']) if self.newbuilddir: newbuilddir = os.path.join(self.newbuilddir, 'build' + suffix) else: @@ -86,16 +102,29 @@ class OESelftestTestContext(OETestContext): oe.path.copytree(builddir + "/cache", newbuilddir + "/cache") oe.path.copytree(selftestdir, newselftestdir) + subprocess.check_output("git init && git add * && git commit -a -m 'initial'", cwd=newselftestdir, shell=True) + + # Tried to used bitbake-layers add/remove but it requires recipe parsing and hence is too slow + subprocess.check_output("sed %s/conf/bblayers.conf -i -e 's#%s#%s#g'" % (newbuilddir, selftestdir, newselftestdir), cwd=newbuilddir, shell=True) + + # Relative paths in BBLAYERS only works when the new build dir share the same ascending node + if self.newbuilddir: + bblayers = subprocess.check_output("bitbake-getvar --value BBLAYERS | tail -1", cwd=builddir, shell=True, text=True) + if '..' in bblayers: + bblayers_abspath = [os.path.abspath(path) for path in bblayers.split()] + with open("%s/conf/bblayers.conf" % newbuilddir, "a") as f: + newbblayers = "# new bblayers to be used by selftest in the new build dir '%s'\n" % newbuilddir + newbblayers += 'BBLAYERS = "%s"\n' % ' '.join(bblayers_abspath) + f.write(newbblayers) + for e in os.environ: if builddir + "/" in os.environ[e]: os.environ[e] = os.environ[e].replace(builddir + "/", newbuilddir + "/") if os.environ[e].endswith(builddir): os.environ[e] = os.environ[e].replace(builddir, newbuilddir) - subprocess.check_output("git init; git add *; git commit -a -m 'initial'", cwd=newselftestdir, shell=True) - - # Tried to used bitbake-layers add/remove but it requires recipe parsing and hence is too slow - subprocess.check_output("sed %s/conf/bblayers.conf -i -e 's#%s#%s#g'" % (newbuilddir, selftestdir, newselftestdir), cwd=newbuilddir, shell=True) + # Set SSTATE_DIR to match the parent SSTATE_DIR + subprocess.check_output("echo 'SSTATE_DIR ?= \"%s\"' >> %s/conf/local.conf" % (sstatedir, newbuilddir), cwd=newbuilddir, shell=True) os.chdir(newbuilddir) @@ -124,17 +153,11 @@ class OESelftestTestContext(OETestContext): if processes: from oeqa.core.utils.concurrencytest import ConcurrentTestSuite - return ConcurrentTestSuite(suites, processes, self.setup_builddir, self.removebuilddir) + return ConcurrentTestSuite(suites, processes, self.setup_builddir, self.removebuilddir, self.bb_vars) else: - return NonConcurrentTestSuite(suites, processes, self.setup_builddir, self.removebuilddir) + return NonConcurrentTestSuite(suites, processes, self.setup_builddir, self.removebuilddir, self.bb_vars) def runTests(self, processes=None, machine=None, skips=[]): - if machine: - self.custommachine = machine - if machine == 'random': - self.custommachine = choice(self.machines) - self.logger.info('Run tests with custom MACHINE set to: %s' % \ - self.custommachine) return super(OESelftestTestContext, self).runTests(processes, skips) def listTests(self, display_type, machine=None): @@ -154,9 +177,6 @@ class OESelftestTestContextExecutor(OETestContextExecutor): group.add_argument('-a', '--run-all-tests', default=False, action="store_true", dest="run_all_tests", help='Run all (unhidden) tests') - group.add_argument('-R', '--skip-tests', required=False, action='store', - nargs='+', dest="skips", default=None, - help='Run all (unhidden) tests except the ones specified. Format should be <module>[.<class>[.<test_method>]]') group.add_argument('-r', '--run-tests', required=False, action='store', nargs='+', dest="run_tests", default=None, help='Select what tests to run (modules, classes or test methods). Format should be: <module>.<class>.<test_method>') @@ -171,11 +191,26 @@ class OESelftestTestContextExecutor(OETestContextExecutor): action="store_true", default=False, help='List all available tests.') - parser.add_argument('-j', '--num-processes', dest='processes', action='store', - type=int, help="number of processes to execute in parallel with") + parser.add_argument('-R', '--skip-tests', required=False, action='store', + nargs='+', dest="skips", default=None, + help='Skip the tests specified. Format should be <module>[.<class>[.<test_method>]]') + + def check_parallel_support(parameter): + if not parameter.isdigit(): + import argparse + raise argparse.ArgumentTypeError("argument -j/--num-processes: invalid int value: '%s' " % str(parameter)) + + processes = int(parameter) + if processes: + try: + import testtools, subunit + except ImportError: + print("Failed to import testtools or subunit, the testcases will run serially") + processes = None + return processes - parser.add_argument('--machine', required=False, choices=['random', 'all'], - help='Run tests on different machines (random/all).') + parser.add_argument('-j', '--num-processes', dest='processes', action='store', + type=check_parallel_support, help="number of processes to execute in parallel with") parser.add_argument('-t', '--select-tag', dest="select_tags", action='append', default=None, @@ -191,20 +226,6 @@ class OESelftestTestContextExecutor(OETestContextExecutor): parser.add_argument('-v', '--verbose', action='store_true') parser.set_defaults(func=self.run) - def _get_available_machines(self): - machines = [] - - bbpath = self.tc_kwargs['init']['td']['BBPATH'].split(':') - - for path in bbpath: - found_machines = glob.glob(os.path.join(path, 'conf', 'machine', '*.conf')) - if found_machines: - for i in found_machines: - # eg: '/home/<user>/poky/meta-intel/conf/machine/intel-core2-32.conf' - machines.append(os.path.splitext(os.path.basename(i))[0]) - - return machines - def _get_cases_paths(self, bbpath): cases_paths = [] for layer in bbpath: @@ -235,11 +256,10 @@ class OESelftestTestContextExecutor(OETestContextExecutor): args.list_tests = 'name' self.tc_kwargs['init']['td'] = bbvars - self.tc_kwargs['init']['machines'] = self._get_available_machines() builddir = os.environ.get("BUILDDIR") self.tc_kwargs['init']['config_paths'] = {} - self.tc_kwargs['init']['config_paths']['testlayer_path'] = get_test_layer() + self.tc_kwargs['init']['config_paths']['testlayer_path'] = get_test_layer(bbvars["BBLAYERS"]) self.tc_kwargs['init']['config_paths']['builddir'] = builddir self.tc_kwargs['init']['config_paths']['localconf'] = os.path.join(builddir, "conf/local.conf") self.tc_kwargs['init']['config_paths']['bblayers'] = os.path.join(builddir, "conf/bblayers.conf") @@ -275,14 +295,14 @@ class OESelftestTestContextExecutor(OETestContextExecutor): os.chdir(builddir) if not "meta-selftest" in self.tc.td["BBLAYERS"]: - self.tc.logger.warning("meta-selftest layer not found in BBLAYERS, adding it") + self.tc.logger.info("meta-selftest layer not found in BBLAYERS, adding it") meta_selftestdir = os.path.join( self.tc.td["BBLAYERS_FETCH_DIR"], 'meta-selftest') if os.path.isdir(meta_selftestdir): - runCmd("bitbake-layers add-layer %s" %meta_selftestdir) + runCmd("bitbake-layers add-layer %s" % meta_selftestdir) # reload data is needed because a meta-selftest layer was add self.tc.td = get_bb_vars() - self.tc.config_paths['testlayer_path'] = get_test_layer() + self.tc.config_paths['testlayer_path'] = get_test_layer(self.tc.td["BBLAYERS"]) else: self.tc.logger.error("could not locate meta-selftest in:\n%s" % meta_selftestdir) raise OEQAPreRun @@ -320,8 +340,15 @@ class OESelftestTestContextExecutor(OETestContextExecutor): _add_layer_libs() - self.tc.logger.info("Running bitbake -e to test the configuration is valid/parsable") - runCmd("bitbake -e") + self.tc.logger.info("Checking base configuration is valid/parsable") + + with bb.tinfoil.Tinfoil(tracking=True) as tinfoil: + tinfoil.prepare(quiet=2, config_only=True) + d = tinfoil.config_data + vars = {} + vars['SSTATE_DIR'] = str(d.getVar('SSTATE_DIR')) + vars['BBLAYERS'] = str(d.getVar('BBLAYERS')) + self.tc.set_variables(vars) def get_json_result_dir(self, args): json_result_dir = os.path.join(self.tc.td["LOG_DIR"], 'oeqa') @@ -334,12 +361,14 @@ class OESelftestTestContextExecutor(OETestContextExecutor): import platform from oeqa.utils.metadata import metadata_from_bb metadata = metadata_from_bb() + oeselftest_metadata = get_oeselftest_metadata(args) configuration = {'TEST_TYPE': 'oeselftest', 'STARTTIME': args.test_start_time, 'MACHINE': self.tc.td["MACHINE"], 'HOST_DISTRO': oe.lsb.distro_identifier().replace(' ', '-'), 'HOST_NAME': metadata['hostname'], - 'LAYERS': metadata['layers']} + 'LAYERS': metadata['layers'], + 'OESELFTEST_METADATA': oeselftest_metadata} return configuration def get_result_id(self, configuration): @@ -374,37 +403,14 @@ class OESelftestTestContextExecutor(OETestContextExecutor): rc = None try: - if args.machine: - logger.info('Custom machine mode enabled. MACHINE set to %s' % - args.machine) - - if args.machine == 'all': - results = [] - for m in self.tc_kwargs['init']['machines']: - self.tc_kwargs['run']['machine'] = m - results.append(self._internal_run(logger, args)) - - # XXX: the oe-selftest script only needs to know if one - # machine run fails - for r in results: - rc = r - if not r.wasSuccessful(): - break - - else: - self.tc_kwargs['run']['machine'] = args.machine - return self._internal_run(logger, args) - - else: - self.tc_kwargs['run']['machine'] = args.machine - rc = self._internal_run(logger, args) + rc = self._internal_run(logger, args) finally: config_paths = self.tc_kwargs['init']['config_paths'] output_link = os.path.join(os.path.dirname(args.output_log), "%s-results.log" % self.name) if os.path.lexists(output_link): - os.remove(output_link) + os.unlink(output_link) os.symlink(args.output_log, output_link) return rc diff --git a/meta/lib/oeqa/targetcontrol.py b/meta/lib/oeqa/targetcontrol.py index 1fdff82889..6e8b781973 100644 --- a/meta/lib/oeqa/targetcontrol.py +++ b/meta/lib/oeqa/targetcontrol.py @@ -7,18 +7,14 @@ # This module is used by testimage.bbclass for setting up and controlling a target machine. import os -import shutil import subprocess import bb -import traceback -import sys import logging from oeqa.utils.sshcontrol import SSHControl from oeqa.utils.qemurunner import QemuRunner from oeqa.utils.qemutinyrunner import QemuTinyRunner from oeqa.utils.dump import TargetDumper from oeqa.utils.dump import MonitorDumper -from oeqa.controllers.testtargetloader import TestTargetLoader from abc import ABCMeta, abstractmethod class BaseTarget(object, metaclass=ABCMeta): @@ -42,7 +38,7 @@ class BaseTarget(object, metaclass=ABCMeta): if os.path.islink(sshloglink): os.unlink(sshloglink) os.symlink(self.sshlog, sshloglink) - self.logger.info("SSH log file: %s" % self.sshlog) + self.logger.info("SSH log file: %s" % self.sshlog) @abstractmethod def start(self, params=None, ssh=True, extra_bootparams=None): @@ -107,8 +103,6 @@ class QemuTarget(BaseTarget): self.rootfs = os.path.join(d.getVar("DEPLOY_DIR_IMAGE"), d.getVar("IMAGE_LINK_NAME") + '.' + self.image_fstype) self.kernel = os.path.join(d.getVar("DEPLOY_DIR_IMAGE"), d.getVar("KERNEL_IMAGETYPE", False) + '-' + d.getVar('MACHINE', False) + '.bin') self.qemulog = os.path.join(self.testdir, "qemu_boot_log.%s" % self.datetime) - dump_target_cmds = d.getVar("testimage_dump_target") - dump_host_cmds = d.getVar("testimage_dump_host") dump_monitor_cmds = d.getVar("testimage_dump_monitor") dump_dir = d.getVar("TESTIMAGE_DUMP_DIR") if not dump_dir: @@ -145,12 +139,10 @@ class QemuTarget(BaseTarget): boottime = int(d.getVar("TEST_QEMUBOOT_TIMEOUT")), use_kvm = use_kvm, dump_dir = dump_dir, - dump_host_cmds = d.getVar("testimage_dump_host"), logger = logger, tmpfsdir = d.getVar("RUNQEMU_TMPFS_DIR"), serial_ports = len(d.getVar("SERIAL_CONSOLES").split())) - self.target_dumper = TargetDumper(dump_target_cmds, dump_dir, self.runner) self.monitor_dumper = MonitorDumper(dump_monitor_cmds, dump_dir, self.runner) if (self.monitor_dumper): self.monitor_dumper.create_dir("qmp") @@ -163,7 +155,7 @@ class QemuTarget(BaseTarget): os.unlink(qemuloglink) os.symlink(self.qemulog, qemuloglink) - self.logger.info("rootfs file: %s" % self.rootfs) + self.logger.info("rootfs file: %s" % self.rootfs) self.logger.info("Qemu log file: %s" % self.qemulog) super(QemuTarget, self).deploy() @@ -205,7 +197,7 @@ class QemuTarget(BaseTarget): self.server_ip = self.runner.server_ip self.connection = SSHControl(ip=self.ip, logfile=self.sshlog) else: - raise RuntimError("%s - FAILED to re-start qemu - check the task log and the boot log" % self.pn) + raise RuntimeError("%s - FAILED to re-start qemu - check the task log and the boot log" % self.pn) def run_serial(self, command, timeout=60): return self.runner.run_serial(command, timeout=timeout) diff --git a/meta/lib/oeqa/utils/__init__.py b/meta/lib/oeqa/utils/__init__.py index 6d1ec4cb99..53bdcbf266 100644 --- a/meta/lib/oeqa/utils/__init__.py +++ b/meta/lib/oeqa/utils/__init__.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # # Enable other layers to have modules in the same named directory @@ -88,3 +90,10 @@ def load_test_components(logger, executor): "_executor_class defined." % (comp_name, comp_context)) return components + +def get_json_result_dir(d): + json_result_dir = os.path.join(d.getVar("LOG_DIR"), 'oeqa') + custom_json_result_dir = d.getVar("OEQA_JSON_RESULT_DIR") + if custom_json_result_dir: + json_result_dir = custom_json_result_dir + return json_result_dir
\ No newline at end of file diff --git a/meta/lib/oeqa/utils/commands.py b/meta/lib/oeqa/utils/commands.py index 024261410e..575e380017 100644 --- a/meta/lib/oeqa/utils/commands.py +++ b/meta/lib/oeqa/utils/commands.py @@ -8,11 +8,8 @@ # This module is mainly used by scripts/oe-selftest and modules under meta/oeqa/selftest # It provides a class and methods for running commands on the host in a convienent way for tests. - - import os import sys -import signal import subprocess import threading import time @@ -21,6 +18,7 @@ from oeqa.utils import CommandError from oeqa.utils import ftools import re import contextlib +import errno # Export test doesn't require bb try: import bb @@ -85,7 +83,7 @@ class Command(object): except OSError as ex: # It's not an error when the command does not consume all # of our data. subprocess.communicate() also ignores that. - if ex.errno != EPIPE: + if ex.errno != errno.EPIPE: raise # We write in a separate thread because then we can read @@ -117,7 +115,7 @@ class Command(object): else: deadline = time.time() + self.timeout for thread in self.threads: - timeout = deadline - time.time() + timeout = deadline - time.time() if timeout < 0: timeout = 0 thread.join(timeout) @@ -168,15 +166,22 @@ class Result(object): def runCmd(command, ignore_status=False, timeout=None, assert_error=True, sync=True, - native_sysroot=None, limit_exc_output=0, output_log=None, **options): + native_sysroot=None, target_sys=None, limit_exc_output=0, output_log=None, **options): result = Result() if native_sysroot: - extra_paths = "%s/sbin:%s/usr/sbin:%s/usr/bin" % \ - (native_sysroot, native_sysroot, native_sysroot) - nenv = dict(options.get('env', os.environ)) - nenv['PATH'] = extra_paths + ':' + nenv.get('PATH', '') - options['env'] = nenv + new_env = dict(options.get('env', os.environ)) + paths = new_env["PATH"].split(":") + paths = [ + os.path.join(native_sysroot, "bin"), + os.path.join(native_sysroot, "sbin"), + os.path.join(native_sysroot, "usr", "bin"), + os.path.join(native_sysroot, "usr", "sbin"), + ] + paths + if target_sys: + paths = [os.path.join(native_sysroot, "usr", "bin", target_sys)] + paths + new_env["PATH"] = ":".join(paths) + options['env'] = new_env cmd = Command(command, timeout=timeout, output_log=output_log, **options) cmd.run() @@ -280,8 +285,10 @@ def get_bb_vars(variables=None, target=None, postconfig=None): def get_bb_var(var, target=None, postconfig=None): return get_bb_vars([var], target, postconfig)[var] -def get_test_layer(): - layers = get_bb_var("BBLAYERS").split() +def get_test_layer(bblayers=None): + if bblayers is None: + bblayers = get_bb_var("BBLAYERS") + layers = bblayers.split() testlayer = None for l in layers: if '~' in l: @@ -293,6 +300,7 @@ def get_test_layer(): def create_temp_layer(templayerdir, templayername, priority=999, recipepathspec='recipes-*/*'): os.makedirs(os.path.join(templayerdir, 'conf')) + corenames = get_bb_var('LAYERSERIES_CORENAMES') with open(os.path.join(templayerdir, 'conf', 'layer.conf'), 'w') as f: f.write('BBPATH .= ":${LAYERDIR}"\n') f.write('BBFILES += "${LAYERDIR}/%s/*.bb \\' % recipepathspec) @@ -301,7 +309,7 @@ def create_temp_layer(templayerdir, templayername, priority=999, recipepathspec= f.write('BBFILE_PATTERN_%s = "^${LAYERDIR}/"\n' % templayername) f.write('BBFILE_PRIORITY_%s = "%d"\n' % (templayername, priority)) f.write('BBFILE_PATTERN_IGNORE_EMPTY_%s = "1"\n' % templayername) - f.write('LAYERSERIES_COMPAT_%s = "${LAYERSERIES_COMPAT_core}"\n' % templayername) + f.write('LAYERSERIES_COMPAT_%s = "%s"\n' % (templayername, corenames)) @contextlib.contextmanager def runqemu(pn, ssh=True, runqemuparams='', image_fstype=None, launch_cmd=None, qemuparams=None, overrides={}, discard_writes=True): diff --git a/meta/lib/oeqa/utils/decorators.py b/meta/lib/oeqa/utils/decorators.py index aabf4110cb..ea90164e5e 100644 --- a/meta/lib/oeqa/utils/decorators.py +++ b/meta/lib/oeqa/utils/decorators.py @@ -16,91 +16,6 @@ import threading import signal from functools import wraps -#get the "result" object from one of the upper frames provided that one of these upper frames is a unittest.case frame -class getResults(object): - def __init__(self): - #dynamically determine the unittest.case frame and use it to get the name of the test method - ident = threading.current_thread().ident - upperf = sys._current_frames()[ident] - while (upperf.f_globals['__name__'] != 'unittest.case'): - upperf = upperf.f_back - - def handleList(items): - ret = [] - # items is a list of tuples, (test, failure) or (_ErrorHandler(), Exception()) - for i in items: - s = i[0].id() - #Handle the _ErrorHolder objects from skipModule failures - if "setUpModule (" in s: - ret.append(s.replace("setUpModule (", "").replace(")","")) - else: - ret.append(s) - # Append also the test without the full path - testname = s.split('.')[-1] - if testname: - ret.append(testname) - return ret - self.faillist = handleList(upperf.f_locals['result'].failures) - self.errorlist = handleList(upperf.f_locals['result'].errors) - self.skiplist = handleList(upperf.f_locals['result'].skipped) - - def getFailList(self): - return self.faillist - - def getErrorList(self): - return self.errorlist - - def getSkipList(self): - return self.skiplist - -class skipIfFailure(object): - - def __init__(self,testcase): - self.testcase = testcase - - def __call__(self,f): - @wraps(f) - def wrapped_f(*args, **kwargs): - res = getResults() - if self.testcase in (res.getFailList() or res.getErrorList()): - raise unittest.SkipTest("Testcase dependency not met: %s" % self.testcase) - return f(*args, **kwargs) - wrapped_f.__name__ = f.__name__ - return wrapped_f - -class skipIfSkipped(object): - - def __init__(self,testcase): - self.testcase = testcase - - def __call__(self,f): - @wraps(f) - def wrapped_f(*args, **kwargs): - res = getResults() - if self.testcase in res.getSkipList(): - raise unittest.SkipTest("Testcase dependency not met: %s" % self.testcase) - return f(*args, **kwargs) - wrapped_f.__name__ = f.__name__ - return wrapped_f - -class skipUnlessPassed(object): - - def __init__(self,testcase): - self.testcase = testcase - - def __call__(self,f): - @wraps(f) - def wrapped_f(*args, **kwargs): - res = getResults() - if self.testcase in res.getSkipList() or \ - self.testcase in res.getFailList() or \ - self.testcase in res.getErrorList(): - raise unittest.SkipTest("Testcase dependency not met: %s" % self.testcase) - return f(*args, **kwargs) - wrapped_f.__name__ = f.__name__ - wrapped_f._depends_on = self.testcase - return wrapped_f - class testcase(object): def __init__(self, test_case): self.test_case = test_case diff --git a/meta/lib/oeqa/utils/dump.py b/meta/lib/oeqa/utils/dump.py index 95a79a571c..d4d271369f 100644 --- a/meta/lib/oeqa/utils/dump.py +++ b/meta/lib/oeqa/utils/dump.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -49,9 +51,7 @@ class BaseDumper(object): self.dump_dir = dump_dir def _construct_filename(self, command): - if isinstance(self, HostDumper): - prefix = "host" - elif isinstance(self, TargetDumper): + if isinstance(self, TargetDumper): prefix = "target" elif isinstance(self, MonitorDumper): prefix = "qmp" @@ -74,54 +74,56 @@ class BaseDumper(object): with open(fullname, 'w') as dump_file: dump_file.write(output) -class HostDumper(BaseDumper): - """ Class to get dumps from the host running the tests """ - - def __init__(self, cmds, parent_dir): - super(HostDumper, self).__init__(cmds, parent_dir) - - def dump_host(self, dump_dir=""): - if dump_dir: - self.dump_dir = dump_dir - env = os.environ.copy() - env['PATH'] = '/usr/sbin:/sbin:/usr/bin:/bin' - env['COLUMNS'] = '9999' - for cmd in self.cmds: - result = runCmd(cmd, ignore_status=True, env=env) - self._write_dump(cmd.split()[0], result.output) - class TargetDumper(BaseDumper): - """ Class to get dumps from target, it only works with QemuRunner """ + """ Class to get dumps from target, it only works with QemuRunner. + Will give up permanently after 5 errors from running commands over + serial console. This helps to end testing when target is really dead, hanging + or unresponsive. + """ def __init__(self, cmds, parent_dir, runner): super(TargetDumper, self).__init__(cmds, parent_dir) self.runner = runner + self.errors = 0 def dump_target(self, dump_dir=""): + if self.errors >= 5: + print("Too many errors when dumping data from target, assuming it is dead! Will not dump data anymore!") + return if dump_dir: self.dump_dir = dump_dir for cmd in self.cmds: # We can continue with the testing if serial commands fail try: (status, output) = self.runner.run_serial(cmd) + if status == 0: + self.errors = self.errors + 1 self._write_dump(cmd.split()[0], output) except: + self.errors = self.errors + 1 print("Tried to dump info from target but " "serial console failed") print("Failed CMD: %s" % (cmd)) class MonitorDumper(BaseDumper): - """ Class to get dumps via the Qemu Monitor, it only works with QemuRunner """ + """ Class to get dumps via the Qemu Monitor, it only works with QemuRunner + Will stop completely if there are more than 5 errors when dumping monitor data. + This helps to end testing when target is really dead, hanging or unresponsive. + """ def __init__(self, cmds, parent_dir, runner): super(MonitorDumper, self).__init__(cmds, parent_dir) self.runner = runner + self.errors = 0 def dump_monitor(self, dump_dir=""): if self.runner is None: return if dump_dir: self.dump_dir = dump_dir + if self.errors >= 5: + print("Too many errors when dumping data from qemu monitor, assuming it is dead! Will not dump data anymore!") + return for cmd in self.cmds: cmd_name = cmd.split()[0] try: @@ -135,4 +137,5 @@ class MonitorDumper(BaseDumper): output = self.runner.run_monitor(cmd_name) self._write_dump(cmd_name, output) except Exception as e: + self.errors = self.errors + 1 print("Failed to dump QMP CMD: %s with\nException: %s" % (cmd_name, e)) diff --git a/meta/lib/oeqa/utils/ftools.py b/meta/lib/oeqa/utils/ftools.py index 3093419cc7..a50aaa84c2 100644 --- a/meta/lib/oeqa/utils/ftools.py +++ b/meta/lib/oeqa/utils/ftools.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/utils/gitarchive.py b/meta/lib/oeqa/utils/gitarchive.py index 6e8040eb5c..10cb267dfa 100644 --- a/meta/lib/oeqa/utils/gitarchive.py +++ b/meta/lib/oeqa/utils/gitarchive.py @@ -100,9 +100,44 @@ def git_commit_data(repo, data_dir, branch, message, exclude, notes, log): if os.path.exists(tmp_index): os.unlink(tmp_index) +def get_tags(repo, log, pattern=None, url=None): + """ Fetch remote tags from current repository + + A pattern can be provided to filter returned tags list + An URL can be provided if local repository has no valid remote configured + """ + + base_cmd = ['ls-remote', '--refs', '--tags', '-q'] + cmd = base_cmd.copy() + + # First try to fetch tags from repository configured remote + cmd.append('origin') + if pattern: + cmd.append("refs/tags/"+pattern) + try: + tags_refs = repo.run_cmd(cmd) + tags = ["".join(d.split()[1].split('/', 2)[2:]) for d in tags_refs.splitlines()] + except GitError as e: + # If it fails, retry with repository url if one is provided + if url: + log.info("No remote repository configured, use provided url") + cmd = base_cmd.copy() + cmd.append(url) + if pattern: + cmd.append(pattern) + tags_refs = repo.run_cmd(cmd) + tags = ["".join(d.split()[1].split('/', 2)[2:]) for d in tags_refs.splitlines()] + else: + log.info("Read local tags only, some remote tags may be missed") + cmd = ["tag"] + if pattern: + cmd += ["-l", pattern] + tags = repo.run_cmd(cmd).splitlines() + + return tags def expand_tag_strings(repo, name_pattern, msg_subj_pattern, msg_body_pattern, - keywords): + url, log, keywords): """Generate tag name and message, with support for running id number""" keyws = keywords.copy() # Tag number is handled specially: if not defined, we autoincrement it @@ -116,7 +151,7 @@ def expand_tag_strings(repo, name_pattern, msg_subj_pattern, msg_body_pattern, tag_re = tag_re.format(tag_number='(?P<tag_number>[0-9]{1,5})') keyws['tag_number'] = 0 - for existing_tag in repo.run_cmd('tag').splitlines(): + for existing_tag in get_tags(repo, log, url=url): match = re.match(tag_re, existing_tag) if match and int(match.group('tag_number')) >= keyws['tag_number']: @@ -143,7 +178,8 @@ def gitarchive(data_dir, git_dir, no_create, bare, commit_msg_subject, commit_ms if not no_tag and tagname: tag_name, tag_msg = expand_tag_strings(data_repo, tagname, tag_msg_subject, - tag_msg_body, keywords) + tag_msg_body, + push, log, keywords) # Commit data commit = git_commit_data(data_repo, data_dir, branch_name, @@ -181,7 +217,7 @@ def get_test_runs(log, repo, tag_name, **kwargs): # Get a list of all matching tags tag_pattern = tag_name.format(**str_fields) - tags = repo.run_cmd(['tag', '-l', tag_pattern]).splitlines() + tags = get_tags(repo, log, pattern=tag_pattern) log.debug("Found %d tags matching pattern '%s'", len(tags), tag_pattern) # Parse undefined fields from tag names @@ -199,6 +235,8 @@ def get_test_runs(log, repo, tag_name, **kwargs): revs = [] for tag in tags: m = tag_re.match(tag) + if not m: + continue groups = m.groupdict() revs.append([groups[f] for f in undef_fields] + [tag]) @@ -219,7 +257,15 @@ def get_test_revs(log, repo, tag_name, **kwargs): if not commit in revs: revs[commit] = TestedRev(commit, commit_num, [tag]) else: - assert commit_num == revs[commit].commit_number, "Commit numbers do not match" + if commit_num != revs[commit].commit_number: + # Historically we have incorrect commit counts of '1' in the repo so fix these up + if int(revs[commit].commit_number) < 5: + tags = revs[commit].tags + revs[commit] = TestedRev(commit, commit_num, [tags]) + elif int(commit_num) < 5: + pass + else: + sys.exit("Commit numbers for commit %s don't match (%s vs %s)" % (commit, commit_num, revs[commit].commit_number)) revs[commit].tags.append(tag) # Return in sorted table diff --git a/meta/lib/oeqa/utils/httpserver.py b/meta/lib/oeqa/utils/httpserver.py index 58d3c3b3f8..80752c1377 100644 --- a/meta/lib/oeqa/utils/httpserver.py +++ b/meta/lib/oeqa/utils/httpserver.py @@ -1,11 +1,13 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # import http.server +import logging import multiprocessing import os -import traceback import signal from socketserver import ThreadingMixIn @@ -13,20 +15,24 @@ class HTTPServer(ThreadingMixIn, http.server.HTTPServer): def server_start(self, root_dir, logger): os.chdir(root_dir) + self.logger = logger self.serve_forever() class HTTPRequestHandler(http.server.SimpleHTTPRequestHandler): def log_message(self, format_str, *args): - pass + self.server.logger.info(format_str, *args) -class HTTPService(object): +class HTTPService: def __init__(self, root_dir, host='', port=0, logger=None): self.root_dir = root_dir self.host = host self.port = port - self.logger = logger + if logger: + self.logger = logger.getChild("HTTPService") + else: + self.logger = logging.getLogger("HTTPService") def start(self): if not os.path.exists(self.root_dir): @@ -38,6 +44,12 @@ class HTTPService(object): self.port = self.server.server_port self.process = multiprocessing.Process(target=self.server.server_start, args=[self.root_dir, self.logger]) + def handle_error(self, request, client_address): + import traceback + exception = traceback.format_exc() + self.logger.warn("Exception when handling %s: %s" % (request, exception)) + self.server.handle_error = handle_error + # The signal handler from testimage.bbclass can cause deadlocks here # if the HTTPServer is terminated before it can restore the standard #signal behaviour @@ -47,7 +59,7 @@ class HTTPService(object): signal.signal(signal.SIGTERM, orig) if self.logger: - self.logger.info("Started HTTPService on %s:%s" % (self.host, self.port)) + self.logger.info("Started HTTPService for %s on %s:%s" % (self.root_dir, self.host, self.port)) def stop(self): @@ -59,3 +71,10 @@ class HTTPService(object): if self.logger: self.logger.info("Stopped HTTPService on %s:%s" % (self.host, self.port)) +if __name__ == "__main__": + import sys, logging + + logger = logging.getLogger(__name__) + logging.basicConfig(level=logging.DEBUG) + httpd = HTTPService(sys.argv[1], port=8888, logger=logger) + httpd.start() diff --git a/meta/lib/oeqa/utils/logparser.py b/meta/lib/oeqa/utils/logparser.py index 879aefca33..496d9e0c90 100644 --- a/meta/lib/oeqa/utils/logparser.py +++ b/meta/lib/oeqa/utils/logparser.py @@ -1,8 +1,10 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # -import sys +import enum import os import re @@ -42,6 +44,8 @@ class PtestParser(object): result = section_regex['begin'].search(line) if result: current_section['name'] = result.group(1) + if current_section['name'] not in self.results: + self.results[current_section['name']] = {} continue result = section_regex['end'].search(line) @@ -73,9 +77,10 @@ class PtestParser(object): for t in test_regex: result = test_regex[t].search(line) if result: - if current_section['name'] not in self.results: - self.results[current_section['name']] = {} - self.results[current_section['name']][result.group(1).strip()] = t + try: + self.results[current_section['name']][result.group(1).strip()] = t + except KeyError: + bb.warn("Result with no section: %s - %s" % (t, result.group(1).strip())) # Python performance for repeatedly joining long strings is poor, do it all at once at the end. # For 2.1 million lines in a log this reduces 18 hours to 12s. @@ -101,30 +106,48 @@ class PtestParser(object): f.write(status + ": " + test_name + "\n") -# ltp log parsing -class LtpParser(object): - def __init__(self): - self.results = {} - self.section = {'duration': "", 'log': ""} - +class LtpParser: + """ + Parse the machine-readable LTP log output into a ptest-friendly data structure. + """ def parse(self, logfile): - test_regex = {} - test_regex['PASSED'] = re.compile(r"PASS") - test_regex['FAILED'] = re.compile(r"FAIL") - test_regex['SKIPPED'] = re.compile(r"SKIP") - - with open(logfile, errors='replace') as f: + results = {} + # Aaccumulate the duration here but as the log rounds quick tests down + # to 0 seconds this is very much a lower bound. The caller can replace + # the value. + section = {"duration": 0, "log": ""} + + class LtpExitCode(enum.IntEnum): + # Exit codes as defined in ltp/include/tst_res_flags.h + TPASS = 0 # Test passed flag + TFAIL = 1 # Test failed flag + TBROK = 2 # Test broken flag + TWARN = 4 # Test warning flag + TINFO = 16 # Test information flag + TCONF = 32 # Test not appropriate for configuration flag + + with open(logfile, errors="replace") as f: + # Lines look like this: + # tag=cfs_bandwidth01 stime=1689762564 dur=0 exit=exited stat=32 core=no cu=0 cs=0 for line in f: - for t in test_regex: - result = test_regex[t].search(line) - if result: - self.results[line.split()[0].strip()] = t - - for test in self.results: - result = self.results[test] - self.section['log'] = self.section['log'] + ("%s: %s\n" % (result.strip()[:-2], test.strip())) + if not line.startswith("tag="): + continue - return self.results, self.section + values = dict(s.split("=") for s in line.strip().split()) + + section["duration"] += int(values["dur"]) + exitcode = int(values["stat"]) + if values["exit"] == "exited" and exitcode == LtpExitCode.TCONF: + # Exited normally with the "invalid configuration" code + results[values["tag"]] = "SKIPPED" + elif exitcode == LtpExitCode.TPASS: + # Successful exit + results[values["tag"]] = "PASSED" + else: + # Other exit + results[values["tag"]] = "FAILED" + + return results, section # ltp Compliance log parsing diff --git a/meta/lib/oeqa/utils/metadata.py b/meta/lib/oeqa/utils/metadata.py index 8013aa684d..15ec190c4a 100644 --- a/meta/lib/oeqa/utils/metadata.py +++ b/meta/lib/oeqa/utils/metadata.py @@ -27,9 +27,9 @@ def metadata_from_bb(): data_dict = get_bb_vars() # Distro information - info_dict['distro'] = {'id': data_dict['DISTRO'], - 'version_id': data_dict['DISTRO_VERSION'], - 'pretty_name': '%s %s' % (data_dict['DISTRO'], data_dict['DISTRO_VERSION'])} + info_dict['distro'] = {'id': data_dict.get('DISTRO', 'NODISTRO'), + 'version_id': data_dict.get('DISTRO_VERSION', 'NO_DISTRO_VERSION'), + 'pretty_name': '%s %s' % (data_dict.get('DISTRO', 'NODISTRO'), data_dict.get('DISTRO_VERSION', 'NO_DISTRO_VERSION'))} # Host distro information os_release = get_os_release() diff --git a/meta/lib/oeqa/utils/network.py b/meta/lib/oeqa/utils/network.py index 59d01723a1..da4ffda9a9 100644 --- a/meta/lib/oeqa/utils/network.py +++ b/meta/lib/oeqa/utils/network.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/utils/nfs.py b/meta/lib/oeqa/utils/nfs.py index a37686c914..903469bfee 100644 --- a/meta/lib/oeqa/utils/nfs.py +++ b/meta/lib/oeqa/utils/nfs.py @@ -1,4 +1,8 @@ +# +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT +# import os import sys import tempfile @@ -8,7 +12,7 @@ from oeqa.utils.commands import bitbake, get_bb_var, Command from oeqa.utils.network import get_free_port @contextlib.contextmanager -def unfs_server(directory, logger = None): +def unfs_server(directory, logger = None, udp = True): unfs_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "unfs3-native") if not os.path.exists(os.path.join(unfs_sysroot, "usr", "bin", "unfsd")): # build native tool @@ -22,11 +26,11 @@ def unfs_server(directory, logger = None): exports.write("{0} (rw,no_root_squash,no_all_squash,insecure)\n".format(directory).encode()) # find some ports for the server - nfsport, mountport = get_free_port(udp = True), get_free_port(udp = True) + nfsport, mountport = get_free_port(udp), get_free_port(udp) nenv = dict(os.environ) nenv['PATH'] = "{0}/sbin:{0}/usr/sbin:{0}/usr/bin:".format(unfs_sysroot) + nenv.get('PATH', '') - cmd = Command(["unfsd", "-d", "-p", "-N", "-e", exports.name, "-n", str(nfsport), "-m", str(mountport)], + cmd = Command(["unfsd", "-d", "-p", "-e", exports.name, "-n", str(nfsport), "-m", str(mountport)], bg = True, env = nenv, output_log = logger) cmd.run() yield nfsport, mountport diff --git a/meta/lib/oeqa/utils/package_manager.py b/meta/lib/oeqa/utils/package_manager.py index 6b67f22fdd..db799b64d6 100644 --- a/meta/lib/oeqa/utils/package_manager.py +++ b/meta/lib/oeqa/utils/package_manager.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # diff --git a/meta/lib/oeqa/utils/postactions.py b/meta/lib/oeqa/utils/postactions.py new file mode 100644 index 0000000000..ecdddd2d40 --- /dev/null +++ b/meta/lib/oeqa/utils/postactions.py @@ -0,0 +1,98 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +# Run a set of actions after tests. The runner provides internal data +# dictionary as well as test context to any action to run. + +from oeqa.utils import get_json_result_dir + +def create_artifacts_directory(d, tc): + import shutil + + local_artifacts_dir = os.path.join(get_json_result_dir(d), "artifacts") + if os.path.isdir(local_artifacts_dir): + shutil.rmtree(local_artifacts_dir) + + os.makedirs(local_artifacts_dir) + +################################################################## +# Host/target statistics +################################################################## + +def get_target_disk_usage(d, tc): + output_file = os.path.join(get_json_result_dir(d), "artifacts", "target_disk_usage.txt") + try: + (status, output) = tc.target.run('df -h') + with open(output_file, 'w') as f: + f.write(output) + f.write("\n") + except Exception as e: + bb.warn(f"Can not get target disk usage: {e}") + +def get_host_disk_usage(d, tc): + import subprocess + + output_file = os.path.join(get_json_result_dir(d), "artifacts", "host_disk_usage.txt") + try: + with open(output_file, 'w') as f: + output = subprocess.run(['df', '-hl'], check=True, text=True, stdout=f, env={}) + except Exception as e: + bb.warn(f"Can not get host disk usage: {e}") + +################################################################## +# Artifacts retrieval +################################################################## + +def get_artifacts_list(target, raw_list): + result = [] + # Passed list may contains patterns in paths, expand them directly on target + for raw_path in raw_list.split(): + cmd = f"for p in {raw_path}; do if [ -e $p ]; then echo $p; fi; done" + try: + status, output = target.run(cmd) + if status != 0 or not output: + raise Exception() + result += output.split() + except: + bb.note(f"No file/directory matching path {raw_path}") + + return result + +def retrieve_test_artifacts(target, artifacts_list, target_dir): + local_artifacts_dir = os.path.join(target_dir, "artifacts") + for artifact_path in artifacts_list: + if not os.path.isabs(artifact_path): + bb.warn(f"{artifact_path} is not an absolute path") + continue + try: + dest_dir = os.path.join(local_artifacts_dir, os.path.dirname(artifact_path[1:])) + os.makedirs(dest_dir, exist_ok=True) + target.copyFrom(artifact_path, dest_dir) + except Exception as e: + bb.warn(f"Can not retrieve {artifact_path} from test target: {e}") + +def list_and_fetch_failed_tests_artifacts(d, tc): + artifacts_list = get_artifacts_list(tc.target, d.getVar("TESTIMAGE_FAILED_QA_ARTIFACTS")) + if not artifacts_list: + bb.warn("Could not load artifacts list, skip artifacts retrieval") + else: + retrieve_test_artifacts(tc.target, artifacts_list, get_json_result_dir(d)) + + +################################################################## +# General post actions runner +################################################################## + +def run_failed_tests_post_actions(d, tc): + post_actions=[ + create_artifacts_directory, + list_and_fetch_failed_tests_artifacts, + get_target_disk_usage, + get_host_disk_usage + ] + + for action in post_actions: + action(d, tc) diff --git a/meta/lib/oeqa/utils/qemurunner.py b/meta/lib/oeqa/utils/qemurunner.py index 76296d50cd..cda43aad8c 100644 --- a/meta/lib/oeqa/utils/qemurunner.py +++ b/meta/lib/oeqa/utils/qemurunner.py @@ -19,11 +19,11 @@ import errno import string import threading import codecs -import logging import tempfile -from oeqa.utils.dump import HostDumper from collections import defaultdict +from contextlib import contextmanager import importlib +import traceback # Get Unicode non printable control chars control_range = list(range(0,32))+list(range(127,160)) @@ -31,10 +31,19 @@ control_chars = [chr(x) for x in control_range if chr(x) not in string.printable] re_control_char = re.compile('[%s]' % re.escape("".join(control_chars))) +def getOutput(o): + import fcntl + fl = fcntl.fcntl(o, fcntl.F_GETFL) + fcntl.fcntl(o, fcntl.F_SETFL, fl | os.O_NONBLOCK) + try: + return os.read(o.fileno(), 1000000).decode("utf-8") + except BlockingIOError: + return "" + class QemuRunner: - def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, boottime, dump_dir, dump_host_cmds, - use_kvm, logger, use_slirp=False, serial_ports=2, boot_patterns = defaultdict(str), use_ovmf=False, workdir=None, tmpfsdir=None): + def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, boottime, dump_dir, use_kvm, logger, use_slirp=False, + serial_ports=2, boot_patterns = defaultdict(str), use_ovmf=False, workdir=None, tmpfsdir=None): # Popen object for runqemu self.runqemu = None @@ -57,6 +66,7 @@ class QemuRunner: self.boottime = boottime self.logged = False self.thread = None + self.threadsock = None self.use_kvm = use_kvm self.use_ovmf = use_ovmf self.use_slirp = use_slirp @@ -69,7 +79,6 @@ class QemuRunner: if not workdir: workdir = os.getcwd() self.qemu_pidfile = workdir + '/pidfile_' + str(os.getpid()) - self.host_dumper = HostDumper(dump_host_cmds, dump_dir) self.monitorpipe = None self.logger = logger @@ -85,7 +94,7 @@ class QemuRunner: accepted_patterns = ['search_reached_prompt', 'send_login_user', 'search_login_succeeded', 'search_cmd_finished'] default_boot_patterns = defaultdict(str) # Default to the usual paterns used to communicate with the target - default_boot_patterns['search_reached_prompt'] = b' login:' + default_boot_patterns['search_reached_prompt'] = ' login:' default_boot_patterns['send_login_user'] = 'root\n' default_boot_patterns['search_login_succeeded'] = r"root@[a-zA-Z0-9\-]+:~#" default_boot_patterns['search_cmd_finished'] = r"[a-zA-Z0-9]+@[a-zA-Z0-9\-]+:~#" @@ -99,6 +108,7 @@ class QemuRunner: try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.setblocking(0) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) sock.bind(("127.0.0.1",0)) sock.listen(2) port = sock.getsockname()[1] @@ -109,33 +119,24 @@ class QemuRunner: sock.close() raise - def log(self, msg): - if self.logfile: - # It is needed to sanitize the data received from qemu - # because is possible to have control characters - msg = msg.decode("utf-8", errors='ignore') - msg = re_control_char.sub('', msg) - self.msg += msg - with codecs.open(self.logfile, "a", encoding="utf-8") as f: - f.write("%s" % msg) - - def getOutput(self, o): - import fcntl - fl = fcntl.fcntl(o, fcntl.F_GETFL) - fcntl.fcntl(o, fcntl.F_SETFL, fl | os.O_NONBLOCK) - try: - return os.read(o.fileno(), 1000000).decode("utf-8") - except BlockingIOError: - return "" + def decode_qemulog(self, todecode): + # Sanitize the data received from qemu as it may contain control characters + msg = todecode.decode("utf-8", errors='backslashreplace') + msg = re_control_char.sub('', msg) + return msg + def log(self, msg, extension=""): + if self.logfile: + with codecs.open(self.logfile + extension, "ab") as f: + f.write(msg) + self.msg += self.decode_qemulog(msg) def handleSIGCHLD(self, signum, frame): if self.runqemu and self.runqemu.poll(): if self.runqemu.returncode: self.logger.error('runqemu exited with code %d' % self.runqemu.returncode) - self.logger.error('Output from runqemu:\n%s' % self.getOutput(self.runqemu.stdout)) + self.logger.error('Output from runqemu:\n%s' % getOutput(self.runqemu.stdout)) self.stop() - self._dump_host() def start(self, qemuparams = None, get_ip = True, extra_bootparams = None, runqemuparams='', launch_cmd=None, discard_writes=True): env = os.environ.copy() @@ -174,28 +175,32 @@ class QemuRunner: launch_cmd += ' slirp' if self.use_ovmf: launch_cmd += ' ovmf' - launch_cmd += ' %s %s %s' % (runqemuparams, self.machine, self.rootfs) + launch_cmd += ' %s %s' % (runqemuparams, self.machine) + if self.rootfs.endswith('.vmdk'): + self.logger.debug('Bypassing VMDK rootfs for runqemu') + else: + launch_cmd += ' %s' % (self.rootfs) return self.launch(launch_cmd, qemuparams=qemuparams, get_ip=get_ip, extra_bootparams=extra_bootparams, env=env) def launch(self, launch_cmd, get_ip = True, qemuparams = None, extra_bootparams = None, env = None): # use logfile to determine the recipe-sysroot-native path and # then add in the site-packages path components and add that - # to the python sys.path so qmp.py can be found. + # to the python sys.path so the qmp module can be found. python_path = os.path.dirname(os.path.dirname(self.logfile)) python_path += "/recipe-sysroot-native/usr/lib/qemu-python" sys.path.append(python_path) importlib.invalidate_caches() try: qmp = importlib.import_module("qmp") - except: - self.logger.error("qemurunner: qmp.py missing, please ensure it's installed") + except Exception as e: + self.logger.error("qemurunner: qmp module missing, please ensure it's installed in %s (%s)" % (python_path, str(e))) return False # Path relative to tmpdir used as cwd for qemu below to avoid unix socket path length issues qmp_file = "." + next(tempfile._get_candidate_names()) qmp_param = ' -S -qmp unix:./%s,server,wait' % (qmp_file) qmp_port = self.tmpdir + "/" + qmp_file - # Create a second socket connection for debugging use, + # Create a second socket connection for debugging use, # note this will NOT cause qemu to block waiting for the connection qmp_file2 = "." + next(tempfile._get_candidate_names()) qmp_param += ' -qmp unix:./%s,server,nowait' % (qmp_file2) @@ -231,7 +236,7 @@ class QemuRunner: self.origchldhandler = signal.getsignal(signal.SIGCHLD) signal.signal(signal.SIGCHLD, self.handleSIGCHLD) - self.logger.debug('launchcmd=%s'%(launch_cmd)) + self.logger.debug('launchcmd=%s' % (launch_cmd)) # FIXME: We pass in stdin=subprocess.PIPE here to work around stty # blocking at the end of the runqemu script when using this within @@ -268,7 +273,7 @@ class QemuRunner: os._exit(0) self.logger.debug("runqemu started, pid is %s" % self.runqemu.pid) - self.logger.debug("waiting at most %s seconds for qemu pid (%s)" % + self.logger.debug("waiting at most %d seconds for qemu pid (%s)" % (self.runqemutime, time.strftime("%D %H:%M:%S"))) endtime = time.time() + self.runqemutime while not self.is_alive() and time.time() < endtime: @@ -279,8 +284,7 @@ class QemuRunner: if self.runqemu.returncode: # No point waiting any longer self.logger.warning('runqemu exited with code %d' % self.runqemu.returncode) - self._dump_host() - self.logger.warning("Output from runqemu:\n%s" % self.getOutput(output)) + self.logger.warning("Output from runqemu:\n%s" % getOutput(output)) self.stop() return False time.sleep(0.5) @@ -292,7 +296,7 @@ class QemuRunner: self.logger.warning('runqemu exited with code %d' % self.runqemu.returncode) if not self.is_alive(): - self.logger.error("Qemu pid didn't appear in %s seconds (%s)" % + self.logger.error("Qemu pid didn't appear in %d seconds (%s)" % (self.runqemutime, time.strftime("%D %H:%M:%S"))) qemu_pid = None @@ -307,8 +311,7 @@ class QemuRunner: ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,pri,ni,command '], stdout=subprocess.PIPE).communicate()[0] processes = ps.decode("utf-8") self.logger.debug("Running processes:\n%s" % processes) - self._dump_host() - op = self.getOutput(output) + op = getOutput(output) self.stop() if op: self.logger.error("Output from runqemu:\n%s" % op) @@ -325,7 +328,8 @@ class QemuRunner: try: os.chdir(os.path.dirname(qmp_port)) try: - self.qmp = qmp.QEMUMonitorProtocol(os.path.basename(qmp_port)) + from qmp.legacy import QEMUMonitorProtocol + self.qmp = QEMUMonitorProtocol(os.path.basename(qmp_port)) except OSError as msg: self.logger.warning("Failed to initialize qemu monitor socket: %s File: %s" % (msg, msg.filename)) return False @@ -335,22 +339,24 @@ class QemuRunner: self.logger.debug("QMP Port does not exist waiting for it to be created") endtime = time.time() + self.runqemutime while not os.path.exists(qmp_port) and self.is_alive() and time.time() < endtime: - self.logger.info("QMP port does not exist yet!") - time.sleep(0.5) + self.logger.info("QMP port does not exist yet!") + time.sleep(0.5) if not os.path.exists(qmp_port) and self.is_alive(): self.logger.warning("QMP Port still does not exist but QEMU is alive") return False try: + # set timeout value for all QMP calls + self.qmp.settimeout(self.runqemutime) self.qmp.connect() connect_time = time.time() - self.logger.info("QMP connected to QEMU at %s and took %s seconds" % + self.logger.info("QMP connected to QEMU at %s and took %.2f seconds" % (time.strftime("%D %H:%M:%S"), time.time() - launch_time)) except OSError as msg: self.logger.warning("Failed to connect qemu monitor socket: %s File: %s" % (msg, msg.filename)) return False - except qmp.QMPConnectError as msg: + except qmp.legacy.QMPError as msg: self.logger.warning("Failed to communicate with qemu monitor: %s" % (msg)) return False finally: @@ -378,14 +384,14 @@ class QemuRunner: # Release the qemu process to continue running self.run_monitor('cont') - self.logger.info("QMP released QEMU at %s and took %s seconds from connect" % + self.logger.info("QMP released QEMU at %s and took %.2f seconds from connect" % (time.strftime("%D %H:%M:%S"), time.time() - connect_time)) # We are alive: qemu is running - out = self.getOutput(output) + out = getOutput(output) netconf = False # network configuration is not required by default - self.logger.debug("qemu started in %s seconds - qemu procces pid is %s (%s)" % + self.logger.debug("qemu started in %.2f seconds - qemu procces pid is %s (%s)" % (time.time() - (endtime - self.runqemutime), self.qemupid, time.strftime("%D %H:%M:%S"))) cmdline = '' @@ -397,9 +403,10 @@ class QemuRunner: cmdline = re_control_char.sub(' ', cmdline) try: if self.use_slirp: - tcp_ports = cmdline.split("hostfwd=tcp::")[1] + tcp_ports = cmdline.split("hostfwd=tcp:")[1] + ip, tcp_ports = tcp_ports.split(":")[:2] host_port = tcp_ports[:tcp_ports.find('-')] - self.ip = "localhost:%s" % host_port + self.ip = "%s:%s" % (ip, host_port) else: ips = re.findall(r"((?:[0-9]{1,3}\.){3}[0-9]{1,3})", cmdline.split("ip=")[1]) self.ip = ips[0] @@ -408,7 +415,7 @@ class QemuRunner: except (IndexError, ValueError): # Try to get network configuration from runqemu output match = re.match(r'.*Network configuration: (?:ip=)*([0-9.]+)::([0-9.]+):([0-9.]+).*', - out, re.MULTILINE|re.DOTALL) + out, re.MULTILINE | re.DOTALL) if match: self.ip, self.server_ip, self.netmask = match.groups() # network configuration is required as we couldn't get it @@ -419,16 +426,16 @@ class QemuRunner: self.logger.error("Couldn't get ip from qemu command line and runqemu output! " "Here is the qemu command line used:\n%s\n" "and output from runqemu:\n%s" % (cmdline, out)) - self._dump_host() self.stop() return False self.logger.debug("Target IP: %s" % self.ip) self.logger.debug("Server IP: %s" % self.server_ip) + self.thread = LoggingThread(self.log, self.threadsock, self.logger, self.runqemu.stdout) + self.thread.start() + if self.serial_ports >= 2: - self.thread = LoggingThread(self.log, self.threadsock, self.logger) - self.thread.start() if not self.thread.connection_established.wait(self.boottime): self.logger.error("Didn't receive a console connection from qemu. " "Here is the qemu command line used:\n%s\nand " @@ -440,7 +447,7 @@ class QemuRunner: self.logger.debug("Waiting at most %d seconds for login banner (%s)" % (self.boottime, time.strftime("%D %H:%M:%S"))) endtime = time.time() + self.boottime - socklist = [self.server_socket] + filelist = [self.server_socket] reachedlogin = False stopread = False qemusock = None @@ -448,38 +455,48 @@ class QemuRunner: data = b'' while time.time() < endtime and not stopread: try: - sread, swrite, serror = select.select(socklist, [], [], 5) + sread, swrite, serror = select.select(filelist, [], [], 5) except InterruptedError: continue - for sock in sread: - if sock is self.server_socket: + for file in sread: + if file is self.server_socket: qemusock, addr = self.server_socket.accept() - qemusock.setblocking(0) - socklist.append(qemusock) - socklist.remove(self.server_socket) + qemusock.setblocking(False) + filelist.append(qemusock) + filelist.remove(self.server_socket) self.logger.debug("Connection from %s:%s" % addr) else: - data = data + sock.recv(1024) + # try to avoid reading only a single character at a time + time.sleep(0.1) + if hasattr(file, 'read'): + read = file.read(1024) + elif hasattr(file, 'recv'): + read = file.recv(1024) + else: + self.logger.error('Invalid file type: %s\n%s' % (file)) + read = b'' + + self.logger.debug2('Partial boot log:\n%s' % (read.decode('utf-8', errors='backslashreplace'))) + data = data + read if data: bootlog += data - if self.serial_ports < 2: - # this socket has mixed console/kernel data, log it to logfile - self.log(data) - + self.log(data, extension = ".2") data = b'' - if self.boot_patterns['search_reached_prompt'] in bootlog: + + if bytes(self.boot_patterns['search_reached_prompt'], 'utf-8') in bootlog: + self.server_socket.close() self.server_socket = qemusock stopread = True reachedlogin = True - self.logger.debug("Reached login banner in %s seconds (%s)" % + self.logger.debug("Reached login banner in %.2f seconds (%s)" % (time.time() - (endtime - self.boottime), time.strftime("%D %H:%M:%S"))) else: # no need to check if reachedlogin unless we support multiple connections self.logger.debug("QEMU socket disconnected before login banner reached. (%s)" % time.strftime("%D %H:%M:%S")) - socklist.remove(sock) - sock.close() + filelist.remove(file) + file.close() stopread = True if not reachedlogin: @@ -487,21 +504,33 @@ class QemuRunner: self.logger.warning("Target didn't reach login banner in %d seconds (%s)" % (self.boottime, time.strftime("%D %H:%M:%S"))) tail = lambda l: "\n".join(l.splitlines()[-25:]) - bootlog = bootlog.decode("utf-8") - # in case bootlog is empty, use tail qemu log store at self.msg - lines = tail(bootlog if bootlog else self.msg) - self.logger.warning("Last 25 lines of text:\n%s" % lines) + bootlog = self.decode_qemulog(bootlog) + self.logger.warning("Last 25 lines of login console (%d):\n%s" % (len(bootlog), tail(bootlog))) + self.logger.warning("Last 25 lines of all logging (%d):\n%s" % (len(self.msg), tail(self.msg))) self.logger.warning("Check full boot log: %s" % self.logfile) - self._dump_host() self.stop() + data = True + while data: + try: + time.sleep(1) + data = qemusock.recv(1024) + self.log(data, extension = ".2") + self.logger.warning('Extra log data read: %s\n' % (data.decode('utf-8', errors='backslashreplace'))) + except Exception as e: + self.logger.warning('Extra log data exception %s' % repr(e)) + data = None + self.thread.serial_lock.release() return False + with self.thread.serial_lock: + self.thread.set_serialsock(self.server_socket) + # If we are not able to login the tests can continue try: (status, output) = self.run_serial(self.boot_patterns['send_login_user'], raw=True, timeout=120) if re.search(self.boot_patterns['search_login_succeeded'], output): self.logged = True - self.logger.debug("Logged as root in serial console") + self.logger.debug("Logged in as %s in serial console" % self.boot_patterns['send_login_user'].replace("\n", "")) if netconf: # configure guest networking cmd = "ifconfig eth0 %s netmask %s up\n" % (self.ip, self.netmask) @@ -512,7 +541,7 @@ class QemuRunner: self.logger.debug("Couldn't configure guest networking") else: self.logger.warning("Couldn't login into serial console" - " as root using blank password") + " as %s using blank password" % self.boot_patterns['send_login_user'].replace("\n", "")) self.logger.warning("The output:\n%s" % output) except: self.logger.warning("Serial console failed while trying to login") @@ -532,14 +561,17 @@ class QemuRunner: except OSError as e: if e.errno != errno.ESRCH: raise - endtime = time.time() + self.runqemutime - while self.runqemu.poll() is None and time.time() < endtime: - time.sleep(1) - if self.runqemu.poll() is None: + try: + outs, errs = self.runqemu.communicate(timeout=self.runqemutime) + if outs: + self.logger.info("Output from runqemu:\n%s", outs.decode("utf-8")) + if errs: + self.logger.info("Stderr from runqemu:\n%s", errs.decode("utf-8")) + except subprocess.TimeoutExpired: self.logger.debug("Sending SIGKILL to runqemu") os.killpg(os.getpgid(self.runqemu.pid), signal.SIGKILL) if not self.runqemu.stdout.closed: - self.logger.info("Output from runqemu:\n%s" % self.getOutput(self.runqemu.stdout)) + self.logger.info("Output from runqemu:\n%s" % getOutput(self.runqemu.stdout)) self.runqemu.stdin.close() self.runqemu.stdout.close() self.runqemu_exited = True @@ -598,8 +630,12 @@ class QemuRunner: # so it's possible that the file has been created but the content is empty pidfile_timeout = time.time() + 3 while time.time() < pidfile_timeout: - with open(self.qemu_pidfile, 'r') as f: - qemu_pid = f.read().strip() + try: + with open(self.qemu_pidfile, 'r') as f: + qemu_pid = f.read().strip() + except FileNotFoundError: + # Can be used to detect shutdown so the pid file can disappear + return False # file created but not yet written contents if not qemu_pid: time.sleep(0.5) @@ -612,41 +648,47 @@ class QemuRunner: def run_monitor(self, command, args=None, timeout=60): if hasattr(self, 'qmp') and self.qmp: + self.qmp.settimeout(timeout) if args is not None: - return self.qmp.cmd(command, args) + return self.qmp.cmd_raw(command, args) else: - return self.qmp.cmd(command) + return self.qmp.cmd_raw(command) def run_serial(self, command, raw=False, timeout=60): + # Returns (status, output) where status is 1 on success and 0 on error + # We assume target system have echo to get command status if not raw: command = "%s; echo $?\n" % command data = '' status = 0 - self.server_socket.sendall(command.encode('utf-8')) - start = time.time() - end = start + timeout - while True: - now = time.time() - if now >= end: - data += "<<< run_serial(): command timed out after %d seconds without output >>>\r\n\r\n" % timeout - break - try: - sread, _, _ = select.select([self.server_socket],[],[], end - now) - except InterruptedError: - continue - if sread: - answer = self.server_socket.recv(1024) - if answer: - data += answer.decode('utf-8') - # Search the prompt to stop - if re.search(self.boot_patterns['search_cmd_finished'], data): - break - else: - if self.canexit: - return (1, "") - raise Exception("No data on serial console socket, connection closed?") + with self.thread.serial_lock: + self.server_socket.sendall(command.encode('utf-8')) + start = time.time() + end = start + timeout + while True: + now = time.time() + if now >= end: + data += "<<< run_serial(): command timed out after %d seconds without output >>>\r\n\r\n" % timeout + break + try: + sread, _, _ = select.select([self.server_socket],[],[], end - now) + except InterruptedError: + continue + if sread: + # try to avoid reading single character at a time + time.sleep(0.1) + answer = self.server_socket.recv(1024) + if answer: + data += answer.decode('utf-8') + # Search the prompt to stop + if re.search(self.boot_patterns['search_cmd_finished'], data): + break + else: + if self.canexit: + return (1, "") + raise Exception("No data on serial console socket, connection closed?") if data: if raw: @@ -665,21 +707,27 @@ class QemuRunner: status = 1 return (status, str(data)) - - def _dump_host(self): - self.host_dumper.create_dir("qemu") - self.logger.warning("Qemu ended unexpectedly, dump data from host" - " is in %s" % self.host_dumper.dump_dir) - self.host_dumper.dump_host() +@contextmanager +def nonblocking_lock(lock): + locked = lock.acquire(False) + try: + yield locked + finally: + if locked: + lock.release() # This class is for reading data from a socket and passing it to logfunc # to be processed. It's completely event driven and has a straightforward # event loop. The mechanism for stopping the thread is a simple pipe which # will wake up the poll and allow for tearing everything down. class LoggingThread(threading.Thread): - def __init__(self, logfunc, sock, logger): + def __init__(self, logfunc, sock, logger, qemuoutput): self.connection_established = threading.Event() + self.serial_lock = threading.Lock() + self.serversock = sock + self.serialsock = None + self.qemuoutput = qemuoutput self.logfunc = logfunc self.logger = logger self.readsock = None @@ -691,9 +739,14 @@ class LoggingThread(threading.Thread): threading.Thread.__init__(self, target=self.threadtarget) + def set_serialsock(self, serialsock): + self.serialsock = serialsock + def threadtarget(self): try: self.eventloop() + except Exception as e: + self.logger.warning("Exception %s in logging thread" % traceback.format_exception(e)) finally: self.teardown() @@ -709,7 +762,8 @@ class LoggingThread(threading.Thread): def teardown(self): self.logger.debug("Tearing down logging thread") - self.close_socket(self.serversock) + if self.serversock: + self.close_socket(self.serversock) if self.readsock is not None: self.close_socket(self.readsock) @@ -724,27 +778,31 @@ class LoggingThread(threading.Thread): def eventloop(self): poll = select.poll() event_read_mask = self.errorevents | self.readevents - poll.register(self.serversock.fileno()) + if self.serversock: + poll.register(self.serversock.fileno()) + serial_registered = False + poll.register(self.qemuoutput.fileno()) poll.register(self.readpipe, event_read_mask) breakout = False self.running = True self.logger.debug("Starting thread event loop") while not breakout: - events = poll.poll() - for event in events: + events = poll.poll(2) + for fd, event in events: + # An error occurred, bail out - if event[1] & self.errorevents: - raise Exception(self.stringify_event(event[1])) + if event & self.errorevents: + raise Exception(self.stringify_event(event)) # Event to stop the thread - if self.readpipe == event[0]: + if self.readpipe == fd: self.logger.debug("Stop event received") breakout = True break # A connection request was received - elif self.serversock.fileno() == event[0]: + elif self.serversock and self.serversock.fileno() == fd: self.logger.debug("Connection request received") self.readsock, _ = self.serversock.accept() self.readsock.setblocking(0) @@ -755,15 +813,35 @@ class LoggingThread(threading.Thread): self.connection_established.set() # Actual data to be logged - elif self.readsock.fileno() == event[0]: - data = self.recv(1024) + elif self.readsock and self.readsock.fileno() == fd: + data = self.recv(1024, self.readsock) self.logfunc(data) + elif self.qemuoutput.fileno() == fd: + data = self.qemuoutput.read() + self.logger.debug("Data received on qemu stdout %s" % data) + self.logfunc(data, ".stdout") + elif self.serialsock and self.serialsock.fileno() == fd: + if self.serial_lock.acquire(blocking=False): + data = self.recv(1024, self.serialsock) + self.logger.debug("Data received serial thread %s" % data.decode('utf-8', 'replace')) + self.logfunc(data, ".2") + self.serial_lock.release() + else: + serial_registered = False + poll.unregister(self.serialsock.fileno()) + + if not serial_registered and self.serialsock: + with nonblocking_lock(self.serial_lock) as l: + if l: + serial_registered = True + poll.register(self.serialsock.fileno(), event_read_mask) + # Since the socket is non-blocking make sure to honor EAGAIN # and EWOULDBLOCK. - def recv(self, count): + def recv(self, count, sock): try: - data = self.readsock.recv(count) + data = sock.recv(count) except socket.error as e: if e.errno == errno.EAGAIN or e.errno == errno.EWOULDBLOCK: return b'' @@ -791,6 +869,9 @@ class LoggingThread(threading.Thread): val = 'POLLHUP' elif select.POLLNVAL == event: val = 'POLLNVAL' + else: + val = "0x%x" % (event) + return val def close_socket(self, sock): diff --git a/meta/lib/oeqa/utils/subprocesstweak.py b/meta/lib/oeqa/utils/subprocesstweak.py index b47975a4bc..3e43ed547b 100644 --- a/meta/lib/oeqa/utils/subprocesstweak.py +++ b/meta/lib/oeqa/utils/subprocesstweak.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # import subprocess diff --git a/meta/lib/patchtest/README.md b/meta/lib/patchtest/README.md new file mode 100644 index 0000000000..f66613c0c1 --- /dev/null +++ b/meta/lib/patchtest/README.md @@ -0,0 +1,20 @@ +# patchtest selftests for openembedded-core + +This directory provides a test suite and selftest script for use with the +patchtest repository: https://git.yoctoproject.org/patchtest/ + +To setup for use: + +1. Clone https://git.openembedded.org/openembedded-core (this repo) and https://git.openembedded.org/bitbake/ +2. Clone https://git.yoctoproject.org/patchtest +3. Install the necessary Python modules: in meta/lib/patchtest or the patchtest + repo, do `pip install -r requirements.txt` +4. Add patchtest to PATH: `export PATH=/path/to/patchtest/repo:$PATH` +5. Initialize the environment: `source oe-init-build-env` +6. Add meta-selftest to bblayers.conf: `bitbake-layers add-layer + /path/to/meta-selftest/` (the selftests use this layer's recipes as test + targets) +7. Finally, run the selftest script: `./meta/lib/patchtest/selftest/selftest` + +For more information on using patchtest, see the patchtest repo at +https://git.yoctoproject.org/patchtest/. diff --git a/meta/lib/patchtest/data.py b/meta/lib/patchtest/data.py new file mode 100644 index 0000000000..356259921d --- /dev/null +++ b/meta/lib/patchtest/data.py @@ -0,0 +1,86 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# patchtestdata: module used to share command line arguments between +# patchtest & test suite and a data store between test cases +# +# Copyright (C) 2016 Intel Corporation +# +# SPDX-License-Identifier: GPL-2.0-only +# +# NOTE: Strictly speaking, unit test should be isolated from outside, +# but patchtest test suites uses command line input data and +# pretest and test test cases may use the datastore defined +# on this module + +import os +import argparse +import collections +import logging + +logger=logging.getLogger('patchtest') +info=logger.info + +default_testdir = os.path.abspath(os.path.dirname(__file__) + "/tests") +default_repodir = os.path.abspath(os.path.dirname(__file__) + "/../../..") + +# Data store commonly used to share values between pre and post-merge tests +PatchTestDataStore = collections.defaultdict(str) + +class PatchTestInput(object): + """Abstract the patchtest argument parser""" + + @classmethod + def set_namespace(cls): + parser = cls.get_parser() + parser.parse_args(namespace=cls) + + @classmethod + def get_parser(cls): + parser = argparse.ArgumentParser() + + target_patch_group = parser.add_mutually_exclusive_group(required=True) + + target_patch_group.add_argument('--patch', metavar='PATCH', dest='patch_path', + help='The patch to be tested') + + target_patch_group.add_argument('--directory', metavar='DIRECTORY', dest='patch_path', + help='The directory containing patches to be tested') + + parser.add_argument('--repodir', metavar='REPO', + default=default_repodir, + help="Name of the repository where patch is merged") + + parser.add_argument('--testdir', metavar='TESTDIR', + default=default_testdir, + help="Directory where test cases are located") + + parser.add_argument('--top-level-directory', '-t', + dest='topdir', + default=None, + help="Top level directory of project (defaults to start directory)") + + parser.add_argument('--pattern', '-p', + dest='pattern', + default='test*.py', + help="Pattern to match test files") + + parser.add_argument('--base-branch', '-b', + dest='basebranch', + help="Branch name used by patchtest to branch from. By default, it uses the current one.") + + parser.add_argument('--base-commit', '-c', + dest='basecommit', + help="Commit ID used by patchtest to branch from. By default, it uses HEAD.") + + parser.add_argument('--debug', '-d', + action='store_true', + help='Enable debug output') + + parser.add_argument('--log-results', + action='store_true', + help='Enable logging to a file matching the target patch name with ".testresult" appended') + + + return parser + diff --git a/meta/lib/patchtest/patch.py b/meta/lib/patchtest/patch.py new file mode 100644 index 0000000000..baf6283873 --- /dev/null +++ b/meta/lib/patchtest/patch.py @@ -0,0 +1,62 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# patchtestpatch: PatchTestPatch class which abstracts a patch file +# +# Copyright (C) 2016 Intel Corporation +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import logging +import utils + +logger = logging.getLogger('patchtest') + +class PatchTestPatch(object): + MERGE_STATUS_INVALID = 'INVALID' + MERGE_STATUS_NOT_MERGED = 'NOTMERGED' + MERGE_STATUS_MERGED_SUCCESSFULL = 'PASS' + MERGE_STATUS_MERGED_FAIL = 'FAIL' + MERGE_STATUS = (MERGE_STATUS_INVALID, + MERGE_STATUS_NOT_MERGED, + MERGE_STATUS_MERGED_SUCCESSFULL, + MERGE_STATUS_MERGED_FAIL) + + def __init__(self, path, forcereload=False): + self._path = path + self._forcereload = forcereload + + self._contents = None + self._branch = None + self._merge_status = PatchTestPatch.MERGE_STATUS_NOT_MERGED + + @property + def contents(self): + if self._forcereload or (not self._contents): + logger.debug('Reading %s contents' % self._path) + try: + with open(self._path, newline='') as _f: + self._contents = _f.read() + except IOError: + logger.warn("Reading the mbox %s failed" % self.resource) + return self._contents + + @property + def path(self): + return self._path + + @property + def branch(self): + if not self._branch: + self._branch = utils.get_branch(self._path) + return self._branch + + def setmergestatus(self, status): + self._merge_status = status + + def getmergestatus(self): + return self._merge_status + + merge_status = property(getmergestatus, setmergestatus) + diff --git a/meta/lib/patchtest/repo.py b/meta/lib/patchtest/repo.py new file mode 100644 index 0000000000..d3788f466d --- /dev/null +++ b/meta/lib/patchtest/repo.py @@ -0,0 +1,174 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# patchtestrepo: PatchTestRepo class used mainly to control a git repo from patchtest +# +# Copyright (C) 2016 Intel Corporation +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import os +import utils +import logging +from patch import PatchTestPatch + +logger = logging.getLogger('patchtest') +info=logger.info + +class PatchTestRepo(object): + + # prefixes used for temporal branches/stashes + prefix = 'patchtest' + + def __init__(self, patch, repodir, commit=None, branch=None): + self._repodir = repodir + self._patch = PatchTestPatch(patch) + self._current_branch = self._get_current_branch() + + # targeted branch defined on the patch may be invalid, so make sure there + # is a corresponding remote branch + valid_patch_branch = None + if self._patch.branch in self.upstream_branches(): + valid_patch_branch = self._patch.branch + + # Target Branch + # Priority (top has highest priority): + # 1. branch given at cmd line + # 2. branch given at the patch + # 3. current branch + self._branch = branch or valid_patch_branch or self._current_branch + + # Target Commit + # Priority (top has highest priority): + # 1. commit given at cmd line + # 2. branch given at cmd line + # 3. branch given at the patch + # 3. current HEAD + self._commit = self._get_commitid(commit) or \ + self._get_commitid(branch) or \ + self._get_commitid(valid_patch_branch) or \ + self._get_commitid('HEAD') + + self._workingbranch = "%s_%s" % (PatchTestRepo.prefix, os.getpid()) + + # create working branch + self._exec({'cmd': ['git', 'checkout', '-b', self._workingbranch, self._commit]}) + + self._patchmerged = False + + # Check if patch can be merged using git-am + self._patchcanbemerged = True + try: + self._exec({'cmd': ['git', 'am', '--keep-cr'], 'input': self._patch.contents}) + except utils.CmdException as ce: + self._exec({'cmd': ['git', 'am', '--abort']}) + self._patchcanbemerged = False + finally: + # if patch was applied, remove it + if self._patchcanbemerged: + self._exec({'cmd':['git', 'reset', '--hard', self._commit]}) + + # for debugging purposes, print all repo parameters + logger.debug("Parameters") + logger.debug("\tRepository : %s" % self._repodir) + logger.debug("\tTarget Commit : %s" % self._commit) + logger.debug("\tTarget Branch : %s" % self._branch) + logger.debug("\tWorking branch : %s" % self._workingbranch) + logger.debug("\tPatch : %s" % self._patch) + + @property + def patch(self): + return self._patch.path + + @property + def branch(self): + return self._branch + + @property + def commit(self): + return self._commit + + @property + def ismerged(self): + return self._patchmerged + + @property + def canbemerged(self): + return self._patchcanbemerged + + def _exec(self, cmds): + _cmds = [] + if isinstance(cmds, dict): + _cmds.append(cmds) + elif isinstance(cmds, list): + _cmds = cmds + else: + raise utils.CmdException({'cmd':str(cmds)}) + + results = [] + cmdfailure = False + try: + results = utils.exec_cmds(_cmds, self._repodir) + except utils.CmdException as ce: + cmdfailure = True + raise ce + finally: + if cmdfailure: + for cmd in _cmds: + logger.debug("CMD: %s" % ' '.join(cmd['cmd'])) + else: + for result in results: + cmd, rc, stdout, stderr = ' '.join(result['cmd']), result['returncode'], result['stdout'], result['stderr'] + logger.debug("CMD: %s RCODE: %s STDOUT: %s STDERR: %s" % (cmd, rc, stdout, stderr)) + + return results + + def _get_current_branch(self, commit='HEAD'): + cmd = {'cmd':['git', 'rev-parse', '--abbrev-ref', commit]} + cb = self._exec(cmd)[0]['stdout'] + if cb == commit: + logger.warning('You may be detached so patchtest will checkout to master after execution') + cb = 'master' + return cb + + def _get_commitid(self, commit): + + if not commit: + return None + + try: + cmd = {'cmd':['git', 'rev-parse', '--short', commit]} + return self._exec(cmd)[0]['stdout'] + except utils.CmdException as ce: + # try getting the commit under any remotes + cmd = {'cmd':['git', 'remote']} + remotes = self._exec(cmd)[0]['stdout'] + for remote in remotes.splitlines(): + cmd = {'cmd':['git', 'rev-parse', '--short', '%s/%s' % (remote, commit)]} + try: + return self._exec(cmd)[0]['stdout'] + except utils.CmdException: + pass + + return None + + def upstream_branches(self): + cmd = {'cmd':['git', 'branch', '--remotes']} + remote_branches = self._exec(cmd)[0]['stdout'] + + # just get the names, without the remote name + branches = set(branch.split('/')[-1] for branch in remote_branches.splitlines()) + return branches + + def merge(self): + if self._patchcanbemerged: + self._exec({'cmd': ['git', 'am', '--keep-cr'], + 'input': self._patch.contents, + 'updateenv': {'PTRESOURCE':self._patch.path}}) + self._patchmerged = True + + def clean(self): + self._exec({'cmd':['git', 'checkout', '%s' % self._current_branch]}) + self._exec({'cmd':['git', 'branch', '-D', self._workingbranch]}) + self._patchmerged = False diff --git a/meta/lib/patchtest/requirements.txt b/meta/lib/patchtest/requirements.txt new file mode 100644 index 0000000000..ba55ff905e --- /dev/null +++ b/meta/lib/patchtest/requirements.txt @@ -0,0 +1,6 @@ +boto3 +git-pw>=2.5.0 +jinja2 +pylint +pyparsing>=3.0.9 +unidiff diff --git a/meta/lib/patchtest/selftest/files/TestMbox.test_author_valid.1.fail b/meta/lib/patchtest/selftest/files/TestMbox.test_author_valid.1.fail new file mode 100644 index 0000000000..0c40cdc1b6 --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestMbox.test_author_valid.1.fail @@ -0,0 +1,32 @@ +From 1fbb446d1849b1208012cbdae5d85d228cdbe4a6 Mon Sep 17 00:00:00 2001 +From: First Last <first.last@example.com> +Date: Tue, 29 Aug 2023 13:32:24 -0400 +Subject: [PATCH] selftest-hello: add a summary + +This patch should fail the selftests because the author address is from the +invalid "example.com". + +Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> +--- + .../recipes-test/selftest-hello/selftest-hello_1.0.bb | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +index 547587bef4..491f0a3df7 100644 +--- a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb ++++ b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +@@ -1,3 +1,4 @@ ++SUMMARY = "A cool sample" + DESCRIPTION = "Simple helloworld application -- selftest variant" + SECTION = "examples" + LICENSE = "MIT" +@@ -16,4 +17,4 @@ do_install() { + install -m 0755 helloworld ${D}${bindir} + } + +-BBCLASSEXTEND = "native nativesdk" +\ No newline at end of file ++BBCLASSEXTEND = "native nativesdk" +-- +2.41.0 + diff --git a/meta/lib/patchtest/selftest/files/TestMbox.test_author_valid.1.pass b/meta/lib/patchtest/selftest/files/TestMbox.test_author_valid.1.pass new file mode 100644 index 0000000000..cbb8ef2cef --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestMbox.test_author_valid.1.pass @@ -0,0 +1,31 @@ +From 1fbb446d1849b1208012cbdae5d85d228cdbe4a6 Mon Sep 17 00:00:00 2001 +From: First Last <first.last@address.com> +Date: Tue, 29 Aug 2023 13:32:24 -0400 +Subject: [PATCH] selftest-hello: add a summary + +This patch should pass the selftests because the author address is in a valid format. + +Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> +--- + .../recipes-test/selftest-hello/selftest-hello_1.0.bb | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +index 547587bef4..491f0a3df7 100644 +--- a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb ++++ b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +@@ -1,3 +1,4 @@ ++SUMMARY = "A cool sample" + DESCRIPTION = "Simple helloworld application -- selftest variant" + SECTION = "examples" + LICENSE = "MIT" +@@ -16,4 +17,4 @@ do_install() { + install -m 0755 helloworld ${D}${bindir} + } + +-BBCLASSEXTEND = "native nativesdk" +\ No newline at end of file ++BBCLASSEXTEND = "native nativesdk" +-- +2.41.0 + diff --git a/meta/lib/patchtest/selftest/files/TestMbox.test_author_valid.2.fail b/meta/lib/patchtest/selftest/files/TestMbox.test_author_valid.2.fail new file mode 100644 index 0000000000..3e2b81bca1 --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestMbox.test_author_valid.2.fail @@ -0,0 +1,31 @@ +From 1fbb446d1849b1208012cbdae5d85d228cdbe4a6 Mon Sep 17 00:00:00 2001 +From: Upgrade Helper <auh@auh.yoctoproject.org> +Date: Tue, 29 Aug 2023 13:32:24 -0400 +Subject: [PATCH] selftest-hello: add a summary + +This patch should fail the selftests because AUH is an invalid sender. + +Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> +--- + .../recipes-test/selftest-hello/selftest-hello_1.0.bb | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +index 547587bef4..491f0a3df7 100644 +--- a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb ++++ b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +@@ -1,3 +1,4 @@ ++SUMMARY = "A cool sample" + DESCRIPTION = "Simple helloworld application -- selftest variant" + SECTION = "examples" + LICENSE = "MIT" +@@ -16,4 +17,4 @@ do_install() { + install -m 0755 helloworld ${D}${bindir} + } + +-BBCLASSEXTEND = "native nativesdk" +\ No newline at end of file ++BBCLASSEXTEND = "native nativesdk" +-- +2.41.0 + diff --git a/meta/lib/patchtest/selftest/files/TestMbox.test_author_valid.2.pass b/meta/lib/patchtest/selftest/files/TestMbox.test_author_valid.2.pass new file mode 100644 index 0000000000..f84e1265a7 --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestMbox.test_author_valid.2.pass @@ -0,0 +1,31 @@ +From 1fbb446d1849b1208012cbdae5d85d228cdbe4a6 Mon Sep 17 00:00:00 2001 +From: First Last <averylongemailaddressthatishardtoread.from@address.com> +Date: Tue, 29 Aug 2023 13:32:24 -0400 +Subject: [PATCH] selftest-hello: add a summary + +This patch should pass the selftests because the author address is in a valid format. + +Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> +--- + .../recipes-test/selftest-hello/selftest-hello_1.0.bb | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +index 547587bef4..491f0a3df7 100644 +--- a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb ++++ b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +@@ -1,3 +1,4 @@ ++SUMMARY = "A cool sample" + DESCRIPTION = "Simple helloworld application -- selftest variant" + SECTION = "examples" + LICENSE = "MIT" +@@ -16,4 +17,4 @@ do_install() { + install -m 0755 helloworld ${D}${bindir} + } + +-BBCLASSEXTEND = "native nativesdk" +\ No newline at end of file ++BBCLASSEXTEND = "native nativesdk" +-- +2.41.0 + diff --git a/meta/lib/patchtest/selftest/files/TestMbox.test_bugzilla_entry_format.fail b/meta/lib/patchtest/selftest/files/TestMbox.test_bugzilla_entry_format.fail new file mode 100644 index 0000000000..80f409e952 --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestMbox.test_bugzilla_entry_format.fail @@ -0,0 +1,25 @@ +From fdfd605e565d874502522c4b70b786c8c5aa0bad Mon Sep 17 00:00:00 2001 +From: name@somedomain.com <email@address.com> +Date: Fri, 17 Feb 2017 16:29:21 -0600 +Subject: [PATCH] README: adds 'foo' to the header + +This test patch adds 'foo' to the header + +[YOCTO 1234] + +Signed-off-by: Daniela Plascencia <daniela.plascencia@linux.intel.com> +--- + README | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/README b/README +index 521916cd4f..cdf29dcea3 100644 +--- a/README ++++ b/README +@@ -1,3 +1,4 @@ ++**** FOO **** + OpenEmbedded-Core + ================= + +-- +2.11.0 diff --git a/meta/lib/patchtest/selftest/files/TestMbox.test_bugzilla_entry_format.pass b/meta/lib/patchtest/selftest/files/TestMbox.test_bugzilla_entry_format.pass new file mode 100644 index 0000000000..2648b03364 --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestMbox.test_bugzilla_entry_format.pass @@ -0,0 +1,25 @@ +From fdfd605e565d874502522c4b70b786c8c5aa0bad Mon Sep 17 00:00:00 2001 +From: name@somedomain.com <email@address.com> +Date: Fri, 17 Feb 2017 16:29:21 -0600 +Subject: [PATCH] README: adds 'foo' to the header + +This test patch adds 'foo' to the header + +[YOCTO #1234] + +Signed-off-by: Daniela Plascencia <daniela.plascencia@linux.intel.com> +--- + README | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/README b/README +index 521916cd4f..cdf29dcea3 100644 +--- a/README ++++ b/README +@@ -1,3 +1,4 @@ ++**** FOO **** + OpenEmbedded-Core + ================= + +-- +2.11.0 diff --git a/meta/lib/patchtest/selftest/files/TestMbox.test_commit_message_presence.fail b/meta/lib/patchtest/selftest/files/TestMbox.test_commit_message_presence.fail new file mode 100644 index 0000000000..93ca0f9119 --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestMbox.test_commit_message_presence.fail @@ -0,0 +1,22 @@ +From 0a52a62c9430c05d22cb7f46380488f2280b69bb Mon Sep 17 00:00:00 2001 +From: Trevor Gamblin <tgamblin@baylibre.com> +Date: Fri, 1 Sep 2023 08:56:14 -0400 +Subject: [PATCH] README.OE-Core.md: add foo + +Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> +--- + README.OE-Core.md | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/README.OE-Core.md b/README.OE-Core.md +index 2f2127fb03..48464252c8 100644 +--- a/README.OE-Core.md ++++ b/README.OE-Core.md +@@ -1,3 +1,4 @@ ++** FOO ** + OpenEmbedded-Core + ================= + +-- +2.41.0 + diff --git a/meta/lib/patchtest/selftest/files/TestMbox.test_commit_message_presence.pass b/meta/lib/patchtest/selftest/files/TestMbox.test_commit_message_presence.pass new file mode 100644 index 0000000000..5e3dcbd58b --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestMbox.test_commit_message_presence.pass @@ -0,0 +1,24 @@ +From 0a52a62c9430c05d22cb7f46380488f2280b69bb Mon Sep 17 00:00:00 2001 +From: Trevor Gamblin <tgamblin@baylibre.com> +Date: Fri, 1 Sep 2023 08:56:14 -0400 +Subject: [PATCH] README.OE-Core.md: add foo + +This is a commit message + +Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> +--- + README.OE-Core.md | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/README.OE-Core.md b/README.OE-Core.md +index 2f2127fb03..48464252c8 100644 +--- a/README.OE-Core.md ++++ b/README.OE-Core.md +@@ -1,3 +1,4 @@ ++** FOO ** + OpenEmbedded-Core + ================= + +-- +2.41.0 + diff --git a/meta/lib/patchtest/selftest/files/TestMbox.test_mbox_format.1.fail b/meta/lib/patchtest/selftest/files/TestMbox.test_mbox_format.1.fail new file mode 100644 index 0000000000..9cc4aab38a --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestMbox.test_mbox_format.1.fail @@ -0,0 +1,36 @@ +From d12db4cfa913b0e7a4b5bd858d3019acc53ce426 Mon Sep 17 00:00:00 2001 +From: Trevor Gamblin <tgamblin@baylibre.com> +Date: Wed, 30 Aug 2023 12:15:00 -0400 +Subject: [PATCH] selftest-hello: upgrade 1.0 -> 1.1 + +This test should fail the mbox formatting test and the merge on head +test. + +Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> +--- + .../{selftest-hello_1.0.bb => selftest-hello_1.1.bb} | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + rename meta-selftest/recipes-test/selftest-hello/{selftest-hello_1.0.bb => selftest-hello_1.1.bb} (88%) + +diff --git a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.1.bb +similarity index 88% +rename from meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +rename to meta-selftest/recipes-test/selftest-hello/selftest-hello_1.1.bb +index 547587bef4..acc388ec2c 100644 +--- a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb ++++ b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.1.bb +@@ -1,3 +1,4 @@ +%+SUMMARY = "Hello!" + DESCRIPTION = "Simple helloworld application -- selftest variant" + SECTION = "examples" + LICENSE = "MIT" +@@ -16,4 +17,4 @@ do_install() { + install -m 0755 helloworld ${D}${bindir} + } + +-BBCLASSEXTEND = "native nativesdk" +\ No newline at end of file ++BBCLASSEXTEND = "native nativesdk" +-- +2.41.0 + diff --git a/meta/lib/patchtest/selftest/files/TestMbox.test_mbox_format.2.fail b/meta/lib/patchtest/selftest/files/TestMbox.test_mbox_format.2.fail new file mode 100644 index 0000000000..eca1c60085 --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestMbox.test_mbox_format.2.fail @@ -0,0 +1,35 @@ +From d12db4cfa913b0e7a4b5bd858d3019acc53ce426 Mon Sep 17 00:00:00 2001 +From: Trevor Gamblin <tgamblin@baylibre.com> +Date: Wed, 30 Aug 2023 12:15:00 -0400 +Subject: [PATCH] selftest-hello: upgrade 1.0 -> 1.1 + +This test should fail the merge-on-head and mbox formatting tests. + +Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> +--- + .../{selftest-hello_1.0.bb => selftest-hello_1.1.bb} | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + rename meta-selftest/recipes-test/selftest-hello/{selftest-hello_1.0.bb => selftest-hello_1.1.bb} (88%) + +diff --git a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.1.bb +similarity index 88% +rename from meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +rename to meta-selftest/recipes-test/selftest-hello/selftest-hello_1.1.bb +index 547587bef4..acc388ec2c 100644 +--- a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb ++++ b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.1.bb +@@ -1,3 +1,4 @@ +%+SUMMARY = "Hello!" + DESCRIPTION = "Simple helloworld application -- selftest variant" + SECTION = "examples" + LICENSE = "MIT" +@@ -16,4 +17,4 @@ do_install() { + install -m 0755 helloworld ${D}${bindir} + } + +-BBCLASSEXTEND = "native nativesdk" +\ No newline at end of file ++BBCLASSEXTEND = "native nativesdk" +-- +2.41.0 + diff --git a/meta/lib/patchtest/selftest/files/TestMbox.test_mbox_format.pass b/meta/lib/patchtest/selftest/files/TestMbox.test_mbox_format.pass new file mode 100644 index 0000000000..33940adffc --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestMbox.test_mbox_format.pass @@ -0,0 +1,33 @@ +From d12db4cfa913b0e7a4b5bd858d3019acc53ce426 Mon Sep 17 00:00:00 2001 +From: Trevor Gamblin <tgamblin@baylibre.com> +Date: Wed, 30 Aug 2023 12:15:00 -0400 +Subject: [PATCH] selftest-hello: upgrade 1.0 -> 1.1 + +Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> +--- + .../{selftest-hello_1.0.bb => selftest-hello_1.1.bb} | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + rename meta-selftest/recipes-test/selftest-hello/{selftest-hello_1.0.bb => selftest-hello_1.1.bb} (88%) + +diff --git a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.1.bb +similarity index 88% +rename from meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +rename to meta-selftest/recipes-test/selftest-hello/selftest-hello_1.1.bb +index 547587bef4..acc388ec2c 100644 +--- a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb ++++ b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.1.bb +@@ -1,3 +1,4 @@ ++SUMMARY = "Hello!" + DESCRIPTION = "Simple helloworld application -- selftest variant" + SECTION = "examples" + LICENSE = "MIT" +@@ -16,4 +17,4 @@ do_install() { + install -m 0755 helloworld ${D}${bindir} + } + +-BBCLASSEXTEND = "native nativesdk" +\ No newline at end of file ++BBCLASSEXTEND = "native nativesdk" +-- +2.41.0 + diff --git a/meta/lib/patchtest/selftest/files/TestMbox.test_series_merge_on_head.1.skip b/meta/lib/patchtest/selftest/files/TestMbox.test_series_merge_on_head.1.skip new file mode 100644 index 0000000000..2a72457878 --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestMbox.test_series_merge_on_head.1.skip @@ -0,0 +1,35 @@ +From d12db4cfa913b0e7a4b5bd858d3019acc53ce426 Mon Sep 17 00:00:00 2001 +From: Trevor Gamblin <tgamblin@baylibre.com> +Date: Wed, 30 Aug 2023 12:15:00 -0400 +Subject: [PATCH] selftest-hello: upgrade 1.0 -> 1.1 + +This file should pass the test_series_merge_on_head test. + +Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> +--- + .../{selftest-hello_1.0.bb => selftest-hello_1.1.bb} | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + rename meta-selftest/recipes-test/selftest-hello/{selftest-hello_1.0.bb => selftest-hello_1.1.bb} (88%) + +diff --git a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.1.bb +similarity index 88% +rename from meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +rename to meta-selftest/recipes-test/selftest-hello/selftest-hello_1.1.bb +index 547587bef4..acc388ec2c 100644 +--- a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb ++++ b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.1.bb +@@ -1,3 +1,4 @@ ++SUMMARY = "Hello!" + DESCRIPTION = "Simple helloworld application -- selftest variant" + SECTION = "examples" + LICENSE = "MIT" +@@ -16,4 +17,4 @@ do_install() { + install -m 0755 helloworld ${D}${bindir} + } + +-BBCLASSEXTEND = "native nativesdk" +\ No newline at end of file ++BBCLASSEXTEND = "native nativesdk" +-- +2.41.0 + diff --git a/meta/lib/patchtest/selftest/files/TestMbox.test_series_merge_on_head.2.skip b/meta/lib/patchtest/selftest/files/TestMbox.test_series_merge_on_head.2.skip new file mode 100644 index 0000000000..49bd1f8ede --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestMbox.test_series_merge_on_head.2.skip @@ -0,0 +1,41 @@ +From 55208224f492af0ad929555ffc9b95ff1d301c5f Mon Sep 17 00:00:00 2001 +From: Trevor Gamblin <tgamblin@baylibre.com> +Date: Thu, 17 Aug 2023 15:02:38 -0400 +Subject: [PATCH] python3-dtc: upgrade 1.6.1 -> 1.7.0 + +Changelog: https://kernel.googlesource.com/pub/scm/utils/dtc/dtc/+log/039a99414e778332d8f9c04cbd3072e1dcc62798 + +Remove custom PV from the recipe since the relevant functionality is in +1.7.0: + +[tgamblin@megalith dtc]$ git tag --contains c001fc01a43e7a06447c06ea3d50bd60641322b8 +v1.7.0 + +Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> +Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org> +--- + .../python/{python3-dtc_1.6.1.bb => python3-dtc_1.7.0.bb} | 3 +-- + 1 file changed, 1 insertion(+), 2 deletions(-) + rename meta/recipes-devtools/python/{python3-dtc_1.6.1.bb => python3-dtc_1.7.0.bb} (92%) + +diff --git a/meta/recipes-devtools/python/python3-dtc_1.6.1.bb b/meta/recipes-devtools/python/python3-dtc_1.7.0.bb +similarity index 92% +rename from meta/recipes-devtools/python/python3-dtc_1.6.1.bb +rename to meta/recipes-devtools/python/python3-dtc_1.7.0.bb +index 95ab0be474..85e48d4694 100644 +--- a/meta/recipes-devtools/python/python3-dtc_1.6.1.bb ++++ b/meta/recipes-devtools/python/python3-dtc_1.7.0.bb +@@ -14,9 +14,8 @@ UPSTREAM_CHECK_GITTAGREGEX = "v(?P<pver>\d+(\.\d+)+)" + + LIC_FILES_CHKSUM = "file://pylibfdt/libfdt.i;beginline=1;endline=6;md5=afda088c974174a29108c8d80b5dce90" + +-SRCREV = "c001fc01a43e7a06447c06ea3d50bd60641322b8" ++SRCREV = "039a99414e778332d8f9c04cbd3072e1dcc62798" + +-PV = "1.6.1+git" + S = "${WORKDIR}/git" + + PYPA_WHEEL = "${S}/dist/libfdt-1.6.2*.whl" +-- +2.41.0 + diff --git a/meta/lib/patchtest/selftest/files/TestMbox.test_shortlog_format.fail b/meta/lib/patchtest/selftest/files/TestMbox.test_shortlog_format.fail new file mode 100644 index 0000000000..cdbbc61b61 --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestMbox.test_shortlog_format.fail @@ -0,0 +1,73 @@ +From 35ccee3cee96fb29514475279248078d88907231 Mon Sep 17 00:00:00 2001 +From: Trevor Gamblin <tgamblin@baylibre.com> +Date: Tue, 29 Aug 2023 14:12:27 -0400 +Subject: [PATCH] selftest-hello% fix CVE-1234-56789 + +CVE: CVE-1234-56789 + +Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> +--- + .../files/0001-Fix-CVE-1234-56789.patch | 27 +++++++++++++++++++ + .../selftest-hello/selftest-hello_1.0.bb | 6 +++-- + 2 files changed, 31 insertions(+), 2 deletions(-) + create mode 100644 meta-selftest/recipes-test/selftest-hello/files/0001-Fix-CVE-1234-56789.patch + +diff --git a/meta-selftest/recipes-test/selftest-hello/files/0001-Fix-CVE-1234-56789.patch b/meta-selftest/recipes-test/selftest-hello/files/0001-Fix-CVE-1234-56789.patch +new file mode 100644 +index 0000000000..9219b8db62 +--- /dev/null ++++ b/meta-selftest/recipes-test/selftest-hello/files/0001-Fix-CVE-1234-56789.patch +@@ -0,0 +1,27 @@ ++From b26a31186e6ee2eb1f506d5f2f9394d327a0df2f Mon Sep 17 00:00:00 2001 ++From: Trevor Gamblin <tgamblin@baylibre.com> ++Date: Tue, 29 Aug 2023 14:08:20 -0400 ++Subject: [PATCH] Fix CVE-NOT-REAL ++ ++CVE: CVE-1234-56789 ++Upstream-Status: Backport(http://example.com/example) ++ ++Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> ++--- ++ strlen.c | 1 + ++ 1 file changed, 1 insertion(+) ++ ++diff --git a/strlen.c b/strlen.c ++index 1788f38..83d7918 100644 ++--- a/strlen.c +++++ b/strlen.c ++@@ -8,6 +8,7 @@ int main() { ++ ++ printf("%d\n", str_len(string1)); ++ printf("%d\n", str_len(string2)); +++ printf("CVE FIXED!!!\n"); ++ ++ return 0; ++ } ++-- ++2.41.0 ++ +diff --git a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +index 547587bef4..76975a6729 100644 +--- a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb ++++ b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +@@ -3,7 +3,9 @@ SECTION = "examples" + LICENSE = "MIT" + LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302" + +-SRC_URI = "file://helloworld.c" ++SRC_URI = "file://helloworld.c \ ++ file://CVE-1234-56789.patch \ ++ " + + S = "${WORKDIR}" + +@@ -16,4 +18,4 @@ do_install() { + install -m 0755 helloworld ${D}${bindir} + } + +-BBCLASSEXTEND = "native nativesdk" +\ No newline at end of file ++BBCLASSEXTEND = "native nativesdk" +-- +2.41.0 + diff --git a/meta/lib/patchtest/selftest/files/TestMbox.test_shortlog_format.pass b/meta/lib/patchtest/selftest/files/TestMbox.test_shortlog_format.pass new file mode 100644 index 0000000000..ef6017037c --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestMbox.test_shortlog_format.pass @@ -0,0 +1,73 @@ +From 35ccee3cee96fb29514475279248078d88907231 Mon Sep 17 00:00:00 2001 +From: Trevor Gamblin <tgamblin@baylibre.com> +Date: Tue, 29 Aug 2023 14:12:27 -0400 +Subject: [PATCH] selftest-hello: fix CVE-1234-56789 + +CVE: CVE-1234-56789 + +Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> +--- + .../files/0001-Fix-CVE-1234-56789.patch | 27 +++++++++++++++++++ + .../selftest-hello/selftest-hello_1.0.bb | 6 +++-- + 2 files changed, 31 insertions(+), 2 deletions(-) + create mode 100644 meta-selftest/recipes-test/selftest-hello/files/0001-Fix-CVE-1234-56789.patch + +diff --git a/meta-selftest/recipes-test/selftest-hello/files/0001-Fix-CVE-1234-56789.patch b/meta-selftest/recipes-test/selftest-hello/files/0001-Fix-CVE-1234-56789.patch +new file mode 100644 +index 0000000000..9219b8db62 +--- /dev/null ++++ b/meta-selftest/recipes-test/selftest-hello/files/0001-Fix-CVE-1234-56789.patch +@@ -0,0 +1,27 @@ ++From b26a31186e6ee2eb1f506d5f2f9394d327a0df2f Mon Sep 17 00:00:00 2001 ++From: Trevor Gamblin <tgamblin@baylibre.com> ++Date: Tue, 29 Aug 2023 14:08:20 -0400 ++Subject: [PATCH] Fix CVE-NOT-REAL ++ ++CVE: CVE-1234-56789 ++Upstream-Status: Backport(http://example.com/example) ++ ++Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> ++--- ++ strlen.c | 1 + ++ 1 file changed, 1 insertion(+) ++ ++diff --git a/strlen.c b/strlen.c ++index 1788f38..83d7918 100644 ++--- a/strlen.c +++++ b/strlen.c ++@@ -8,6 +8,7 @@ int main() { ++ ++ printf("%d\n", str_len(string1)); ++ printf("%d\n", str_len(string2)); +++ printf("CVE FIXED!!!\n"); ++ ++ return 0; ++ } ++-- ++2.41.0 ++ +diff --git a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +index 547587bef4..76975a6729 100644 +--- a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb ++++ b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +@@ -3,7 +3,9 @@ SECTION = "examples" + LICENSE = "MIT" + LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302" + +-SRC_URI = "file://helloworld.c" ++SRC_URI = "file://helloworld.c \ ++ file://CVE-1234-56789.patch \ ++ " + + S = "${WORKDIR}" + +@@ -16,4 +18,4 @@ do_install() { + install -m 0755 helloworld ${D}${bindir} + } + +-BBCLASSEXTEND = "native nativesdk" +\ No newline at end of file ++BBCLASSEXTEND = "native nativesdk" +-- +2.41.0 + diff --git a/meta/lib/patchtest/selftest/files/TestMbox.test_shortlog_length.fail b/meta/lib/patchtest/selftest/files/TestMbox.test_shortlog_length.fail new file mode 100644 index 0000000000..629e78540b --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestMbox.test_shortlog_length.fail @@ -0,0 +1,73 @@ +From 35ccee3cee96fb29514475279248078d88907231 Mon Sep 17 00:00:00 2001 +From: Trevor Gamblin <tgamblin@baylibre.com> +Date: Tue, 29 Aug 2023 14:12:27 -0400 +Subject: [PATCH] selftest-hello: this is a very long commit shortlog with way too many words included in it to pass the test + +CVE: CVE-1234-56789 + +Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> +--- + .../files/0001-Fix-CVE-1234-56789.patch | 27 +++++++++++++++++++ + .../selftest-hello/selftest-hello_1.0.bb | 6 +++-- + 2 files changed, 31 insertions(+), 2 deletions(-) + create mode 100644 meta-selftest/recipes-test/selftest-hello/files/0001-Fix-CVE-1234-56789.patch + +diff --git a/meta-selftest/recipes-test/selftest-hello/files/0001-Fix-CVE-1234-56789.patch b/meta-selftest/recipes-test/selftest-hello/files/0001-Fix-CVE-1234-56789.patch +new file mode 100644 +index 0000000000..9219b8db62 +--- /dev/null ++++ b/meta-selftest/recipes-test/selftest-hello/files/0001-Fix-CVE-1234-56789.patch +@@ -0,0 +1,27 @@ ++From b26a31186e6ee2eb1f506d5f2f9394d327a0df2f Mon Sep 17 00:00:00 2001 ++From: Trevor Gamblin <tgamblin@baylibre.com> ++Date: Tue, 29 Aug 2023 14:08:20 -0400 ++Subject: [PATCH] Fix CVE-NOT-REAL ++ ++CVE: CVE-1234-56789 ++Upstream-Status: Backport(http://example.com/example) ++ ++Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> ++--- ++ strlen.c | 1 + ++ 1 file changed, 1 insertion(+) ++ ++diff --git a/strlen.c b/strlen.c ++index 1788f38..83d7918 100644 ++--- a/strlen.c +++++ b/strlen.c ++@@ -8,6 +8,7 @@ int main() { ++ ++ printf("%d\n", str_len(string1)); ++ printf("%d\n", str_len(string2)); +++ printf("CVE FIXED!!!\n"); ++ ++ return 0; ++ } ++-- ++2.41.0 ++ +diff --git a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +index 547587bef4..76975a6729 100644 +--- a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb ++++ b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +@@ -3,7 +3,9 @@ SECTION = "examples" + LICENSE = "MIT" + LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302" + +-SRC_URI = "file://helloworld.c" ++SRC_URI = "file://helloworld.c \ ++ file://0001-Fix-CVE-1234-56789.patch \ ++ " + + S = "${WORKDIR}" + +@@ -16,4 +18,4 @@ do_install() { + install -m 0755 helloworld ${D}${bindir} + } + +-BBCLASSEXTEND = "native nativesdk" +\ No newline at end of file ++BBCLASSEXTEND = "native nativesdk" +-- +2.41.0 + diff --git a/meta/lib/patchtest/selftest/files/TestMbox.test_shortlog_length.pass b/meta/lib/patchtest/selftest/files/TestMbox.test_shortlog_length.pass new file mode 100644 index 0000000000..ef6017037c --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestMbox.test_shortlog_length.pass @@ -0,0 +1,73 @@ +From 35ccee3cee96fb29514475279248078d88907231 Mon Sep 17 00:00:00 2001 +From: Trevor Gamblin <tgamblin@baylibre.com> +Date: Tue, 29 Aug 2023 14:12:27 -0400 +Subject: [PATCH] selftest-hello: fix CVE-1234-56789 + +CVE: CVE-1234-56789 + +Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> +--- + .../files/0001-Fix-CVE-1234-56789.patch | 27 +++++++++++++++++++ + .../selftest-hello/selftest-hello_1.0.bb | 6 +++-- + 2 files changed, 31 insertions(+), 2 deletions(-) + create mode 100644 meta-selftest/recipes-test/selftest-hello/files/0001-Fix-CVE-1234-56789.patch + +diff --git a/meta-selftest/recipes-test/selftest-hello/files/0001-Fix-CVE-1234-56789.patch b/meta-selftest/recipes-test/selftest-hello/files/0001-Fix-CVE-1234-56789.patch +new file mode 100644 +index 0000000000..9219b8db62 +--- /dev/null ++++ b/meta-selftest/recipes-test/selftest-hello/files/0001-Fix-CVE-1234-56789.patch +@@ -0,0 +1,27 @@ ++From b26a31186e6ee2eb1f506d5f2f9394d327a0df2f Mon Sep 17 00:00:00 2001 ++From: Trevor Gamblin <tgamblin@baylibre.com> ++Date: Tue, 29 Aug 2023 14:08:20 -0400 ++Subject: [PATCH] Fix CVE-NOT-REAL ++ ++CVE: CVE-1234-56789 ++Upstream-Status: Backport(http://example.com/example) ++ ++Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> ++--- ++ strlen.c | 1 + ++ 1 file changed, 1 insertion(+) ++ ++diff --git a/strlen.c b/strlen.c ++index 1788f38..83d7918 100644 ++--- a/strlen.c +++++ b/strlen.c ++@@ -8,6 +8,7 @@ int main() { ++ ++ printf("%d\n", str_len(string1)); ++ printf("%d\n", str_len(string2)); +++ printf("CVE FIXED!!!\n"); ++ ++ return 0; ++ } ++-- ++2.41.0 ++ +diff --git a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +index 547587bef4..76975a6729 100644 +--- a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb ++++ b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +@@ -3,7 +3,9 @@ SECTION = "examples" + LICENSE = "MIT" + LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302" + +-SRC_URI = "file://helloworld.c" ++SRC_URI = "file://helloworld.c \ ++ file://CVE-1234-56789.patch \ ++ " + + S = "${WORKDIR}" + +@@ -16,4 +18,4 @@ do_install() { + install -m 0755 helloworld ${D}${bindir} + } + +-BBCLASSEXTEND = "native nativesdk" +\ No newline at end of file ++BBCLASSEXTEND = "native nativesdk" +-- +2.41.0 + diff --git a/meta/lib/patchtest/selftest/files/TestMbox.test_signed_off_by_presence.1.fail b/meta/lib/patchtest/selftest/files/TestMbox.test_signed_off_by_presence.1.fail new file mode 100644 index 0000000000..35d92aeed7 --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestMbox.test_signed_off_by_presence.1.fail @@ -0,0 +1,71 @@ +From 14d72f6973270f78455a8628143f2cff90e8f41e Mon Sep 17 00:00:00 2001 +From: Trevor Gamblin <tgamblin@baylibre.com> +Date: Tue, 29 Aug 2023 14:12:27 -0400 +Subject: [PATCH] selftest-hello: fix CVE-1234-56789 + +CVE: CVE-1234-56789 + +--- + .../selftest-hello/files/CVE-1234-56789.patch | 27 +++++++++++++++++++ + .../selftest-hello/selftest-hello_1.0.bb | 6 +++-- + 2 files changed, 31 insertions(+), 2 deletions(-) + create mode 100644 meta-selftest/recipes-test/selftest-hello/files/CVE-1234-56789.patch + +diff --git a/meta-selftest/recipes-test/selftest-hello/files/CVE-1234-56789.patch b/meta-selftest/recipes-test/selftest-hello/files/CVE-1234-56789.patch +new file mode 100644 +index 0000000000..869cfb6fe5 +--- /dev/null ++++ b/meta-selftest/recipes-test/selftest-hello/files/CVE-1234-56789.patch +@@ -0,0 +1,27 @@ ++From b26a31186e6ee2eb1f506d5f2f9394d327a0df2f Mon Sep 17 00:00:00 2001 ++From: Trevor Gamblin <tgamblin@baylibre.com> ++Date: Tue, 29 Aug 2023 14:08:20 -0400 ++Subject: [PATCH] Fix CVE-NOT-REAL ++ ++CVE: CVE-1234-56789 ++Upstream-Status: Backport(http://example.com/example) ++ ++Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> ++--- ++ strlen.c | 1 + ++ 1 file changed, 1 insertion(+) ++ ++diff --git a/strlen.c b/strlen.c ++index 1788f38..83d7918 100644 ++--- a/strlen.c +++++ b/strlen.c ++@@ -8,6 +8,7 @@ int main() { ++ ++ printf("%d\n", str_len(string1)); ++ printf("%d\n", str_len(string2)); +++ printf("CVE FIXED!!!\n"); ++ ++ return 0; ++ } ++-- ++2.41.0 +diff --git a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +index 547587bef4..76975a6729 100644 +--- a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb ++++ b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +@@ -3,7 +3,9 @@ SECTION = "examples" + LICENSE = "MIT" + LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302" + +-SRC_URI = "file://helloworld.c" ++SRC_URI = "file://helloworld.c \ ++ file://CVE-1234-56789.patch \ ++ " + + S = "${WORKDIR}" + +@@ -16,4 +18,4 @@ do_install() { + install -m 0755 helloworld ${D}${bindir} + } + +-BBCLASSEXTEND = "native nativesdk" +\ No newline at end of file ++BBCLASSEXTEND = "native nativesdk" +-- +2.41.0 + diff --git a/meta/lib/patchtest/selftest/files/TestMbox.test_signed_off_by_presence.2.fail b/meta/lib/patchtest/selftest/files/TestMbox.test_signed_off_by_presence.2.fail new file mode 100644 index 0000000000..68f38dee06 --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestMbox.test_signed_off_by_presence.2.fail @@ -0,0 +1,72 @@ +From 14d72f6973270f78455a8628143f2cff90e8f41e Mon Sep 17 00:00:00 2001 +From: Trevor Gamblin <tgamblin@baylibre.com> +Date: Tue, 29 Aug 2023 14:12:27 -0400 +Subject: [PATCH] selftest-hello: fix CVE-1234-56789 + +CVE: CVE-1234-56789 + +Approved: Trevor Gamblin <tgamblin@baylibre.com> +--- + .../selftest-hello/files/CVE-1234-56789.patch | 27 +++++++++++++++++++ + .../selftest-hello/selftest-hello_1.0.bb | 6 +++-- + 2 files changed, 31 insertions(+), 2 deletions(-) + create mode 100644 meta-selftest/recipes-test/selftest-hello/files/CVE-1234-56789.patch + +diff --git a/meta-selftest/recipes-test/selftest-hello/files/CVE-1234-56789.patch b/meta-selftest/recipes-test/selftest-hello/files/CVE-1234-56789.patch +new file mode 100644 +index 0000000000..869cfb6fe5 +--- /dev/null ++++ b/meta-selftest/recipes-test/selftest-hello/files/CVE-1234-56789.patch +@@ -0,0 +1,27 @@ ++From b26a31186e6ee2eb1f506d5f2f9394d327a0df2f Mon Sep 17 00:00:00 2001 ++From: Trevor Gamblin <tgamblin@baylibre.com> ++Date: Tue, 29 Aug 2023 14:08:20 -0400 ++Subject: [PATCH] Fix CVE-NOT-REAL ++ ++CVE: CVE-1234-56789 ++Upstream-Status: Backport(http://example.com/example) ++ ++Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> ++--- ++ strlen.c | 1 + ++ 1 file changed, 1 insertion(+) ++ ++diff --git a/strlen.c b/strlen.c ++index 1788f38..83d7918 100644 ++--- a/strlen.c +++++ b/strlen.c ++@@ -8,6 +8,7 @@ int main() { ++ ++ printf("%d\n", str_len(string1)); ++ printf("%d\n", str_len(string2)); +++ printf("CVE FIXED!!!\n"); ++ ++ return 0; ++ } ++-- ++2.41.0 +diff --git a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +index 547587bef4..76975a6729 100644 +--- a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb ++++ b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +@@ -3,7 +3,9 @@ SECTION = "examples" + LICENSE = "MIT" + LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302" + +-SRC_URI = "file://helloworld.c" ++SRC_URI = "file://helloworld.c \ ++ file://CVE-1234-56789.patch \ ++ " + + S = "${WORKDIR}" + +@@ -16,4 +18,4 @@ do_install() { + install -m 0755 helloworld ${D}${bindir} + } + +-BBCLASSEXTEND = "native nativesdk" +\ No newline at end of file ++BBCLASSEXTEND = "native nativesdk" +-- +2.41.0 + diff --git a/meta/lib/patchtest/selftest/files/TestMbox.test_signed_off_by_presence.pass b/meta/lib/patchtest/selftest/files/TestMbox.test_signed_off_by_presence.pass new file mode 100644 index 0000000000..ea34c76f0d --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestMbox.test_signed_off_by_presence.pass @@ -0,0 +1,72 @@ +From 14d72f6973270f78455a8628143f2cff90e8f41e Mon Sep 17 00:00:00 2001 +From: Trevor Gamblin <tgamblin@baylibre.com> +Date: Tue, 29 Aug 2023 14:12:27 -0400 +Subject: [PATCH] selftest-hello: fix CVE-1234-56789 + +CVE: CVE-1234-56789 + +Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> +--- + .../selftest-hello/files/CVE-1234-56789.patch | 27 +++++++++++++++++++ + .../selftest-hello/selftest-hello_1.0.bb | 6 +++-- + 2 files changed, 31 insertions(+), 2 deletions(-) + create mode 100644 meta-selftest/recipes-test/selftest-hello/files/CVE-1234-56789.patch + +diff --git a/meta-selftest/recipes-test/selftest-hello/files/CVE-1234-56789.patch b/meta-selftest/recipes-test/selftest-hello/files/CVE-1234-56789.patch +new file mode 100644 +index 0000000000..869cfb6fe5 +--- /dev/null ++++ b/meta-selftest/recipes-test/selftest-hello/files/CVE-1234-56789.patch +@@ -0,0 +1,27 @@ ++From b26a31186e6ee2eb1f506d5f2f9394d327a0df2f Mon Sep 17 00:00:00 2001 ++From: Trevor Gamblin <tgamblin@baylibre.com> ++Date: Tue, 29 Aug 2023 14:08:20 -0400 ++Subject: [PATCH] Fix CVE-NOT-REAL ++ ++CVE: CVE-1234-56789 ++Upstream-Status: Backport(http://example.com/example) ++ ++Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> ++--- ++ strlen.c | 1 + ++ 1 file changed, 1 insertion(+) ++ ++diff --git a/strlen.c b/strlen.c ++index 1788f38..83d7918 100644 ++--- a/strlen.c +++++ b/strlen.c ++@@ -8,6 +8,7 @@ int main() { ++ ++ printf("%d\n", str_len(string1)); ++ printf("%d\n", str_len(string2)); +++ printf("CVE FIXED!!!\n"); ++ ++ return 0; ++ } ++-- ++2.41.0 +diff --git a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +index 547587bef4..76975a6729 100644 +--- a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb ++++ b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +@@ -3,7 +3,9 @@ SECTION = "examples" + LICENSE = "MIT" + LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302" + +-SRC_URI = "file://helloworld.c" ++SRC_URI = "file://helloworld.c \ ++ file://CVE-1234-56789.patch \ ++ " + + S = "${WORKDIR}" + +@@ -16,4 +18,4 @@ do_install() { + install -m 0755 helloworld ${D}${bindir} + } + +-BBCLASSEXTEND = "native nativesdk" +\ No newline at end of file ++BBCLASSEXTEND = "native nativesdk" +-- +2.41.0 + diff --git a/meta/lib/patchtest/selftest/files/TestMetadata.test_cve_check_ignore.fail b/meta/lib/patchtest/selftest/files/TestMetadata.test_cve_check_ignore.fail new file mode 100644 index 0000000000..3574463ade --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestMetadata.test_cve_check_ignore.fail @@ -0,0 +1,30 @@ +From c4ca86b9cca3643097db0328e2f34dccffbba309 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Simone=20Wei=C3=9F?= <simone.p.weiss@posteo.com> +Date: Sat, 10 Feb 2024 13:18:44 +0100 +Subject: [PATCH] selftest-hello: add CVE_CHECK_IGNORE +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +This should fail the test_cve_tag_format selftest. + +Signed-off-by: Simone Weiß <simone.p.weiss@posteo.com> +--- + .../recipes-test/selftest-hello/selftest-hello_1.0.bb | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +index 547587bef4..3ef9b87c34 100644 +--- a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb ++++ b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +@@ -16,4 +16,5 @@ do_install() { + install -m 0755 helloworld ${D}${bindir} + } + +-BBCLASSEXTEND = "native nativesdk" +\ No newline at end of file ++CVE_CHECK_IGNORE = "CVE-2024-12345" ++BBCLASSEXTEND = "native nativesdk" +-- +2.39.2 + diff --git a/meta/lib/patchtest/selftest/files/TestMetadata.test_cve_check_ignore.pass b/meta/lib/patchtest/selftest/files/TestMetadata.test_cve_check_ignore.pass new file mode 100644 index 0000000000..10f942a6eb --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestMetadata.test_cve_check_ignore.pass @@ -0,0 +1,31 @@ +From 7d4d3fee0c7111830ee9b2b049ae3ce265b26030 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Simone=20Wei=C3=9F?= <simone.p.weiss@posteo.com> +Date: Sat, 10 Feb 2024 13:23:56 +0100 +Subject: [PATCH] selftest-hello: add CVE_STATUS +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +This should pass the test_cve_tag_format selftest. + +Signed-off-by: Simone Weiß <simone.p.weiss@posteo.com> +--- + .../recipes-test/selftest-hello/selftest-hello_1.0.bb | 4 +++- + 1 file changed, 3 insertions(+), 1 deletion(-) + +diff --git a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +index 547587bef4..9908b3b417 100644 +--- a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb ++++ b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +@@ -16,4 +16,6 @@ do_install() { + install -m 0755 helloworld ${D}${bindir} + } + +-BBCLASSEXTEND = "native nativesdk" +\ No newline at end of file ++CVE_STATUS[CVE-2024-12345] = "not-applicable-platform: Issue only applies on Windows" ++ ++BBCLASSEXTEND = "native nativesdk" +-- +2.39.2 + diff --git a/meta/lib/patchtest/selftest/files/TestMetadata.test_lic_files_chksum_modified_not_mentioned.fail b/meta/lib/patchtest/selftest/files/TestMetadata.test_lic_files_chksum_modified_not_mentioned.fail new file mode 100644 index 0000000000..ab6c52c374 --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestMetadata.test_lic_files_chksum_modified_not_mentioned.fail @@ -0,0 +1,37 @@ +From f89919ea86d38404dd621521680a0162367bb965 Mon Sep 17 00:00:00 2001 +From: Trevor Gamblin <tgamblin@baylibre.com> +Date: Wed, 6 Sep 2023 09:09:27 -0400 +Subject: [PATCH] selftest-hello: update LIC_FILES_CHKSUM + +This test should fail the +test_metadata_lic_files_chksum.LicFilesChkSum.test_lic_files_chksum_modified_not_mentioned +test. + +Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> +--- + .../recipes-test/selftest-hello/selftest-hello_1.0.bb | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +index 547587bef4..65dda40aba 100644 +--- a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb ++++ b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +@@ -1,7 +1,7 @@ + DESCRIPTION = "Simple helloworld application -- selftest variant" + SECTION = "examples" + LICENSE = "MIT" +-LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302" ++LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f303" + + SRC_URI = "file://helloworld.c" + +@@ -16,4 +16,4 @@ do_install() { + install -m 0755 helloworld ${D}${bindir} + } + +-BBCLASSEXTEND = "native nativesdk" +\ No newline at end of file ++BBCLASSEXTEND = "native nativesdk" +-- +2.41.0 + diff --git a/meta/lib/patchtest/selftest/files/TestMetadata.test_lic_files_chksum_modified_not_mentioned.pass b/meta/lib/patchtest/selftest/files/TestMetadata.test_lic_files_chksum_modified_not_mentioned.pass new file mode 100644 index 0000000000..99d9f144da --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestMetadata.test_lic_files_chksum_modified_not_mentioned.pass @@ -0,0 +1,39 @@ +From f89919ea86d38404dd621521680a0162367bb965 Mon Sep 17 00:00:00 2001 +From: Trevor Gamblin <tgamblin@baylibre.com> +Date: Wed, 6 Sep 2023 09:09:27 -0400 +Subject: [PATCH] selftest-hello: update LIC_FILES_CHKSUM + +License-Update: Fix checksum + +This test should pass the +test_metadata_lic_files_chksum.LicFilesChkSum.test_lic_files_chksum_modified_not_mentioned +test. + +Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> +--- + .../recipes-test/selftest-hello/selftest-hello_1.0.bb | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +index 547587bef4..65dda40aba 100644 +--- a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb ++++ b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +@@ -1,7 +1,7 @@ + DESCRIPTION = "Simple helloworld application -- selftest variant" + SECTION = "examples" + LICENSE = "MIT" +-LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302" ++LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f303" + + SRC_URI = "file://helloworld.c" + +@@ -16,4 +16,4 @@ do_install() { + install -m 0755 helloworld ${D}${bindir} + } + +-BBCLASSEXTEND = "native nativesdk" +\ No newline at end of file ++BBCLASSEXTEND = "native nativesdk" +-- +2.41.0 + diff --git a/meta/lib/patchtest/selftest/files/TestMetadata.test_lic_files_chksum_presence.fail b/meta/lib/patchtest/selftest/files/TestMetadata.test_lic_files_chksum_presence.fail new file mode 100644 index 0000000000..e14d644bb2 --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestMetadata.test_lic_files_chksum_presence.fail @@ -0,0 +1,53 @@ +From 66430e7c6fbd5187b66560909a510e136fed91c0 Mon Sep 17 00:00:00 2001 +From: Daniela Plascencia <daniela.plascencia@linux.intel.com> +Date: Thu, 23 Feb 2017 10:34:27 -0600 +Subject: [PATCH] meta: adding hello-yocto recipe + +This is a sample recipe + +Signed-off-by: Daniela Plascencia <daniela.plascencia@linux.intel.com> +--- + .../hello-world/hello-world/hello_world.c | 5 +++++ + .../hello-world/hello-world_1.0.bb | 18 ++++++++++++++++++ + 2 files changed, 23 insertions(+) + create mode 100644 meta/recipes-devtools/hello-world/hello-world/hello_world.c + create mode 100644 meta/recipes-devtools/hello-world/hello-world_1.0.bb + +diff --git a/meta/recipes-devtools/hello-world/hello-world/hello_world.c b/meta/recipes-devtools/hello-world/hello-world/hello_world.c +new file mode 100644 +index 0000000000..0d59f57d4c +--- /dev/null ++++ b/meta/recipes-devtools/hello-world/hello-world/hello_world.c +@@ -0,0 +1,5 @@ ++#include <stdio.h> ++ ++int main(){ ++ printf("Hello World\n"); ++} +diff --git a/meta/recipes-devtools/hello-world/hello-world_1.0.bb b/meta/recipes-devtools/hello-world/hello-world_1.0.bb +new file mode 100644 +index 0000000000..3c990c108a +--- /dev/null ++++ b/meta/recipes-devtools/hello-world/hello-world_1.0.bb +@@ -0,0 +1,18 @@ ++SUMMARY = "This is a sample summary" ++DESCRIPTION = "This is a sample description" ++HOMEPAGE = "https://sample.com/this-is-a-sample" ++LICENSE = "MIT" ++ ++SRC_URI += "file://hello_world.c" ++ ++SRC_URI[md5sum] = "4ee21e9dcc9b5b6012c23038734e1632" ++SRC_URI[sha256sum] = "edef2bbde0fbf0d88232782a0eded323f483a0519d6fde9a3b1809056fd35f3e" ++ ++do_compile(){ ++ ${CC} -o hello_world ../hello_world.c ++} ++ ++do_install(){ ++ install -d ${D}${bindir} ++ install -m +x hello_world ${D}${bindir}/hello_world ++} +-- +2.41.0 + diff --git a/meta/lib/patchtest/selftest/files/TestMetadata.test_lic_files_chksum_presence.pass b/meta/lib/patchtest/selftest/files/TestMetadata.test_lic_files_chksum_presence.pass new file mode 100644 index 0000000000..b8da16dfe5 --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestMetadata.test_lic_files_chksum_presence.pass @@ -0,0 +1,54 @@ +From 5144d2ba1aa763312c047dd5f8901368cff79da6 Mon Sep 17 00:00:00 2001 +From: Daniela Plascencia <daniela.plascencia@linux.intel.com> +Date: Thu, 23 Feb 2017 10:34:27 -0600 +Subject: [PATCH] meta: adding hello-yocto recipe + +This is a sample recipe + +Signed-off-by: Daniela Plascencia <daniela.plascencia@linux.intel.com> +--- + .../hello-world/hello-world/hello_world.c | 5 +++++ + .../hello-world/hello-world_1.0.bb | 19 +++++++++++++++++++ + 2 files changed, 24 insertions(+) + create mode 100644 meta/recipes-devtools/hello-world/hello-world/hello_world.c + create mode 100644 meta/recipes-devtools/hello-world/hello-world_1.0.bb + +diff --git a/meta/recipes-devtools/hello-world/hello-world/hello_world.c b/meta/recipes-devtools/hello-world/hello-world/hello_world.c +new file mode 100644 +index 0000000000..0d59f57d4c +--- /dev/null ++++ b/meta/recipes-devtools/hello-world/hello-world/hello_world.c +@@ -0,0 +1,5 @@ ++#include <stdio.h> ++ ++int main(){ ++ printf("Hello World\n"); ++} +diff --git a/meta/recipes-devtools/hello-world/hello-world_1.0.bb b/meta/recipes-devtools/hello-world/hello-world_1.0.bb +new file mode 100644 +index 0000000000..44d888c82a +--- /dev/null ++++ b/meta/recipes-devtools/hello-world/hello-world_1.0.bb +@@ -0,0 +1,19 @@ ++SUMMARY = "This is a sample summary" ++DESCRIPTION = "This is a sample description" ++HOMEPAGE = "https://sample.com/this-is-a-sample" ++LICENSE = "MIT" ++LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302" ++ ++SRC_URI += "file://hello_world.c" ++ ++SRC_URI[md5sum] = "4ee21e9dcc9b5b6012c23038734e1632" ++SRC_URI[sha256sum] = "edef2bbde0fbf0d88232782a0eded323f483a0519d6fde9a3b1809056fd35f3e" ++ ++do_compile(){ ++ ${CC} -o hello_world ../hello_world.c ++} ++ ++do_install(){ ++ install -d ${D}${bindir} ++ install -m +x hello_world ${D}${bindir}/hello_world ++} +-- +2.41.0 + diff --git a/meta/lib/patchtest/selftest/files/TestMetadata.test_src_uri_left_files.fail b/meta/lib/patchtest/selftest/files/TestMetadata.test_src_uri_left_files.fail new file mode 100644 index 0000000000..983b6e0c2b --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestMetadata.test_src_uri_left_files.fail @@ -0,0 +1,35 @@ +From 4ab06b5f81455249cd5e89d2cce9863803b5ecb5 Mon Sep 17 00:00:00 2001 +From: Trevor Gamblin <tgamblin@baylibre.com> +Date: Fri, 8 Sep 2023 14:41:00 -0400 +Subject: [PATCH] selftest-hello: remove helloworld.c + +This should fail the test_src_uri_left_files selftest. + +Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> +--- + .../recipes-test/selftest-hello/selftest-hello_1.0.bb | 4 +--- + 1 file changed, 1 insertion(+), 3 deletions(-) + +diff --git a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +index 547587bef4..f6817f05bc 100644 +--- a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb ++++ b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +@@ -3,8 +3,6 @@ SECTION = "examples" + LICENSE = "MIT" + LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302" + +-SRC_URI = "file://helloworld.c" +- + S = "${WORKDIR}" + + do_compile() { +@@ -16,4 +14,4 @@ do_install() { + install -m 0755 helloworld ${D}${bindir} + } + +-BBCLASSEXTEND = "native nativesdk" +\ No newline at end of file ++BBCLASSEXTEND = "native nativesdk" +-- +2.41.0 + diff --git a/meta/lib/patchtest/selftest/files/TestMetadata.test_src_uri_left_files.pass b/meta/lib/patchtest/selftest/files/TestMetadata.test_src_uri_left_files.pass new file mode 100644 index 0000000000..1f1a77e581 --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestMetadata.test_src_uri_left_files.pass @@ -0,0 +1,51 @@ +From 6c7ac367a873bf827c19b81085c943eace917a99 Mon Sep 17 00:00:00 2001 +From: Trevor Gamblin <tgamblin@baylibre.com> +Date: Fri, 8 Sep 2023 14:41:00 -0400 +Subject: [PATCH] selftest-hello: remove helloworld.c + +This should pass the test_src_uri_left_files selftest. + +Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> +--- + .../recipes-test/selftest-hello/files/helloworld.c | 8 -------- + .../recipes-test/selftest-hello/selftest-hello_1.0.bb | 4 +--- + 2 files changed, 1 insertion(+), 11 deletions(-) + delete mode 100644 meta-selftest/recipes-test/selftest-hello/files/helloworld.c + +diff --git a/meta-selftest/recipes-test/selftest-hello/files/helloworld.c b/meta-selftest/recipes-test/selftest-hello/files/helloworld.c +deleted file mode 100644 +index fc7169b7b8..0000000000 +--- a/meta-selftest/recipes-test/selftest-hello/files/helloworld.c ++++ /dev/null +@@ -1,8 +0,0 @@ +-#include <stdio.h> +- +-int main(void) +-{ +- printf("Hello world!\n"); +- +- return 0; +-} +diff --git a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +index 547587bef4..f6817f05bc 100644 +--- a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb ++++ b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +@@ -3,8 +3,6 @@ SECTION = "examples" + LICENSE = "MIT" + LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302" + +-SRC_URI = "file://helloworld.c" +- + S = "${WORKDIR}" + + do_compile() { +@@ -16,4 +14,4 @@ do_install() { + install -m 0755 helloworld ${D}${bindir} + } + +-BBCLASSEXTEND = "native nativesdk" +\ No newline at end of file ++BBCLASSEXTEND = "native nativesdk" +-- +2.41.0 + diff --git a/meta/lib/patchtest/selftest/files/TestMetadata.test_summary_presence.fail b/meta/lib/patchtest/selftest/files/TestMetadata.test_summary_presence.fail new file mode 100644 index 0000000000..2d2b4e683d --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestMetadata.test_summary_presence.fail @@ -0,0 +1,46 @@ +From e29da5faa74409be394caa09d9f3b7b60f8592b9 Mon Sep 17 00:00:00 2001 +From: Daniela Plascencia <daniela.plascencia@linux.intel.com> +Date: Thu, 23 Feb 2017 10:34:27 -0600 +Subject: [PATCH] meta: adding hello-yocto recipe + +This is a sample recipe + +Signed-off-by: Daniela Plascencia <daniela.plascencia@linux.intel.com> +--- + meta/recipes-devtools/hello-world/hello-world/hello_world.c | 5 +++++ + meta/recipes-devtools/hello-world/hello-world_1.0.bb | 12 ++++++++++++ + 2 files changed, 17 insertions(+) + create mode 100644 meta/recipes-devtools/hello-world/hello-world/hello_world.c + create mode 100644 meta/recipes-devtools/hello-world/hello-world_1.0.bb + +diff --git a/meta/recipes-devtools/hello-world/hello-world/hello_world.c b/meta/recipes-devtools/hello-world/hello-world/hello_world.c +new file mode 100644 +index 0000000000..0d59f57d4c +--- /dev/null ++++ b/meta/recipes-devtools/hello-world/hello-world/hello_world.c +@@ -0,0 +1,5 @@ ++#include <stdio.h> ++ ++int main(){ ++ printf("Hello World\n"); ++} +diff --git a/meta/recipes-devtools/hello-world/hello-world_1.0.bb b/meta/recipes-devtools/hello-world/hello-world_1.0.bb +new file mode 100644 +index 0000000000..c4e1359217 +--- /dev/null ++++ b/meta/recipes-devtools/hello-world/hello-world_1.0.bb +@@ -0,0 +1,12 @@ ++LICENSE = "CLOSED" ++ ++SRC_URI += "file://hello_world.c" ++ ++do_compile(){ ++ ${CC} -o hello_world ../hello_world.c ++} ++ ++do_install(){ ++ install -d ${D}${bindir} ++ install -m +x hello_world ${D}${bindir}/hello_world ++} +-- +2.11.0 diff --git a/meta/lib/patchtest/selftest/files/TestMetadata.test_summary_presence.pass b/meta/lib/patchtest/selftest/files/TestMetadata.test_summary_presence.pass new file mode 100644 index 0000000000..55f0309b3f --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestMetadata.test_summary_presence.pass @@ -0,0 +1,49 @@ +From 0cd2fed12ce4b7b071edde12aec4481ad7a6f107 Mon Sep 17 00:00:00 2001 +From: Daniela Plascencia <daniela.plascencia@linux.intel.com> +Date: Thu, 23 Feb 2017 10:34:27 -0600 +Subject: [PATCH] meta: adding hello-yocto recipe + +This is a sample recipe + +Signed-off-by: Daniela Plascencia <daniela.plascencia@linux.intel.com> +--- + .../hello-world/hello-world/hello_world.c | 5 +++++ + meta/recipes-devtools/hello-world/hello-world_1.0.bb | 15 +++++++++++++++ + 2 files changed, 20 insertions(+) + create mode 100644 meta/recipes-devtools/hello-world/hello-world/hello_world.c + create mode 100644 meta/recipes-devtools/hello-world/hello-world_1.0.bb + +diff --git a/meta/recipes-devtools/hello-world/hello-world/hello_world.c b/meta/recipes-devtools/hello-world/hello-world/hello_world.c +new file mode 100644 +index 0000000000..0d59f57d4c +--- /dev/null ++++ b/meta/recipes-devtools/hello-world/hello-world/hello_world.c +@@ -0,0 +1,5 @@ ++#include <stdio.h> ++ ++int main(){ ++ printf("Hello World\n"); ++} +diff --git a/meta/recipes-devtools/hello-world/hello-world_1.0.bb b/meta/recipes-devtools/hello-world/hello-world_1.0.bb +new file mode 100644 +index 0000000000..c54283eece +--- /dev/null ++++ b/meta/recipes-devtools/hello-world/hello-world_1.0.bb +@@ -0,0 +1,15 @@ ++SUMMARY = "This is a sample summary" ++DESCRIPTION = "This is a sample description" ++HOMEPAGE = "https://sample.com/this-is-a-sample" ++LICENSE = "CLOSED" ++ ++SRC_URI += "file://hello_world.c" ++ ++do_compile(){ ++ ${CC} -o hello_world ../hello_world.c ++} ++ ++do_install(){ ++ install -d ${D}${bindir} ++ install -m +x hello_world ${D}${bindir}/hello_world ++} +-- +2.11.0 diff --git a/meta/lib/patchtest/selftest/files/TestPatch.test_cve_tag_format.fail b/meta/lib/patchtest/selftest/files/TestPatch.test_cve_tag_format.fail new file mode 100644 index 0000000000..c763a7506e --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestPatch.test_cve_tag_format.fail @@ -0,0 +1,73 @@ +From 35ccee3cee96fb29514475279248078d88907231 Mon Sep 17 00:00:00 2001 +From: Trevor Gamblin <tgamblin@baylibre.com> +Date: Tue, 29 Aug 2023 14:12:27 -0400 +Subject: [PATCH] selftest-hello: fix CVE-1234-56789 + +CVE: CVE-BAD-FORMAT + +Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> +--- + .../files/0001-Fix-CVE-1234-56789.patch | 27 +++++++++++++++++++ + .../selftest-hello/selftest-hello_1.0.bb | 6 +++-- + 2 files changed, 31 insertions(+), 2 deletions(-) + create mode 100644 meta-selftest/recipes-test/selftest-hello/files/0001-Fix-CVE-1234-56789.patch + +diff --git a/meta-selftest/recipes-test/selftest-hello/files/0001-Fix-CVE-1234-56789.patch b/meta-selftest/recipes-test/selftest-hello/files/0001-Fix-CVE-1234-56789.patch +new file mode 100644 +index 0000000000..9219b8db62 +--- /dev/null ++++ b/meta-selftest/recipes-test/selftest-hello/files/0001-Fix-CVE-1234-56789.patch +@@ -0,0 +1,27 @@ ++From b26a31186e6ee2eb1f506d5f2f9394d327a0df2f Mon Sep 17 00:00:00 2001 ++From: Trevor Gamblin <tgamblin@baylibre.com> ++Date: Tue, 29 Aug 2023 14:08:20 -0400 ++Subject: [PATCH] Fix CVE-NOT-REAL ++ ++CVE: CVE-BAD-FORMAT ++Upstream-Status: Backport(http://example.com/example) ++ ++Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> ++--- ++ strlen.c | 1 + ++ 1 file changed, 1 insertion(+) ++ ++diff --git a/strlen.c b/strlen.c ++index 1788f38..83d7918 100644 ++--- a/strlen.c +++++ b/strlen.c ++@@ -8,6 +8,7 @@ int main() { ++ ++ printf("%d\n", str_len(string1)); ++ printf("%d\n", str_len(string2)); +++ printf("CVE FIXED!!!\n"); ++ ++ return 0; ++ } ++-- ++2.41.0 ++ +diff --git a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +index 547587bef4..76975a6729 100644 +--- a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb ++++ b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +@@ -3,7 +3,9 @@ SECTION = "examples" + LICENSE = "MIT" + LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302" + +-SRC_URI = "file://helloworld.c" ++SRC_URI = "file://helloworld.c \ ++ file://CVE-1234-56789.patch \ ++ " + + S = "${WORKDIR}" + +@@ -16,4 +18,4 @@ do_install() { + install -m 0755 helloworld ${D}${bindir} + } + +-BBCLASSEXTEND = "native nativesdk" +\ No newline at end of file ++BBCLASSEXTEND = "native nativesdk" +-- +2.41.0 + diff --git a/meta/lib/patchtest/selftest/files/TestPatch.test_cve_tag_format.pass b/meta/lib/patchtest/selftest/files/TestPatch.test_cve_tag_format.pass new file mode 100644 index 0000000000..ef6017037c --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestPatch.test_cve_tag_format.pass @@ -0,0 +1,73 @@ +From 35ccee3cee96fb29514475279248078d88907231 Mon Sep 17 00:00:00 2001 +From: Trevor Gamblin <tgamblin@baylibre.com> +Date: Tue, 29 Aug 2023 14:12:27 -0400 +Subject: [PATCH] selftest-hello: fix CVE-1234-56789 + +CVE: CVE-1234-56789 + +Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> +--- + .../files/0001-Fix-CVE-1234-56789.patch | 27 +++++++++++++++++++ + .../selftest-hello/selftest-hello_1.0.bb | 6 +++-- + 2 files changed, 31 insertions(+), 2 deletions(-) + create mode 100644 meta-selftest/recipes-test/selftest-hello/files/0001-Fix-CVE-1234-56789.patch + +diff --git a/meta-selftest/recipes-test/selftest-hello/files/0001-Fix-CVE-1234-56789.patch b/meta-selftest/recipes-test/selftest-hello/files/0001-Fix-CVE-1234-56789.patch +new file mode 100644 +index 0000000000..9219b8db62 +--- /dev/null ++++ b/meta-selftest/recipes-test/selftest-hello/files/0001-Fix-CVE-1234-56789.patch +@@ -0,0 +1,27 @@ ++From b26a31186e6ee2eb1f506d5f2f9394d327a0df2f Mon Sep 17 00:00:00 2001 ++From: Trevor Gamblin <tgamblin@baylibre.com> ++Date: Tue, 29 Aug 2023 14:08:20 -0400 ++Subject: [PATCH] Fix CVE-NOT-REAL ++ ++CVE: CVE-1234-56789 ++Upstream-Status: Backport(http://example.com/example) ++ ++Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> ++--- ++ strlen.c | 1 + ++ 1 file changed, 1 insertion(+) ++ ++diff --git a/strlen.c b/strlen.c ++index 1788f38..83d7918 100644 ++--- a/strlen.c +++++ b/strlen.c ++@@ -8,6 +8,7 @@ int main() { ++ ++ printf("%d\n", str_len(string1)); ++ printf("%d\n", str_len(string2)); +++ printf("CVE FIXED!!!\n"); ++ ++ return 0; ++ } ++-- ++2.41.0 ++ +diff --git a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +index 547587bef4..76975a6729 100644 +--- a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb ++++ b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +@@ -3,7 +3,9 @@ SECTION = "examples" + LICENSE = "MIT" + LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302" + +-SRC_URI = "file://helloworld.c" ++SRC_URI = "file://helloworld.c \ ++ file://CVE-1234-56789.patch \ ++ " + + S = "${WORKDIR}" + +@@ -16,4 +18,4 @@ do_install() { + install -m 0755 helloworld ${D}${bindir} + } + +-BBCLASSEXTEND = "native nativesdk" +\ No newline at end of file ++BBCLASSEXTEND = "native nativesdk" +-- +2.41.0 + diff --git a/meta/lib/patchtest/selftest/files/TestPatch.test_signed_off_by_presence.fail b/meta/lib/patchtest/selftest/files/TestPatch.test_signed_off_by_presence.fail new file mode 100644 index 0000000000..ce8bf7b7d1 --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestPatch.test_signed_off_by_presence.fail @@ -0,0 +1,71 @@ +From 5a2d0ac780a0f4c046fb1a3c3463d3e726f191cb Mon Sep 17 00:00:00 2001 +From: Trevor Gamblin <tgamblin@baylibre.com> +Date: Tue, 29 Aug 2023 14:12:27 -0400 +Subject: [PATCH] selftest-hello: fix CVE-1234-56789 + +CVE: CVE-1234-56789 + +Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> +--- + .../selftest-hello/files/CVE-1234-56789.patch | 26 +++++++++++++++++++ + .../selftest-hello/selftest-hello_1.0.bb | 6 +++-- + 2 files changed, 30 insertions(+), 2 deletions(-) + create mode 100644 meta-selftest/recipes-test/selftest-hello/files/CVE-1234-56789.patch + +diff --git a/meta-selftest/recipes-test/selftest-hello/files/CVE-1234-56789.patch b/meta-selftest/recipes-test/selftest-hello/files/CVE-1234-56789.patch +new file mode 100644 +index 0000000000..92a5b65a53 +--- /dev/null ++++ b/meta-selftest/recipes-test/selftest-hello/files/CVE-1234-56789.patch +@@ -0,0 +1,26 @@ ++From b26a31186e6ee2eb1f506d5f2f9394d327a0df2f Mon Sep 17 00:00:00 2001 ++From: Trevor Gamblin <tgamblin@baylibre.com> ++Date: Tue, 29 Aug 2023 14:08:20 -0400 ++Subject: [PATCH] Fix CVE-NOT-REAL ++ ++CVE: CVE-1234-56789 ++Upstream-Status: Backport(http://example.com/example) ++ ++--- ++ strlen.c | 1 + ++ 1 file changed, 1 insertion(+) ++ ++diff --git a/strlen.c b/strlen.c ++index 1788f38..83d7918 100644 ++--- a/strlen.c +++++ b/strlen.c ++@@ -8,6 +8,7 @@ int main() { ++ ++ printf("%d\n", str_len(string1)); ++ printf("%d\n", str_len(string2)); +++ printf("CVE FIXED!!!\n"); ++ ++ return 0; ++ } ++-- ++2.41.0 +diff --git a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +index 547587bef4..76975a6729 100644 +--- a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb ++++ b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +@@ -3,7 +3,9 @@ SECTION = "examples" + LICENSE = "MIT" + LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302" + +-SRC_URI = "file://helloworld.c" ++SRC_URI = "file://helloworld.c \ ++ file://CVE-1234-56789.patch \ ++ " + + S = "${WORKDIR}" + +@@ -16,4 +18,4 @@ do_install() { + install -m 0755 helloworld ${D}${bindir} + } + +-BBCLASSEXTEND = "native nativesdk" +\ No newline at end of file ++BBCLASSEXTEND = "native nativesdk" +-- +2.41.0 + diff --git a/meta/lib/patchtest/selftest/files/TestPatch.test_signed_off_by_presence.pass b/meta/lib/patchtest/selftest/files/TestPatch.test_signed_off_by_presence.pass new file mode 100644 index 0000000000..ea34c76f0d --- /dev/null +++ b/meta/lib/patchtest/selftest/files/TestPatch.test_signed_off_by_presence.pass @@ -0,0 +1,72 @@ +From 14d72f6973270f78455a8628143f2cff90e8f41e Mon Sep 17 00:00:00 2001 +From: Trevor Gamblin <tgamblin@baylibre.com> +Date: Tue, 29 Aug 2023 14:12:27 -0400 +Subject: [PATCH] selftest-hello: fix CVE-1234-56789 + +CVE: CVE-1234-56789 + +Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> +--- + .../selftest-hello/files/CVE-1234-56789.patch | 27 +++++++++++++++++++ + .../selftest-hello/selftest-hello_1.0.bb | 6 +++-- + 2 files changed, 31 insertions(+), 2 deletions(-) + create mode 100644 meta-selftest/recipes-test/selftest-hello/files/CVE-1234-56789.patch + +diff --git a/meta-selftest/recipes-test/selftest-hello/files/CVE-1234-56789.patch b/meta-selftest/recipes-test/selftest-hello/files/CVE-1234-56789.patch +new file mode 100644 +index 0000000000..869cfb6fe5 +--- /dev/null ++++ b/meta-selftest/recipes-test/selftest-hello/files/CVE-1234-56789.patch +@@ -0,0 +1,27 @@ ++From b26a31186e6ee2eb1f506d5f2f9394d327a0df2f Mon Sep 17 00:00:00 2001 ++From: Trevor Gamblin <tgamblin@baylibre.com> ++Date: Tue, 29 Aug 2023 14:08:20 -0400 ++Subject: [PATCH] Fix CVE-NOT-REAL ++ ++CVE: CVE-1234-56789 ++Upstream-Status: Backport(http://example.com/example) ++ ++Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com> ++--- ++ strlen.c | 1 + ++ 1 file changed, 1 insertion(+) ++ ++diff --git a/strlen.c b/strlen.c ++index 1788f38..83d7918 100644 ++--- a/strlen.c +++++ b/strlen.c ++@@ -8,6 +8,7 @@ int main() { ++ ++ printf("%d\n", str_len(string1)); ++ printf("%d\n", str_len(string2)); +++ printf("CVE FIXED!!!\n"); ++ ++ return 0; ++ } ++-- ++2.41.0 +diff --git a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +index 547587bef4..76975a6729 100644 +--- a/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb ++++ b/meta-selftest/recipes-test/selftest-hello/selftest-hello_1.0.bb +@@ -3,7 +3,9 @@ SECTION = "examples" + LICENSE = "MIT" + LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302" + +-SRC_URI = "file://helloworld.c" ++SRC_URI = "file://helloworld.c \ ++ file://CVE-1234-56789.patch \ ++ " + + S = "${WORKDIR}" + +@@ -16,4 +18,4 @@ do_install() { + install -m 0755 helloworld ${D}${bindir} + } + +-BBCLASSEXTEND = "native nativesdk" +\ No newline at end of file ++BBCLASSEXTEND = "native nativesdk" +-- +2.41.0 + diff --git a/meta/lib/patchtest/selftest/selftest b/meta/lib/patchtest/selftest/selftest new file mode 100755 index 0000000000..6fad50ce61 --- /dev/null +++ b/meta/lib/patchtest/selftest/selftest @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 + +# Test every patch from files folder and output error on failure +# +# Copyright (C) 2016 Intel Corporation +# +# SPDX-License-Identifier: GPL-2.0-only + +import os +import subprocess +import sys + +currentdir = os.path.dirname(os.path.abspath(__file__)) +patchesdir = os.path.join(currentdir, 'files') +topdir = os.path.dirname(currentdir) +parentdir = os.path.dirname(topdir) + +# path to the repo root +repodir = os.path.dirname(os.path.dirname(parentdir)) + +def print_results(passcount, failcount, skipcount, xpasscount, xfailcount, xskipcount, errorcount): + total = passcount + skipcount + failcount + xpasscount + xfailcount + xskipcount + errorcount + print("============================================================================") + print("Testsuite summary for %s" % os.path.basename(topdir)) + print("============================================================================") + print("# TOTAL: %s" % str(total)) + print("# XPASS: %s" % str(xpasscount)) + print("# XFAIL: %s" % str(xfailcount)) + print("# XSKIP: %s" % str(xskipcount)) + print("# PASS: %s" % str(passcount)) + print("# FAIL: %s" % str(failcount)) + print("# SKIP: %s" % str(skipcount)) + print("# ERROR: %s" % str(errorcount)) + print("============================================================================") + +# Once the tests are in oe-core, we can remove the testdir param and use os.path.dirname to get relative paths +def test(root, patch): + res = True + patchpath = os.path.abspath(os.path.join(root, patch)) + + cmd = 'patchtest --repodir %s --testdir %s/tests --patch %s' % (repodir, topdir, patchpath) + results = subprocess.check_output(cmd, stderr=subprocess.STDOUT, universal_newlines=True, shell=True) + + return results + +if __name__ == '__main__': + passcount = 0 + failcount = 0 + skipcount = 0 + xpasscount = 0 + xfailcount = 0 + xskipcount = 0 + errorcount = 0 + + results = None + + for root, dirs, patches in os.walk(patchesdir): + for patch in patches: + results = test(root, patch) + + a = patch.split('.') + klass, testname = a[0], a[1] + expected_result = a[-1] + testid = ".%s.%s" % (klass,testname) + + for resultline in results.splitlines(): + if testid in resultline: + result, _ = resultline.split(':', 1) + + if expected_result.upper() == "FAIL" and result.upper() == "FAIL": + xfailcount = xfailcount + 1 + print("XFAIL: %s (file: %s)" % (testid.strip("."), os.path.basename(patch))) + elif expected_result.upper() == "PASS" and result.upper() == "PASS": + xpasscount = xpasscount + 1 + print("XPASS: %s (file: %s)" % (testid.strip("."), os.path.basename(patch))) + elif expected_result.upper() == "SKIP" and result.upper() == "SKIP": + xskipcount = xskipcount + 1 + print("XSKIP: %s (file: %s)" % (testid.strip("."), os.path.basename(patch))) + else: + print("%s: %s (%s)" % (result.upper(), testid.strip("."), os.path.basename(patch))) + if result.upper() == "PASS": + passcount = passcount + 1 + elif result.upper() == "FAIL": + failcount = failcount + 1 + elif result.upper() == "SKIP": + skipcount = skipcount + 1 + else: + print("Bad result on test %s against %s" % (testid.strip("."), os.path.basename(patch))) + errorcount = errorcount + 1 + break + else: + print ("No test for=%s" % patch) + + print_results(passcount, failcount, skipcount, xpasscount, xfailcount, xskipcount, errorcount) diff --git a/meta/lib/patchtest/tests/__init__.py b/meta/lib/patchtest/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/meta/lib/patchtest/tests/__init__.py diff --git a/meta/lib/patchtest/tests/base.py b/meta/lib/patchtest/tests/base.py new file mode 100644 index 0000000000..424e61b5be --- /dev/null +++ b/meta/lib/patchtest/tests/base.py @@ -0,0 +1,239 @@ +# Base class to be used by all test cases defined in the suite +# +# Copyright (C) 2016 Intel Corporation +# +# SPDX-License-Identifier: GPL-2.0-only + +import unittest +import logging +import json +import unidiff +from data import PatchTestInput +import mailbox +import collections +import sys +import os +import re + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'pyparsing')) + +logger = logging.getLogger('patchtest') +debug=logger.debug +info=logger.info +warn=logger.warn +error=logger.error + +Commit = collections.namedtuple('Commit', ['author', 'subject', 'commit_message', 'shortlog', 'payload']) + +class PatchtestOEError(Exception): + """Exception for handling patchtest-oe errors""" + def __init__(self, message, exitcode=1): + super().__init__(message) + self.exitcode = exitcode + +class Base(unittest.TestCase): + # if unit test fails, fail message will throw at least the following JSON: {"id": <testid>} + + endcommit_messages_regex = re.compile(r'\(From \w+-\w+ rev:|(?<!\S)Signed-off-by|(?<!\S)---\n') + patchmetadata_regex = re.compile(r'-{3} \S+|\+{3} \S+|@{2} -\d+,\d+ \+\d+,\d+ @{2} \S+') + + + @staticmethod + def msg_to_commit(msg): + payload = msg.get_payload() + return Commit(subject=msg['subject'].replace('\n', ' ').replace(' ', ' '), + author=msg.get('From'), + shortlog=Base.shortlog(msg['subject']), + commit_message=Base.commit_message(payload), + payload=payload) + + @staticmethod + def commit_message(payload): + commit_message = payload.__str__() + match = Base.endcommit_messages_regex.search(payload) + if match: + commit_message = payload[:match.start()] + return commit_message + + @staticmethod + def shortlog(shlog): + # remove possible prefix (between brackets) before colon + start = shlog.find(']', 0, shlog.find(':')) + # remove also newlines and spaces at both sides + return shlog[start + 1:].replace('\n', '').strip() + + @classmethod + def setUpClass(cls): + + # General objects: mailbox.mbox and patchset + cls.mbox = mailbox.mbox(PatchTestInput.repo.patch) + + # Patch may be malformed, so try parsing it + cls.unidiff_parse_error = '' + cls.patchset = None + try: + cls.patchset = unidiff.PatchSet.from_filename(PatchTestInput.repo.patch, encoding=u'UTF-8') + except unidiff.UnidiffParseError as upe: + cls.patchset = [] + cls.unidiff_parse_error = str(upe) + + # Easy to iterate list of commits + cls.commits = [] + for msg in cls.mbox: + if msg['subject'] and msg.get_payload(): + cls.commits.append(Base.msg_to_commit(msg)) + + cls.setUpClassLocal() + + @classmethod + def tearDownClass(cls): + cls.tearDownClassLocal() + + @classmethod + def setUpClassLocal(cls): + pass + + @classmethod + def tearDownClassLocal(cls): + pass + + def fail(self, issue, fix=None, commit=None, data=None): + """ Convert to a JSON string failure data""" + value = {'id': self.id(), + 'issue': issue} + + if fix: + value['fix'] = fix + if commit: + value['commit'] = {'subject': commit.subject, + 'shortlog': commit.shortlog} + + # extend return value with other useful info + if data: + value['data'] = data + + return super(Base, self).fail(json.dumps(value)) + + def skip(self, issue, data=None): + """ Convert the skip string to JSON""" + value = {'id': self.id(), + 'issue': issue} + + # extend return value with other useful info + if data: + value['data'] = data + + return super(Base, self).skipTest(json.dumps(value)) + + def shortid(self): + return self.id().split('.')[-1] + + def __str__(self): + return json.dumps({'id': self.id()}) + +class Metadata(Base): + @classmethod + def setUpClassLocal(cls): + cls.tinfoil = cls.setup_tinfoil() + + # get info about added/modified/remove recipes + cls.added, cls.modified, cls.removed = cls.get_metadata_stats(cls.patchset) + + @classmethod + def tearDownClassLocal(cls): + cls.tinfoil.shutdown() + + @classmethod + def setup_tinfoil(cls, config_only=False): + """Initialize tinfoil api from bitbake""" + + # import relevant libraries + try: + scripts_path = os.path.join(PatchTestInput.repodir, 'scripts', 'lib') + if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) + import scriptpath + scriptpath.add_bitbake_lib_path() + import bb.tinfoil + except ImportError: + raise PatchtestOEError('Could not import tinfoil module') + + orig_cwd = os.path.abspath(os.curdir) + + # Load tinfoil + tinfoil = None + try: + builddir = os.environ.get('BUILDDIR') + if not builddir: + logger.warn('Bitbake environment not loaded?') + return tinfoil + os.chdir(builddir) + tinfoil = bb.tinfoil.Tinfoil() + tinfoil.prepare(config_only=config_only) + except bb.tinfoil.TinfoilUIException as te: + if tinfoil: + tinfoil.shutdown() + raise PatchtestOEError('Could not prepare properly tinfoil (TinfoilUIException)') + except Exception as e: + if tinfoil: + tinfoil.shutdown() + raise e + finally: + os.chdir(orig_cwd) + + return tinfoil + + @classmethod + def get_metadata_stats(cls, patchset): + """Get lists of added, modified and removed metadata files""" + + def find_pn(data, path): + """Find the PN from data""" + pn = None + pn_native = None + for _path, _pn in data: + if path in _path: + if 'native' in _pn: + # store the native PN but look for the non-native one first + pn_native = _pn + else: + pn = _pn + break + else: + # sent the native PN if found previously + if pn_native: + return pn_native + + # on renames (usually upgrades), we need to check (FILE) base names + # because the unidiff library does not provided the new filename, just the modified one + # and tinfoil datastore, once the patch is merged, will contain the new filename + path_basename = path.split('_')[0] + for _path, _pn in data: + _path_basename = _path.split('_')[0] + if path_basename == _path_basename: + pn = _pn + return pn + + if not cls.tinfoil: + cls.tinfoil = cls.setup_tinfoil() + + added_paths, modified_paths, removed_paths = [], [], [] + added, modified, removed = [], [], [] + + # get metadata filename additions, modification and removals + for patch in patchset: + if patch.path.endswith('.bb') or patch.path.endswith('.bbappend') or patch.path.endswith('.inc'): + if patch.is_added_file: + added_paths.append(os.path.join(os.path.abspath(PatchTestInput.repodir), patch.path)) + elif patch.is_modified_file: + modified_paths.append(os.path.join(os.path.abspath(PatchTestInput.repodir), patch.path)) + elif patch.is_removed_file: + removed_paths.append(os.path.join(os.path.abspath(PatchTestInput.repodir), patch.path)) + + data = cls.tinfoil.cooker.recipecaches[''].pkg_fn.items() + + added = [find_pn(data,path) for path in added_paths] + modified = [find_pn(data,path) for path in modified_paths] + removed = [find_pn(data,path) for path in removed_paths] + + return [a for a in added if a], [m for m in modified if m], [r for r in removed if r] diff --git a/meta/lib/patchtest/tests/pyparsing/common.py b/meta/lib/patchtest/tests/pyparsing/common.py new file mode 100644 index 0000000000..cbce4c38bc --- /dev/null +++ b/meta/lib/patchtest/tests/pyparsing/common.py @@ -0,0 +1,26 @@ +# common pyparsing variables +# +# Copyright (C) 2016 Intel Corporation +# +# SPDX-License-Identifier: GPL-2.0-only + +import pyparsing + +# general +colon = pyparsing.Literal(":") +start = pyparsing.LineStart() +end = pyparsing.LineEnd() +at = pyparsing.Literal("@") +lessthan = pyparsing.Literal("<") +greaterthan = pyparsing.Literal(">") +opensquare = pyparsing.Literal("[") +closesquare = pyparsing.Literal("]") +inappropriate = pyparsing.CaselessLiteral("Inappropriate") +submitted = pyparsing.CaselessLiteral("Submitted") + +# word related +nestexpr = pyparsing.nestedExpr(opener='[', closer=']') +inappropriateinfo = pyparsing.Literal("Inappropriate") + nestexpr +submittedinfo = pyparsing.Literal("Submitted") + nestexpr +word = pyparsing.Word(pyparsing.alphas) +worddot = pyparsing.Word(pyparsing.alphas+".") diff --git a/meta/lib/patchtest/tests/pyparsing/parse_cve_tags.py b/meta/lib/patchtest/tests/pyparsing/parse_cve_tags.py new file mode 100644 index 0000000000..f7fb82ec2b --- /dev/null +++ b/meta/lib/patchtest/tests/pyparsing/parse_cve_tags.py @@ -0,0 +1,18 @@ +# signed-off-by pyparsing definition +# +# Copyright (C) 2016 Intel Corporation +# +# SPDX-License-Identifier: GPL-2.0-only + + +import pyparsing +import common + +name = pyparsing.Regex('\S+.*(?= <)') +username = pyparsing.OneOrMore(common.worddot) +domain = pyparsing.OneOrMore(common.worddot) +cve = pyparsing.Regex('CVE\-\d{4}\-\d+') +cve_mark = pyparsing.Literal("CVE:") + +cve_tag = pyparsing.AtLineStart(cve_mark + cve) +patch_cve_tag = pyparsing.AtLineStart("+" + cve_mark + cve) diff --git a/meta/lib/patchtest/tests/pyparsing/parse_shortlog.py b/meta/lib/patchtest/tests/pyparsing/parse_shortlog.py new file mode 100644 index 0000000000..30d3ab35b3 --- /dev/null +++ b/meta/lib/patchtest/tests/pyparsing/parse_shortlog.py @@ -0,0 +1,14 @@ +# subject pyparsing definition +# +# Copyright (C) 2016 Intel Corporation +# +# SPDX-License-Identifier: GPL-2.0-only + +# NOTE:This is an oversimplified syntax of the mbox's summary + +import pyparsing +import common + +target = pyparsing.OneOrMore(pyparsing.Word(pyparsing.printables.replace(':',''))) +summary = pyparsing.OneOrMore(pyparsing.Word(pyparsing.printables)) +shortlog = common.start + target + common.colon + summary + common.end diff --git a/meta/lib/patchtest/tests/pyparsing/parse_signed_off_by.py b/meta/lib/patchtest/tests/pyparsing/parse_signed_off_by.py new file mode 100644 index 0000000000..692ebec3ff --- /dev/null +++ b/meta/lib/patchtest/tests/pyparsing/parse_signed_off_by.py @@ -0,0 +1,22 @@ +# signed-off-by pyparsing definition +# +# Copyright (C) 2016 Intel Corporation +# +# SPDX-License-Identifier: GPL-2.0-only + + +import pyparsing +import common + +name = pyparsing.Regex('\S+.*(?= <)') +username = pyparsing.OneOrMore(common.worddot) +domain = pyparsing.OneOrMore(common.worddot) + +# taken from https://pyparsing-public.wikispaces.com/Helpful+Expressions +email = pyparsing.Regex(r"(?P<user>[A-Za-z0-9._%+-]+)@(?P<hostname>[A-Za-z0-9.-]+)\.(?P<domain>[A-Za-z]{2,})") + +email_enclosed = common.lessthan + email + common.greaterthan + +signed_off_by_mark = pyparsing.Literal("Signed-off-by:") +signed_off_by = pyparsing.AtLineStart(signed_off_by_mark + name + email_enclosed) +patch_signed_off_by = pyparsing.AtLineStart("+" + signed_off_by_mark + name + email_enclosed) diff --git a/meta/lib/patchtest/tests/pyparsing/parse_upstream_status.py b/meta/lib/patchtest/tests/pyparsing/parse_upstream_status.py new file mode 100644 index 0000000000..bc6c427c4c --- /dev/null +++ b/meta/lib/patchtest/tests/pyparsing/parse_upstream_status.py @@ -0,0 +1,24 @@ +# upstream-status pyparsing definition +# +# Copyright (C) 2016 Intel Corporation +# +# SPDX-License-Identifier: GPL-2.0-only + + +import common +import pyparsing + +upstream_status_literal_valid_status = ["Pending", "Backport", "Denied", "Inappropriate", "Submitted"] +upstream_status_nonliteral_valid_status = ["Pending", "Backport", "Denied", "Inappropriate [reason]", "Submitted [where]"] + +upstream_status_valid_status = pyparsing.Or( + [pyparsing.Literal(status) for status in upstream_status_literal_valid_status] +) + +upstream_status_mark = pyparsing.Literal("Upstream-Status") +inappropriate_status_mark = common.inappropriate +submitted_status_mark = common.submitted + +upstream_status = common.start + upstream_status_mark + common.colon + upstream_status_valid_status +upstream_status_inappropriate_info = common.start + upstream_status_mark + common.colon + common.inappropriateinfo +upstream_status_submitted_info = common.start + upstream_status_mark + common.colon + common.submittedinfo diff --git a/meta/lib/patchtest/tests/test_mbox.py b/meta/lib/patchtest/tests/test_mbox.py new file mode 100644 index 0000000000..0b623b7d17 --- /dev/null +++ b/meta/lib/patchtest/tests/test_mbox.py @@ -0,0 +1,159 @@ +# Checks related to the patch's author +# +# Copyright (C) 2016 Intel Corporation +# +# SPDX-License-Identifier: GPL-2.0-only + +import base +import collections +import parse_shortlog +import parse_signed_off_by +import pyparsing +import subprocess +from data import PatchTestInput + +def headlog(): + output = subprocess.check_output( + "cd %s; git log --pretty='%%h#%%aN#%%cD:#%%s' -1" % PatchTestInput.repodir, + universal_newlines=True, + shell=True + ) + return output.split('#') + +class TestMbox(base.Base): + + auh_email = 'auh@auh.yoctoproject.org' + + invalids = [pyparsing.Regex("^Upgrade Helper.+"), + pyparsing.Regex(auh_email), + pyparsing.Regex("uh@not\.set"), + pyparsing.Regex("\S+@example\.com")] + + rexp_detect = pyparsing.Regex('\[\s?YOCTO.*\]') + rexp_validation = pyparsing.Regex('\[(\s?YOCTO\s?#\s?(\d+)\s?,?)+\]') + revert_shortlog_regex = pyparsing.Regex('Revert\s+".*"') + signoff_prog = parse_signed_off_by.signed_off_by + revert_shortlog_regex = pyparsing.Regex('Revert\s+".*"') + maxlength = 90 + + # base paths of main yocto project sub-projects + paths = { + 'oe-core': ['meta-selftest', 'meta-skeleton', 'meta', 'scripts'], + 'bitbake': ['bitbake'], + 'documentation': ['documentation'], + 'poky': ['meta-poky','meta-yocto-bsp'], + 'oe': ['meta-gpe', 'meta-gnome', 'meta-efl', 'meta-networking', 'meta-multimedia','meta-initramfs', 'meta-ruby', 'contrib', 'meta-xfce', 'meta-filesystems', 'meta-perl', 'meta-webserver', 'meta-systemd', 'meta-oe', 'meta-python'] + } + + # scripts folder is a mix of oe-core and poky, most is oe-core code except: + poky_scripts = ['scripts/yocto-bsp', 'scripts/yocto-kernel', 'scripts/yocto-layer', 'scripts/lib/bsp'] + + Project = collections.namedtuple('Project', ['name', 'listemail', 'gitrepo', 'paths']) + + bitbake = Project(name='Bitbake', listemail='bitbake-devel@lists.openembedded.org', gitrepo='http://git.openembedded.org/bitbake/', paths=paths['bitbake']) + doc = Project(name='Documentantion', listemail='yocto@yoctoproject.org', gitrepo='http://git.yoctoproject.org/cgit/cgit.cgi/yocto-docs/', paths=paths['documentation']) + poky = Project(name='Poky', listemail='poky@yoctoproject.org', gitrepo='http://git.yoctoproject.org/cgit/cgit.cgi/poky/', paths=paths['poky']) + oe = Project(name='oe', listemail='openembedded-devel@lists.openembedded.org', gitrepo='http://git.openembedded.org/meta-openembedded/', paths=paths['oe']) + + + def test_signed_off_by_presence(self): + for commit in TestMbox.commits: + # skip those patches that revert older commits, these do not required the tag presence + if self.revert_shortlog_regex.search_string(commit.shortlog): + continue + if not self.signoff_prog.search_string(commit.payload): + self.fail('Mbox is missing Signed-off-by. Add it manually or with "git commit --amend -s"', + commit=commit) + + def test_shortlog_format(self): + for commit in TestMbox.commits: + shortlog = commit.shortlog + if not shortlog.strip(): + self.skip('Empty shortlog, no reason to execute shortlog format test') + else: + # no reason to re-check on revert shortlogs + if shortlog.startswith('Revert "'): + continue + try: + parse_shortlog.shortlog.parseString(shortlog) + except pyparsing.ParseException as pe: + self.fail('Commit shortlog (first line of commit message) should follow the format "<target>: <summary>"', + commit=commit) + + def test_shortlog_length(self): + for commit in TestMbox.commits: + # no reason to re-check on revert shortlogs + shortlog = commit.shortlog + if shortlog.startswith('Revert "'): + continue + l = len(shortlog) + if l > self.maxlength: + self.fail('Edit shortlog so that it is %d characters or less (currently %d characters)' % (self.maxlength, l), + commit=commit) + + def test_series_merge_on_head(self): + self.skip("Merge test is disabled for now") + if PatchTestInput.repo.branch != "master": + self.skip("Skipping merge test since patch is not intended for master branch. Target detected is %s" % PatchTestInput.repo.branch) + if not PatchTestInput.repo.ismerged: + commithash, author, date, shortlog = headlog() + self.fail('Series does not apply on top of target branch %s' % PatchTestInput.repo.branch, + data=[('Targeted branch', '%s (currently at %s)' % (PatchTestInput.repo.branch, commithash))]) + + def test_target_mailing_list(self): + """In case of merge failure, check for other targeted projects""" + if PatchTestInput.repo.ismerged: + self.skip('Series merged, no reason to check other mailing lists') + + # a meta project may be indicted in the message subject, if this is the case, just fail + # TODO: there may be other project with no-meta prefix, we also need to detect these + project_regex = pyparsing.Regex("\[(?P<project>meta-.+)\]") + for commit in TestMbox.commits: + match = project_regex.search_string(commit.subject) + if match: + self.fail('Series sent to the wrong mailing list or some patches from the series correspond to different mailing lists', + commit=commit) + + for patch in self.patchset: + folders = patch.path.split('/') + base_path = folders[0] + for project in [self.bitbake, self.doc, self.oe, self.poky]: + if base_path in project.paths: + self.fail('Series sent to the wrong mailing list or some patches from the series correspond to different mailing lists', + data=[('Suggested ML', '%s [%s]' % (project.listemail, project.gitrepo)), + ('Patch\'s path:', patch.path)]) + + # check for poky's scripts code + if base_path.startswith('scripts'): + for poky_file in self.poky_scripts: + if patch.path.startswith(poky_file): + self.fail('Series sent to the wrong mailing list or some patches from the series correspond to different mailing lists', + data=[('Suggested ML', '%s [%s]' % (self.poky.listemail, self.poky.gitrepo)),('Patch\'s path:', patch.path)]) + + def test_mbox_format(self): + if self.unidiff_parse_error: + self.fail('Series has malformed diff lines. Create the series again using git-format-patch and ensure it applies using git am', + data=[('Diff line',self.unidiff_parse_error)]) + + def test_commit_message_presence(self): + for commit in TestMbox.commits: + if not commit.commit_message.strip(): + self.fail('Please include a commit message on your patch explaining the change', commit=commit) + + def test_bugzilla_entry_format(self): + for commit in TestMbox.commits: + if not self.rexp_detect.search_string(commit.commit_message): + self.skip("No bug ID found") + elif not self.rexp_validation.search_string(commit.commit_message): + self.fail('Bugzilla issue ID is not correctly formatted - specify it with format: "[YOCTO #<bugzilla ID>]"', commit=commit) + + def test_author_valid(self): + for commit in self.commits: + for invalid in self.invalids: + if invalid.search_string(commit.author): + self.fail('Invalid author %s. Resend the series with a valid patch author' % commit.author, commit=commit) + + def test_non_auh_upgrade(self): + for commit in self.commits: + if self.auh_email in commit.payload: + self.fail('Invalid author %s. Resend the series with a valid patch author' % self.auh_email, commit=commit) diff --git a/meta/lib/patchtest/tests/test_metadata.py b/meta/lib/patchtest/tests/test_metadata.py new file mode 100644 index 0000000000..f5dbcf01ed --- /dev/null +++ b/meta/lib/patchtest/tests/test_metadata.py @@ -0,0 +1,197 @@ +# Checks related to the patch's LIC_FILES_CHKSUM metadata variable +# +# Copyright (C) 2016 Intel Corporation +# +# SPDX-License-Identifier: GPL-2.0-only + +import base +import os +import pyparsing +from data import PatchTestInput, PatchTestDataStore + +class TestMetadata(base.Metadata): + metadata_lic = 'LICENSE' + invalid_license = 'PATCHTESTINVALID' + metadata_chksum = 'LIC_FILES_CHKSUM' + license_var = 'LICENSE' + closed = 'CLOSED' + lictag_re = pyparsing.AtLineStart("License-Update:") + lic_chksum_added = pyparsing.AtLineStart("+" + metadata_chksum) + lic_chksum_removed = pyparsing.AtLineStart("-" + metadata_chksum) + add_mark = pyparsing.Regex('\\+ ') + max_length = 200 + metadata_src_uri = 'SRC_URI' + md5sum = 'md5sum' + sha256sum = 'sha256sum' + git_regex = pyparsing.Regex('^git\\:\\/\\/.*') + metadata_summary = 'SUMMARY' + cve_check_ignore_var = 'CVE_CHECK_IGNORE' + cve_status_var = 'CVE_STATUS' + + def test_license_presence(self): + if not self.added: + self.skip('No added recipes, skipping test') + + # TODO: this is a workaround so we can parse the recipe not + # containing the LICENSE var: add some default license instead + # of INVALID into auto.conf, then remove this line at the end + auto_conf = os.path.join(os.environ.get('BUILDDIR'), 'conf', 'auto.conf') + open_flag = 'w' + if os.path.exists(auto_conf): + open_flag = 'a' + with open(auto_conf, open_flag) as fd: + for pn in self.added: + fd.write('LICENSE ??= "%s"\n' % self.invalid_license) + + no_license = False + for pn in self.added: + rd = self.tinfoil.parse_recipe(pn) + license = rd.getVar(self.metadata_lic) + if license == self.invalid_license: + no_license = True + break + + # remove auto.conf line or the file itself + if open_flag == 'w': + os.remove(auto_conf) + else: + fd = open(auto_conf, 'r') + lines = fd.readlines() + fd.close() + with open(auto_conf, 'w') as fd: + fd.write(''.join(lines[:-1])) + + if no_license: + self.fail('Recipe does not have the LICENSE field set.') + + def test_lic_files_chksum_presence(self): + if not self.added: + self.skip('No added recipes, skipping test') + + for pn in self.added: + rd = self.tinfoil.parse_recipe(pn) + pathname = rd.getVar('FILE') + # we are not interested in images + if '/images/' in pathname: + continue + lic_files_chksum = rd.getVar(self.metadata_chksum) + if rd.getVar(self.license_var) == self.closed: + continue + if not lic_files_chksum: + self.fail('%s is missing in newly added recipe' % self.metadata_chksum) + + def test_lic_files_chksum_modified_not_mentioned(self): + if not self.modified: + self.skip('No modified recipes, skipping test') + + for patch in self.patchset: + # for the moment, we are just interested in metadata + if patch.path.endswith('.patch'): + continue + payload = str(patch) + if (self.lic_chksum_added.search_string(payload) or self.lic_chksum_removed.search_string(payload)): + # if any patch on the series contain reference on the metadata, fail + for commit in self.commits: + if self.lictag_re.search_string(commit.commit_message): + break + else: + self.fail('LIC_FILES_CHKSUM changed without "License-Update:" tag and description in commit message') + + def test_max_line_length(self): + for patch in self.patchset: + # for the moment, we are just interested in metadata + if patch.path.endswith('.patch'): + continue + payload = str(patch) + for line in payload.splitlines(): + if self.add_mark.search_string(line): + current_line_length = len(line[1:]) + if current_line_length > self.max_length: + self.fail('Patch line too long (current length %s, maximum is %s)' % (current_line_length, self.max_length), + data=[('Patch', patch.path), ('Line', '%s ...' % line[0:80])]) + + def pretest_src_uri_left_files(self): + # these tests just make sense on patches that can be merged + if not PatchTestInput.repo.canbemerged: + self.skip('Patch cannot be merged') + if not self.modified: + self.skip('No modified recipes, skipping pretest') + + # get the proper metadata values + for pn in self.modified: + # we are not interested in images + if 'core-image' in pn: + continue + rd = self.tinfoil.parse_recipe(pn) + PatchTestDataStore['%s-%s-%s' % (self.shortid(), self.metadata_src_uri, pn)] = rd.getVar(self.metadata_src_uri) + + def test_src_uri_left_files(self): + # these tests just make sense on patches that can be merged + if not PatchTestInput.repo.canbemerged: + self.skip('Patch cannot be merged') + if not self.modified: + self.skip('No modified recipes, skipping pretest') + + # get the proper metadata values + for pn in self.modified: + # we are not interested in images + if 'core-image' in pn: + continue + rd = self.tinfoil.parse_recipe(pn) + PatchTestDataStore['%s-%s-%s' % (self.shortid(), self.metadata_src_uri, pn)] = rd.getVar(self.metadata_src_uri) + + for pn in self.modified: + pretest_src_uri = PatchTestDataStore['pre%s-%s-%s' % (self.shortid(), self.metadata_src_uri, pn)].split() + test_src_uri = PatchTestDataStore['%s-%s-%s' % (self.shortid(), self.metadata_src_uri, pn)].split() + + pretest_files = set([os.path.basename(patch) for patch in pretest_src_uri if patch.startswith('file://')]) + test_files = set([os.path.basename(patch) for patch in test_src_uri if patch.startswith('file://')]) + + # check if files were removed + if len(test_files) < len(pretest_files): + + # get removals from patchset + filesremoved_from_patchset = set() + for patch in self.patchset: + if patch.is_removed_file: + filesremoved_from_patchset.add(os.path.basename(patch.path)) + + # get the deleted files from the SRC_URI + filesremoved_from_usr_uri = pretest_files - test_files + + # finally, get those patches removed at SRC_URI and not removed from the patchset + # TODO: we are not taking into account renames, so test may raise false positives + not_removed = filesremoved_from_usr_uri - filesremoved_from_patchset + if not_removed: + self.fail('Patches not removed from tree. Remove them and amend the submitted mbox', + data=[('Patch', f) for f in not_removed]) + + def test_summary_presence(self): + if not self.added: + self.skip('No added recipes, skipping test') + + for pn in self.added: + # we are not interested in images + if 'core-image' in pn: + continue + rd = self.tinfoil.parse_recipe(pn) + summary = rd.getVar(self.metadata_summary) + + # "${PN} version ${PN}-${PR}" is the default, so fail if default + if summary.startswith('%s version' % pn): + self.fail('%s is missing in newly added recipe' % self.metadata_summary) + + def test_cve_check_ignore(self): + # Skip if we neither modified a recipe or target branches are not + # Nanbield and newer. CVE_CHECK_IGNORE was first deprecated in Nanbield. + if not self.modified or PatchTestInput.repo.branch == "kirkstone" or PatchTestInput.repo.branch == "dunfell": + self.skip('No modified recipes or older target branch, skipping test') + for pn in self.modified: + # we are not interested in images + if 'core-image' in pn: + continue + rd = self.tinfoil.parse_recipe(pn) + cve_check_ignore = rd.getVar(self.cve_check_ignore_var) + + if cve_check_ignore is not None: + self.fail('%s is deprecated and should be replaced by %s' % (self.cve_check_ignore_var, self.cve_status_var)) diff --git a/meta/lib/patchtest/tests/test_patch.py b/meta/lib/patchtest/tests/test_patch.py new file mode 100644 index 0000000000..d7187a0cb1 --- /dev/null +++ b/meta/lib/patchtest/tests/test_patch.py @@ -0,0 +1,103 @@ +# Checks related to the patch's CVE lines +# +# Copyright (C) 2016 Intel Corporation +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import base +import os +import parse_signed_off_by +import parse_upstream_status +import pyparsing + +class TestPatch(base.Base): + + re_cve_pattern = pyparsing.Regex("CVE\-\d{4}\-\d+") + re_cve_payload_tag = pyparsing.Regex("\+CVE:(\s+CVE\-\d{4}\-\d+)+") + upstream_status_regex = pyparsing.AtLineStart("+" + "Upstream-Status") + + @classmethod + def setUpClassLocal(cls): + cls.newpatches = [] + # get just those relevant patches: new software patches + for patch in cls.patchset: + if patch.path.endswith('.patch') and patch.is_added_file: + cls.newpatches.append(patch) + + cls.mark = str(parse_signed_off_by.signed_off_by_mark).strip('"') + + # match PatchSignedOffBy.mark with '+' preceding it + cls.prog = parse_signed_off_by.patch_signed_off_by + + def setUp(self): + if self.unidiff_parse_error: + self.skip('Parse error %s' % self.unidiff_parse_error) + + self.valid_status = ', '.join(parse_upstream_status.upstream_status_nonliteral_valid_status) + self.standard_format = 'Upstream-Status: <Valid status>' + + # we are just interested in series that introduce CVE patches, thus discard other + # possibilities: modification to current CVEs, patch directly introduced into the + # recipe, upgrades already including the CVE, etc. + new_cves = [p for p in self.patchset if p.path.endswith('.patch') and p.is_added_file] + if not new_cves: + self.skip('No new CVE patches introduced') + + def test_upstream_status_presence_format(self): + if not TestPatch.newpatches: + self.skip("There are no new software patches, no reason to test Upstream-Status presence/format") + + for newpatch in TestPatch.newpatches: + payload = newpatch.__str__() + if not self.upstream_status_regex.search_string(payload): + self.fail('Added patch file is missing Upstream-Status: <Valid status> in the commit message', + data=[('Standard format', self.standard_format), ('Valid status', self.valid_status)]) + for line in payload.splitlines(): + if self.patchmetadata_regex.match(line): + continue + if self.upstream_status_regex.search_string(line): + if parse_upstream_status.inappropriate_status_mark.searchString(line): + try: + parse_upstream_status.upstream_status_inappropriate_info.parseString(line.lstrip('+')) + except pyparsing.ParseException as pe: + self.fail('Upstream-Status is Inappropriate, but no reason was provided', + data=[('Current', pe.pstr), ('Standard format', 'Upstream-Status: Inappropriate [reason]')]) + elif parse_upstream_status.submitted_status_mark.searchString(line): + try: + parse_upstream_status.upstream_status_submitted_info.parseString(line.lstrip('+')) + except pyparsing.ParseException as pe: + self.fail('Upstream-Status is Submitted, but it is not mentioned where', + data=[('Current', pe.pstr), ('Standard format', 'Upstream-Status: Submitted [where]')]) + else: + try: + parse_upstream_status.upstream_status.parseString(line.lstrip('+')) + except pyparsing.ParseException as pe: + self.fail('Upstream-Status is in incorrect format', + data=[('Current', pe.pstr), ('Standard format', self.standard_format), ('Valid status', self.valid_status)]) + + def test_signed_off_by_presence(self): + if not TestPatch.newpatches: + self.skip("There are no new software patches, no reason to test %s presence" % PatchSignedOffBy.mark) + + for newpatch in TestPatch.newpatches: + payload = newpatch.__str__() + for line in payload.splitlines(): + if self.patchmetadata_regex.match(line): + continue + if TestPatch.prog.search_string(payload): + break + else: + self.fail('A patch file has been added without a Signed-off-by tag: \'%s\'' % os.path.basename(newpatch.path)) + + def test_cve_tag_format(self): + for commit in TestPatch.commits: + if self.re_cve_pattern.search_string(commit.shortlog) or self.re_cve_pattern.search_string(commit.commit_message): + tag_found = False + for line in commit.payload.splitlines(): + if self.re_cve_payload_tag.search_string(line): + tag_found = True + break + if not tag_found: + self.fail('Missing or incorrectly formatted CVE tag in patch file. Correct or include the CVE tag in the patch with format: "CVE: CVE-YYYY-XXXX"', + commit=commit) diff --git a/meta/lib/patchtest/tests/test_python_pylint.py b/meta/lib/patchtest/tests/test_python_pylint.py new file mode 100644 index 0000000000..ef315e591c --- /dev/null +++ b/meta/lib/patchtest/tests/test_python_pylint.py @@ -0,0 +1,65 @@ +# Checks related to the python code done with pylint +# +# Copyright (C) 2016 Intel Corporation +# +# SPDX-License-Identifier: GPL-2.0-only + +import base +from io import StringIO +from data import PatchTestInput +from pylint.reporters.text import TextReporter +import pylint.lint as lint + + +class PyLint(base.Base): + pythonpatches = [] + pylint_pretest = {} + pylint_test = {} + pylint_options = " -E --disable='E0611, E1101, F0401, E0602' --msg-template='L:{line} F:{module} I:{msg}'" + + @classmethod + def setUpClassLocal(cls): + # get just those patches touching python files + cls.pythonpatches = [] + for patch in cls.patchset: + if patch.path.endswith('.py'): + if not patch.is_removed_file: + cls.pythonpatches.append(patch) + + def setUp(self): + if self.unidiff_parse_error: + self.skip('Python-unidiff parse error') + if not PyLint.pythonpatches: + self.skip('No python related patches, skipping test') + + def pretest_pylint(self): + for pythonpatch in self.pythonpatches: + if pythonpatch.is_modified_file: + pylint_output = StringIO() + reporter = TextReporter(pylint_output) + lint.Run([self.pylint_options, pythonpatch.path], reporter=reporter, exit=False) + for line in pylint_output.readlines(): + if not '*' in line: + if line.strip(): + self.pylint_pretest[line.strip().split(' ',1)[0]] = line.strip().split(' ',1)[1] + + def test_pylint(self): + for pythonpatch in self.pythonpatches: + # a condition checking whether a file is renamed or not + # unidiff doesn't support this yet + if pythonpatch.target_file is not pythonpatch.path: + path = pythonpatch.target_file[2:] + else: + path = pythonpatch.path + pylint_output = StringIO() + reporter = TextReporter(pylint_output) + lint.Run([self.pylint_options, pythonpatch.path], reporter=reporter, exit=False) + for line in pylint_output.readlines(): + if not '*' in line: + if line.strip(): + self.pylint_test[line.strip().split(' ',1)[0]] = line.strip().split(' ',1)[1] + + for issue in self.pylint_test: + if self.pylint_test[issue] not in self.pylint_pretest.values(): + self.fail('Errors in your Python code were encountered. Please check your code with a linter and resubmit', + data=[('Output', 'Please, fix the listed issues:'), ('', issue + ' ' + self.pylint_test[issue])]) diff --git a/meta/lib/patchtest/utils.py b/meta/lib/patchtest/utils.py new file mode 100644 index 0000000000..dd0abc22d9 --- /dev/null +++ b/meta/lib/patchtest/utils.py @@ -0,0 +1,168 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# utils: common methods used by the patchtest framework +# +# Copyright (C) 2016 Intel Corporation +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import os +import subprocess +import logging +import re +import mailbox + +class CmdException(Exception): + """ Simple exception class where its attributes are the ones passed when instantiated """ + def __init__(self, cmd): + self._cmd = cmd + def __getattr__(self, name): + value = None + if self._cmd.has_key(name): + value = self._cmd[name] + return value + +def exec_cmd(cmd, cwd, ignore_error=False, input=None, strip=True, updateenv={}): + """ + Input: + + cmd: dict containing the following keys: + + cmd : the command itself as an array of strings + ignore_error: if False, no exception is raised + strip: indicates if strip is done on the output (stdout and stderr) + input: input data to the command (stdin) + updateenv: environment variables to be appended to the current + process environment variables + + NOTE: keys 'ignore_error' and 'input' are optional; if not included, + the defaults are the ones specify in the arguments + cwd: directory where commands are executed + ignore_error: raise CmdException if command fails to execute and + this value is False + input: input data (stdin) for the command + + Output: dict containing the following keys: + + cmd: the same as input + ignore_error: the same as input + strip: the same as input + input: the same as input + stdout: Standard output after command's execution + stderr: Standard error after command's execution + returncode: Return code after command's execution + + """ + cmddefaults = { + 'cmd':'', + 'ignore_error':ignore_error, + 'strip':strip, + 'input':input, + 'updateenv':updateenv, + } + + # update input values if necessary + cmddefaults.update(cmd) + + _cmd = cmddefaults + + if not _cmd['cmd']: + raise CmdException({'cmd':None, 'stderr':'no command given'}) + + # update the environment + env = os.environ + env.update(_cmd['updateenv']) + + _command = [e for e in _cmd['cmd']] + p = subprocess.Popen(_command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + cwd=cwd, + env=env) + + # execute the command and strip output + (_stdout, _stderr) = p.communicate(_cmd['input']) + if _cmd['strip']: + _stdout, _stderr = map(str.strip, [_stdout, _stderr]) + + # generate the result + result = _cmd + result.update({'cmd':_command,'stdout':_stdout,'stderr':_stderr,'returncode':p.returncode}) + + # launch exception if necessary + if not _cmd['ignore_error'] and p.returncode: + raise CmdException(result) + + return result + +def exec_cmds(cmds, cwd): + """ Executes commands + + Input: + cmds: Array of commands + cwd: directory where commands are executed + + Output: Array of output commands + """ + results = [] + _cmds = cmds + + for cmd in _cmds: + result = exec_cmd(cmd, cwd) + results.append(result) + + return results + +def logger_create(name): + logger = logging.getLogger(name) + loggerhandler = logging.StreamHandler() + loggerhandler.setFormatter(logging.Formatter("%(message)s")) + logger.addHandler(loggerhandler) + logger.setLevel(logging.INFO) + return logger + +def get_subject_prefix(path): + prefix = "" + mbox = mailbox.mbox(path) + + if len(mbox): + subject = mbox[0]['subject'] + if subject: + pattern = re.compile(r"(\[.*\])", re.DOTALL) + match = pattern.search(subject) + if match: + prefix = match.group(1) + + return prefix + +def valid_branch(branch): + """ Check if branch is valid name """ + lbranch = branch.lower() + + invalid = lbranch.startswith('patch') or \ + lbranch.startswith('rfc') or \ + lbranch.startswith('resend') or \ + re.search(r'^v\d+', lbranch) or \ + re.search(r'^\d+/\d+', lbranch) + + return not invalid + +def get_branch(path): + """ Get the branch name from mbox """ + fullprefix = get_subject_prefix(path) + branch, branches, valid_branches = None, [], [] + + if fullprefix: + prefix = fullprefix.strip('[]') + branches = [ b.strip() for b in prefix.split(',')] + valid_branches = [b for b in branches if valid_branch(b)] + + if len(valid_branches): + branch = valid_branches[0] + + return branch + diff --git a/meta/lib/rootfspostcommands.py b/meta/lib/rootfspostcommands.py index fdb9f5b850..5386eea409 100644 --- a/meta/lib/rootfspostcommands.py +++ b/meta/lib/rootfspostcommands.py @@ -1,16 +1,19 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # import os -def sort_file(filename, mapping): +def sort_shadowutils_file(filename, mapping): """ Sorts a passwd or group file based on the numeric ID in the third column. If a mapping is given, the name from the first column is mapped via that dictionary instead (necessary for /etc/shadow and /etc/gshadow). If not, a new mapping is created on the fly and returned. """ + new_mapping = {} with open(filename, 'rb+') as f: lines = f.readlines() @@ -31,30 +34,57 @@ def sort_file(filename, mapping): # We overwrite the entire file, i.e. no truncate() necessary. f.seek(0) f.write(b''.join(lines)) + return new_mapping -def remove_backup(filename): +def sort_shadowutils_files(sysconfdir): """ - Removes the backup file for files like /etc/passwd. + Sorts shadow-utils 'passwd' and 'group' files in a rootfs' /etc directory + by ID. """ - backup_filename = filename + '-' - if os.path.exists(backup_filename): - os.unlink(backup_filename) -def sort_passwd(sysconfdir): - """ - Sorts passwd and group files in a rootfs /etc directory by ID. - Backup files are sometimes are inconsistent and then cannot be - sorted (YOCTO #11043), and more importantly, are not needed in - the initial rootfs, so they get deleted. - """ for main, shadow in (('passwd', 'shadow'), ('group', 'gshadow')): filename = os.path.join(sysconfdir, main) - remove_backup(filename) if os.path.exists(filename): - mapping = sort_file(filename, None) + mapping = sort_shadowutils_file(filename, None) filename = os.path.join(sysconfdir, shadow) - remove_backup(filename) if os.path.exists(filename): - sort_file(filename, mapping) + sort_shadowutils_file(filename, mapping) + +def remove_shadowutils_backup_file(filename): + """ + Remove shadow-utils backup file for files like /etc/passwd. + """ + + backup_filename = filename + '-' + if os.path.exists(backup_filename): + os.unlink(backup_filename) + +def remove_shadowutils_backup_files(sysconfdir): + """ + Remove shadow-utils backup files in a rootfs /etc directory. They are not + needed in the initial root filesystem and sorting them can be inconsistent + (YOCTO #11043). + """ + + for filename in ( + 'group', + 'gshadow', + 'passwd', + 'shadow', + 'subgid', + 'subuid', + ): + filepath = os.path.join(sysconfdir, filename) + remove_shadowutils_backup_file(filepath) + +def tidy_shadowutils_files(sysconfdir): + """ + Tidy up shadow-utils files. + """ + + remove_shadowutils_backup_files(sysconfdir) + sort_shadowutils_files(sysconfdir) + + return True |