diff options
Diffstat (limited to 'scripts/lib')
52 files changed, 5444 insertions, 1108 deletions
diff --git a/scripts/lib/argparse_oe.py b/scripts/lib/argparse_oe.py index 94a4ac5011..176b732bbc 100644 --- a/scripts/lib/argparse_oe.py +++ b/scripts/lib/argparse_oe.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/scripts/lib/build_perf/report.py b/scripts/lib/build_perf/report.py index 4e8e2a8a93..ab77424cc7 100644 --- a/scripts/lib/build_perf/report.py +++ b/scripts/lib/build_perf/report.py @@ -4,7 +4,8 @@ # SPDX-License-Identifier: GPL-2.0-only # """Handling of build perf test reports""" -from collections import OrderedDict, Mapping, namedtuple +from collections import OrderedDict, namedtuple +from collections.abc import Mapping from datetime import datetime, timezone from numbers import Number from statistics import mean, stdev, variance diff --git a/scripts/lib/buildstats.py b/scripts/lib/buildstats.py index c69b5bf4d7..6db60d5bcf 100644 --- a/scripts/lib/buildstats.py +++ b/scripts/lib/buildstats.py @@ -8,7 +8,7 @@ import json import logging import os import re -from collections import namedtuple,OrderedDict +from collections import namedtuple from statistics import mean @@ -79,8 +79,8 @@ class BSTask(dict): return self['rusage']['ru_oublock'] @classmethod - def from_file(cls, buildstat_file): - """Read buildstat text file""" + def from_file(cls, buildstat_file, fallback_end=0): + """Read buildstat text file. fallback_end is an optional end time for tasks that are not recorded as finishing.""" bs_task = cls() log.debug("Reading task buildstats from %s", buildstat_file) end_time = None @@ -108,7 +108,10 @@ class BSTask(dict): bs_task[ru_type][ru_key] = val elif key == 'Status': bs_task['status'] = val - if end_time is not None and start_time is not None: + # If the task didn't finish, fill in the fallback end time if specified + if start_time and not end_time and fallback_end: + end_time = fallback_end + if start_time and end_time: bs_task['elapsed_time'] = end_time - start_time else: raise BSError("{} looks like a invalid buildstats file".format(buildstat_file)) @@ -226,25 +229,44 @@ class BuildStats(dict): epoch = match.group('epoch') return name, epoch, version, revision + @staticmethod + def parse_top_build_stats(path): + """ + Parse the top-level build_stats file for build-wide start and duration. + """ + start = elapsed = 0 + with open(path) as fobj: + for line in fobj.readlines(): + key, val = line.split(':', 1) + val = val.strip() + if key == 'Build Started': + start = float(val) + elif key == "Elapsed time": + elapsed = float(val.split()[0]) + return start, elapsed + @classmethod def from_dir(cls, path): """Load buildstats from a buildstats directory""" - if not os.path.isfile(os.path.join(path, 'build_stats')): + top_stats = os.path.join(path, 'build_stats') + if not os.path.isfile(top_stats): raise BSError("{} does not look like a buildstats directory".format(path)) log.debug("Reading buildstats directory %s", path) - buildstats = cls() + build_started, build_elapsed = buildstats.parse_top_build_stats(top_stats) + build_end = build_started + build_elapsed + subdirs = os.listdir(path) for dirname in subdirs: recipe_dir = os.path.join(path, dirname) - if not os.path.isdir(recipe_dir): + if dirname == "reduced_proc_pressure" or not os.path.isdir(recipe_dir): continue name, epoch, version, revision = cls.split_nevr(dirname) bsrecipe = BSRecipe(name, epoch, version, revision) for task in os.listdir(recipe_dir): bsrecipe.tasks[task] = BSTask.from_file( - os.path.join(recipe_dir, task)) + os.path.join(recipe_dir, task), build_end) if name in buildstats: raise BSError("Cannot handle multiple versions of the same " "package ({})".format(name)) diff --git a/scripts/lib/checklayer/__init__.py b/scripts/lib/checklayer/__init__.py index fe545607bb..62ecdfe390 100644 --- a/scripts/lib/checklayer/__init__.py +++ b/scripts/lib/checklayer/__init__.py @@ -16,6 +16,7 @@ class LayerType(Enum): BSP = 0 DISTRO = 1 SOFTWARE = 2 + CORE = 3 ERROR_NO_LAYER_CONF = 98 ERROR_BSP_DISTRO = 99 @@ -43,7 +44,7 @@ def _get_layer_collections(layer_path, lconf=None, data=None): ldata.setVar('LAYERDIR', layer_path) try: - ldata = bb.parse.handle(lconf, ldata, include=True) + ldata = bb.parse.handle(lconf, ldata, include=True, baseconfig=True) except: raise RuntimeError("Parsing of layer.conf from layer: %s failed" % layer_path) ldata.expandVarref('LAYERDIR') @@ -106,7 +107,13 @@ def _detect_layer(layer_path): if distros: is_distro = True - if is_bsp and is_distro: + layer['collections'] = _get_layer_collections(layer['path']) + + if layer_name == "meta" and "core" in layer['collections']: + layer['type'] = LayerType.CORE + layer['conf']['machines'] = machines + layer['conf']['distros'] = distros + elif is_bsp and is_distro: layer['type'] = LayerType.ERROR_BSP_DISTRO elif is_bsp: layer['type'] = LayerType.BSP @@ -117,8 +124,6 @@ def _detect_layer(layer_path): else: layer['type'] = LayerType.SOFTWARE - layer['collections'] = _get_layer_collections(layer['path']) - return layer def detect_layers(layer_directories, no_auto): @@ -146,7 +151,7 @@ def detect_layers(layer_directories, no_auto): return layers -def _find_layer_depends(depend, layers): +def _find_layer(depend, layers): for layer in layers: if 'collections' not in layer: continue @@ -156,7 +161,28 @@ def _find_layer_depends(depend, layers): return layer return None -def add_layer_dependencies(bblayersconf, layer, layers, logger): +def sanity_check_layers(layers, logger): + """ + Check that we didn't find duplicate collection names, as the layer that will + be used is non-deterministic. The precise check is duplicate collections + with different patterns, as the same pattern being repeated won't cause + problems. + """ + import collections + + passed = True + seen = collections.defaultdict(set) + for layer in layers: + for name, data in layer.get("collections", {}).items(): + seen[name].add(data["pattern"]) + + for name, patterns in seen.items(): + if len(patterns) > 1: + passed = False + logger.error("Collection %s found multiple times: %s" % (name, ", ".join(patterns))) + return passed + +def get_layer_dependencies(layer, layers, logger): def recurse_dependencies(depends, layer, layers, logger, ret = []): logger.debug('Processing dependencies %s for layer %s.' % \ (depends, layer['name'])) @@ -166,7 +192,7 @@ def add_layer_dependencies(bblayersconf, layer, layers, logger): if depend == 'core': continue - layer_depend = _find_layer_depends(depend, layers) + layer_depend = _find_layer(depend, layers) if not layer_depend: logger.error('Layer %s depends on %s and isn\'t found.' % \ (layer['name'], depend)) @@ -203,6 +229,11 @@ def add_layer_dependencies(bblayersconf, layer, layers, logger): layer_depends = recurse_dependencies(depends, layer, layers, logger, layer_depends) # Note: [] (empty) is allowed, None is not! + return layer_depends + +def add_layer_dependencies(bblayersconf, layer, layers, logger): + + layer_depends = get_layer_dependencies(layer, layers, logger) if layer_depends is None: return False else: @@ -256,7 +287,7 @@ def check_command(error_msg, cmd, cwd=None): raise RuntimeError(msg) return output -def get_signatures(builddir, failsafe=False, machine=None): +def get_signatures(builddir, failsafe=False, machine=None, extravars=None): import re # some recipes needs to be excluded like meta-world-pkgdata @@ -267,13 +298,16 @@ def get_signatures(builddir, failsafe=False, machine=None): sigs = {} tune2tasks = {} - cmd = 'BB_ENV_EXTRAWHITE="$BB_ENV_EXTRAWHITE BB_SIGNATURE_HANDLER" BB_SIGNATURE_HANDLER="OEBasicHash" ' + cmd = 'BB_ENV_PASSTHROUGH_ADDITIONS="$BB_ENV_PASSTHROUGH_ADDITIONS BB_SIGNATURE_HANDLER" BB_SIGNATURE_HANDLER="OEBasicHash" ' + if extravars: + cmd += extravars + cmd += ' ' if machine: cmd += 'MACHINE=%s ' % machine cmd += 'bitbake ' if failsafe: cmd += '-k ' - cmd += '-S none world' + cmd += '-S lockedsigs world' sigs_file = os.path.join(builddir, 'locked-sigs.inc') if os.path.exists(sigs_file): os.unlink(sigs_file) @@ -290,8 +324,8 @@ def get_signatures(builddir, failsafe=False, machine=None): else: raise - sig_regex = re.compile("^(?P<task>.*:.*):(?P<hash>.*) .$") - tune_regex = re.compile("(^|\s)SIGGEN_LOCKEDSIGS_t-(?P<tune>\S*)\s*=\s*") + sig_regex = re.compile(r"^(?P<task>.*:.*):(?P<hash>.*) .$") + tune_regex = re.compile(r"(^|\s)SIGGEN_LOCKEDSIGS_t-(?P<tune>\S*)\s*=\s*") current_tune = None with open(sigs_file, 'r') as f: for line in f.readlines(): diff --git a/scripts/lib/checklayer/cases/bsp.py b/scripts/lib/checklayer/cases/bsp.py index 7fd56f5d36..b76163fb56 100644 --- a/scripts/lib/checklayer/cases/bsp.py +++ b/scripts/lib/checklayer/cases/bsp.py @@ -11,7 +11,7 @@ from checklayer.case import OECheckLayerTestCase class BSPCheckLayer(OECheckLayerTestCase): @classmethod def setUpClass(self): - if self.tc.layer['type'] != LayerType.BSP: + if self.tc.layer['type'] not in (LayerType.BSP, LayerType.CORE): raise unittest.SkipTest("BSPCheckLayer: Layer %s isn't BSP one." %\ self.tc.layer['name']) @@ -153,7 +153,7 @@ class BSPCheckLayer(OECheckLayerTestCase): # do_build can be ignored: it is know to have # different signatures in some cases, for example in # the allarch ca-certificates due to RDEPENDS=openssl. - # That particular dependency is whitelisted via + # That particular dependency is marked via # SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS, but still shows up # in the sstate signature hash because filtering it # out would be hard and running do_build multiple diff --git a/scripts/lib/checklayer/cases/common.py b/scripts/lib/checklayer/cases/common.py index b82304e361..97b16f78c8 100644 --- a/scripts/lib/checklayer/cases/common.py +++ b/scripts/lib/checklayer/cases/common.py @@ -6,15 +6,19 @@ import glob import os import unittest +import re from checklayer import get_signatures, LayerType, check_command, get_depgraph, compare_signatures from checklayer.case import OECheckLayerTestCase class CommonCheckLayer(OECheckLayerTestCase): def test_readme(self): + if self.tc.layer['type'] == LayerType.CORE: + raise unittest.SkipTest("Core layer's README is top level") + # The top-level README file may have a suffix (like README.rst or README.txt). readme_files = glob.glob(os.path.join(self.tc.layer['path'], '[Rr][Ee][Aa][Dd][Mm][Ee]*')) self.assertTrue(len(readme_files) > 0, - msg="Layer doesn't contains README file.") + msg="Layer doesn't contain a README file.") # There might be more than one file matching the file pattern above # (for example, README.rst and README-COPYING.rst). The one with the shortest @@ -26,6 +30,16 @@ class CommonCheckLayer(OECheckLayerTestCase): self.assertTrue(data, msg="Layer contains a README file but it is empty.") + # If a layer's README references another README, then the checks below are not valid + if re.search('README', data, re.IGNORECASE): + return + + self.assertIn('maintainer', data.lower()) + self.assertIn('patch', data.lower()) + # Check that there is an email address in the README + email_regex = re.compile(r"[^@]+@[^@]+") + self.assertTrue(email_regex.match(data)) + def test_parse(self): check_command('Layer %s failed to parse.' % self.tc.layer['name'], 'bitbake -p') @@ -43,6 +57,36 @@ class CommonCheckLayer(OECheckLayerTestCase): ''' get_signatures(self.td['builddir'], failsafe=False) + def test_world_inherit_class(self): + ''' + This also does "bitbake -S none world" along with inheriting "yocto-check-layer" + class, which can do additional per-recipe test cases. + ''' + msg = [] + try: + get_signatures(self.td['builddir'], failsafe=False, machine=None, extravars='BB_ENV_PASSTHROUGH_ADDITIONS="$BB_ENV_PASSTHROUGH_ADDITIONS INHERIT" INHERIT="yocto-check-layer"') + except RuntimeError as ex: + msg.append(str(ex)) + if msg: + msg.insert(0, 'Layer %s failed additional checks from yocto-check-layer.bbclass\nSee below log for specific recipe parsing errors:\n' % \ + self.tc.layer['name']) + self.fail('\n'.join(msg)) + + @unittest.expectedFailure + def test_patches_upstream_status(self): + import sys + sys.path.append(os.path.join(sys.path[0], '../../../../meta/lib/')) + import oe.qa + patches = [] + for dirpath, dirs, files in os.walk(self.tc.layer['path']): + for filename in files: + if filename.endswith(".patch"): + ppath = os.path.join(dirpath, filename) + if oe.qa.check_upstream_status(ppath): + patches.append(ppath) + self.assertEqual(len(patches), 0 , \ + msg="Found following patches with malformed or missing upstream status:\n%s" % '\n'.join([str(patch) for patch in patches])) + def test_signatures(self): if self.tc.layer['type'] == LayerType.SOFTWARE and \ not self.tc.test_software_layer_signatures: diff --git a/scripts/lib/checklayer/cases/distro.py b/scripts/lib/checklayer/cases/distro.py index f0bee5493c..a35332451c 100644 --- a/scripts/lib/checklayer/cases/distro.py +++ b/scripts/lib/checklayer/cases/distro.py @@ -11,7 +11,7 @@ from checklayer.case import OECheckLayerTestCase class DistroCheckLayer(OECheckLayerTestCase): @classmethod def setUpClass(self): - if self.tc.layer['type'] != LayerType.DISTRO: + if self.tc.layer['type'] not in (LayerType.DISTRO, LayerType.CORE): raise unittest.SkipTest("DistroCheckLayer: Layer %s isn't Distro one." %\ self.tc.layer['name']) diff --git a/scripts/lib/devtool/__init__.py b/scripts/lib/devtool/__init__.py index 702db669de..6133c1c5b4 100644 --- a/scripts/lib/devtool/__init__.py +++ b/scripts/lib/devtool/__init__.py @@ -78,12 +78,15 @@ def exec_fakeroot(d, cmd, **kwargs): """Run a command under fakeroot (pseudo, in fact) so that it picks up the appropriate file permissions""" # Grab the command and check it actually exists fakerootcmd = d.getVar('FAKEROOTCMD') + fakerootenv = d.getVar('FAKEROOTENV') + exec_fakeroot_no_d(fakerootcmd, fakerootenv, cmd, kwargs) + +def exec_fakeroot_no_d(fakerootcmd, fakerootenv, cmd, **kwargs): if not os.path.exists(fakerootcmd): logger.error('pseudo executable %s could not be found - have you run a build yet? pseudo-native should install this and if you have run any build then that should have been built') return 2 # Set up the appropriate environment newenv = dict(os.environ) - fakerootenv = d.getVar('FAKEROOTENV') for varvalue in fakerootenv.split(): if '=' in varvalue: splitval = varvalue.split('=', 1) @@ -233,6 +236,28 @@ def setup_git_repo(repodir, version, devbranch, basetag='devtool-base', d=None): bb.process.run('git checkout -b %s' % devbranch, cwd=repodir) bb.process.run('git tag -f %s' % basetag, cwd=repodir) + # if recipe unpacks another git repo inside S, we need to declare it as a regular git submodule now, + # so we will be able to tag branches on it and extract patches when doing finish/update on the recipe + stdout, _ = bb.process.run("git status --porcelain", cwd=repodir) + found = False + for line in stdout.splitlines(): + if line.endswith("/"): + new_dir = line.split()[1] + for root, dirs, files in os.walk(os.path.join(repodir, new_dir)): + if ".git" in dirs + files: + (stdout, _) = bb.process.run('git remote', cwd=root) + remote = stdout.splitlines()[0] + (stdout, _) = bb.process.run('git remote get-url %s' % remote, cwd=root) + remote_url = stdout.splitlines()[0] + logger.error(os.path.relpath(os.path.join(root, ".."), root)) + bb.process.run('git submodule add %s %s' % (remote_url, os.path.relpath(root, os.path.join(root, ".."))), cwd=os.path.join(root, "..")) + found = True + if found: + oe.patch.GitApplyTree.commitIgnored("Add additional submodule from SRC_URI", dir=os.path.join(root, ".."), d=d) + found = False + if os.path.exists(os.path.join(repodir, '.gitmodules')): + bb.process.run('git submodule foreach --recursive "git tag -f %s"' % basetag, cwd=repodir) + def recipe_to_append(recipefile, config, wildcard=False): """ Convert a recipe file to a bbappend file path within the workspace. diff --git a/scripts/lib/devtool/build_image.py b/scripts/lib/devtool/build_image.py index 9388abbacf..980f90ddd6 100644 --- a/scripts/lib/devtool/build_image.py +++ b/scripts/lib/devtool/build_image.py @@ -113,7 +113,7 @@ def build_image_task(config, basepath, workspace, image, add_packages=None, task with open(appendfile, 'w') as afile: if packages: # include packages from workspace recipes into the image - afile.write('IMAGE_INSTALL_append = " %s"\n' % ' '.join(packages)) + afile.write('IMAGE_INSTALL:append = " %s"\n' % ' '.join(packages)) if not task: logger.info('Building image %s with the following ' 'additional packages: %s', image, ' '.join(packages)) diff --git a/scripts/lib/devtool/build_sdk.py b/scripts/lib/devtool/build_sdk.py index 6fe02fff2a..1cd4831d2b 100644 --- a/scripts/lib/devtool/build_sdk.py +++ b/scripts/lib/devtool/build_sdk.py @@ -13,7 +13,7 @@ import shutil import errno import sys import tempfile -from devtool import exec_build_env_command, setup_tinfoil, parse_recipe, DevtoolError +from devtool import DevtoolError from devtool import build_image logger = logging.getLogger('devtool') diff --git a/scripts/lib/devtool/deploy.py b/scripts/lib/devtool/deploy.py index e5af2c95ae..b5ca8f2c2f 100644 --- a/scripts/lib/devtool/deploy.py +++ b/scripts/lib/devtool/deploy.py @@ -16,7 +16,7 @@ import bb.utils import argparse_oe import oe.types -from devtool import exec_fakeroot, setup_tinfoil, check_workspace_recipe, DevtoolError +from devtool import exec_fakeroot_no_d, setup_tinfoil, check_workspace_recipe, DevtoolError logger = logging.getLogger('devtool') @@ -133,16 +133,38 @@ def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=Fals return '\n'.join(lines) - - def deploy(args, config, basepath, workspace): """Entry point for the devtool 'deploy' subcommand""" - import math - import oe.recipeutils - import oe.package + import oe.utils check_workspace_recipe(workspace, args.recipename, checksrc=False) + tinfoil = setup_tinfoil(basepath=basepath) + try: + try: + rd = tinfoil.parse_recipe(args.recipename) + except Exception as e: + raise DevtoolError('Exception parsing recipe %s: %s' % + (args.recipename, e)) + + srcdir = rd.getVar('D') + workdir = rd.getVar('WORKDIR') + path = rd.getVar('PATH') + strip_cmd = rd.getVar('STRIP') + libdir = rd.getVar('libdir') + base_libdir = rd.getVar('base_libdir') + max_process = oe.utils.get_bb_number_threads(rd) + fakerootcmd = rd.getVar('FAKEROOTCMD') + fakerootenv = rd.getVar('FAKEROOTENV') + finally: + tinfoil.shutdown() + + return deploy_no_d(srcdir, workdir, path, strip_cmd, libdir, base_libdir, max_process, fakerootcmd, fakerootenv, args) + +def deploy_no_d(srcdir, workdir, path, strip_cmd, libdir, base_libdir, max_process, fakerootcmd, fakerootenv, args): + import math + import oe.package + try: host, destdir = args.target.split(':') except ValueError: @@ -152,118 +174,108 @@ def deploy(args, config, basepath, workspace): if not destdir.endswith('/'): destdir += '/' - tinfoil = setup_tinfoil(basepath=basepath) - try: - try: - rd = tinfoil.parse_recipe(args.recipename) - except Exception as e: - raise DevtoolError('Exception parsing recipe %s: %s' % - (args.recipename, e)) - recipe_outdir = rd.getVar('D') - if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir): - raise DevtoolError('No files to deploy - have you built the %s ' - 'recipe? If so, the install step has not installed ' - 'any files.' % args.recipename) - - if args.strip and not args.dry_run: - # Fakeroot copy to new destination - srcdir = recipe_outdir - recipe_outdir = os.path.join(rd.getVar('WORKDIR'), 'deploy-target-stripped') - if os.path.isdir(recipe_outdir): - bb.utils.remove(recipe_outdir, True) - exec_fakeroot(rd, "cp -af %s %s" % (os.path.join(srcdir, '.'), recipe_outdir), shell=True) - os.environ['PATH'] = ':'.join([os.environ['PATH'], rd.getVar('PATH') or '']) - oe.package.strip_execs(args.recipename, recipe_outdir, rd.getVar('STRIP'), rd.getVar('libdir'), - rd.getVar('base_libdir'), rd) - - filelist = [] - inodes = set({}) - ftotalsize = 0 - for root, _, files in os.walk(recipe_outdir): - for fn in files: - fstat = os.lstat(os.path.join(root, fn)) - # Get the size in kiB (since we'll be comparing it to the output of du -k) - # MUST use lstat() here not stat() or getfilesize() since we don't want to - # dereference symlinks - if fstat.st_ino in inodes: - fsize = 0 - else: - fsize = int(math.ceil(float(fstat.st_size)/1024)) - inodes.add(fstat.st_ino) - ftotalsize += fsize - # The path as it would appear on the target - fpath = os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn) - filelist.append((fpath, fsize)) - - if args.dry_run: - print('Files to be deployed for %s on target %s:' % (args.recipename, args.target)) - for item, _ in filelist: - print(' %s' % item) - return 0 - - extraoptions = '' - if args.no_host_check: - extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' - if not args.show_status: - extraoptions += ' -q' - - scp_sshexec = '' - ssh_sshexec = 'ssh' - if args.ssh_exec: - scp_sshexec = "-S %s" % args.ssh_exec - ssh_sshexec = args.ssh_exec - scp_port = '' - ssh_port = '' - if args.port: - scp_port = "-P %s" % args.port - ssh_port = "-p %s" % args.port - - if args.key: - extraoptions += ' -i %s' % args.key - - # In order to delete previously deployed files and have the manifest file on - # the target, we write out a shell script and then copy it to the target - # so we can then run it (piping tar output to it). - # (We cannot use scp here, because it doesn't preserve symlinks.) - tmpdir = tempfile.mkdtemp(prefix='devtool') - try: - tmpscript = '/tmp/devtool_deploy.sh' - tmpfilelist = os.path.join(os.path.dirname(tmpscript), 'devtool_deploy.list') - shellscript = _prepare_remote_script(deploy=True, - verbose=args.show_status, - nopreserve=args.no_preserve, - nocheckspace=args.no_check_space) - # Write out the script to a file - with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f: - f.write(shellscript) - # Write out the file list - with open(os.path.join(tmpdir, os.path.basename(tmpfilelist)), 'w') as f: - f.write('%d\n' % ftotalsize) - for fpath, fsize in filelist: - f.write('%s %d\n' % (fpath, fsize)) - # Copy them to the target - ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True) - if ret != 0: - raise DevtoolError('Failed to copy script to %s - rerun with -s to ' - 'get a complete error message' % args.target) - finally: - shutil.rmtree(tmpdir) + recipe_outdir = srcdir + if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir): + raise DevtoolError('No files to deploy - have you built the %s ' + 'recipe? If so, the install step has not installed ' + 'any files.' % args.recipename) + + if args.strip and not args.dry_run: + # Fakeroot copy to new destination + srcdir = recipe_outdir + recipe_outdir = os.path.join(workdir, 'devtool-deploy-target-stripped') + if os.path.isdir(recipe_outdir): + exec_fakeroot_no_d(fakerootcmd, fakerootenv, "rm -rf %s" % recipe_outdir, shell=True) + exec_fakeroot_no_d(fakerootcmd, fakerootenv, "cp -af %s %s" % (os.path.join(srcdir, '.'), recipe_outdir), shell=True) + os.environ['PATH'] = ':'.join([os.environ['PATH'], path or '']) + oe.package.strip_execs(args.recipename, recipe_outdir, strip_cmd, libdir, base_libdir, max_process) + + filelist = [] + inodes = set({}) + ftotalsize = 0 + for root, _, files in os.walk(recipe_outdir): + for fn in files: + fstat = os.lstat(os.path.join(root, fn)) + # Get the size in kiB (since we'll be comparing it to the output of du -k) + # MUST use lstat() here not stat() or getfilesize() since we don't want to + # dereference symlinks + if fstat.st_ino in inodes: + fsize = 0 + else: + fsize = int(math.ceil(float(fstat.st_size)/1024)) + inodes.add(fstat.st_ino) + ftotalsize += fsize + # The path as it would appear on the target + fpath = os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn) + filelist.append((fpath, fsize)) + + if args.dry_run: + print('Files to be deployed for %s on target %s:' % (args.recipename, args.target)) + for item, _ in filelist: + print(' %s' % item) + return 0 - # Now run the script - ret = exec_fakeroot(rd, 'tar cf - . | %s %s %s %s \'sh %s %s %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, args.target, tmpscript, args.recipename, destdir, tmpfilelist), cwd=recipe_outdir, shell=True) - if ret != 0: - raise DevtoolError('Deploy failed - rerun with -s to get a complete ' - 'error message') + extraoptions = '' + if args.no_host_check: + extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' + if not args.show_status: + extraoptions += ' -q' - logger.info('Successfully deployed %s' % recipe_outdir) + scp_sshexec = '' + ssh_sshexec = 'ssh' + if args.ssh_exec: + scp_sshexec = "-S %s" % args.ssh_exec + ssh_sshexec = args.ssh_exec + scp_port = '' + ssh_port = '' + if args.port: + scp_port = "-P %s" % args.port + ssh_port = "-p %s" % args.port + + if args.key: + extraoptions += ' -i %s' % args.key - files_list = [] - for root, _, files in os.walk(recipe_outdir): - for filename in files: - filename = os.path.relpath(os.path.join(root, filename), recipe_outdir) - files_list.append(os.path.join(destdir, filename)) + # In order to delete previously deployed files and have the manifest file on + # the target, we write out a shell script and then copy it to the target + # so we can then run it (piping tar output to it). + # (We cannot use scp here, because it doesn't preserve symlinks.) + tmpdir = tempfile.mkdtemp(prefix='devtool') + try: + tmpscript = '/tmp/devtool_deploy.sh' + tmpfilelist = os.path.join(os.path.dirname(tmpscript), 'devtool_deploy.list') + shellscript = _prepare_remote_script(deploy=True, + verbose=args.show_status, + nopreserve=args.no_preserve, + nocheckspace=args.no_check_space) + # Write out the script to a file + with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f: + f.write(shellscript) + # Write out the file list + with open(os.path.join(tmpdir, os.path.basename(tmpfilelist)), 'w') as f: + f.write('%d\n' % ftotalsize) + for fpath, fsize in filelist: + f.write('%s %d\n' % (fpath, fsize)) + # Copy them to the target + ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True) + if ret != 0: + raise DevtoolError('Failed to copy script to %s - rerun with -s to ' + 'get a complete error message' % args.target) finally: - tinfoil.shutdown() + shutil.rmtree(tmpdir) + + # Now run the script + ret = exec_fakeroot_no_d(fakerootcmd, fakerootenv, 'tar cf - . | %s %s %s %s \'sh %s %s %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, args.target, tmpscript, args.recipename, destdir, tmpfilelist), cwd=recipe_outdir, shell=True) + if ret != 0: + raise DevtoolError('Deploy failed - rerun with -s to get a complete ' + 'error message') + + logger.info('Successfully deployed %s' % recipe_outdir) + + files_list = [] + for root, _, files in os.walk(recipe_outdir): + for filename in files: + filename = os.path.relpath(os.path.join(root, filename), recipe_outdir) + files_list.append(os.path.join(destdir, filename)) return 0 diff --git a/scripts/lib/devtool/ide_plugins/__init__.py b/scripts/lib/devtool/ide_plugins/__init__.py new file mode 100644 index 0000000000..19c2f61c5f --- /dev/null +++ b/scripts/lib/devtool/ide_plugins/__init__.py @@ -0,0 +1,282 @@ +# +# Copyright (C) 2023-2024 Siemens AG +# +# SPDX-License-Identifier: GPL-2.0-only +# +"""Devtool ide-sdk IDE plugin interface definition and helper functions""" + +import errno +import json +import logging +import os +import stat +from enum import Enum, auto +from devtool import DevtoolError +from bb.utils import mkdirhier + +logger = logging.getLogger('devtool') + + +class BuildTool(Enum): + UNDEFINED = auto() + CMAKE = auto() + MESON = auto() + + @property + def is_c_ccp(self): + if self is BuildTool.CMAKE: + return True + if self is BuildTool.MESON: + return True + return False + + +class GdbCrossConfig: + """Base class defining the GDB configuration generator interface + + Generate a GDB configuration for a binary on the target device. + Only one instance per binary is allowed. This allows to assign unique port + numbers for all gdbserver instances. + """ + _gdbserver_port_next = 1234 + _binaries = [] + + def __init__(self, image_recipe, modified_recipe, binary, gdbserver_multi=True): + self.image_recipe = image_recipe + self.modified_recipe = modified_recipe + self.gdb_cross = modified_recipe.gdb_cross + self.binary = binary + if binary in GdbCrossConfig._binaries: + raise DevtoolError( + "gdbserver config for binary %s is already generated" % binary) + GdbCrossConfig._binaries.append(binary) + self.script_dir = modified_recipe.ide_sdk_scripts_dir + self.gdbinit_dir = os.path.join(self.script_dir, 'gdbinit') + self.gdbserver_multi = gdbserver_multi + self.binary_pretty = self.binary.replace(os.sep, '-').lstrip('-') + self.gdbserver_port = GdbCrossConfig._gdbserver_port_next + GdbCrossConfig._gdbserver_port_next += 1 + self.id_pretty = "%d_%s" % (self.gdbserver_port, self.binary_pretty) + # gdbserver start script + gdbserver_script_file = 'gdbserver_' + self.id_pretty + if self.gdbserver_multi: + gdbserver_script_file += "_m" + self.gdbserver_script = os.path.join( + self.script_dir, gdbserver_script_file) + # gdbinit file + self.gdbinit = os.path.join( + self.gdbinit_dir, 'gdbinit_' + self.id_pretty) + # gdb start script + self.gdb_script = os.path.join( + self.script_dir, 'gdb_' + self.id_pretty) + + def _gen_gdbserver_start_script(self): + """Generate a shell command starting the gdbserver on the remote device via ssh + + GDB supports two modes: + multi: gdbserver remains running over several debug sessions + once: gdbserver terminates after the debugged process terminates + """ + cmd_lines = ['#!/bin/sh'] + if self.gdbserver_multi: + temp_dir = "TEMP_DIR=/tmp/gdbserver_%s; " % self.id_pretty + gdbserver_cmd_start = temp_dir + gdbserver_cmd_start += "test -f \\$TEMP_DIR/pid && exit 0; " + gdbserver_cmd_start += "mkdir -p \\$TEMP_DIR; " + gdbserver_cmd_start += "%s --multi :%s > \\$TEMP_DIR/log 2>&1 & " % ( + self.gdb_cross.gdbserver_path, self.gdbserver_port) + gdbserver_cmd_start += "echo \\$! > \\$TEMP_DIR/pid;" + + gdbserver_cmd_stop = temp_dir + gdbserver_cmd_stop += "test -f \\$TEMP_DIR/pid && kill \\$(cat \\$TEMP_DIR/pid); " + gdbserver_cmd_stop += "rm -rf \\$TEMP_DIR; " + + gdbserver_cmd_l = [] + gdbserver_cmd_l.append('if [ "$1" = "stop" ]; then') + gdbserver_cmd_l.append(' shift') + gdbserver_cmd_l.append(" %s %s %s %s 'sh -c \"%s\"'" % ( + self.gdb_cross.target_device.ssh_sshexec, self.gdb_cross.target_device.ssh_port, self.gdb_cross.target_device.extraoptions, self.gdb_cross.target_device.target, gdbserver_cmd_stop)) + gdbserver_cmd_l.append('else') + gdbserver_cmd_l.append(" %s %s %s %s 'sh -c \"%s\"'" % ( + self.gdb_cross.target_device.ssh_sshexec, self.gdb_cross.target_device.ssh_port, self.gdb_cross.target_device.extraoptions, self.gdb_cross.target_device.target, gdbserver_cmd_start)) + gdbserver_cmd_l.append('fi') + gdbserver_cmd = os.linesep.join(gdbserver_cmd_l) + else: + gdbserver_cmd_start = "%s --once :%s %s" % ( + self.gdb_cross.gdbserver_path, self.gdbserver_port, self.binary) + gdbserver_cmd = "%s %s %s %s 'sh -c \"%s\"'" % ( + self.gdb_cross.target_device.ssh_sshexec, self.gdb_cross.target_device.ssh_port, self.gdb_cross.target_device.extraoptions, self.gdb_cross.target_device.target, gdbserver_cmd_start) + cmd_lines.append(gdbserver_cmd) + GdbCrossConfig.write_file(self.gdbserver_script, cmd_lines, True) + + def _gen_gdbinit_config(self): + """Generate a gdbinit file for this binary and the corresponding gdbserver configuration""" + gdbinit_lines = ['# This file is generated by devtool ide-sdk'] + if self.gdbserver_multi: + target_help = '# gdbserver --multi :%d' % self.gdbserver_port + remote_cmd = 'target extended-remote' + else: + target_help = '# gdbserver :%d %s' % ( + self.gdbserver_port, self.binary) + remote_cmd = 'target remote' + gdbinit_lines.append('# On the remote target:') + gdbinit_lines.append(target_help) + gdbinit_lines.append('# On the build machine:') + gdbinit_lines.append('# cd ' + self.modified_recipe.real_srctree) + gdbinit_lines.append( + '# ' + self.gdb_cross.gdb + ' -ix ' + self.gdbinit) + + gdbinit_lines.append('set sysroot ' + self.modified_recipe.d) + gdbinit_lines.append('set substitute-path "/usr/include" "' + + os.path.join(self.modified_recipe.recipe_sysroot, 'usr', 'include') + '"') + # Disable debuginfod for now, the IDE configuration uses rootfs-dbg from the image workdir. + gdbinit_lines.append('set debuginfod enabled off') + if self.image_recipe.rootfs_dbg: + gdbinit_lines.append( + 'set solib-search-path "' + self.modified_recipe.solib_search_path_str(self.image_recipe) + '"') + # First: Search for sources of this recipe in the workspace folder + if self.modified_recipe.pn in self.modified_recipe.target_dbgsrc_dir: + gdbinit_lines.append('set substitute-path "%s" "%s"' % + (self.modified_recipe.target_dbgsrc_dir, self.modified_recipe.real_srctree)) + else: + logger.error( + "TARGET_DBGSRC_DIR must contain the recipe name PN.") + # Second: Search for sources of other recipes in the rootfs-dbg + if self.modified_recipe.target_dbgsrc_dir.startswith("/usr/src/debug"): + gdbinit_lines.append('set substitute-path "/usr/src/debug" "%s"' % os.path.join( + self.image_recipe.rootfs_dbg, "usr", "src", "debug")) + else: + logger.error( + "TARGET_DBGSRC_DIR must start with /usr/src/debug.") + else: + logger.warning( + "Cannot setup debug symbols configuration for GDB. IMAGE_GEN_DEBUGFS is not enabled.") + gdbinit_lines.append( + '%s %s:%d' % (remote_cmd, self.gdb_cross.host, self.gdbserver_port)) + gdbinit_lines.append('set remote exec-file ' + self.binary) + gdbinit_lines.append( + 'run ' + os.path.join(self.modified_recipe.d, self.binary)) + + GdbCrossConfig.write_file(self.gdbinit, gdbinit_lines) + + def _gen_gdb_start_script(self): + """Generate a script starting GDB with the corresponding gdbinit configuration.""" + cmd_lines = ['#!/bin/sh'] + cmd_lines.append('cd ' + self.modified_recipe.real_srctree) + cmd_lines.append(self.gdb_cross.gdb + ' -ix ' + + self.gdbinit + ' "$@"') + GdbCrossConfig.write_file(self.gdb_script, cmd_lines, True) + + def initialize(self): + self._gen_gdbserver_start_script() + self._gen_gdbinit_config() + self._gen_gdb_start_script() + + @staticmethod + def write_file(script_file, cmd_lines, executable=False): + script_dir = os.path.dirname(script_file) + mkdirhier(script_dir) + with open(script_file, 'w') as script_f: + script_f.write(os.linesep.join(cmd_lines)) + script_f.write(os.linesep) + if executable: + st = os.stat(script_file) + os.chmod(script_file, st.st_mode | stat.S_IEXEC) + logger.info("Created: %s" % script_file) + + +class IdeBase: + """Base class defining the interface for IDE plugins""" + + def __init__(self): + self.ide_name = 'undefined' + self.gdb_cross_configs = [] + + @classmethod + def ide_plugin_priority(cls): + """Used to find the default ide handler if --ide is not passed""" + return 10 + + def setup_shared_sysroots(self, shared_env): + logger.warn("Shared sysroot mode is not supported for IDE %s" % + self.ide_name) + + def setup_modified_recipe(self, args, image_recipe, modified_recipe): + logger.warn("Modified recipe mode is not supported for IDE %s" % + self.ide_name) + + def initialize_gdb_cross_configs(self, image_recipe, modified_recipe, gdb_cross_config_class=GdbCrossConfig): + binaries = modified_recipe.find_installed_binaries() + for binary in binaries: + gdb_cross_config = gdb_cross_config_class( + image_recipe, modified_recipe, binary) + gdb_cross_config.initialize() + self.gdb_cross_configs.append(gdb_cross_config) + + @staticmethod + def gen_oe_scrtips_sym_link(modified_recipe): + # create a sym-link from sources to the scripts directory + if os.path.isdir(modified_recipe.ide_sdk_scripts_dir): + IdeBase.symlink_force(modified_recipe.ide_sdk_scripts_dir, + os.path.join(modified_recipe.real_srctree, 'oe-scripts')) + + @staticmethod + def update_json_file(json_dir, json_file, update_dict): + """Update a json file + + By default it uses the dict.update function. If this is not sutiable + the update function might be passed via update_func parameter. + """ + json_path = os.path.join(json_dir, json_file) + logger.info("Updating IDE config file: %s (%s)" % + (json_file, json_path)) + if not os.path.exists(json_dir): + os.makedirs(json_dir) + try: + with open(json_path) as f: + orig_dict = json.load(f) + except json.decoder.JSONDecodeError: + logger.info( + "Decoding %s failed. Probably because of comments in the json file" % json_path) + orig_dict = {} + except FileNotFoundError: + orig_dict = {} + orig_dict.update(update_dict) + with open(json_path, 'w') as f: + json.dump(orig_dict, f, indent=4) + + @staticmethod + def symlink_force(tgt, dst): + try: + os.symlink(tgt, dst) + except OSError as err: + if err.errno == errno.EEXIST: + if os.readlink(dst) != tgt: + os.remove(dst) + os.symlink(tgt, dst) + else: + raise err + + +def get_devtool_deploy_opts(args): + """Filter args for devtool deploy-target args""" + if not args.target: + return None + devtool_deploy_opts = [args.target] + if args.no_host_check: + devtool_deploy_opts += ["-c"] + if args.show_status: + devtool_deploy_opts += ["-s"] + if args.no_preserve: + devtool_deploy_opts += ["-p"] + if args.no_check_space: + devtool_deploy_opts += ["--no-check-space"] + if args.ssh_exec: + devtool_deploy_opts += ["-e", args.ssh.exec] + if args.port: + devtool_deploy_opts += ["-P", args.port] + if args.key: + devtool_deploy_opts += ["-I", args.key] + if args.strip is False: + devtool_deploy_opts += ["--no-strip"] + return devtool_deploy_opts diff --git a/scripts/lib/devtool/ide_plugins/ide_code.py b/scripts/lib/devtool/ide_plugins/ide_code.py new file mode 100644 index 0000000000..a62b93224e --- /dev/null +++ b/scripts/lib/devtool/ide_plugins/ide_code.py @@ -0,0 +1,463 @@ +# +# Copyright (C) 2023-2024 Siemens AG +# +# SPDX-License-Identifier: GPL-2.0-only +# +"""Devtool ide-sdk IDE plugin for VSCode and VSCodium""" + +import json +import logging +import os +import shutil +from devtool.ide_plugins import BuildTool, IdeBase, GdbCrossConfig, get_devtool_deploy_opts + +logger = logging.getLogger('devtool') + + +class GdbCrossConfigVSCode(GdbCrossConfig): + def __init__(self, image_recipe, modified_recipe, binary): + super().__init__(image_recipe, modified_recipe, binary, False) + + def initialize(self): + self._gen_gdbserver_start_script() + + +class IdeVSCode(IdeBase): + """Manage IDE configurations for VSCode + + Modified recipe mode: + - cmake: use the cmake-preset generated by devtool ide-sdk + - meson: meson is called via a wrapper script generated by devtool ide-sdk + + Shared sysroot mode: + In shared sysroot mode, the cross tool-chain is exported to the user's global configuration. + A workspace cannot be created because there is no recipe that defines how a workspace could + be set up. + - cmake: adds a cmake-kit to .local/share/CMakeTools/cmake-tools-kits.json + The cmake-kit uses the environment script and the tool-chain file + generated by meta-ide-support. + - meson: Meson needs manual workspace configuration. + """ + + @classmethod + def ide_plugin_priority(cls): + """If --ide is not passed this is the default plugin""" + if shutil.which('code'): + return 100 + return 0 + + def setup_shared_sysroots(self, shared_env): + """Expose the toolchain of the shared sysroots SDK""" + datadir = shared_env.ide_support.datadir + deploy_dir_image = shared_env.ide_support.deploy_dir_image + real_multimach_target_sys = shared_env.ide_support.real_multimach_target_sys + standalone_sysroot_native = shared_env.build_sysroots.standalone_sysroot_native + vscode_ws_path = os.path.join( + os.environ['HOME'], '.local', 'share', 'CMakeTools') + cmake_kits_path = os.path.join(vscode_ws_path, 'cmake-tools-kits.json') + oecmake_generator = "Ninja" + env_script = os.path.join( + deploy_dir_image, 'environment-setup-' + real_multimach_target_sys) + + if not os.path.isdir(vscode_ws_path): + os.makedirs(vscode_ws_path) + cmake_kits_old = [] + if os.path.exists(cmake_kits_path): + with open(cmake_kits_path, 'r', encoding='utf-8') as cmake_kits_file: + cmake_kits_old = json.load(cmake_kits_file) + cmake_kits = cmake_kits_old.copy() + + cmake_kit_new = { + "name": "OE " + real_multimach_target_sys, + "environmentSetupScript": env_script, + "toolchainFile": standalone_sysroot_native + datadir + "/cmake/OEToolchainConfig.cmake", + "preferredGenerator": { + "name": oecmake_generator + } + } + + def merge_kit(cmake_kits, cmake_kit_new): + i = 0 + while i < len(cmake_kits): + if 'environmentSetupScript' in cmake_kits[i] and \ + cmake_kits[i]['environmentSetupScript'] == cmake_kit_new['environmentSetupScript']: + cmake_kits[i] = cmake_kit_new + return + i += 1 + cmake_kits.append(cmake_kit_new) + merge_kit(cmake_kits, cmake_kit_new) + + if cmake_kits != cmake_kits_old: + logger.info("Updating: %s" % cmake_kits_path) + with open(cmake_kits_path, 'w', encoding='utf-8') as cmake_kits_file: + json.dump(cmake_kits, cmake_kits_file, indent=4) + else: + logger.info("Already up to date: %s" % cmake_kits_path) + + cmake_native = os.path.join( + shared_env.build_sysroots.standalone_sysroot_native, 'usr', 'bin', 'cmake') + if os.path.isfile(cmake_native): + logger.info('cmake-kits call cmake by default. If the cmake provided by this SDK should be used, please add the following line to ".vscode/settings.json" file: "cmake.cmakePath": "%s"' % cmake_native) + else: + logger.error("Cannot find cmake native at: %s" % cmake_native) + + def dot_code_dir(self, modified_recipe): + return os.path.join(modified_recipe.srctree, '.vscode') + + def __vscode_settings_meson(self, settings_dict, modified_recipe): + if modified_recipe.build_tool is not BuildTool.MESON: + return + settings_dict["mesonbuild.mesonPath"] = modified_recipe.meson_wrapper + + confopts = modified_recipe.mesonopts.split() + confopts += modified_recipe.meson_cross_file.split() + confopts += modified_recipe.extra_oemeson.split() + settings_dict["mesonbuild.configureOptions"] = confopts + settings_dict["mesonbuild.buildFolder"] = modified_recipe.b + + def __vscode_settings_cmake(self, settings_dict, modified_recipe): + """Add cmake specific settings to settings.json. + + Note: most settings are passed to the cmake preset. + """ + if modified_recipe.build_tool is not BuildTool.CMAKE: + return + settings_dict["cmake.configureOnOpen"] = True + settings_dict["cmake.sourceDirectory"] = modified_recipe.real_srctree + + def vscode_settings(self, modified_recipe, image_recipe): + files_excludes = { + "**/.git/**": True, + "**/oe-logs/**": True, + "**/oe-workdir/**": True, + "**/source-date-epoch/**": True + } + python_exclude = [ + "**/.git/**", + "**/oe-logs/**", + "**/oe-workdir/**", + "**/source-date-epoch/**" + ] + files_readonly = { + modified_recipe.recipe_sysroot + '/**': True, + modified_recipe.recipe_sysroot_native + '/**': True, + } + if image_recipe.rootfs_dbg is not None: + files_readonly[image_recipe.rootfs_dbg + '/**'] = True + settings_dict = { + "files.watcherExclude": files_excludes, + "files.exclude": files_excludes, + "files.readonlyInclude": files_readonly, + "python.analysis.exclude": python_exclude + } + self.__vscode_settings_cmake(settings_dict, modified_recipe) + self.__vscode_settings_meson(settings_dict, modified_recipe) + + settings_file = 'settings.json' + IdeBase.update_json_file( + self.dot_code_dir(modified_recipe), settings_file, settings_dict) + + def __vscode_extensions_cmake(self, modified_recipe, recommendations): + if modified_recipe.build_tool is not BuildTool.CMAKE: + return + recommendations += [ + "twxs.cmake", + "ms-vscode.cmake-tools", + "ms-vscode.cpptools", + "ms-vscode.cpptools-extension-pack", + "ms-vscode.cpptools-themes" + ] + + def __vscode_extensions_meson(self, modified_recipe, recommendations): + if modified_recipe.build_tool is not BuildTool.MESON: + return + recommendations += [ + 'mesonbuild.mesonbuild', + "ms-vscode.cpptools", + "ms-vscode.cpptools-extension-pack", + "ms-vscode.cpptools-themes" + ] + + def vscode_extensions(self, modified_recipe): + recommendations = [] + self.__vscode_extensions_cmake(modified_recipe, recommendations) + self.__vscode_extensions_meson(modified_recipe, recommendations) + extensions_file = 'extensions.json' + IdeBase.update_json_file( + self.dot_code_dir(modified_recipe), extensions_file, {"recommendations": recommendations}) + + def vscode_c_cpp_properties(self, modified_recipe): + properties_dict = { + "name": modified_recipe.recipe_id_pretty, + } + if modified_recipe.build_tool is BuildTool.CMAKE: + properties_dict["configurationProvider"] = "ms-vscode.cmake-tools" + elif modified_recipe.build_tool is BuildTool.MESON: + properties_dict["configurationProvider"] = "mesonbuild.mesonbuild" + properties_dict["compilerPath"] = os.path.join(modified_recipe.staging_bindir_toolchain, modified_recipe.cxx.split()[0]) + else: # no C/C++ build + return + + properties_dicts = { + "configurations": [ + properties_dict + ], + "version": 4 + } + prop_file = 'c_cpp_properties.json' + IdeBase.update_json_file( + self.dot_code_dir(modified_recipe), prop_file, properties_dicts) + + def vscode_launch_bin_dbg(self, gdb_cross_config): + modified_recipe = gdb_cross_config.modified_recipe + + launch_config = { + "name": gdb_cross_config.id_pretty, + "type": "cppdbg", + "request": "launch", + "program": os.path.join(modified_recipe.d, gdb_cross_config.binary.lstrip('/')), + "stopAtEntry": True, + "cwd": "${workspaceFolder}", + "environment": [], + "externalConsole": False, + "MIMode": "gdb", + "preLaunchTask": gdb_cross_config.id_pretty, + "miDebuggerPath": modified_recipe.gdb_cross.gdb, + "miDebuggerServerAddress": "%s:%d" % (modified_recipe.gdb_cross.host, gdb_cross_config.gdbserver_port) + } + + # Search for header files in recipe-sysroot. + src_file_map = { + "/usr/include": os.path.join(modified_recipe.recipe_sysroot, "usr", "include") + } + # First of all search for not stripped binaries in the image folder. + # These binaries are copied (and optionally stripped) by deploy-target + setup_commands = [ + { + "description": "sysroot", + "text": "set sysroot " + modified_recipe.d + } + ] + + if gdb_cross_config.image_recipe.rootfs_dbg: + launch_config['additionalSOLibSearchPath'] = modified_recipe.solib_search_path_str( + gdb_cross_config.image_recipe) + # First: Search for sources of this recipe in the workspace folder + if modified_recipe.pn in modified_recipe.target_dbgsrc_dir: + src_file_map[modified_recipe.target_dbgsrc_dir] = "${workspaceFolder}" + else: + logger.error( + "TARGET_DBGSRC_DIR must contain the recipe name PN.") + # Second: Search for sources of other recipes in the rootfs-dbg + if modified_recipe.target_dbgsrc_dir.startswith("/usr/src/debug"): + src_file_map["/usr/src/debug"] = os.path.join( + gdb_cross_config.image_recipe.rootfs_dbg, "usr", "src", "debug") + else: + logger.error( + "TARGET_DBGSRC_DIR must start with /usr/src/debug.") + else: + logger.warning( + "Cannot setup debug symbols configuration for GDB. IMAGE_GEN_DEBUGFS is not enabled.") + + launch_config['sourceFileMap'] = src_file_map + launch_config['setupCommands'] = setup_commands + return launch_config + + def vscode_launch(self, modified_recipe): + """GDB Launch configuration for binaries (elf files)""" + + configurations = [] + for gdb_cross_config in self.gdb_cross_configs: + if gdb_cross_config.modified_recipe is modified_recipe: + configurations.append(self.vscode_launch_bin_dbg(gdb_cross_config)) + launch_dict = { + "version": "0.2.0", + "configurations": configurations + } + launch_file = 'launch.json' + IdeBase.update_json_file( + self.dot_code_dir(modified_recipe), launch_file, launch_dict) + + def vscode_tasks_cpp(self, args, modified_recipe): + run_install_deploy = modified_recipe.gen_install_deploy_script(args) + install_task_name = "install && deploy-target %s" % modified_recipe.recipe_id_pretty + tasks_dict = { + "version": "2.0.0", + "tasks": [ + { + "label": install_task_name, + "type": "shell", + "command": run_install_deploy, + "problemMatcher": [] + } + ] + } + for gdb_cross_config in self.gdb_cross_configs: + if gdb_cross_config.modified_recipe is not modified_recipe: + continue + tasks_dict['tasks'].append( + { + "label": gdb_cross_config.id_pretty, + "type": "shell", + "isBackground": True, + "dependsOn": [ + install_task_name + ], + "command": gdb_cross_config.gdbserver_script, + "problemMatcher": [ + { + "pattern": [ + { + "regexp": ".", + "file": 1, + "location": 2, + "message": 3 + } + ], + "background": { + "activeOnStart": True, + "beginsPattern": ".", + "endsPattern": ".", + } + } + ] + }) + tasks_file = 'tasks.json' + IdeBase.update_json_file( + self.dot_code_dir(modified_recipe), tasks_file, tasks_dict) + + def vscode_tasks_fallback(self, args, modified_recipe): + oe_init_dir = modified_recipe.oe_init_dir + oe_init = ". %s %s > /dev/null && " % (modified_recipe.oe_init_build_env, modified_recipe.topdir) + dt_build = "devtool build " + dt_build_label = dt_build + modified_recipe.recipe_id_pretty + dt_build_cmd = dt_build + modified_recipe.bpn + clean_opt = " --clean" + dt_build_clean_label = dt_build + modified_recipe.recipe_id_pretty + clean_opt + dt_build_clean_cmd = dt_build + modified_recipe.bpn + clean_opt + dt_deploy = "devtool deploy-target " + dt_deploy_label = dt_deploy + modified_recipe.recipe_id_pretty + dt_deploy_cmd = dt_deploy + modified_recipe.bpn + dt_build_deploy_label = "devtool build & deploy-target %s" % modified_recipe.recipe_id_pretty + deploy_opts = ' '.join(get_devtool_deploy_opts(args)) + tasks_dict = { + "version": "2.0.0", + "tasks": [ + { + "label": dt_build_label, + "type": "shell", + "command": "bash", + "linux": { + "options": { + "cwd": oe_init_dir + } + }, + "args": [ + "--login", + "-c", + "%s%s" % (oe_init, dt_build_cmd) + ], + "problemMatcher": [] + }, + { + "label": dt_deploy_label, + "type": "shell", + "command": "bash", + "linux": { + "options": { + "cwd": oe_init_dir + } + }, + "args": [ + "--login", + "-c", + "%s%s %s" % ( + oe_init, dt_deploy_cmd, deploy_opts) + ], + "problemMatcher": [] + }, + { + "label": dt_build_deploy_label, + "dependsOrder": "sequence", + "dependsOn": [ + dt_build_label, + dt_deploy_label + ], + "problemMatcher": [], + "group": { + "kind": "build", + "isDefault": True + } + }, + { + "label": dt_build_clean_label, + "type": "shell", + "command": "bash", + "linux": { + "options": { + "cwd": oe_init_dir + } + }, + "args": [ + "--login", + "-c", + "%s%s" % (oe_init, dt_build_clean_cmd) + ], + "problemMatcher": [] + } + ] + } + if modified_recipe.gdb_cross: + for gdb_cross_config in self.gdb_cross_configs: + if gdb_cross_config.modified_recipe is not modified_recipe: + continue + tasks_dict['tasks'].append( + { + "label": gdb_cross_config.id_pretty, + "type": "shell", + "isBackground": True, + "dependsOn": [ + dt_build_deploy_label + ], + "command": gdb_cross_config.gdbserver_script, + "problemMatcher": [ + { + "pattern": [ + { + "regexp": ".", + "file": 1, + "location": 2, + "message": 3 + } + ], + "background": { + "activeOnStart": True, + "beginsPattern": ".", + "endsPattern": ".", + } + } + ] + }) + tasks_file = 'tasks.json' + IdeBase.update_json_file( + self.dot_code_dir(modified_recipe), tasks_file, tasks_dict) + + def vscode_tasks(self, args, modified_recipe): + if modified_recipe.build_tool.is_c_ccp: + self.vscode_tasks_cpp(args, modified_recipe) + else: + self.vscode_tasks_fallback(args, modified_recipe) + + def setup_modified_recipe(self, args, image_recipe, modified_recipe): + self.vscode_settings(modified_recipe, image_recipe) + self.vscode_extensions(modified_recipe) + self.vscode_c_cpp_properties(modified_recipe) + if args.target: + self.initialize_gdb_cross_configs( + image_recipe, modified_recipe, gdb_cross_config_class=GdbCrossConfigVSCode) + self.vscode_launch(modified_recipe) + self.vscode_tasks(args, modified_recipe) + + +def register_ide_plugin(ide_plugins): + ide_plugins['code'] = IdeVSCode diff --git a/scripts/lib/devtool/ide_plugins/ide_none.py b/scripts/lib/devtool/ide_plugins/ide_none.py new file mode 100644 index 0000000000..f106c5a026 --- /dev/null +++ b/scripts/lib/devtool/ide_plugins/ide_none.py @@ -0,0 +1,53 @@ +# +# Copyright (C) 2023-2024 Siemens AG +# +# SPDX-License-Identifier: GPL-2.0-only +# +"""Devtool ide-sdk generic IDE plugin""" + +import os +import logging +from devtool.ide_plugins import IdeBase, GdbCrossConfig + +logger = logging.getLogger('devtool') + + +class IdeNone(IdeBase): + """Generate some generic helpers for other IDEs + + Modified recipe mode: + Generate some helper scripts for remote debugging with GDB + + Shared sysroot mode: + A wrapper for bitbake meta-ide-support and bitbake build-sysroots + """ + + def __init__(self): + super().__init__() + + def setup_shared_sysroots(self, shared_env): + real_multimach_target_sys = shared_env.ide_support.real_multimach_target_sys + deploy_dir_image = shared_env.ide_support.deploy_dir_image + env_script = os.path.join( + deploy_dir_image, 'environment-setup-' + real_multimach_target_sys) + logger.info( + "To use this SDK please source this: %s" % env_script) + + def setup_modified_recipe(self, args, image_recipe, modified_recipe): + """generate some helper scripts and config files + + - Execute the do_install task + - Execute devtool deploy-target + - Generate a gdbinit file per executable + - Generate the oe-scripts sym-link + """ + script_path = modified_recipe.gen_install_deploy_script(args) + logger.info("Created: %s" % script_path) + + self.initialize_gdb_cross_configs(image_recipe, modified_recipe) + + IdeBase.gen_oe_scrtips_sym_link(modified_recipe) + + +def register_ide_plugin(ide_plugins): + ide_plugins['none'] = IdeNone diff --git a/scripts/lib/devtool/ide_sdk.py b/scripts/lib/devtool/ide_sdk.py new file mode 100755 index 0000000000..7807b322b3 --- /dev/null +++ b/scripts/lib/devtool/ide_sdk.py @@ -0,0 +1,1070 @@ +# Development tool - ide-sdk command plugin +# +# Copyright (C) 2023-2024 Siemens AG +# +# SPDX-License-Identifier: GPL-2.0-only +# +"""Devtool ide-sdk plugin""" + +import json +import logging +import os +import re +import shutil +import stat +import subprocess +import sys +from argparse import RawTextHelpFormatter +from enum import Enum + +import scriptutils +import bb +from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, DevtoolError, parse_recipe +from devtool.standard import get_real_srctree +from devtool.ide_plugins import BuildTool + + +logger = logging.getLogger('devtool') + +# dict of classes derived from IdeBase +ide_plugins = {} + + +class DevtoolIdeMode(Enum): + """Different modes are supported by the ide-sdk plugin. + + The enum might be extended by more advanced modes in the future. Some ideas: + - auto: modified if all recipes are modified, shared if none of the recipes is modified. + - mixed: modified mode for modified recipes, shared mode for all other recipes. + """ + + modified = 'modified' + shared = 'shared' + + +class TargetDevice: + """SSH remote login parameters""" + + def __init__(self, args): + self.extraoptions = '' + if args.no_host_check: + self.extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' + self.ssh_sshexec = 'ssh' + if args.ssh_exec: + self.ssh_sshexec = args.ssh_exec + self.ssh_port = '' + if args.port: + self.ssh_port = "-p %s" % args.port + if args.key: + self.extraoptions += ' -i %s' % args.key + + self.target = args.target + target_sp = args.target.split('@') + if len(target_sp) == 1: + self.login = "" + self.host = target_sp[0] + elif len(target_sp) == 2: + self.login = target_sp[0] + self.host = target_sp[1] + else: + logger.error("Invalid target argument: %s" % args.target) + + +class RecipeNative: + """Base class for calling bitbake to provide a -native recipe""" + + def __init__(self, name, target_arch=None): + self.name = name + self.target_arch = target_arch + self.bootstrap_tasks = [self.name + ':do_addto_recipe_sysroot'] + self.staging_bindir_native = None + self.target_sys = None + self.__native_bin = None + + def _initialize(self, config, workspace, tinfoil): + """Get the parsed recipe""" + recipe_d = parse_recipe( + config, tinfoil, self.name, appends=True, filter_workspace=False) + if not recipe_d: + raise DevtoolError("Parsing %s recipe failed" % self.name) + self.staging_bindir_native = os.path.realpath( + recipe_d.getVar('STAGING_BINDIR_NATIVE')) + self.target_sys = recipe_d.getVar('TARGET_SYS') + return recipe_d + + def initialize(self, config, workspace, tinfoil): + """Basic initialization that can be overridden by a derived class""" + self._initialize(config, workspace, tinfoil) + + @property + def native_bin(self): + if not self.__native_bin: + raise DevtoolError("native binary name is not defined.") + return self.__native_bin + + +class RecipeGdbCross(RecipeNative): + """Handle handle gdb-cross on the host and the gdbserver on the target device""" + + def __init__(self, args, target_arch, target_device): + super().__init__('gdb-cross-' + target_arch, target_arch) + self.target_device = target_device + self.gdb = None + self.gdbserver_port_next = int(args.gdbserver_port_start) + self.config_db = {} + + def __find_gdbserver(self, config, tinfoil): + """Absolute path of the gdbserver""" + recipe_d_gdb = parse_recipe( + config, tinfoil, 'gdb', appends=True, filter_workspace=False) + if not recipe_d_gdb: + raise DevtoolError("Parsing gdb recipe failed") + return os.path.join(recipe_d_gdb.getVar('bindir'), 'gdbserver') + + def initialize(self, config, workspace, tinfoil): + super()._initialize(config, workspace, tinfoil) + gdb_bin = self.target_sys + '-gdb' + gdb_path = os.path.join( + self.staging_bindir_native, self.target_sys, gdb_bin) + self.gdb = gdb_path + self.gdbserver_path = self.__find_gdbserver(config, tinfoil) + + @property + def host(self): + return self.target_device.host + + +class RecipeImage: + """Handle some image recipe related properties + + Most workflows require firmware that runs on the target device. + This firmware must be consistent with the setup of the host system. + In particular, the debug symbols must be compatible. For this, the + rootfs must be created as part of the SDK. + """ + + def __init__(self, name): + self.combine_dbg_image = False + self.gdbserver_missing = False + self.name = name + self.rootfs = None + self.__rootfs_dbg = None + self.bootstrap_tasks = [self.name + ':do_build'] + + def initialize(self, config, tinfoil): + image_d = parse_recipe( + config, tinfoil, self.name, appends=True, filter_workspace=False) + if not image_d: + raise DevtoolError( + "Parsing image recipe %s failed" % self.name) + + self.combine_dbg_image = bb.data.inherits_class( + 'image-combined-dbg', image_d) + + workdir = image_d.getVar('WORKDIR') + self.rootfs = os.path.join(workdir, 'rootfs') + if image_d.getVar('IMAGE_GEN_DEBUGFS') == "1": + self.__rootfs_dbg = os.path.join(workdir, 'rootfs-dbg') + + self.gdbserver_missing = 'gdbserver' not in image_d.getVar( + 'IMAGE_INSTALL') + + @property + def debug_support(self): + return bool(self.rootfs_dbg) + + @property + def rootfs_dbg(self): + if self.__rootfs_dbg and os.path.isdir(self.__rootfs_dbg): + return self.__rootfs_dbg + return None + + +class RecipeMetaIdeSupport: + """For the shared sysroots mode meta-ide-support is needed + + For use cases where just a cross tool-chain is required but + no recipe is used, devtool ide-sdk abstracts calling bitbake meta-ide-support + and bitbake build-sysroots. This also allows to expose the cross-toolchains + to IDEs. For example VSCode support different tool-chains with e.g. cmake-kits. + """ + + def __init__(self): + self.bootstrap_tasks = ['meta-ide-support:do_build'] + self.topdir = None + self.datadir = None + self.deploy_dir_image = None + self.build_sys = None + # From toolchain-scripts + self.real_multimach_target_sys = None + + def initialize(self, config, tinfoil): + meta_ide_support_d = parse_recipe( + config, tinfoil, 'meta-ide-support', appends=True, filter_workspace=False) + if not meta_ide_support_d: + raise DevtoolError("Parsing meta-ide-support recipe failed") + + self.topdir = meta_ide_support_d.getVar('TOPDIR') + self.datadir = meta_ide_support_d.getVar('datadir') + self.deploy_dir_image = meta_ide_support_d.getVar( + 'DEPLOY_DIR_IMAGE') + self.build_sys = meta_ide_support_d.getVar('BUILD_SYS') + self.real_multimach_target_sys = meta_ide_support_d.getVar( + 'REAL_MULTIMACH_TARGET_SYS') + + +class RecipeBuildSysroots: + """For the shared sysroots mode build-sysroots is needed""" + + def __init__(self): + self.standalone_sysroot = None + self.standalone_sysroot_native = None + self.bootstrap_tasks = [ + 'build-sysroots:do_build_target_sysroot', + 'build-sysroots:do_build_native_sysroot' + ] + + def initialize(self, config, tinfoil): + build_sysroots_d = parse_recipe( + config, tinfoil, 'build-sysroots', appends=True, filter_workspace=False) + if not build_sysroots_d: + raise DevtoolError("Parsing build-sysroots recipe failed") + self.standalone_sysroot = build_sysroots_d.getVar( + 'STANDALONE_SYSROOT') + self.standalone_sysroot_native = build_sysroots_d.getVar( + 'STANDALONE_SYSROOT_NATIVE') + + +class SharedSysrootsEnv: + """Handle the shared sysroots based workflow + + Support the workflow with just a tool-chain without a recipe. + It's basically like: + bitbake some-dependencies + bitbake meta-ide-support + bitbake build-sysroots + Use the environment-* file found in the deploy folder + """ + + def __init__(self): + self.ide_support = None + self.build_sysroots = None + + def initialize(self, ide_support, build_sysroots): + self.ide_support = ide_support + self.build_sysroots = build_sysroots + + def setup_ide(self, ide): + ide.setup(self) + + +class RecipeNotModified: + """Handling of recipes added to the Direct DSK shared sysroots.""" + + def __init__(self, name): + self.name = name + self.bootstrap_tasks = [name + ':do_populate_sysroot'] + + +class RecipeModified: + """Handling of recipes in the workspace created by devtool modify""" + OE_INIT_BUILD_ENV = 'oe-init-build-env' + + VALID_BASH_ENV_NAME_CHARS = re.compile(r"^[a-zA-Z0-9_]*$") + + def __init__(self, name): + self.name = name + self.bootstrap_tasks = [name + ':do_install'] + self.gdb_cross = None + # workspace + self.real_srctree = None + self.srctree = None + self.ide_sdk_dir = None + self.ide_sdk_scripts_dir = None + self.bbappend = None + # recipe variables from d.getVar + self.b = None + self.base_libdir = None + self.bblayers = None + self.bpn = None + self.d = None + self.fakerootcmd = None + self.fakerootenv = None + self.libdir = None + self.max_process = None + self.package_arch = None + self.package_debug_split_style = None + self.path = None + self.pn = None + self.recipe_sysroot = None + self.recipe_sysroot_native = None + self.staging_incdir = None + self.strip_cmd = None + self.target_arch = None + self.target_dbgsrc_dir = None + self.topdir = None + self.workdir = None + self.recipe_id = None + # replicate bitbake build environment + self.exported_vars = None + self.cmd_compile = None + self.__oe_init_dir = None + # main build tool used by this recipe + self.build_tool = BuildTool.UNDEFINED + # build_tool = cmake + self.oecmake_generator = None + self.cmake_cache_vars = None + # build_tool = meson + self.meson_buildtype = None + self.meson_wrapper = None + self.mesonopts = None + self.extra_oemeson = None + self.meson_cross_file = None + + def initialize(self, config, workspace, tinfoil): + recipe_d = parse_recipe( + config, tinfoil, self.name, appends=True, filter_workspace=False) + if not recipe_d: + raise DevtoolError("Parsing %s recipe failed" % self.name) + + # Verify this recipe is built as externalsrc setup by devtool modify + workspacepn = check_workspace_recipe( + workspace, self.name, bbclassextend=True) + self.srctree = workspace[workspacepn]['srctree'] + # Need to grab this here in case the source is within a subdirectory + self.real_srctree = get_real_srctree( + self.srctree, recipe_d.getVar('S'), recipe_d.getVar('WORKDIR')) + self.bbappend = workspace[workspacepn]['bbappend'] + + self.ide_sdk_dir = os.path.join( + config.workspace_path, 'ide-sdk', self.name) + if os.path.exists(self.ide_sdk_dir): + shutil.rmtree(self.ide_sdk_dir) + self.ide_sdk_scripts_dir = os.path.join(self.ide_sdk_dir, 'scripts') + + self.b = recipe_d.getVar('B') + self.base_libdir = recipe_d.getVar('base_libdir') + self.bblayers = recipe_d.getVar('BBLAYERS').split() + self.bpn = recipe_d.getVar('BPN') + self.cxx = recipe_d.getVar('CXX') + self.d = recipe_d.getVar('D') + self.fakerootcmd = recipe_d.getVar('FAKEROOTCMD') + self.fakerootenv = recipe_d.getVar('FAKEROOTENV') + self.libdir = recipe_d.getVar('libdir') + self.max_process = int(recipe_d.getVar( + "BB_NUMBER_THREADS") or os.cpu_count() or 1) + self.package_arch = recipe_d.getVar('PACKAGE_ARCH') + self.package_debug_split_style = recipe_d.getVar( + 'PACKAGE_DEBUG_SPLIT_STYLE') + self.path = recipe_d.getVar('PATH') + self.pn = recipe_d.getVar('PN') + self.recipe_sysroot = os.path.realpath( + recipe_d.getVar('RECIPE_SYSROOT')) + self.recipe_sysroot_native = os.path.realpath( + recipe_d.getVar('RECIPE_SYSROOT_NATIVE')) + self.staging_bindir_toolchain = os.path.realpath( + recipe_d.getVar('STAGING_BINDIR_TOOLCHAIN')) + self.staging_incdir = os.path.realpath( + recipe_d.getVar('STAGING_INCDIR')) + self.strip_cmd = recipe_d.getVar('STRIP') + self.target_arch = recipe_d.getVar('TARGET_ARCH') + self.target_dbgsrc_dir = recipe_d.getVar('TARGET_DBGSRC_DIR') + self.topdir = recipe_d.getVar('TOPDIR') + self.workdir = os.path.realpath(recipe_d.getVar('WORKDIR')) + + self.__init_exported_variables(recipe_d) + + if bb.data.inherits_class('cmake', recipe_d): + self.oecmake_generator = recipe_d.getVar('OECMAKE_GENERATOR') + self.__init_cmake_preset_cache(recipe_d) + self.build_tool = BuildTool.CMAKE + elif bb.data.inherits_class('meson', recipe_d): + self.meson_buildtype = recipe_d.getVar('MESON_BUILDTYPE') + self.mesonopts = recipe_d.getVar('MESONOPTS') + self.extra_oemeson = recipe_d.getVar('EXTRA_OEMESON') + self.meson_cross_file = recipe_d.getVar('MESON_CROSS_FILE') + self.build_tool = BuildTool.MESON + + # Recipe ID is the identifier for IDE config sections + self.recipe_id = self.bpn + "-" + self.package_arch + self.recipe_id_pretty = self.bpn + ": " + self.package_arch + + def append_to_bbappend(self, append_text): + with open(self.bbappend, 'a') as bbap: + bbap.write(append_text) + + def remove_from_bbappend(self, append_text): + with open(self.bbappend, 'r') as bbap: + text = bbap.read() + new_text = text.replace(append_text, '') + with open(self.bbappend, 'w') as bbap: + bbap.write(new_text) + + @staticmethod + def is_valid_shell_variable(var): + """Skip strange shell variables like systemd + + prevent from strange bugs because of strange variables which + are not used in this context but break various tools. + """ + if RecipeModified.VALID_BASH_ENV_NAME_CHARS.match(var): + bb.debug(1, "ignoring variable: %s" % var) + return True + return False + + def debug_build_config(self, args): + """Explicitely set for example CMAKE_BUILD_TYPE to Debug if not defined otherwise""" + if self.build_tool is BuildTool.CMAKE: + append_text = os.linesep + \ + 'OECMAKE_ARGS:append = " -DCMAKE_BUILD_TYPE:STRING=Debug"' + os.linesep + if args.debug_build_config and not 'CMAKE_BUILD_TYPE' in self.cmake_cache_vars: + self.cmake_cache_vars['CMAKE_BUILD_TYPE'] = { + "type": "STRING", + "value": "Debug", + } + self.append_to_bbappend(append_text) + elif 'CMAKE_BUILD_TYPE' in self.cmake_cache_vars: + del self.cmake_cache_vars['CMAKE_BUILD_TYPE'] + self.remove_from_bbappend(append_text) + elif self.build_tool is BuildTool.MESON: + append_text = os.linesep + 'MESON_BUILDTYPE = "debug"' + os.linesep + if args.debug_build_config and self.meson_buildtype != "debug": + self.mesonopts.replace( + '--buildtype ' + self.meson_buildtype, '--buildtype debug') + self.append_to_bbappend(append_text) + elif self.meson_buildtype == "debug": + self.mesonopts.replace( + '--buildtype debug', '--buildtype plain') + self.remove_from_bbappend(append_text) + elif args.debug_build_config: + logger.warn( + "--debug-build-config is not implemented for this build tool yet.") + + def solib_search_path(self, image): + """Search for debug symbols in the rootfs and rootfs-dbg + + The debug symbols of shared libraries which are provided by other packages + are grabbed from the -dbg packages in the rootfs-dbg. + + But most cross debugging tools like gdb, perf, and systemtap need to find + executable/library first and through it debuglink note find corresponding + symbols file. Therefore the library paths from the rootfs are added as well. + + Note: For the devtool modified recipe compiled from the IDE, the debug + symbols are taken from the unstripped binaries in the image folder. + Also, devtool deploy-target takes the files from the image folder. + debug symbols in the image folder refer to the corresponding source files + with absolute paths of the build machine. Debug symbols found in the + rootfs-dbg are relocated and contain paths which refer to the source files + installed on the target device e.g. /usr/src/... + """ + base_libdir = self.base_libdir.lstrip('/') + libdir = self.libdir.lstrip('/') + so_paths = [ + # debug symbols for package_debug_split_style: debug-with-srcpkg or .debug + os.path.join(image.rootfs_dbg, base_libdir, ".debug"), + os.path.join(image.rootfs_dbg, libdir, ".debug"), + # debug symbols for package_debug_split_style: debug-file-directory + os.path.join(image.rootfs_dbg, "usr", "lib", "debug"), + + # The binaries are required as well, the debug packages are not enough + # With image-combined-dbg.bbclass the binaries are copied into rootfs-dbg + os.path.join(image.rootfs_dbg, base_libdir), + os.path.join(image.rootfs_dbg, libdir), + # Without image-combined-dbg.bbclass the binaries are only in rootfs. + # Note: Stepping into source files located in rootfs-dbg does not + # work without image-combined-dbg.bbclass yet. + os.path.join(image.rootfs, base_libdir), + os.path.join(image.rootfs, libdir) + ] + return so_paths + + def solib_search_path_str(self, image): + """Return a : separated list of paths usable by GDB's set solib-search-path""" + return ':'.join(self.solib_search_path(image)) + + def __init_exported_variables(self, d): + """Find all variables with export flag set. + + This allows to generate IDE configurations which compile with the same + environment as bitbake does. That's at least a reasonable default behavior. + """ + exported_vars = {} + + vars = (key for key in d.keys() if not key.startswith( + "__") and not d.getVarFlag(key, "func", False)) + for var in vars: + func = d.getVarFlag(var, "func", False) + if d.getVarFlag(var, 'python', False) and func: + continue + export = d.getVarFlag(var, "export", False) + unexport = d.getVarFlag(var, "unexport", False) + if not export and not unexport and not func: + continue + if unexport: + continue + + val = d.getVar(var) + if val is None: + continue + if set(var) & set("-.{}+"): + logger.warn( + "Warning: Found invalid character in variable name %s", str(var)) + continue + varExpanded = d.expand(var) + val = str(val) + + if not RecipeModified.is_valid_shell_variable(varExpanded): + continue + + if func: + code_line = "line: {0}, file: {1}\n".format( + d.getVarFlag(var, "lineno", False), + d.getVarFlag(var, "filename", False)) + val = val.rstrip('\n') + logger.warn("Warning: exported shell function %s() is not exported (%s)" % + (varExpanded, code_line)) + continue + + if export: + exported_vars[varExpanded] = val.strip() + continue + + self.exported_vars = exported_vars + + def __init_cmake_preset_cache(self, d): + """Get the arguments passed to cmake + + Replicate the cmake configure arguments with all details to + share on build folder between bitbake and SDK. + """ + site_file = os.path.join(self.workdir, 'site-file.cmake') + if os.path.exists(site_file): + print("Warning: site-file.cmake is not supported") + + cache_vars = {} + oecmake_args = d.getVar('OECMAKE_ARGS').split() + extra_oecmake = d.getVar('EXTRA_OECMAKE').split() + for param in oecmake_args + extra_oecmake: + d_pref = "-D" + if param.startswith(d_pref): + param = param[len(d_pref):] + else: + print("Error: expected a -D") + param_s = param.split('=', 1) + param_nt = param_s[0].split(':', 1) + + def handle_undefined_variable(var): + if var.startswith('${') and var.endswith('}'): + return '' + else: + return var + # Example: FOO=ON + if len(param_nt) == 1: + cache_vars[param_s[0]] = handle_undefined_variable(param_s[1]) + # Example: FOO:PATH=/tmp + elif len(param_nt) == 2: + cache_vars[param_nt[0]] = { + "type": param_nt[1], + "value": handle_undefined_variable(param_s[1]), + } + else: + print("Error: cannot parse %s" % param) + self.cmake_cache_vars = cache_vars + + def cmake_preset(self): + """Create a preset for cmake that mimics how bitbake calls cmake""" + toolchain_file = os.path.join(self.workdir, 'toolchain.cmake') + cmake_executable = os.path.join( + self.recipe_sysroot_native, 'usr', 'bin', 'cmake') + self.cmd_compile = cmake_executable + " --build --preset " + self.recipe_id + + preset_dict_configure = { + "name": self.recipe_id, + "displayName": self.recipe_id_pretty, + "description": "Bitbake build environment for the recipe %s compiled for %s" % (self.bpn, self.package_arch), + "binaryDir": self.b, + "generator": self.oecmake_generator, + "toolchainFile": toolchain_file, + "cacheVariables": self.cmake_cache_vars, + "environment": self.exported_vars, + "cmakeExecutable": cmake_executable + } + + preset_dict_build = { + "name": self.recipe_id, + "displayName": self.recipe_id_pretty, + "description": "Bitbake build environment for the recipe %s compiled for %s" % (self.bpn, self.package_arch), + "configurePreset": self.recipe_id, + "inheritConfigureEnvironment": True + } + + preset_dict_test = { + "name": self.recipe_id, + "displayName": self.recipe_id_pretty, + "description": "Bitbake build environment for the recipe %s compiled for %s" % (self.bpn, self.package_arch), + "configurePreset": self.recipe_id, + "inheritConfigureEnvironment": True + } + + preset_dict = { + "version": 3, # cmake 3.21, backward compatible with kirkstone + "configurePresets": [preset_dict_configure], + "buildPresets": [preset_dict_build], + "testPresets": [preset_dict_test] + } + + # Finally write the json file + json_file = 'CMakeUserPresets.json' + json_path = os.path.join(self.real_srctree, json_file) + logger.info("Updating CMake preset: %s (%s)" % (json_file, json_path)) + if not os.path.exists(self.real_srctree): + os.makedirs(self.real_srctree) + try: + with open(json_path) as f: + orig_dict = json.load(f) + except json.decoder.JSONDecodeError: + logger.info( + "Decoding %s failed. Probably because of comments in the json file" % json_path) + orig_dict = {} + except FileNotFoundError: + orig_dict = {} + + # Add or update the presets for the recipe and keep other presets + for k, v in preset_dict.items(): + if isinstance(v, list): + update_preset = v[0] + preset_added = False + if k in orig_dict: + for index, orig_preset in enumerate(orig_dict[k]): + if 'name' in orig_preset: + if orig_preset['name'] == update_preset['name']: + logger.debug("Updating preset: %s" % + orig_preset['name']) + orig_dict[k][index] = update_preset + preset_added = True + break + else: + logger.debug("keeping preset: %s" % + orig_preset['name']) + else: + logger.warn("preset without a name found") + if not preset_added: + if not k in orig_dict: + orig_dict[k] = [] + orig_dict[k].append(update_preset) + logger.debug("Added preset: %s" % + update_preset['name']) + else: + orig_dict[k] = v + + with open(json_path, 'w') as f: + json.dump(orig_dict, f, indent=4) + + def gen_meson_wrapper(self): + """Generate a wrapper script to call meson with the cross environment""" + bb.utils.mkdirhier(self.ide_sdk_scripts_dir) + meson_wrapper = os.path.join(self.ide_sdk_scripts_dir, 'meson') + meson_real = os.path.join( + self.recipe_sysroot_native, 'usr', 'bin', 'meson.real') + with open(meson_wrapper, 'w') as mwrap: + mwrap.write("#!/bin/sh" + os.linesep) + for var, val in self.exported_vars.items(): + mwrap.write('export %s="%s"' % (var, val) + os.linesep) + mwrap.write("unset CC CXX CPP LD AR NM STRIP" + os.linesep) + private_temp = os.path.join(self.b, "meson-private", "tmp") + mwrap.write('mkdir -p "%s"' % private_temp + os.linesep) + mwrap.write('export TMPDIR="%s"' % private_temp + os.linesep) + mwrap.write('exec "%s" "$@"' % meson_real + os.linesep) + st = os.stat(meson_wrapper) + os.chmod(meson_wrapper, st.st_mode | stat.S_IEXEC) + self.meson_wrapper = meson_wrapper + self.cmd_compile = meson_wrapper + " compile -C " + self.b + + def which(self, executable): + bin_path = shutil.which(executable, path=self.path) + if not bin_path: + raise DevtoolError( + 'Cannot find %s. Probably the recipe %s is not built yet.' % (executable, self.bpn)) + return bin_path + + @staticmethod + def is_elf_file(file_path): + with open(file_path, "rb") as f: + data = f.read(4) + if data == b'\x7fELF': + return True + return False + + def find_installed_binaries(self): + """find all executable elf files in the image directory""" + binaries = [] + d_len = len(self.d) + re_so = re.compile(r'.*\.so[.0-9]*$') + for root, _, files in os.walk(self.d, followlinks=False): + for file in files: + if os.path.islink(file): + continue + if re_so.match(file): + continue + abs_name = os.path.join(root, file) + if os.access(abs_name, os.X_OK) and RecipeModified.is_elf_file(abs_name): + binaries.append(abs_name[d_len:]) + return sorted(binaries) + + def gen_delete_package_dirs(self): + """delete folders of package tasks + + This is a workaround for and issue with recipes having their sources + downloaded as file:// + This likely breaks pseudo like: + path mismatch [3 links]: ino 79147802 db + .../build/tmp/.../cmake-example/1.0/package/usr/src/debug/ + cmake-example/1.0-r0/oe-local-files/cpp-example-lib.cpp + .../build/workspace/sources/cmake-example/oe-local-files/cpp-example-lib.cpp + Since the files are anyway outdated lets deleted them (also from pseudo's db) to workaround this issue. + """ + cmd_lines = ['#!/bin/sh'] + + # Set up the appropriate environment + newenv = dict(os.environ) + for varvalue in self.fakerootenv.split(): + if '=' in varvalue: + splitval = varvalue.split('=', 1) + newenv[splitval[0]] = splitval[1] + + # Replicate the environment variables from bitbake + for var, val in newenv.items(): + if not RecipeModified.is_valid_shell_variable(var): + continue + cmd_lines.append('%s="%s"' % (var, val)) + cmd_lines.append('export %s' % var) + + # Delete the folders + pkg_dirs = ' '.join([os.path.join(self.workdir, d) for d in [ + "package", "packages-split", "pkgdata", "sstate-install-package", "debugsources.list", "*.spec"]]) + cmd = "%s rm -rf %s" % (self.fakerootcmd, pkg_dirs) + cmd_lines.append('%s || { "%s failed"; exit 1; }' % (cmd, cmd)) + + return self.write_script(cmd_lines, 'delete_package_dirs') + + def gen_deploy_target_script(self, args): + """Generate a script which does what devtool deploy-target does + + This script is much quicker than devtool target-deploy. Because it + does not need to start a bitbake server. All information from tinfoil + is hard-coded in the generated script. + """ + cmd_lines = ['#!%s' % str(sys.executable)] + cmd_lines.append('import sys') + cmd_lines.append('devtool_sys_path = %s' % str(sys.path)) + cmd_lines.append('devtool_sys_path.reverse()') + cmd_lines.append('for p in devtool_sys_path:') + cmd_lines.append(' if p not in sys.path:') + cmd_lines.append(' sys.path.insert(0, p)') + cmd_lines.append('from devtool.deploy import deploy_no_d') + args_filter = ['debug', 'dry_run', 'key', 'no_check_space', 'no_host_check', + 'no_preserve', 'port', 'show_status', 'ssh_exec', 'strip', 'target'] + filtered_args_dict = {key: value for key, value in vars( + args).items() if key in args_filter} + cmd_lines.append('filtered_args_dict = %s' % str(filtered_args_dict)) + cmd_lines.append('class Dict2Class(object):') + cmd_lines.append(' def __init__(self, my_dict):') + cmd_lines.append(' for key in my_dict:') + cmd_lines.append(' setattr(self, key, my_dict[key])') + cmd_lines.append('filtered_args = Dict2Class(filtered_args_dict)') + cmd_lines.append( + 'setattr(filtered_args, "recipename", "%s")' % self.bpn) + cmd_lines.append('deploy_no_d("%s", "%s", "%s", "%s", "%s", "%s", %d, "%s", "%s", filtered_args)' % + (self.d, self.workdir, self.path, self.strip_cmd, + self.libdir, self.base_libdir, self.max_process, + self.fakerootcmd, self.fakerootenv)) + return self.write_script(cmd_lines, 'deploy_target') + + def gen_install_deploy_script(self, args): + """Generate a script which does install and deploy""" + cmd_lines = ['#!/bin/bash'] + + cmd_lines.append(self.gen_delete_package_dirs()) + + # . oe-init-build-env $BUILDDIR + # Note: Sourcing scripts with arguments requires bash + cmd_lines.append('cd "%s" || { echo "cd %s failed"; exit 1; }' % ( + self.oe_init_dir, self.oe_init_dir)) + cmd_lines.append('. "%s" "%s" || { echo ". %s %s failed"; exit 1; }' % ( + self.oe_init_build_env, self.topdir, self.oe_init_build_env, self.topdir)) + + # bitbake -c install + cmd_lines.append( + 'bitbake %s -c install --force || { echo "bitbake %s -c install --force failed"; exit 1; }' % (self.bpn, self.bpn)) + + # Self contained devtool deploy-target + cmd_lines.append(self.gen_deploy_target_script(args)) + + return self.write_script(cmd_lines, 'install_and_deploy') + + def write_script(self, cmd_lines, script_name): + bb.utils.mkdirhier(self.ide_sdk_scripts_dir) + script_name_arch = script_name + '_' + self.recipe_id + script_file = os.path.join(self.ide_sdk_scripts_dir, script_name_arch) + with open(script_file, 'w') as script_f: + script_f.write(os.linesep.join(cmd_lines)) + st = os.stat(script_file) + os.chmod(script_file, st.st_mode | stat.S_IEXEC) + return script_file + + @property + def oe_init_build_env(self): + """Find the oe-init-build-env used for this setup""" + oe_init_dir = self.oe_init_dir + if oe_init_dir: + return os.path.join(oe_init_dir, RecipeModified.OE_INIT_BUILD_ENV) + return None + + @property + def oe_init_dir(self): + """Find the directory where the oe-init-build-env is located + + Assumption: There might be a layer with higher priority than poky + which provides to oe-init-build-env in the layer's toplevel folder. + """ + if not self.__oe_init_dir: + for layer in reversed(self.bblayers): + result = subprocess.run( + ['git', 'rev-parse', '--show-toplevel'], cwd=layer, capture_output=True) + if result.returncode == 0: + oe_init_dir = result.stdout.decode('utf-8').strip() + oe_init_path = os.path.join( + oe_init_dir, RecipeModified.OE_INIT_BUILD_ENV) + if os.path.exists(oe_init_path): + logger.debug("Using %s from: %s" % ( + RecipeModified.OE_INIT_BUILD_ENV, oe_init_path)) + self.__oe_init_dir = oe_init_dir + break + if not self.__oe_init_dir: + logger.error("Cannot find the bitbake top level folder") + return self.__oe_init_dir + + +def ide_setup(args, config, basepath, workspace): + """Generate the IDE configuration for the workspace""" + + # Explicitely passing some special recipes does not make sense + for recipe in args.recipenames: + if recipe in ['meta-ide-support', 'build-sysroots']: + raise DevtoolError("Invalid recipe: %s." % recipe) + + # Collect information about tasks which need to be bitbaked + bootstrap_tasks = [] + bootstrap_tasks_late = [] + tinfoil = setup_tinfoil(config_only=False, basepath=basepath) + try: + # define mode depending on recipes which need to be processed + recipes_image_names = [] + recipes_modified_names = [] + recipes_other_names = [] + for recipe in args.recipenames: + try: + check_workspace_recipe( + workspace, recipe, bbclassextend=True) + recipes_modified_names.append(recipe) + except DevtoolError: + recipe_d = parse_recipe( + config, tinfoil, recipe, appends=True, filter_workspace=False) + if not recipe_d: + raise DevtoolError("Parsing recipe %s failed" % recipe) + if bb.data.inherits_class('image', recipe_d): + recipes_image_names.append(recipe) + else: + recipes_other_names.append(recipe) + + invalid_params = False + if args.mode == DevtoolIdeMode.shared: + if len(recipes_modified_names): + logger.error("In shared sysroots mode modified recipes %s cannot be handled." % str( + recipes_modified_names)) + invalid_params = True + if args.mode == DevtoolIdeMode.modified: + if len(recipes_other_names): + logger.error("Only in shared sysroots mode not modified recipes %s can be handled." % str( + recipes_other_names)) + invalid_params = True + if len(recipes_image_names) != 1: + logger.error( + "One image recipe is required as the rootfs for the remote development.") + invalid_params = True + for modified_recipe_name in recipes_modified_names: + if modified_recipe_name.startswith('nativesdk-') or modified_recipe_name.endswith('-native'): + logger.error( + "Only cross compiled recipes are support. %s is not cross." % modified_recipe_name) + invalid_params = True + + if invalid_params: + raise DevtoolError("Invalid parameters are passed.") + + # For the shared sysroots mode, add all dependencies of all the images to the sysroots + # For the modified mode provide one rootfs and the corresponding debug symbols via rootfs-dbg + recipes_images = [] + for recipes_image_name in recipes_image_names: + logger.info("Using image: %s" % recipes_image_name) + recipe_image = RecipeImage(recipes_image_name) + recipe_image.initialize(config, tinfoil) + bootstrap_tasks += recipe_image.bootstrap_tasks + recipes_images.append(recipe_image) + + # Provide a Direct SDK with shared sysroots + recipes_not_modified = [] + if args.mode == DevtoolIdeMode.shared: + ide_support = RecipeMetaIdeSupport() + ide_support.initialize(config, tinfoil) + bootstrap_tasks += ide_support.bootstrap_tasks + + logger.info("Adding %s to the Direct SDK sysroots." % + str(recipes_other_names)) + for recipe_name in recipes_other_names: + recipe_not_modified = RecipeNotModified(recipe_name) + bootstrap_tasks += recipe_not_modified.bootstrap_tasks + recipes_not_modified.append(recipe_not_modified) + + build_sysroots = RecipeBuildSysroots() + build_sysroots.initialize(config, tinfoil) + bootstrap_tasks_late += build_sysroots.bootstrap_tasks + shared_env = SharedSysrootsEnv() + shared_env.initialize(ide_support, build_sysroots) + + recipes_modified = [] + if args.mode == DevtoolIdeMode.modified: + logger.info("Setting up workspaces for modified recipe: %s" % + str(recipes_modified_names)) + gdbs_cross = {} + for recipe_name in recipes_modified_names: + recipe_modified = RecipeModified(recipe_name) + recipe_modified.initialize(config, workspace, tinfoil) + bootstrap_tasks += recipe_modified.bootstrap_tasks + recipes_modified.append(recipe_modified) + + if recipe_modified.target_arch not in gdbs_cross: + target_device = TargetDevice(args) + gdb_cross = RecipeGdbCross( + args, recipe_modified.target_arch, target_device) + gdb_cross.initialize(config, workspace, tinfoil) + bootstrap_tasks += gdb_cross.bootstrap_tasks + gdbs_cross[recipe_modified.target_arch] = gdb_cross + recipe_modified.gdb_cross = gdbs_cross[recipe_modified.target_arch] + + finally: + tinfoil.shutdown() + + if not args.skip_bitbake: + bb_cmd = 'bitbake ' + if args.bitbake_k: + bb_cmd += "-k " + bb_cmd_early = bb_cmd + ' '.join(bootstrap_tasks) + exec_build_env_command( + config.init_path, basepath, bb_cmd_early, watch=True) + if bootstrap_tasks_late: + bb_cmd_late = bb_cmd + ' '.join(bootstrap_tasks_late) + exec_build_env_command( + config.init_path, basepath, bb_cmd_late, watch=True) + + for recipe_image in recipes_images: + if (recipe_image.gdbserver_missing): + logger.warning( + "gdbserver not installed in image %s. Remote debugging will not be available" % recipe_image) + + if recipe_image.combine_dbg_image is False: + logger.warning( + 'IMAGE_CLASSES += "image-combined-dbg" is missing for image %s. Remote debugging will not find debug symbols from rootfs-dbg.' % recipe_image) + + # Instantiate the active IDE plugin + ide = ide_plugins[args.ide]() + if args.mode == DevtoolIdeMode.shared: + ide.setup_shared_sysroots(shared_env) + elif args.mode == DevtoolIdeMode.modified: + for recipe_modified in recipes_modified: + if recipe_modified.build_tool is BuildTool.CMAKE: + recipe_modified.cmake_preset() + if recipe_modified.build_tool is BuildTool.MESON: + recipe_modified.gen_meson_wrapper() + ide.setup_modified_recipe( + args, recipe_image, recipe_modified) + else: + raise DevtoolError("Must not end up here.") + + +def register_commands(subparsers, context): + """Register devtool subcommands from this plugin""" + + global ide_plugins + + # Search for IDE plugins in all sub-folders named ide_plugins where devtool seraches for plugins. + pluginpaths = [os.path.join(path, 'ide_plugins') + for path in context.pluginpaths] + ide_plugin_modules = [] + for pluginpath in pluginpaths: + scriptutils.load_plugins(logger, ide_plugin_modules, pluginpath) + + for ide_plugin_module in ide_plugin_modules: + if hasattr(ide_plugin_module, 'register_ide_plugin'): + ide_plugin_module.register_ide_plugin(ide_plugins) + # Sort plugins according to their priority. The first entry is the default IDE plugin. + ide_plugins = dict(sorted(ide_plugins.items(), + key=lambda p: p[1].ide_plugin_priority(), reverse=True)) + + parser_ide_sdk = subparsers.add_parser('ide-sdk', group='working', order=50, formatter_class=RawTextHelpFormatter, + help='Setup the SDK and configure the IDE') + parser_ide_sdk.add_argument( + 'recipenames', nargs='+', help='Generate an IDE configuration suitable to work on the given recipes.\n' + 'Depending on the --mode paramter different types of SDKs and IDE configurations are generated.') + parser_ide_sdk.add_argument( + '-m', '--mode', type=DevtoolIdeMode, default=DevtoolIdeMode.modified, + help='Different SDK types are supported:\n' + '- "' + DevtoolIdeMode.modified.name + '" (default):\n' + ' devtool modify creates a workspace to work on the source code of a recipe.\n' + ' devtool ide-sdk builds the SDK and generates the IDE configuration(s) in the workspace directorie(s)\n' + ' Usage example:\n' + ' devtool modify cmake-example\n' + ' devtool ide-sdk cmake-example core-image-minimal\n' + ' Start the IDE in the workspace folder\n' + ' At least one devtool modified recipe plus one image recipe are required:\n' + ' The image recipe is used to generate the target image and the remote debug configuration.\n' + '- "' + DevtoolIdeMode.shared.name + '":\n' + ' Usage example:\n' + ' devtool ide-sdk -m ' + DevtoolIdeMode.shared.name + ' recipe(s)\n' + ' This command generates a cross-toolchain as well as the corresponding shared sysroot directories.\n' + ' To use this tool-chain the environment-* file found in the deploy..image folder needs to be sourced into a shell.\n' + ' In case of VSCode and cmake the tool-chain is also exposed as a cmake-kit') + default_ide = list(ide_plugins.keys())[0] + parser_ide_sdk.add_argument( + '-i', '--ide', choices=ide_plugins.keys(), default=default_ide, + help='Setup the configuration for this IDE (default: %s)' % default_ide) + parser_ide_sdk.add_argument( + '-t', '--target', default='root@192.168.7.2', + help='Live target machine running an ssh server: user@hostname.') + parser_ide_sdk.add_argument( + '-G', '--gdbserver-port-start', default="1234", help='port where gdbserver is listening.') + parser_ide_sdk.add_argument( + '-c', '--no-host-check', help='Disable ssh host key checking', action='store_true') + parser_ide_sdk.add_argument( + '-e', '--ssh-exec', help='Executable to use in place of ssh') + parser_ide_sdk.add_argument( + '-P', '--port', help='Specify ssh port to use for connection to the target') + parser_ide_sdk.add_argument( + '-I', '--key', help='Specify ssh private key for connection to the target') + parser_ide_sdk.add_argument( + '--skip-bitbake', help='Generate IDE configuration but skip calling bibtake to update the SDK.', action='store_true') + parser_ide_sdk.add_argument( + '-k', '--bitbake-k', help='Pass -k parameter to bitbake', action='store_true') + parser_ide_sdk.add_argument( + '--no-strip', help='Do not strip executables prior to deploy', dest='strip', action='store_false') + parser_ide_sdk.add_argument( + '-n', '--dry-run', help='List files to be undeployed only', action='store_true') + parser_ide_sdk.add_argument( + '-s', '--show-status', help='Show progress/status output', action='store_true') + parser_ide_sdk.add_argument( + '-p', '--no-preserve', help='Do not preserve existing files', action='store_true') + parser_ide_sdk.add_argument( + '--no-check-space', help='Do not check for available space before deploying', action='store_true') + parser_ide_sdk.add_argument( + '--debug-build-config', help='Use debug build flags, for example set CMAKE_BUILD_TYPE=Debug', action='store_true') + parser_ide_sdk.set_defaults(func=ide_setup) diff --git a/scripts/lib/devtool/menuconfig.py b/scripts/lib/devtool/menuconfig.py index 95384c5333..18daef30c3 100644 --- a/scripts/lib/devtool/menuconfig.py +++ b/scripts/lib/devtool/menuconfig.py @@ -3,6 +3,8 @@ # Copyright (C) 2018 Xilinx # Written by: Chandana Kalluri <ckalluri@xilinx.com> # +# SPDX-License-Identifier: MIT +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. @@ -43,7 +45,7 @@ def menuconfig(args, config, basepath, workspace): return 1 check_workspace_recipe(workspace, args.component) - pn = rd.getVar('PN', True) + pn = rd.getVar('PN') if not rd.getVarFlag('do_menuconfig','task'): raise DevtoolError("This recipe does not support menuconfig option") diff --git a/scripts/lib/devtool/sdk.py b/scripts/lib/devtool/sdk.py index 3aa42a1466..9aefd7e354 100644 --- a/scripts/lib/devtool/sdk.py +++ b/scripts/lib/devtool/sdk.py @@ -207,7 +207,7 @@ def sdk_update(args, config, basepath, workspace): if not sstate_mirrors: with open(os.path.join(conf_dir, 'site.conf'), 'a') as f: f.write('SCONF_VERSION = "%s"\n' % site_conf_version) - f.write('SSTATE_MIRRORS_append = " file://.* %s/sstate-cache/PATH \\n "\n' % updateserver) + f.write('SSTATE_MIRRORS:append = " file://.* %s/sstate-cache/PATH"\n' % updateserver) finally: shutil.rmtree(tmpsdk_dir) @@ -300,7 +300,8 @@ def sdk_install(args, config, basepath, workspace): return 2 try: - exec_build_env_command(config.init_path, basepath, 'bitbake build-sysroots', watch=True) + exec_build_env_command(config.init_path, basepath, 'bitbake build-sysroots -c build_native_sysroot', watch=True) + exec_build_env_command(config.init_path, basepath, 'bitbake build-sysroots -c build_target_sysroot', watch=True) except bb.process.ExecutionError as e: raise DevtoolError('Failed to bitbake build-sysroots:\n%s' % (str(e))) diff --git a/scripts/lib/devtool/search.py b/scripts/lib/devtool/search.py index d24040df37..70b81cac5e 100644 --- a/scripts/lib/devtool/search.py +++ b/scripts/lib/devtool/search.py @@ -62,10 +62,11 @@ def search(args, config, basepath, workspace): with open(os.path.join(pkgdata_dir, 'runtime', pkg), 'r') as f: for line in f: if ': ' in line: - splitline = line.split(':', 1) + splitline = line.split(': ', 1) key = splitline[0] value = splitline[1].strip() - if key in ['PKG_%s' % pkg, 'DESCRIPTION', 'FILES_INFO'] or key.startswith('FILERPROVIDES_'): + key = key.replace(":" + pkg, "") + if key in ['PKG', 'DESCRIPTION', 'FILES_INFO', 'FILERPROVIDES']: if keyword_rc.search(value): match = True break diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py index 7b62b7e7b8..bd009f44b1 100644 --- a/scripts/lib/devtool/standard.py +++ b/scripts/lib/devtool/standard.py @@ -147,6 +147,8 @@ def add(args, config, basepath, workspace): extracmdopts += ' -a' if args.npm_dev: extracmdopts += ' --npm-dev' + if args.no_pypi: + extracmdopts += ' --no-pypi' if args.mirrors: extracmdopts += ' --mirrors' if args.srcrev: @@ -234,10 +236,14 @@ def add(args, config, basepath, workspace): if args.fetchuri and not args.no_git: setup_git_repo(srctree, args.version, 'devtool', d=tinfoil.config_data) - initial_rev = None + initial_rev = {} if os.path.exists(os.path.join(srctree, '.git')): (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) - initial_rev = stdout.rstrip() + initial_rev["."] = stdout.rstrip() + (stdout, _) = bb.process.run('git submodule --quiet foreach --recursive \'echo `git rev-parse HEAD` $PWD\'', cwd=srctree) + for line in stdout.splitlines(): + (rev, submodule) = line.split() + initial_rev[os.path.relpath(submodule, srctree)] = rev if args.src_subdir: srctree = os.path.join(srctree, args.src_subdir) @@ -251,16 +257,17 @@ def add(args, config, basepath, workspace): if b_is_s: f.write('EXTERNALSRC_BUILD = "%s"\n' % srctree) if initial_rev: - f.write('\n# initial_rev: %s\n' % initial_rev) + for key, value in initial_rev.items(): + f.write('\n# initial_rev %s: %s\n' % (key, value)) if args.binary: - f.write('do_install_append() {\n') + f.write('do_install:append() {\n') f.write(' rm -rf ${D}/.git\n') f.write(' rm -f ${D}/singletask.lock\n') f.write('}\n') if bb.data.inherits_class('npm', rd): - f.write('python do_configure_append() {\n') + f.write('python do_configure:append() {\n') f.write(' pkgdir = d.getVar("NPM_PACKAGE")\n') f.write(' lockfile = os.path.join(pkgdir, "singletask.lock")\n') f.write(' bb.utils.remove(lockfile)\n') @@ -318,10 +325,6 @@ def _check_compatible_recipe(pn, d): raise DevtoolError("The %s recipe is a packagegroup, and therefore is " "not supported by this tool" % pn, 4) - if bb.data.inherits_class('meta', d): - raise DevtoolError("The %s recipe is a meta-recipe, and therefore is " - "not supported by this tool" % pn, 4) - if bb.data.inherits_class('externalsrc', d) and d.getVar('EXTERNALSRC'): # Not an incompatibility error per se, so we don't pass the error code raise DevtoolError("externalsrc is currently enabled for the %s " @@ -357,7 +360,7 @@ def _move_file(src, dst, dry_run_outdir=None, base_outdir=None): bb.utils.mkdirhier(dst_d) shutil.move(src, dst) -def _copy_file(src, dst, dry_run_outdir=None): +def _copy_file(src, dst, dry_run_outdir=None, base_outdir=None): """Copy a file. Creates all the directory components of destination path.""" dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' logger.debug('Copying %s to %s%s' % (src, dst, dry_run_suffix)) @@ -457,7 +460,7 @@ def sync(args, config, basepath, workspace): finally: tinfoil.shutdown() -def symlink_oelocal_files_srctree(rd,srctree): +def symlink_oelocal_files_srctree(rd, srctree): import oe.patch if os.path.abspath(rd.getVar('S')) == os.path.abspath(rd.getVar('WORKDIR')): # If recipe extracts to ${WORKDIR}, symlink the files into the srctree @@ -481,11 +484,7 @@ def symlink_oelocal_files_srctree(rd,srctree): os.symlink('oe-local-files/%s' % fn, destpth) addfiles.append(os.path.join(relpth, fn)) if addfiles: - bb.process.run('git add %s' % ' '.join(addfiles), cwd=srctree) - useroptions = [] - oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=rd) - bb.process.run('git %s commit -m "Committing local file symlinks\n\n%s"' % (' '.join(useroptions), oe.patch.GitApplyTree.ignore_commit_prefix), cwd=srctree) - + oe.patch.GitApplyTree.commitIgnored("Add local file symlinks", dir=srctree, files=addfiles, d=rd) def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, workspace, fixed_setup, d, tinfoil, no_overrides=False): """Extract sources of a recipe""" @@ -523,8 +522,10 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works history = d.varhistory.variable('SRC_URI') for event in history: if not 'flag' in event: - if event['op'].startswith(('_append[', '_prepend[')): - extra_overrides.append(event['op'].split('[')[1].split(']')[0]) + if event['op'].startswith((':append[', ':prepend[')): + override = event['op'].split('[')[1].split(']')[0] + if not override.startswith('pn-'): + extra_overrides.append(override) # We want to remove duplicate overrides. If a recipe had multiple # SRC_URI_override += values it would cause mulitple instances of # overrides. This doesn't play nicely with things like creating a @@ -569,6 +570,9 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works logger.debug('writing append file %s' % appendfile) with open(appendfile, 'a') as f: f.write('###--- _extract_source\n') + f.write('deltask do_recipe_qa\n') + f.write('deltask do_recipe_qa_setscene\n') + f.write('ERROR_QA:remove = "patch-fuzz"\n') f.write('DEVTOOL_TEMPDIR = "%s"\n' % tempdir) f.write('DEVTOOL_DEVBRANCH = "%s"\n' % devbranch) if not is_kernel_yocto: @@ -586,6 +590,7 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works preservestampfile = os.path.join(sstate_manifests, 'preserve-stamps') with open(preservestampfile, 'w') as f: f.write(d.getVar('STAMP')) + tinfoil.modified_files() try: if is_kernel_yocto: # We need to generate the kernel config @@ -648,23 +653,34 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works if os.path.exists(workshareddir) and (not os.listdir(workshareddir) or kernelVersion != staging_kerVer): shutil.rmtree(workshareddir) - oe.path.copyhardlinktree(srcsubdir,workshareddir) + oe.path.copyhardlinktree(srcsubdir, workshareddir) elif not os.path.exists(workshareddir): - oe.path.copyhardlinktree(srcsubdir,workshareddir) + oe.path.copyhardlinktree(srcsubdir, workshareddir) tempdir_localdir = os.path.join(tempdir, 'oe-local-files') srctree_localdir = os.path.join(srctree, 'oe-local-files') if sync: - bb.process.run('git fetch file://' + srcsubdir + ' ' + devbranch + ':' + devbranch, cwd=srctree) - - # Move oe-local-files directory to srctree - # As the oe-local-files is not part of the constructed git tree, - # remove them directly during the synchrounizating might surprise - # the users. Instead, we move it to oe-local-files.bak and remind - # user in the log message. + try: + logger.info('Backing up current %s branch as branch: %s.bak' % (devbranch, devbranch)) + bb.process.run('git branch -f ' + devbranch + '.bak', cwd=srctree) + + # Use git fetch to update the source with the current recipe + # To be able to update the currently checked out branch with + # possibly new history (no fast-forward) git needs to be told + # that's ok + logger.info('Syncing source files including patches to git branch: %s' % devbranch) + bb.process.run('git fetch --update-head-ok --force file://' + srcsubdir + ' ' + devbranch + ':' + devbranch, cwd=srctree) + except bb.process.ExecutionError as e: + raise DevtoolError("Error when syncing source files to local checkout: %s" % str(e)) + + # Move the oe-local-files directory to srctree. + # As oe-local-files is not part of the constructed git tree, + # removing it directly during the synchronization might surprise + # the user. Instead, we move it to oe-local-files.bak and remind + # the user in the log message. if os.path.exists(srctree_localdir + '.bak'): - shutil.rmtree(srctree_localdir, srctree_localdir + '.bak') + shutil.rmtree(srctree_localdir + '.bak') if os.path.exists(srctree_localdir): logger.info('Backing up current local file directory %s' % srctree_localdir) @@ -680,7 +696,7 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works shutil.move(tempdir_localdir, srcsubdir) shutil.move(srcsubdir, srctree) - symlink_oelocal_files_srctree(d,srctree) + symlink_oelocal_files_srctree(d, srctree) if is_kernel_yocto: logger.info('Copying kernel config to srctree') @@ -746,14 +762,14 @@ def _check_preserve(config, recipename): os.remove(removefile) else: tf.write(line) - os.rename(newfile, origfile) + bb.utils.rename(newfile, origfile) def get_staging_kver(srcdir): # Kernel version from work-shared kerver = [] staging_kerVer="" if os.path.exists(srcdir) and os.listdir(srcdir): - with open(os.path.join(srcdir,"Makefile")) as f: + with open(os.path.join(srcdir, "Makefile")) as f: version = [next(f) for x in range(5)][1:4] for word in version: kerver.append(word.split('= ')[1].split('\n')[0]) @@ -763,10 +779,20 @@ def get_staging_kver(srcdir): def get_staging_kbranch(srcdir): staging_kbranch = "" if os.path.exists(srcdir) and os.listdir(srcdir): - (branch, _) = bb.process.run('git branch | grep \* | cut -d \' \' -f2', cwd=srcdir) + (branch, _) = bb.process.run('git branch | grep \\* | cut -d \' \' -f2', cwd=srcdir) staging_kbranch = "".join(branch.split('\n')[0]) return staging_kbranch +def get_real_srctree(srctree, s, workdir): + # Check that recipe isn't using a shared workdir + s = os.path.abspath(s) + workdir = os.path.abspath(workdir) + if s.startswith(workdir) and s != workdir and os.path.dirname(s) != workdir: + # Handle if S is set to a subdirectory of the source + srcsubdir = os.path.relpath(s, workdir).split(os.sep, 1)[1] + srctree = os.path.join(srctree, srcsubdir) + return srctree + def modify(args, config, basepath, workspace): """Entry point for the devtool 'modify' subcommand""" import bb @@ -811,8 +837,8 @@ def modify(args, config, basepath, workspace): _check_compatible_recipe(pn, rd) - initial_rev = None - commits = [] + initial_revs = {} + commits = {} check_commits = False if bb.data.inherits_class('kernel-yocto', rd): @@ -824,10 +850,10 @@ def modify(args, config, basepath, workspace): staging_kerVer = get_staging_kver(srcdir) staging_kbranch = get_staging_kbranch(srcdir) if (os.path.exists(srcdir) and os.listdir(srcdir)) and (kernelVersion in staging_kerVer and staging_kbranch == kbranch): - oe.path.copyhardlinktree(srcdir,srctree) + oe.path.copyhardlinktree(srcdir, srctree) workdir = rd.getVar('WORKDIR') srcsubdir = rd.getVar('S') - localfilesdir = os.path.join(srctree,'oe-local-files') + localfilesdir = os.path.join(srctree, 'oe-local-files') # Move local source files into separate subdir recipe_patches = [os.path.basename(patch) for patch in oe.recipeutils.get_recipe_patches(rd)] local_files = oe.recipeutils.get_recipe_local_files(rd) @@ -851,9 +877,9 @@ def modify(args, config, basepath, workspace): for fname in local_files: _move_file(os.path.join(workdir, fname), os.path.join(srctree, 'oe-local-files', fname)) with open(os.path.join(srctree, 'oe-local-files', '.gitignore'), 'w') as f: - f.write('# Ignore local files, by default. Remove this file ''if you want to commit the directory to Git\n*\n') + f.write('# Ignore local files, by default. Remove this file if you want to commit the directory to Git\n*\n') - symlink_oelocal_files_srctree(rd,srctree) + symlink_oelocal_files_srctree(rd, srctree) task = 'do_configure' res = tinfoil.build_targets(pn, task, handle_events=True) @@ -861,22 +887,30 @@ def modify(args, config, basepath, workspace): # Copy .config to workspace kconfpath = rd.getVar('B') logger.info('Copying kernel config to workspace') - shutil.copy2(os.path.join(kconfpath, '.config'),srctree) + shutil.copy2(os.path.join(kconfpath, '.config'), srctree) # Set this to true, we still need to get initial_rev # by parsing the git repo args.no_extract = True if not args.no_extract: - initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) - if not initial_rev: + initial_revs["."], _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) + if not initial_revs["."]: return 1 logger.info('Source tree extracted to %s' % srctree) + if os.path.exists(os.path.join(srctree, '.git')): # Get list of commits since this revision - (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_rev, cwd=srctree) - commits = stdout.split() + (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_revs["."], cwd=srctree) + commits["."] = stdout.split() check_commits = True + (stdout, _) = bb.process.run('git submodule --quiet foreach --recursive \'echo `git rev-parse devtool-base` $PWD\'', cwd=srctree) + for line in stdout.splitlines(): + (rev, submodule_path) = line.split() + submodule = os.path.relpath(submodule_path, srctree) + initial_revs[submodule] = rev + (stdout, _) = bb.process.run('git rev-list --reverse devtool-base..HEAD', cwd=submodule_path) + commits[submodule] = stdout.split() else: if os.path.exists(os.path.join(srctree, '.git')): # Check if it's a tree previously extracted by us. This is done @@ -893,11 +927,11 @@ def modify(args, config, basepath, workspace): for line in stdout.splitlines(): if line.startswith('*'): (stdout, _) = bb.process.run('git rev-parse devtool-base', cwd=srctree) - initial_rev = stdout.rstrip() - if not initial_rev: + initial_revs["."] = stdout.rstrip() + if "." not in initial_revs: # Otherwise, just grab the head revision (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) - initial_rev = stdout.rstrip() + initial_revs["."] = stdout.rstrip() branch_patches = {} if check_commits: @@ -914,62 +948,81 @@ def modify(args, config, basepath, workspace): seen_patches = [] for branch in branches: branch_patches[branch] = [] - (stdout, _) = bb.process.run('git log devtool-base..%s' % branch, cwd=srctree) - for line in stdout.splitlines(): - line = line.strip() - if line.startswith(oe.patch.GitApplyTree.patch_line_prefix): - origpatch = line[len(oe.patch.GitApplyTree.patch_line_prefix):].split(':', 1)[-1].strip() - if not origpatch in seen_patches: - seen_patches.append(origpatch) - branch_patches[branch].append(origpatch) + (stdout, _) = bb.process.run('git rev-list devtool-base..%s' % branch, cwd=srctree) + for sha1 in stdout.splitlines(): + notes = oe.patch.GitApplyTree.getNotes(srctree, sha1.strip()) + origpatch = notes.get(oe.patch.GitApplyTree.original_patch) + if origpatch and origpatch not in seen_patches: + seen_patches.append(origpatch) + branch_patches[branch].append(origpatch) # Need to grab this here in case the source is within a subdirectory srctreebase = srctree - - # Check that recipe isn't using a shared workdir - s = os.path.abspath(rd.getVar('S')) - workdir = os.path.abspath(rd.getVar('WORKDIR')) - if s.startswith(workdir) and s != workdir and os.path.dirname(s) != workdir: - # Handle if S is set to a subdirectory of the source - srcsubdir = os.path.relpath(s, workdir).split(os.sep, 1)[1] - srctree = os.path.join(srctree, srcsubdir) + srctree = get_real_srctree(srctree, rd.getVar('S'), rd.getVar('WORKDIR')) bb.utils.mkdirhier(os.path.dirname(appendfile)) with open(appendfile, 'w') as f: - f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n') + # if not present, add type=git-dependency to the secondary sources + # (non local files) so they can be extracted correctly when building a recipe after + # doing a devtool modify on it + src_uri = rd.getVar('SRC_URI').split() + src_uri_append = [] + src_uri_remove = [] + + # Assume first entry is main source extracted in ${S} so skip it + src_uri = src_uri[1::] + + # Add "type=git-dependency" to all non local sources + for url in src_uri: + if not url.startswith('file://') and not 'type=' in url: + src_uri_remove.append(url) + src_uri_append.append('%s;type=git-dependency' % url) + + if src_uri_remove: + f.write('SRC_URI:remove = "%s"\n' % ' '.join(src_uri_remove)) + f.write('SRC_URI:append = " %s"\n\n' % ' '.join(src_uri_append)) + + f.write('FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n') # Local files can be modified/tracked in separate subdir under srctree # Mostly useful for packages with S != WORKDIR - f.write('FILESPATH_prepend := "%s:"\n' % + f.write('FILESPATH:prepend := "%s:"\n' % os.path.join(srctreebase, 'oe-local-files')) f.write('# srctreebase: %s\n' % srctreebase) f.write('\ninherit externalsrc\n') f.write('# NOTE: We use pn- overrides here to avoid affecting multiple variants in the case where the recipe uses BBCLASSEXTEND\n') - f.write('EXTERNALSRC_pn-%s = "%s"\n' % (pn, srctree)) + f.write('EXTERNALSRC:pn-%s = "%s"\n' % (pn, srctree)) b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd) if b_is_s: - f.write('EXTERNALSRC_BUILD_pn-%s = "%s"\n' % (pn, srctree)) + f.write('EXTERNALSRC_BUILD:pn-%s = "%s"\n' % (pn, srctree)) if bb.data.inherits_class('kernel', rd): f.write('SRCTREECOVEREDTASKS = "do_validate_branches do_kernel_checkout ' - 'do_fetch do_unpack do_kernel_configme do_kernel_configcheck"\n') + 'do_fetch do_unpack do_kernel_configcheck"\n') f.write('\ndo_patch[noexec] = "1"\n') - f.write('\ndo_configure_append() {\n' + f.write('\ndo_configure:append() {\n' ' cp ${B}/.config ${S}/.config.baseline\n' ' ln -sfT ${B}/.config ${S}/.config.new\n' '}\n') - if rd.getVarFlag('do_menuconfig','task'): - f.write('\ndo_configure_append() {\n' - ' if [ ! ${DEVTOOL_DISABLE_MENUCONFIG} ]; then\n' - ' cp ${B}/.config ${S}/.config.baseline\n' - ' ln -sfT ${B}/.config ${S}/.config.new\n' + f.write('\ndo_kernel_configme:prepend() {\n' + ' if [ -e ${S}/.config ]; then\n' + ' mv ${S}/.config ${S}/.config.old\n' + ' fi\n' + '}\n') + if rd.getVarFlag('do_menuconfig', 'task'): + f.write('\ndo_configure:append() {\n' + ' if [ ${@oe.types.boolean(d.getVar("KCONFIG_CONFIG_ENABLE_MENUCONFIG"))} = True ]; then\n' + ' cp ${KCONFIG_CONFIG_ROOTDIR}/.config ${S}/.config.baseline\n' + ' ln -sfT ${KCONFIG_CONFIG_ROOTDIR}/.config ${S}/.config.new\n' ' fi\n' '}\n') - if initial_rev: - f.write('\n# initial_rev: %s\n' % initial_rev) - for commit in commits: - f.write('# commit: %s\n' % commit) + if initial_revs: + for name, rev in initial_revs.items(): + f.write('\n# initial_rev %s: %s\n' % (name, rev)) + if name in commits: + for commit in commits[name]: + f.write('# commit %s: %s\n' % (name, commit)) if branch_patches: for branch in branch_patches: if branch == args.branch: @@ -1089,10 +1142,10 @@ def rename(args, config, basepath, workspace): # Rename bbappend logger.info('Renaming %s to %s' % (append, newappend)) - os.rename(append, newappend) + bb.utils.rename(append, newappend) # Rename recipe file logger.info('Renaming %s to %s' % (recipefile, newfile)) - os.rename(recipefile, newfile) + bb.utils.rename(recipefile, newfile) # Rename source tree if it's the default path appendmd5 = None @@ -1192,44 +1245,56 @@ def _get_patchset_revs(srctree, recipe_path, initial_rev=None, force_patch_refre branchname = stdout.rstrip() # Parse initial rev from recipe if not specified - commits = [] + commits = {} patches = [] + initial_revs = {} with open(recipe_path, 'r') as f: for line in f: - if line.startswith('# initial_rev:'): - if not initial_rev: - initial_rev = line.split(':')[-1].strip() - elif line.startswith('# commit:') and not force_patch_refresh: - commits.append(line.split(':')[-1].strip()) - elif line.startswith('# patches_%s:' % branchname): - patches = line.split(':')[-1].strip().split(',') - - update_rev = initial_rev - changed_revs = None - if initial_rev: + pattern = r'^#\s.*\s(.*):\s([0-9a-fA-F]+)$' + match = re.search(pattern, line) + if match: + name = match.group(1) + rev = match.group(2) + if line.startswith('# initial_rev'): + if not (name == "." and initial_rev): + initial_revs[name] = rev + elif line.startswith('# commit') and not force_patch_refresh: + if name not in commits: + commits[name] = [rev] + else: + commits[name].append(rev) + elif line.startswith('# patches_%s:' % branchname): + patches = line.split(':')[-1].strip().split(',') + + update_revs = dict(initial_revs) + changed_revs = {} + for name, rev in initial_revs.items(): # Find first actually changed revision stdout, _ = bb.process.run('git rev-list --reverse %s..HEAD' % - initial_rev, cwd=srctree) + rev, cwd=os.path.join(srctree, name)) newcommits = stdout.split() - for i in range(min(len(commits), len(newcommits))): - if newcommits[i] == commits[i]: - update_rev = commits[i] + if name in commits: + for i in range(min(len(commits[name]), len(newcommits))): + if newcommits[i] == commits[name][i]: + update_revs[name] = commits[name][i] try: stdout, _ = bb.process.run('git cherry devtool-patched', - cwd=srctree) + cwd=os.path.join(srctree, name)) except bb.process.ExecutionError as err: stdout = None if stdout is not None and not force_patch_refresh: - changed_revs = [] for line in stdout.splitlines(): if line.startswith('+ '): rev = line.split()[1] if rev in newcommits: - changed_revs.append(rev) + if name not in changed_revs: + changed_revs[name] = [rev] + else: + changed_revs[name].append(rev) - return initial_rev, update_rev, changed_revs, patches + return initial_revs, update_revs, changed_revs, patches def _remove_file_entries(srcuri, filelist): """Remove file:// entries from SRC_URI""" @@ -1284,14 +1349,17 @@ def _remove_source_files(append, files, destpath, no_report_remove=False, dry_ru raise -def _export_patches(srctree, rd, start_rev, destdir, changed_revs=None): +def _export_patches(srctree, rd, start_revs, destdir, changed_revs=None): """Export patches from srctree to given location. Returns three-tuple of dicts: 1. updated - patches that already exist in SRCURI 2. added - new patches that don't exist in SRCURI 3 removed - patches that exist in SRCURI but not in exported patches - In each dict the key is the 'basepath' of the URI and value is the - absolute path to the existing file in recipe space (if any). + In each dict the key is the 'basepath' of the URI and value is: + - for updated and added dicts, a dict with 2 optionnal keys: + - 'path': the absolute path to the existing file in recipe space (if any) + - 'patchdir': the directory in wich the patch should be applied (if any) + - for removed dict, the absolute path to the existing file in recipe space """ import oe.recipeutils from oe.patch import GitApplyTree @@ -1305,54 +1373,60 @@ def _export_patches(srctree, rd, start_rev, destdir, changed_revs=None): # Generate patches from Git, exclude local files directory patch_pathspec = _git_exclude_path(srctree, 'oe-local-files') - GitApplyTree.extractPatches(srctree, start_rev, destdir, patch_pathspec) - - new_patches = sorted(os.listdir(destdir)) - for new_patch in new_patches: - # Strip numbering from patch names. If it's a git sequence named patch, - # the numbers might not match up since we are starting from a different - # revision This does assume that people are using unique shortlog - # values, but they ought to be anyway... - new_basename = seqpatch_re.match(new_patch).group(2) - match_name = None - for old_patch in existing_patches: - old_basename = seqpatch_re.match(old_patch).group(2) - old_basename_splitext = os.path.splitext(old_basename) - if old_basename.endswith(('.gz', '.bz2', '.Z')) and old_basename_splitext[0] == new_basename: - old_patch_noext = os.path.splitext(old_patch)[0] - match_name = old_patch_noext - break - elif new_basename == old_basename: - match_name = old_patch - break - if match_name: - # Rename patch files - if new_patch != match_name: - os.rename(os.path.join(destdir, new_patch), - os.path.join(destdir, match_name)) - # Need to pop it off the list now before checking changed_revs - oldpath = existing_patches.pop(old_patch) - if changed_revs is not None: - # Avoid updating patches that have not actually changed - with open(os.path.join(destdir, match_name), 'r') as f: - firstlineitems = f.readline().split() - # Looking for "From <hash>" line - if len(firstlineitems) > 1 and len(firstlineitems[1]) == 40: - if not firstlineitems[1] in changed_revs: - continue - # Recompress if necessary - if oldpath.endswith(('.gz', '.Z')): - bb.process.run(['gzip', match_name], cwd=destdir) - if oldpath.endswith('.gz'): - match_name += '.gz' - else: - match_name += '.Z' - elif oldpath.endswith('.bz2'): - bb.process.run(['bzip2', match_name], cwd=destdir) - match_name += '.bz2' - updated[match_name] = oldpath - else: - added[new_patch] = None + GitApplyTree.extractPatches(srctree, start_revs, destdir, patch_pathspec) + for dirpath, dirnames, filenames in os.walk(destdir): + new_patches = filenames + reldirpath = os.path.relpath(dirpath, destdir) + for new_patch in new_patches: + # Strip numbering from patch names. If it's a git sequence named patch, + # the numbers might not match up since we are starting from a different + # revision This does assume that people are using unique shortlog + # values, but they ought to be anyway... + new_basename = seqpatch_re.match(new_patch).group(2) + match_name = None + for old_patch in existing_patches: + old_basename = seqpatch_re.match(old_patch).group(2) + old_basename_splitext = os.path.splitext(old_basename) + if old_basename.endswith(('.gz', '.bz2', '.Z')) and old_basename_splitext[0] == new_basename: + old_patch_noext = os.path.splitext(old_patch)[0] + match_name = old_patch_noext + break + elif new_basename == old_basename: + match_name = old_patch + break + if match_name: + # Rename patch files + if new_patch != match_name: + bb.utils.rename(os.path.join(destdir, new_patch), + os.path.join(destdir, match_name)) + # Need to pop it off the list now before checking changed_revs + oldpath = existing_patches.pop(old_patch) + if changed_revs is not None and dirpath in changed_revs: + # Avoid updating patches that have not actually changed + with open(os.path.join(dirpath, match_name), 'r') as f: + firstlineitems = f.readline().split() + # Looking for "From <hash>" line + if len(firstlineitems) > 1 and len(firstlineitems[1]) == 40: + if not firstlineitems[1] in changed_revs[dirpath]: + continue + # Recompress if necessary + if oldpath.endswith(('.gz', '.Z')): + bb.process.run(['gzip', match_name], cwd=destdir) + if oldpath.endswith('.gz'): + match_name += '.gz' + else: + match_name += '.Z' + elif oldpath.endswith('.bz2'): + bb.process.run(['bzip2', match_name], cwd=destdir) + match_name += '.bz2' + updated[match_name] = {'path' : oldpath} + if reldirpath != ".": + updated[match_name]['patchdir'] = reldirpath + else: + added[new_patch] = {} + if reldirpath != ".": + added[new_patch]['patchdir'] = reldirpath + return (updated, added, existing_patches) @@ -1389,8 +1463,10 @@ def _export_local_files(srctree, rd, destdir, srctreebase): 1. updated - files that already exist in SRCURI 2. added - new files files that don't exist in SRCURI 3 removed - files that exist in SRCURI but not in exported files - In each dict the key is the 'basepath' of the URI and value is the - absolute path to the existing file in recipe space (if any). + In each dict the key is the 'basepath' of the URI and value is: + - for updated and added dicts, a dict with 1 optionnal key: + - 'path': the absolute path to the existing file in recipe space (if any) + - for removed dict, the absolute path to the existing file in recipe space """ import oe.recipeutils @@ -1403,6 +1479,18 @@ def _export_local_files(srctree, rd, destdir, srctreebase): updated = OrderedDict() added = OrderedDict() removed = OrderedDict() + + # Get current branch and return early with empty lists + # if on one of the override branches + # (local files are provided only for the main branch and processing + # them against lists from recipe overrides will result in mismatches + # and broken modifications to recipes). + stdout, _ = bb.process.run('git rev-parse --abbrev-ref HEAD', + cwd=srctree) + branchname = stdout.rstrip() + if branchname.startswith(override_branch_prefix): + return (updated, added, removed) + local_files_dir = os.path.join(srctreebase, 'oe-local-files') git_files = _git_ls_tree(srctree) if 'oe-local-files' in git_files: @@ -1460,9 +1548,9 @@ def _export_local_files(srctree, rd, destdir, srctreebase): origpath = existing_files.pop(fname) workpath = os.path.join(local_files_dir, fname) if not filecmp.cmp(origpath, workpath): - updated[fname] = origpath + updated[fname] = {'path' : origpath} elif fname != '.gitignore': - added[fname] = None + added[fname] = {} workdir = rd.getVar('WORKDIR') s = rd.getVar('S') @@ -1479,7 +1567,7 @@ def _export_local_files(srctree, rd, destdir, srctreebase): if os.path.exists(fpath): origpath = existing_files.pop(fname) if not filecmp.cmp(origpath, fpath): - updated[fpath] = origpath + updated[fpath] = {'path' : origpath} removed = existing_files return (updated, added, removed) @@ -1508,6 +1596,12 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi recipedir = os.path.basename(recipefile) logger.info('Updating SRCREV in recipe %s%s' % (recipedir, dry_run_suffix)) + # Get original SRCREV + old_srcrev = rd.getVar('SRCREV') or '' + if old_srcrev == "INVALID": + raise DevtoolError('Update mode srcrev is only valid for recipe fetched from an SCM repository') + old_srcrev = {'.': old_srcrev} + # Get HEAD revision try: stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree) @@ -1534,13 +1628,12 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi if not no_remove: # Find list of existing patches in recipe file patches_dir = tempfile.mkdtemp(dir=tempdir) - old_srcrev = rd.getVar('SRCREV') or '' upd_p, new_p, del_p = _export_patches(srctree, rd, old_srcrev, patches_dir) logger.debug('Patches: update %s, new %s, delete %s' % (dict(upd_p), dict(new_p), dict(del_p))) # Remove deleted local files and "overlapping" patches - remove_files = list(del_f.values()) + list(upd_p.values()) + list(del_p.values()) + remove_files = list(del_f.values()) + [value["path"] for value in upd_p.values() if "path" in value] + [value["path"] for value in del_p.values() if "path" in value] if remove_files: removedentries = _remove_file_entries(srcuri, remove_files)[0] update_srcuri = True @@ -1554,14 +1647,14 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi patchfields['SRC_URI'] = '\\\n '.join(srcuri) if dry_run_outdir: logger.info('Creating bbappend (dry-run)') - else: - appendfile, destpath = oe.recipeutils.bbappend_recipe( - rd, appendlayerdir, files, wildcardver=wildcard_version, - extralines=patchfields, removevalues=removevalues, - redirect_output=dry_run_outdir) + appendfile, destpath = oe.recipeutils.bbappend_recipe( + rd, appendlayerdir, files, wildcardver=wildcard_version, + extralines=patchfields, removevalues=removevalues, + redirect_output=dry_run_outdir) else: files_dir = _determine_files_dir(rd) - for basepath, path in upd_f.items(): + for basepath, param in upd_f.items(): + path = param['path'] logger.info('Updating file %s%s' % (basepath, dry_run_suffix)) if os.path.isabs(basepath): # Original file (probably with subdir pointing inside source tree) @@ -1571,7 +1664,8 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi _move_file(os.path.join(local_files_dir, basepath), path, dry_run_outdir=dry_run_outdir, base_outdir=recipedir) update_srcuri= True - for basepath, path in new_f.items(): + for basepath, param in new_f.items(): + path = param['path'] logger.info('Adding new file %s%s' % (basepath, dry_run_suffix)) _move_file(os.path.join(local_files_dir, basepath), os.path.join(files_dir, basepath), @@ -1603,9 +1697,22 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil if not os.path.exists(append): raise DevtoolError('unable to find workspace bbappend for recipe %s' % recipename) + srctreebase = workspace[recipename]['srctreebase'] + relpatchdir = os.path.relpath(srctreebase, srctree) + if relpatchdir == '.': + patchdir_params = {} + else: + patchdir_params = {'patchdir': relpatchdir} - initial_rev, update_rev, changed_revs, filter_patches = _get_patchset_revs(srctree, append, initial_rev, force_patch_refresh) - if not initial_rev: + def srcuri_entry(basepath, patchdir_params): + if patchdir_params: + paramstr = ';' + ';'.join('%s=%s' % (k,v) for k,v in patchdir_params.items()) + else: + paramstr = '' + return 'file://%s%s' % (basepath, paramstr) + + initial_revs, update_revs, changed_revs, filter_patches = _get_patchset_revs(srctree, append, initial_rev, force_patch_refresh) + if not initial_revs: raise DevtoolError('Unable to find initial revision - please specify ' 'it with --initial-rev') @@ -1619,61 +1726,69 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil tempdir = tempfile.mkdtemp(prefix='devtool') try: local_files_dir = tempfile.mkdtemp(dir=tempdir) - if filter_patches: - upd_f = {} - new_f = {} - del_f = {} - else: - srctreebase = workspace[recipename]['srctreebase'] - upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase) - - remove_files = [] - if not no_remove: - # Get all patches from source tree and check if any should be removed - all_patches_dir = tempfile.mkdtemp(dir=tempdir) - _, _, del_p = _export_patches(srctree, rd, initial_rev, - all_patches_dir) - # Remove deleted local files and patches - remove_files = list(del_f.values()) + list(del_p.values()) + upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase) # Get updated patches from source tree patches_dir = tempfile.mkdtemp(dir=tempdir) - upd_p, new_p, _ = _export_patches(srctree, rd, update_rev, + upd_p, new_p, _ = _export_patches(srctree, rd, update_revs, patches_dir, changed_revs) + # Get all patches from source tree and check if any should be removed + all_patches_dir = tempfile.mkdtemp(dir=tempdir) + _, _, del_p = _export_patches(srctree, rd, initial_revs, + all_patches_dir) logger.debug('Pre-filtering: update: %s, new: %s' % (dict(upd_p), dict(new_p))) if filter_patches: new_p = OrderedDict() upd_p = OrderedDict((k,v) for k,v in upd_p.items() if k in filter_patches) - remove_files = [f for f in remove_files if f in filter_patches] + del_p = OrderedDict((k,v) for k,v in del_p.items() if k in filter_patches) + remove_files = [] + if not no_remove: + # Remove deleted local files and patches + remove_files = list(del_f.values()) + list(del_p.values()) updatefiles = False updaterecipe = False destpath = None srcuri = (rd.getVar('SRC_URI', False) or '').split() + if appendlayerdir: files = OrderedDict((os.path.join(local_files_dir, key), val) for key, val in list(upd_f.items()) + list(new_f.items())) files.update(OrderedDict((os.path.join(patches_dir, key), val) for key, val in list(upd_p.items()) + list(new_p.items()))) + + params = [] + for file, param in files.items(): + patchdir_param = dict(patchdir_params) + patchdir = param.get('patchdir', ".") + if patchdir != "." : + if patchdir_param: + patchdir_param['patchdir'] += patchdir + else: + patchdir_param['patchdir'] = patchdir + params.append(patchdir_param) + if files or remove_files: removevalues = None if remove_files: removedentries, remaining = _remove_file_entries( srcuri, remove_files) if removedentries or remaining: - remaining = ['file://' + os.path.basename(item) for + remaining = [srcuri_entry(os.path.basename(item), patchdir_params) for item in remaining] removevalues = {'SRC_URI': removedentries + remaining} appendfile, destpath = oe.recipeutils.bbappend_recipe( rd, appendlayerdir, files, wildcardver=wildcard_version, removevalues=removevalues, - redirect_output=dry_run_outdir) + redirect_output=dry_run_outdir, + params=params) else: logger.info('No patches or local source files needed updating') else: # Update existing files files_dir = _determine_files_dir(rd) - for basepath, path in upd_f.items(): + for basepath, param in upd_f.items(): + path = param['path'] logger.info('Updating file %s' % basepath) if os.path.isabs(basepath): # Original file (probably with subdir pointing inside source tree) @@ -1684,14 +1799,22 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil _move_file(os.path.join(local_files_dir, basepath), path, dry_run_outdir=dry_run_outdir, base_outdir=recipedir) updatefiles = True - for basepath, path in upd_p.items(): - patchfn = os.path.join(patches_dir, basepath) + for basepath, param in upd_p.items(): + path = param['path'] + patchdir = param.get('patchdir', ".") + if patchdir != "." : + patchdir_param = dict(patchdir_params) + if patchdir_param: + patchdir_param['patchdir'] += patchdir + else: + patchdir_param['patchdir'] = patchdir + patchfn = os.path.join(patches_dir, patchdir, basepath) if os.path.dirname(path) + '/' == dl_dir: # This is a a downloaded patch file - we now need to # replace the entry in SRC_URI with our local version logger.info('Replacing remote patch %s with updated local version' % basepath) path = os.path.join(files_dir, basepath) - _replace_srcuri_entry(srcuri, basepath, 'file://%s' % basepath) + _replace_srcuri_entry(srcuri, basepath, srcuri_entry(basepath, patchdir_param)) updaterecipe = True else: logger.info('Updating patch %s%s' % (basepath, dry_run_suffix)) @@ -1699,21 +1822,29 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil dry_run_outdir=dry_run_outdir, base_outdir=recipedir) updatefiles = True # Add any new files - for basepath, path in new_f.items(): + for basepath, param in new_f.items(): logger.info('Adding new file %s%s' % (basepath, dry_run_suffix)) _move_file(os.path.join(local_files_dir, basepath), os.path.join(files_dir, basepath), dry_run_outdir=dry_run_outdir, base_outdir=recipedir) - srcuri.append('file://%s' % basepath) + srcuri.append(srcuri_entry(basepath, patchdir_params)) updaterecipe = True - for basepath, path in new_p.items(): + for basepath, param in new_p.items(): + patchdir = param.get('patchdir', ".") logger.info('Adding new patch %s%s' % (basepath, dry_run_suffix)) - _move_file(os.path.join(patches_dir, basepath), + _move_file(os.path.join(patches_dir, patchdir, basepath), os.path.join(files_dir, basepath), dry_run_outdir=dry_run_outdir, base_outdir=recipedir) - srcuri.append('file://%s' % basepath) + params = dict(patchdir_params) + if patchdir != "." : + if params: + params['patchdir'] += patchdir + else: + params['patchdir'] = patchdir + + srcuri.append(srcuri_entry(basepath, params)) updaterecipe = True # Update recipe, if needed if _remove_file_entries(srcuri, remove_files)[0]: @@ -1770,6 +1901,8 @@ def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_ver for line in stdout.splitlines(): branchname = line[2:] if line.startswith('* '): + if 'HEAD' in line: + raise DevtoolError('Detached HEAD - please check out a branch, e.g., "devtool"') startbranch = branchname if branchname.startswith(override_branch_prefix): override_branches.append(branchname) @@ -1959,9 +2092,19 @@ def _reset(recipes, no_clean, remove_work, config, basepath, workspace): shutil.rmtree(srctreebase) else: # We don't want to risk wiping out any work in progress - logger.info('Leaving source tree %s as-is; if you no ' - 'longer need it then please delete it manually' - % srctreebase) + if srctreebase.startswith(os.path.join(config.workspace_path, 'sources')): + from datetime import datetime + preservesrc = os.path.join(config.workspace_path, 'attic', 'sources', "{}.{}".format(pn, datetime.now().strftime("%Y%m%d%H%M%S"))) + logger.info('Preserving source tree in %s\nIf you no ' + 'longer need it then please delete it manually.\n' + 'It is also possible to reuse it via devtool source tree argument.' + % preservesrc) + bb.utils.mkdirhier(os.path.dirname(preservesrc)) + shutil.move(srctreebase, preservesrc) + else: + logger.info('Leaving source tree %s as-is; if you no ' + 'longer need it then please delete it manually' + % srctreebase) else: # This is unlikely, but if it's empty we can just remove it os.rmdir(srctreebase) @@ -2221,6 +2364,7 @@ def register_commands(subparsers, context): group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") parser_add.add_argument('--fetch', '-f', help='Fetch the specified URI and extract it to create the source tree (deprecated - pass as positional argument instead)', metavar='URI') parser_add.add_argument('--npm-dev', help='For npm, also fetch devDependencies', action="store_true") + parser_add.add_argument('--no-pypi', help='Do not inherit pypi class', action="store_true") parser_add.add_argument('--version', '-V', help='Version to use within recipe (PV)') parser_add.add_argument('--no-git', '-g', help='If fetching source, do not set up source tree as a git repository', action="store_true") group = parser_add.add_mutually_exclusive_group() diff --git a/scripts/lib/devtool/upgrade.py b/scripts/lib/devtool/upgrade.py index 5a057e95f5..fa5b8ef3c7 100644 --- a/scripts/lib/devtool/upgrade.py +++ b/scripts/lib/devtool/upgrade.py @@ -35,6 +35,8 @@ def _get_srctree(tmpdir): dirs = scriptutils.filter_src_subdirs(tmpdir) if len(dirs) == 1: srctree = os.path.join(tmpdir, dirs[0]) + else: + raise DevtoolError("Cannot determine where the source tree is after unpacking in {}: {}".format(tmpdir,dirs)) return srctree def _copy_source_code(orig, dest): @@ -71,7 +73,8 @@ def _rename_recipe_dirs(oldpv, newpv, path): if oldfile.find(oldpv) != -1: newfile = oldfile.replace(oldpv, newpv) if oldfile != newfile: - os.rename(os.path.join(path, oldfile), os.path.join(path, newfile)) + bb.utils.rename(os.path.join(path, oldfile), + os.path.join(path, newfile)) def _rename_recipe_file(oldrecipe, bpn, oldpv, newpv, path): oldrecipe = os.path.basename(oldrecipe) @@ -87,7 +90,7 @@ def _rename_recipe_files(oldrecipe, bpn, oldpv, newpv, path): _rename_recipe_dirs(oldpv, newpv, path) return _rename_recipe_file(oldrecipe, bpn, oldpv, newpv, path) -def _write_append(rc, srctree, same_dir, no_same_dir, rev, copied, workspace, d): +def _write_append(rc, srctreebase, srctree, same_dir, no_same_dir, revs, copied, workspace, d): """Writes an append file""" if not os.path.exists(rc): raise DevtoolError("bbappend not created because %s does not exist" % rc) @@ -102,36 +105,38 @@ def _write_append(rc, srctree, same_dir, no_same_dir, rev, copied, workspace, d) pn = d.getVar('PN') af = os.path.join(appendpath, '%s.bbappend' % brf) with open(af, 'w') as f: - f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n\n') + f.write('FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n\n') + # Local files can be modified/tracked in separate subdir under srctree + # Mostly useful for packages with S != WORKDIR + f.write('FILESPATH:prepend := "%s:"\n' % + os.path.join(srctreebase, 'oe-local-files')) + f.write('# srctreebase: %s\n' % srctreebase) f.write('inherit externalsrc\n') f.write(('# NOTE: We use pn- overrides here to avoid affecting' 'multiple variants in the case where the recipe uses BBCLASSEXTEND\n')) - f.write('EXTERNALSRC_pn-%s = "%s"\n' % (pn, srctree)) + f.write('EXTERNALSRC:pn-%s = "%s"\n' % (pn, srctree)) b_is_s = use_external_build(same_dir, no_same_dir, d) if b_is_s: - f.write('EXTERNALSRC_BUILD_pn-%s = "%s"\n' % (pn, srctree)) + f.write('EXTERNALSRC_BUILD:pn-%s = "%s"\n' % (pn, srctree)) f.write('\n') - if rev: - f.write('# initial_rev: %s\n' % rev) + if revs: + for name, rev in revs.items(): + f.write('# initial_rev %s: %s\n' % (name, rev)) if copied: f.write('# original_path: %s\n' % os.path.dirname(d.getVar('FILE'))) f.write('# original_files: %s\n' % ' '.join(copied)) return af -def _cleanup_on_error(rf, srctree): - rfp = os.path.split(rf)[0] # recipe folder - rfpp = os.path.split(rfp)[0] # recipes folder - if os.path.exists(rfp): - shutil.rmtree(rfp) - if not len(os.listdir(rfpp)): - os.rmdir(rfpp) +def _cleanup_on_error(rd, srctree): + if os.path.exists(rd): + shutil.rmtree(rd) srctree = os.path.abspath(srctree) if os.path.exists(srctree): shutil.rmtree(srctree) -def _upgrade_error(e, rf, srctree, keep_failure=False, extramsg=None): - if rf and not keep_failure: - _cleanup_on_error(rf, srctree) +def _upgrade_error(e, rd, srctree, keep_failure=False, extramsg=None): + if not keep_failure: + _cleanup_on_error(rd, srctree) logger.error(e) if extramsg: logger.error(extramsg) @@ -178,12 +183,16 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee uri, rev = _get_uri(crd) if srcrev: rev = srcrev + paths = [srctree] if uri.startswith('git://') or uri.startswith('gitsm://'): __run('git fetch') __run('git checkout %s' % rev) __run('git tag -f devtool-base-new') - md5 = None - sha256 = None + __run('git submodule update --recursive') + __run('git submodule foreach \'git tag -f devtool-base-new\'') + (stdout, _) = __run('git submodule --quiet foreach \'echo $sm_path\'') + paths += [os.path.join(srctree, p) for p in stdout.splitlines()] + checksums = {} _, _, _, _, _, params = bb.fetch2.decodeurl(uri) srcsubdir_rel = params.get('destsuffix', 'git') if not srcbranch: @@ -191,14 +200,15 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee get_branch = [x.strip() for x in check_branch.splitlines()] # Remove HEAD reference point and drop remote prefix get_branch = [x.split('/', 1)[1] for x in get_branch if not x.startswith('origin/HEAD')] - if 'master' in get_branch: - # If it is master, we do not need to append 'branch=master' as this is default. - # Even with the case where get_branch has multiple objects, if 'master' is one - # of them, we should default take from 'master' - srcbranch = '' - elif len(get_branch) == 1: - # If 'master' isn't in get_branch and get_branch contains only ONE object, then store result into 'srcbranch' + if len(get_branch) == 1: + # If srcrev is on only ONE branch, then use that branch srcbranch = get_branch[0] + elif 'main' in get_branch: + # If srcrev is on multiple branches, then choose 'main' if it is one of them + srcbranch = 'main' + elif 'master' in get_branch: + # Otherwise choose 'master' if it is one of the branches + srcbranch = 'master' else: # If get_branch contains more than one objects, then display error and exit. mbrch = '\n ' + '\n '.join(get_branch) @@ -215,9 +225,6 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee if ftmpdir and keep_temp: logger.info('Fetch temp directory is %s' % ftmpdir) - md5 = checksums['md5sum'] - sha256 = checksums['sha256sum'] - tmpsrctree = _get_srctree(tmpdir) srctree = os.path.abspath(srctree) srcsubdir_rel = os.path.relpath(tmpsrctree, tmpdir) @@ -251,30 +258,50 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee __run('git %s commit -q -m "Commit of upstream changes at version %s" --allow-empty' % (' '.join(useroptions), newpv)) __run('git tag -f devtool-base-%s' % newpv) - (stdout, _) = __run('git rev-parse HEAD') - rev = stdout.rstrip() + revs = {} + for path in paths: + (stdout, _) = _run('git rev-parse HEAD', cwd=path) + revs[os.path.relpath(path, srctree)] = stdout.rstrip() if no_patch: patches = oe.recipeutils.get_recipe_patches(crd) if patches: logger.warning('By user choice, the following patches will NOT be applied to the new source tree:\n %s' % '\n '.join([os.path.basename(patch) for patch in patches])) else: - __run('git checkout devtool-patched -b %s' % branch) - skiptag = False - try: - __run('git rebase %s' % rev) - except bb.process.ExecutionError as e: - skiptag = True - if 'conflict' in e.stdout: - logger.warning('Command \'%s\' failed:\n%s\n\nYou will need to resolve conflicts in order to complete the upgrade.' % (e.command, e.stdout.rstrip())) - else: - logger.warning('Command \'%s\' failed:\n%s' % (e.command, e.stdout)) - if not skiptag: - if uri.startswith('git://') or uri.startswith('gitsm://'): - suffix = 'new' - else: - suffix = newpv - __run('git tag -f devtool-patched-%s' % suffix) + for path in paths: + _run('git checkout devtool-patched -b %s' % branch, cwd=path) + (stdout, _) = _run('git branch --list devtool-override-*', cwd=path) + branches_to_rebase = [branch] + stdout.split() + target_branch = revs[os.path.relpath(path, srctree)] + + # There is a bug (or feature?) in git rebase where if a commit with + # a note is fully rebased away by being part of an old commit, the + # note is still attached to the old commit. Avoid this by making + # sure all old devtool related commits have a note attached to them + # (this assumes git config notes.rewriteMode is set to ignore). + (stdout, _) = __run('git rev-list devtool-base..%s' % target_branch) + for rev in stdout.splitlines(): + if not oe.patch.GitApplyTree.getNotes(path, rev): + oe.patch.GitApplyTree.addNote(path, rev, "dummy") + + for b in branches_to_rebase: + logger.info("Rebasing {} onto {}".format(b, target_branch)) + _run('git checkout %s' % b, cwd=path) + try: + _run('git rebase %s' % target_branch, cwd=path) + except bb.process.ExecutionError as e: + if 'conflict' in e.stdout: + logger.warning('Command \'%s\' failed:\n%s\n\nYou will need to resolve conflicts in order to complete the upgrade.' % (e.command, e.stdout.rstrip())) + _run('git rebase --abort', cwd=path) + else: + logger.warning('Command \'%s\' failed:\n%s' % (e.command, e.stdout)) + + # Remove any dummy notes added above. + (stdout, _) = __run('git rev-list devtool-base..%s' % target_branch) + for rev in stdout.splitlines(): + oe.patch.GitApplyTree.removeNote(path, rev, "dummy") + + _run('git checkout %s' % branch, cwd=path) if tmpsrctree: if keep_temp: @@ -284,7 +311,7 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee if tmpdir != tmpsrctree: shutil.rmtree(tmpdir) - return (rev, md5, sha256, srcbranch, srcsubdir_rel) + return (revs, checksums, srcbranch, srcsubdir_rel) def _add_license_diff_to_recipe(path, diff): notice_text = """# FIXME: the LIC_FILES_CHKSUM values have been updated by 'devtool upgrade'. @@ -305,7 +332,7 @@ def _add_license_diff_to_recipe(path, diff): f.write("\n#\n\n".encode()) f.write(orig_content) -def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, srcsubdir_old, srcsubdir_new, workspace, tinfoil, rd, license_diff, new_licenses, srctree, keep_failure): +def _create_new_recipe(newpv, checksums, srcrev, srcbranch, srcsubdir_old, srcsubdir_new, workspace, tinfoil, rd, license_diff, new_licenses, srctree, keep_failure): """Creates the new recipe under workspace""" bpn = rd.getVar('BPN') @@ -336,7 +363,10 @@ def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, srcsubdir_old, src replacing = True new_src_uri = [] for entry in src_uri: - scheme, network, path, user, passwd, params = bb.fetch2.decodeurl(entry) + try: + scheme, network, path, user, passwd, params = bb.fetch2.decodeurl(entry) + except bb.fetch2.MalformedUrl as e: + raise DevtoolError("Could not decode SRC_URI: {}".format(e)) if replacing and scheme in ['git', 'gitsm']: branch = params.get('branch', 'master') if rd.expand(branch) != srcbranch: @@ -374,30 +404,39 @@ def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, srcsubdir_old, src addnames.append(params['name']) # Find what's been set in the original recipe oldnames = [] + oldsums = [] noname = False for varflag in rd.getVarFlags('SRC_URI'): - if varflag.endswith(('.md5sum', '.sha256sum')): - name = varflag.rsplit('.', 1)[0] - if name not in oldnames: - oldnames.append(name) - elif varflag in ['md5sum', 'sha256sum']: - noname = True + for checksum in checksums: + if varflag.endswith('.' + checksum): + name = varflag.rsplit('.', 1)[0] + if name not in oldnames: + oldnames.append(name) + oldsums.append(checksum) + elif varflag == checksum: + noname = True + oldsums.append(checksum) # Even if SRC_URI has named entries it doesn't have to actually use the name if noname and addnames and addnames[0] not in oldnames: addnames = [] # Drop any old names (the name actually might include ${PV}) for name in oldnames: if name not in newnames: - newvalues['SRC_URI[%s.md5sum]' % name] = None - newvalues['SRC_URI[%s.sha256sum]' % name] = None + for checksum in oldsums: + newvalues['SRC_URI[%s.%s]' % (name, checksum)] = None - if sha256: - if addnames: - nameprefix = '%s.' % addnames[0] - else: - nameprefix = '' + nameprefix = '%s.' % addnames[0] if addnames else '' + + # md5sum is deprecated, remove any traces of it. If it was the only old + # checksum, then replace it with the default checksums. + if 'md5sum' in oldsums: newvalues['SRC_URI[%smd5sum]' % nameprefix] = None - newvalues['SRC_URI[%ssha256sum]' % nameprefix] = sha256 + oldsums.remove('md5sum') + if not oldsums: + oldsums = ["%ssum" % s for s in bb.fetch2.SHOWN_CHECKSUM_LIST] + + for checksum in oldsums: + newvalues['SRC_URI[%s%s]' % (nameprefix, checksum)] = checksums[checksum] if srcsubdir_new != srcsubdir_old: s_subdir_old = os.path.relpath(os.path.abspath(rd.getVar('S')), rd.getVar('WORKDIR')) @@ -422,10 +461,11 @@ def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, srcsubdir_old, src newvalues["LIC_FILES_CHKSUM"] = newlicchksum _add_license_diff_to_recipe(fullpath, license_diff) + tinfoil.modified_files() try: rd = tinfoil.parse_recipe_file(fullpath, False) except bb.tinfoil.TinfoilCommandFailed as e: - _upgrade_error(e, fullpath, srctree, keep_failure, 'Parsing of upgraded recipe failed') + _upgrade_error(e, os.path.dirname(fullpath), srctree, keep_failure, 'Parsing of upgraded recipe failed') oe.recipeutils.patch_recipe(rd, fullpath, newvalues) return fullpath, copied @@ -434,7 +474,7 @@ def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, srcsubdir_old, src def _check_git_config(): def getconfig(name): try: - value = bb.process.run('git config --global %s' % name)[0].strip() + value = bb.process.run('git config %s' % name)[0].strip() except bb.process.ExecutionError as e: if e.exitcode == 1: value = None @@ -521,6 +561,8 @@ def upgrade(args, config, basepath, workspace): else: srctree = standard.get_default_srctree(config, pn) + srctree_s = standard.get_real_srctree(srctree, rd.getVar('S'), rd.getVar('WORKDIR')) + # try to automatically discover latest version and revision if not provided on command line if not args.version and not args.srcrev: version_info = oe.recipeutils.get_recipe_upstream_version(rd) @@ -550,21 +592,20 @@ def upgrade(args, config, basepath, workspace): try: logger.info('Extracting current version source...') rev1, srcsubdir1 = standard._extract_source(srctree, False, 'devtool-orig', False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) - old_licenses = _extract_licenses(srctree, (rd.getVar('LIC_FILES_CHKSUM') or "")) + old_licenses = _extract_licenses(srctree_s, (rd.getVar('LIC_FILES_CHKSUM') or "")) logger.info('Extracting upgraded version source...') - rev2, md5, sha256, srcbranch, srcsubdir2 = _extract_new_source(args.version, srctree, args.no_patch, + rev2, checksums, srcbranch, srcsubdir2 = _extract_new_source(args.version, srctree, args.no_patch, args.srcrev, args.srcbranch, args.branch, args.keep_temp, tinfoil, rd) - new_licenses = _extract_licenses(srctree, (rd.getVar('LIC_FILES_CHKSUM') or "")) + new_licenses = _extract_licenses(srctree_s, (rd.getVar('LIC_FILES_CHKSUM') or "")) license_diff = _generate_license_diff(old_licenses, new_licenses) - rf, copied = _create_new_recipe(args.version, md5, sha256, args.srcrev, srcbranch, srcsubdir1, srcsubdir2, config.workspace_path, tinfoil, rd, license_diff, new_licenses, srctree, args.keep_failure) - except bb.process.CmdError as e: - _upgrade_error(e, rf, srctree, args.keep_failure) - except DevtoolError as e: - _upgrade_error(e, rf, srctree, args.keep_failure) + rf, copied = _create_new_recipe(args.version, checksums, args.srcrev, srcbranch, srcsubdir1, srcsubdir2, config.workspace_path, tinfoil, rd, license_diff, new_licenses, srctree, args.keep_failure) + except (bb.process.CmdError, DevtoolError) as e: + recipedir = os.path.join(config.workspace_path, 'recipes', rd.getVar('BPN')) + _upgrade_error(e, recipedir, srctree, args.keep_failure) standard._add_md5(config, pn, os.path.dirname(rf)) - af = _write_append(rf, srctree, args.same_dir, args.no_same_dir, rev2, + af = _write_append(rf, srctree, srctree_s, args.same_dir, args.no_same_dir, rev2, copied, config.workspace_path, rd) standard._add_md5(config, pn, af) @@ -574,6 +615,9 @@ def upgrade(args, config, basepath, workspace): logger.info('New recipe is %s' % rf) if license_diff: logger.info('License checksums have been updated in the new recipe; please refer to it for the difference between the old and the new license texts.') + preferred_version = rd.getVar('PREFERRED_VERSION_%s' % rd.getVar('PN')) + if preferred_version: + logger.warning('Version is pinned to %s via PREFERRED_VERSION; it may need adjustment to match the new version before any further steps are taken' % preferred_version) finally: tinfoil.shutdown() return 0 @@ -605,7 +649,7 @@ def check_upgrade_status(args, config, basepath, workspace): for result in results: # pn, update_status, current, latest, maintainer, latest_commit, no_update_reason if args.all or result[1] != 'MATCH': - logger.info("{:25} {:15} {:15} {} {} {}".format( result[0], + print("{:25} {:15} {:15} {} {} {}".format( result[0], result[2], result[1] if result[1] != 'UPDATE' else (result[3] if not result[3].endswith("new-commits-available") else "new commits"), result[4], diff --git a/scripts/lib/recipetool/append.py b/scripts/lib/recipetool/append.py index e9d52bb67b..341e893305 100644 --- a/scripts/lib/recipetool/append.py +++ b/scripts/lib/recipetool/append.py @@ -18,6 +18,7 @@ import shutil import scriptutils import errno from collections import defaultdict +import difflib logger = logging.getLogger('recipetool') @@ -49,7 +50,7 @@ def find_target_file(targetpath, d, pkglist=None): '/etc/group': '/etc/group should be managed through the useradd and extrausers classes', '/etc/shadow': '/etc/shadow should be managed through the useradd and extrausers classes', '/etc/gshadow': '/etc/gshadow should be managed through the useradd and extrausers classes', - '${sysconfdir}/hostname': '${sysconfdir}/hostname contents should be set by setting hostname_pn-base-files = "value" in configuration',} + '${sysconfdir}/hostname': '${sysconfdir}/hostname contents should be set by setting hostname:pn-base-files = "value" in configuration',} for pthspec, message in invalidtargets.items(): if fnmatch.fnmatchcase(targetpath, d.expand(pthspec)): @@ -72,15 +73,15 @@ def find_target_file(targetpath, d, pkglist=None): # This does assume that PN comes before other values, but that's a fairly safe assumption for line in f: if line.startswith('PN:'): - pn = line.split(':', 1)[1].strip() - elif line.startswith('FILES_INFO:'): - val = line.split(':', 1)[1].strip() + pn = line.split(': ', 1)[1].strip() + elif line.startswith('FILES_INFO'): + val = line.split(': ', 1)[1].strip() dictval = json.loads(val) for fullpth in dictval.keys(): if fnmatch.fnmatchcase(fullpth, targetpath): recipes[targetpath].append(pn) - elif line.startswith('pkg_preinst_') or line.startswith('pkg_postinst_'): - scriptval = line.split(':', 1)[1].strip().encode('utf-8').decode('unicode_escape') + elif line.startswith('pkg_preinst:') or line.startswith('pkg_postinst:'): + scriptval = line.split(': ', 1)[1].strip().encode('utf-8').decode('unicode_escape') if 'update-alternatives --install %s ' % targetpath in scriptval: recipes[targetpath].append('?%s' % pn) elif targetpath_re.search(scriptval): @@ -299,7 +300,10 @@ def appendfile(args): if st.st_mode & stat.S_IXUSR: perms = '0755' install = {args.newfile: (args.targetpath, perms)} - oe.recipeutils.bbappend_recipe(rd, args.destlayer, {args.newfile: sourcepath}, install, wildcardver=args.wildcard_version, machine=args.machine) + if sourcepath: + sourcepath = os.path.basename(sourcepath) + oe.recipeutils.bbappend_recipe(rd, args.destlayer, {args.newfile: {'newname' : sourcepath}}, install, wildcardver=args.wildcard_version, machine=args.machine) + tinfoil.modified_files() return 0 else: if alternative_pns: @@ -327,6 +331,7 @@ def appendsrc(args, files, rd, extralines=None): copyfiles = {} extralines = extralines or [] + params = [] for newfile, srcfile in files.items(): src_destdir = os.path.dirname(srcfile) if not args.use_workdir: @@ -337,25 +342,46 @@ def appendsrc(args, files, rd, extralines=None): src_destdir = os.path.join(os.path.relpath(srcdir, workdir), src_destdir) src_destdir = os.path.normpath(src_destdir) - source_uri = 'file://{0}'.format(os.path.basename(srcfile)) if src_destdir and src_destdir != '.': - source_uri += ';subdir={0}'.format(src_destdir) - - simple = bb.fetch.URI(source_uri) - simple.params = {} - simple_str = str(simple) - if simple_str in simplified: - existing = simplified[simple_str] - if source_uri != existing: - logger.warning('{0!r} is already in SRC_URI, with different parameters: {1!r}, not adding'.format(source_uri, existing)) - else: - logger.warning('{0!r} is already in SRC_URI, not adding'.format(source_uri)) + params.append({'subdir': src_destdir}) else: - extralines.append('SRC_URI += {0}'.format(source_uri)) - copyfiles[newfile] = srcfile - - oe.recipeutils.bbappend_recipe(rd, args.destlayer, copyfiles, None, wildcardver=args.wildcard_version, machine=args.machine, extralines=extralines) - + params.append({}) + + copyfiles[newfile] = {'newname' : os.path.basename(srcfile)} + + dry_run_output = None + dry_run_outdir = None + if args.dry_run: + import tempfile + dry_run_output = tempfile.TemporaryDirectory(prefix='devtool') + dry_run_outdir = dry_run_output.name + + appendfile, _ = oe.recipeutils.bbappend_recipe(rd, args.destlayer, copyfiles, None, wildcardver=args.wildcard_version, machine=args.machine, extralines=extralines, params=params, + redirect_output=dry_run_outdir, update_original_recipe=args.update_recipe) + if not appendfile: + return + if args.dry_run: + output = '' + appendfilename = os.path.basename(appendfile) + newappendfile = appendfile + if appendfile and os.path.exists(appendfile): + with open(appendfile, 'r') as f: + oldlines = f.readlines() + else: + appendfile = '/dev/null' + oldlines = [] + + with open(os.path.join(dry_run_outdir, appendfilename), 'r') as f: + newlines = f.readlines() + diff = difflib.unified_diff(oldlines, newlines, appendfile, newappendfile) + difflines = list(diff) + if difflines: + output += ''.join(difflines) + if output: + logger.info('Diff of changed files:\n%s' % output) + else: + logger.info('No changed files') + tinfoil.modified_files() def appendsrcfiles(parser, args): recipedata = _parse_recipe(args.recipe, tinfoil) @@ -435,6 +461,8 @@ def register_commands(subparsers): help='Create/update a bbappend to add or replace source files', description='Creates a bbappend (or updates an existing one) to add or replace the specified file in the recipe sources, either those in WORKDIR or those in the source tree. This command lets you specify multiple files with a destination directory, so cannot specify the destination filename. See the `appendsrcfile` command for the other behavior.') parser.add_argument('-D', '--destdir', help='Destination directory (relative to S or WORKDIR, defaults to ".")', default='', type=destination_path) + parser.add_argument('-u', '--update-recipe', help='Update recipe instead of creating (or updating) a bbapend file. DESTLAYER must contains the recipe to update', action='store_true') + parser.add_argument('-n', '--dry-run', help='Dry run mode', action='store_true') parser.add_argument('files', nargs='+', metavar='FILE', help='File(s) to be added to the recipe sources (WORKDIR or S)', type=existing_path) parser.set_defaults(func=lambda a: appendsrcfiles(parser, a), parserecipes=True) @@ -442,6 +470,8 @@ def register_commands(subparsers): parents=[common_src], help='Create/update a bbappend to add or replace a source file', description='Creates a bbappend (or updates an existing one) to add or replace the specified files in the recipe sources, either those in WORKDIR or those in the source tree. This command lets you specify the destination filename, not just destination directory, but only works for one file. See the `appendsrcfiles` command for the other behavior.') + parser.add_argument('-u', '--update-recipe', help='Update recipe instead of creating (or updating) a bbapend file. DESTLAYER must contains the recipe to update', action='store_true') + parser.add_argument('-n', '--dry-run', help='Dry run mode', action='store_true') parser.add_argument('file', metavar='FILE', help='File to be added to the recipe sources (WORKDIR or S)', type=existing_path) parser.add_argument('destfile', metavar='DESTFILE', nargs='?', help='Destination path (relative to S or WORKDIR, optional)', type=destination_path) parser.set_defaults(func=lambda a: appendsrcfile(parser, a), parserecipes=True) diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py index 566c75369a..8e9ff38db6 100644 --- a/scripts/lib/recipetool/create.py +++ b/scripts/lib/recipetool/create.py @@ -115,8 +115,8 @@ class RecipeHandler(object): for line in f: if line.startswith('PN:'): pn = line.split(':', 1)[-1].strip() - elif line.startswith('FILES_INFO:'): - val = line.split(':', 1)[1].strip() + elif line.startswith('FILES_INFO:%s:' % pkg): + val = line.split(': ', 1)[1].strip() dictval = json.loads(val) for fullpth in sorted(dictval): if fullpth.startswith(includedir) and fullpth.endswith('.h'): @@ -366,7 +366,7 @@ def supports_srcrev(uri): def reformat_git_uri(uri): '''Convert any http[s]://....git URI into git://...;protocol=http[s]''' checkuri = uri.split(';', 1)[0] - if checkuri.endswith('.git') or '/git/' in checkuri or re.match('https?://github.com/[^/]+/[^/]+/?$', checkuri): + if checkuri.endswith('.git') or '/git/' in checkuri or re.match('https?://git(hub|lab).com/[^/]+/[^/]+/?$', checkuri): # Appends scheme if the scheme is missing if not '://' in uri: uri = 'git://' + uri @@ -423,6 +423,36 @@ def create_recipe(args): storeTagName = '' pv_srcpv = False + handled = [] + classes = [] + + # Find all plugins that want to register handlers + logger.debug('Loading recipe handlers') + raw_handlers = [] + for plugin in plugins: + if hasattr(plugin, 'register_recipe_handlers'): + plugin.register_recipe_handlers(raw_handlers) + # Sort handlers by priority + handlers = [] + for i, handler in enumerate(raw_handlers): + if isinstance(handler, tuple): + handlers.append((handler[0], handler[1], i)) + else: + handlers.append((handler, 0, i)) + handlers.sort(key=lambda item: (item[1], -item[2]), reverse=True) + for handler, priority, _ in handlers: + logger.debug('Handler: %s (priority %d)' % (handler.__class__.__name__, priority)) + setattr(handler, '_devtool', args.devtool) + handlers = [item[0] for item in handlers] + + fetchuri = None + for handler in handlers: + if hasattr(handler, 'process_url'): + ret = handler.process_url(args, classes, handled, extravalues) + if 'url' in handled and ret: + fetchuri = ret + break + if os.path.isfile(source): source = 'file://%s' % os.path.abspath(source) @@ -431,11 +461,12 @@ def create_recipe(args): if re.match(r'https?://github.com/[^/]+/[^/]+/archive/.+(\.tar\..*|\.zip)$', source): logger.warning('github archive files are not guaranteed to be stable and may be re-generated over time. If the latter occurs, the checksums will likely change and the recipe will fail at do_fetch. It is recommended that you point to an actual commit or tag in the repository instead (using the repository URL in conjunction with the -S/--srcrev option).') # Fetch a URL - fetchuri = reformat_git_uri(urldefrag(source)[0]) + if not fetchuri: + fetchuri = reformat_git_uri(urldefrag(source)[0]) if args.binary: # Assume the archive contains the directory structure verbatim # so we need to extract to a subdirectory - fetchuri += ';subdir=${BP}' + fetchuri += ';subdir=${BPN}' srcuri = fetchuri rev_re = re.compile(';rev=([^;]+)') res = rev_re.search(srcuri) @@ -478,6 +509,9 @@ def create_recipe(args): storeTagName = params['tag'] params['nobranch'] = '1' del params['tag'] + # Assume 'master' branch if not set + if scheme in ['git', 'gitsm'] and 'branch' not in params and 'nobranch' not in params: + params['branch'] = 'master' fetchuri = bb.fetch2.encodeurl((scheme, network, path, user, passwd, params)) tmpparent = tinfoil.config_data.getVar('BASE_WORKDIR') @@ -527,10 +561,9 @@ def create_recipe(args): # Remove HEAD reference point and drop remote prefix get_branch = [x.split('/', 1)[1] for x in get_branch if not x.startswith('origin/HEAD')] if 'master' in get_branch: - # If it is master, we do not need to append 'branch=master' as this is default. # Even with the case where get_branch has multiple objects, if 'master' is one # of them, we should default take from 'master' - srcbranch = '' + srcbranch = 'master' elif len(get_branch) == 1: # If 'master' isn't in get_branch and get_branch contains only ONE object, then store result into 'srcbranch' srcbranch = get_branch[0] @@ -543,8 +576,8 @@ def create_recipe(args): # Since we might have a value in srcbranch, we need to # recontruct the srcuri to include 'branch' in params. scheme, network, path, user, passwd, params = bb.fetch2.decodeurl(srcuri) - if srcbranch: - params['branch'] = srcbranch + if scheme in ['git', 'gitsm']: + params['branch'] = srcbranch or 'master' if storeTagName and scheme in ['git', 'gitsm']: # Check srcrev using tag and check validity of the tag @@ -603,7 +636,7 @@ def create_recipe(args): splitline = line.split() if len(splitline) > 1: if splitline[0] == 'origin' and scriptutils.is_src_url(splitline[1]): - srcuri = reformat_git_uri(splitline[1]) + srcuri = reformat_git_uri(splitline[1]) + ';branch=master' srcsubdir = 'git' break @@ -636,8 +669,6 @@ def create_recipe(args): # We'll come back and replace this later in handle_license_vars() lines_before.append('##LICENSE_PLACEHOLDER##') - handled = [] - classes = [] # FIXME This is kind of a hack, we probably ought to be using bitbake to do this pn = None @@ -675,8 +706,10 @@ def create_recipe(args): if not srcuri: lines_before.append('# No information for SRC_URI yet (only an external source tree was specified)') lines_before.append('SRC_URI = "%s"' % srcuri) + shown_checksums = ["%ssum" % s for s in bb.fetch2.SHOWN_CHECKSUM_LIST] for key, value in sorted(checksums.items()): - lines_before.append('SRC_URI[%s] = "%s"' % (key, value)) + if key in shown_checksums: + lines_before.append('SRC_URI[%s] = "%s"' % (key, value)) if srcuri and supports_srcrev(srcuri): lines_before.append('') lines_before.append('# Modify these as desired') @@ -688,7 +721,7 @@ def create_recipe(args): srcpvprefix = 'svnr' else: srcpvprefix = scheme - lines_before.append('PV = "%s+%s${SRCPV}"' % (realpv or '1.0', srcpvprefix)) + lines_before.append('PV = "%s+%s"' % (realpv or '1.0', srcpvprefix)) pv_srcpv = True if not args.autorev and srcrev == '${AUTOREV}': if os.path.exists(os.path.join(srctree, '.git')): @@ -710,31 +743,12 @@ def create_recipe(args): lines_after.append('') if args.binary: - lines_after.append('INSANE_SKIP_${PN} += "already-stripped"') + lines_after.append('INSANE_SKIP:${PN} += "already-stripped"') lines_after.append('') if args.npm_dev: extravalues['NPM_INSTALL_DEV'] = 1 - # Find all plugins that want to register handlers - logger.debug('Loading recipe handlers') - raw_handlers = [] - for plugin in plugins: - if hasattr(plugin, 'register_recipe_handlers'): - plugin.register_recipe_handlers(raw_handlers) - # Sort handlers by priority - handlers = [] - for i, handler in enumerate(raw_handlers): - if isinstance(handler, tuple): - handlers.append((handler[0], handler[1], i)) - else: - handlers.append((handler, 0, i)) - handlers.sort(key=lambda item: (item[1], -item[2]), reverse=True) - for handler, priority, _ in handlers: - logger.debug('Handler: %s (priority %d)' % (handler.__class__.__name__, priority)) - setattr(handler, '_devtool', args.devtool) - handlers = [item[0] for item in handlers] - # Apply the handlers if args.binary: classes.append('bin_package') @@ -743,6 +757,10 @@ def create_recipe(args): for handler in handlers: handler.process(srctree_use, classes, lines_before, lines_after, handled, extravalues) + # native and nativesdk classes are special and must be inherited last + # If present, put them at the end of the classes list + classes.sort(key=lambda c: c in ("native", "nativesdk")) + extrafiles = extravalues.pop('extrafiles', {}) extra_pn = extravalues.pop('PN', None) extra_pv = extravalues.pop('PV', None) @@ -867,8 +885,10 @@ def create_recipe(args): outlines.append('') outlines.extend(lines_after) + outlines = [ line.rstrip('\n') +"\n" for line in outlines] + if extravalues: - _, outlines = oe.recipeutils.patch_recipe_lines(outlines, extravalues, trailing_newline=False) + _, outlines = oe.recipeutils.patch_recipe_lines(outlines, extravalues, trailing_newline=True) if args.extract_to: scriptutils.git_convert_standalone_clone(srctree) @@ -884,7 +904,7 @@ def create_recipe(args): log_info_cond('Source extracted to %s' % args.extract_to, args.devtool) if outfile == '-': - sys.stdout.write('\n'.join(outlines) + '\n') + sys.stdout.write(''.join(outlines) + '\n') else: with open(outfile, 'w') as f: lastline = None @@ -892,9 +912,10 @@ def create_recipe(args): if not lastline and not line: # Skip extra blank lines continue - f.write('%s\n' % line) + f.write('%s' % line) lastline = line log_info_cond('Recipe %s has been created; further editing may be required to make it fully functional' % outfile, args.devtool) + tinfoil.modified_files() if tempsrc: if args.keep_temp: @@ -917,6 +938,22 @@ def split_value(value): else: return value +def fixup_license(value): + # Ensure licenses with OR starts and ends with brackets + if '|' in value: + return '(' + value + ')' + return value + +def tidy_licenses(value): + """Flat, split and sort licenses""" + from oe.license import flattened_licenses + def _choose(a, b): + str_a, str_b = sorted((" & ".join(a), " & ".join(b)), key=str.casefold) + return ["(%s | %s)" % (str_a, str_b)] + if not isinstance(value, str): + value = " & ".join(value) + return sorted(list(set(flattened_licenses(value, _choose))), key=str.casefold) + def handle_license_vars(srctree, lines_before, handled, extravalues, d): lichandled = [x for x in handled if x[0] == 'license'] if lichandled: @@ -930,10 +967,13 @@ def handle_license_vars(srctree, lines_before, handled, extravalues, d): lines = [] if licvalues: for licvalue in licvalues: - if not licvalue[0] in licenses: - licenses.append(licvalue[0]) + license = licvalue[0] + lics = tidy_licenses(fixup_license(license)) + lics = [lic for lic in lics if lic not in licenses] + if len(lics): + licenses.extend(lics) lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2])) - if licvalue[0] == 'Unknown': + if license == 'Unknown': lic_unknown.append(licvalue[1]) if lic_unknown: lines.append('#') @@ -942,9 +982,7 @@ def handle_license_vars(srctree, lines_before, handled, extravalues, d): for licfile in lic_unknown: lines.append('# %s' % licfile) - extra_license = split_value(extravalues.pop('LICENSE', [])) - if '&' in extra_license: - extra_license.remove('&') + extra_license = tidy_licenses(extravalues.pop('LICENSE', '')) if extra_license: if licenses == ['Unknown']: licenses = extra_license @@ -985,7 +1023,7 @@ def handle_license_vars(srctree, lines_before, handled, extravalues, d): lines.append('# instead of &. If there is any doubt, check the accompanying documentation') lines.append('# to determine which situation is applicable.') - lines.append('LICENSE = "%s"' % ' & '.join(licenses)) + lines.append('LICENSE = "%s"' % ' & '.join(sorted(licenses, key=str.casefold))) lines.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n '.join(lic_files_chksum)) lines.append('') @@ -1002,118 +1040,170 @@ def handle_license_vars(srctree, lines_before, handled, extravalues, d): handled.append(('license', licvalues)) return licvalues -def get_license_md5sums(d, static_only=False): +def get_license_md5sums(d, static_only=False, linenumbers=False): import bb.utils + import csv md5sums = {} - if not static_only: + if not static_only and not linenumbers: # Gather md5sums of license files in common license dir commonlicdir = d.getVar('COMMON_LICENSE_DIR') for fn in os.listdir(commonlicdir): md5value = bb.utils.md5_file(os.path.join(commonlicdir, fn)) md5sums[md5value] = fn + # The following were extracted from common values in various recipes # (double checking the license against the license file itself, not just # the LICENSE value in the recipe) - md5sums['94d55d512a9ba36caa9b7df079bae19f'] = 'GPLv2' - md5sums['b234ee4d69f5fce4486a80fdaf4a4263'] = 'GPLv2' - md5sums['59530bdf33659b29e73d4adb9f9f6552'] = 'GPLv2' - md5sums['0636e73ff0215e8d672dc4c32c317bb3'] = 'GPLv2' - md5sums['eb723b61539feef013de476e68b5c50a'] = 'GPLv2' - md5sums['751419260aa954499f7abaabaa882bbe'] = 'GPLv2' - md5sums['393a5ca445f6965873eca0259a17f833'] = 'GPLv2' - md5sums['12f884d2ae1ff87c09e5b7ccc2c4ca7e'] = 'GPLv2' - md5sums['8ca43cbc842c2336e835926c2166c28b'] = 'GPLv2' - md5sums['ebb5c50ab7cab4baeffba14977030c07'] = 'GPLv2' - md5sums['c93c0550bd3173f4504b2cbd8991e50b'] = 'GPLv2' - md5sums['9ac2e7cff1ddaf48b6eab6028f23ef88'] = 'GPLv2' - md5sums['4325afd396febcb659c36b49533135d4'] = 'GPLv2' - md5sums['18810669f13b87348459e611d31ab760'] = 'GPLv2' - md5sums['d7810fab7487fb0aad327b76f1be7cd7'] = 'GPLv2' # the Linux kernel's COPYING file - md5sums['bbb461211a33b134d42ed5ee802b37ff'] = 'LGPLv2.1' - md5sums['7fbc338309ac38fefcd64b04bb903e34'] = 'LGPLv2.1' - md5sums['4fbd65380cdd255951079008b364516c'] = 'LGPLv2.1' - md5sums['2d5025d4aa3495befef8f17206a5b0a1'] = 'LGPLv2.1' - md5sums['fbc093901857fcd118f065f900982c24'] = 'LGPLv2.1' - md5sums['a6f89e2100d9b6cdffcea4f398e37343'] = 'LGPLv2.1' - md5sums['d8045f3b8f929c1cb29a1e3fd737b499'] = 'LGPLv2.1' - md5sums['fad9b3332be894bab9bc501572864b29'] = 'LGPLv2.1' - md5sums['3bf50002aefd002f49e7bb854063f7e7'] = 'LGPLv2' - md5sums['9f604d8a4f8e74f4f5140845a21b6674'] = 'LGPLv2' - md5sums['5f30f0716dfdd0d91eb439ebec522ec2'] = 'LGPLv2' - md5sums['55ca817ccb7d5b5b66355690e9abc605'] = 'LGPLv2' - md5sums['252890d9eee26aab7b432e8b8a616475'] = 'LGPLv2' - md5sums['3214f080875748938ba060314b4f727d'] = 'LGPLv2' - md5sums['db979804f025cf55aabec7129cb671ed'] = 'LGPLv2' - md5sums['d32239bcb673463ab874e80d47fae504'] = 'GPLv3' - md5sums['f27defe1e96c2e1ecd4e0c9be8967949'] = 'GPLv3' - md5sums['6a6a8e020838b23406c81b19c1d46df6'] = 'LGPLv3' - md5sums['3b83ef96387f14655fc854ddc3c6bd57'] = 'Apache-2.0' - md5sums['385c55653886acac3821999a3ccd17b3'] = 'Artistic-1.0 | GPL-2.0' # some perl modules - md5sums['54c7042be62e169199200bc6477f04d1'] = 'BSD-3-Clause' - md5sums['bfe1f75d606912a4111c90743d6c7325'] = 'MPL-1.1' + + # Read license md5sums from csv file + scripts_path = os.path.dirname(os.path.realpath(__file__)) + for path in (d.getVar('BBPATH').split(':') + + [os.path.join(scripts_path, '..', '..')]): + csv_path = os.path.join(path, 'lib', 'recipetool', 'licenses.csv') + if os.path.isfile(csv_path): + with open(csv_path, newline='') as csv_file: + fieldnames = ['md5sum', 'license', 'beginline', 'endline', 'md5'] + reader = csv.DictReader(csv_file, delimiter=',', fieldnames=fieldnames) + for row in reader: + if linenumbers: + md5sums[row['md5sum']] = ( + row['license'], row['beginline'], row['endline'], row['md5']) + else: + md5sums[row['md5sum']] = row['license'] + return md5sums -def crunch_license(licfile): +def crunch_known_licenses(d): ''' - Remove non-material text from a license file and then check - its md5sum against a known list. This works well for licenses - which contain a copyright statement, but is also a useful way - to handle people's insistence upon reformatting the license text - slightly (with no material difference to the text of the - license). + Calculate the MD5 checksums for the crunched versions of all common + licenses. Also add additional known checksums. ''' + + crunched_md5sums = {} - import oe.utils + # common licenses + crunched_md5sums['ad4e9d34a2e966dfe9837f18de03266d'] = 'GFDL-1.1-only' + crunched_md5sums['d014fb11a34eb67dc717fdcfc97e60ed'] = 'GFDL-1.2-only' + crunched_md5sums['e020ca655b06c112def28e597ab844f1'] = 'GFDL-1.3-only' - # Note: these are carefully constructed! - license_title_re = re.compile(r'^\(?(#+ *)?(The )?.{1,10} [Ll]icen[sc]e( \(.{1,10}\))?\)?:?$') - license_statement_re = re.compile(r'^(This (project|software) is( free software)? (released|licen[sc]ed)|(Released|Licen[cs]ed)) under the .{1,10} [Ll]icen[sc]e:?$') - copyright_re = re.compile('^(#+)? *Copyright .*$') - - crunched_md5sums = {} # The following two were gleaned from the "forever" npm package crunched_md5sums['0a97f8e4cbaf889d6fa51f84b89a79f6'] = 'ISC' - crunched_md5sums['eecf6429523cbc9693547cf2db790b5c'] = 'MIT' - # https://github.com/vasi/pixz/blob/master/LICENSE - crunched_md5sums['2f03392b40bbe663597b5bd3cc5ebdb9'] = 'BSD-2-Clause' # https://github.com/waffle-gl/waffle/blob/master/LICENSE.txt - crunched_md5sums['e72e5dfef0b1a4ca8a3d26a60587db66'] = 'BSD-2-Clause' + crunched_md5sums['50fab24ce589d69af8964fdbfe414c60'] = 'BSD-2-Clause' # https://github.com/spigwitmer/fakeds1963s/blob/master/LICENSE - crunched_md5sums['8be76ac6d191671f347ee4916baa637e'] = 'GPLv2' - # https://github.com/datto/dattobd/blob/master/COPYING - # http://git.savannah.gnu.org/cgit/freetype/freetype2.git/tree/docs/GPLv2.TXT - crunched_md5sums['1d65c5ad4bf6489f85f4812bf08ae73d'] = 'GPLv2' + crunched_md5sums['88a4355858a1433fea99fae34a44da88'] = 'GPL-2.0-only' # http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt - # http://git.neil.brown.name/?p=mdadm.git;a=blob;f=COPYING;h=d159169d1050894d3ea3b98e1c965c4058208fe1;hb=HEAD - crunched_md5sums['fb530f66a7a89ce920f0e912b5b66d4b'] = 'GPLv2' - # https://github.com/gkos/nrf24/blob/master/COPYING - crunched_md5sums['7b6aaa4daeafdfa6ed5443fd2684581b'] = 'GPLv2' - # https://github.com/josch09/resetusb/blob/master/COPYING - crunched_md5sums['8b8ac1d631a4d220342e83bcf1a1fbc3'] = 'GPLv3' + crunched_md5sums['063b5c3ebb5f3aa4c85a2ed18a31fbe7'] = 'GPL-2.0-only' # https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv2.1 - crunched_md5sums['2ea316ed973ae176e502e2297b574bb3'] = 'LGPLv2.1' + crunched_md5sums['7f5202f4d44ed15dcd4915f5210417d8'] = 'LGPL-2.1-only' # unixODBC-2.3.4 COPYING - crunched_md5sums['1daebd9491d1e8426900b4fa5a422814'] = 'LGPLv2.1' + crunched_md5sums['3debde09238a8c8e1f6a847e1ec9055b'] = 'LGPL-2.1-only' # https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv3 - crunched_md5sums['2ebfb3bb49b9a48a075cc1425e7f4129'] = 'LGPLv3' + crunched_md5sums['f90c613c51aa35da4d79dd55fc724ceb'] = 'LGPL-3.0-only' # https://raw.githubusercontent.com/eclipse/mosquitto/v1.4.14/epl-v10 crunched_md5sums['efe2cb9a35826992b9df68224e3c2628'] = 'EPL-1.0' - # https://raw.githubusercontent.com/eclipse/mosquitto/v1.4.14/edl-v10 - crunched_md5sums['0a9c78c0a398d1bbce4a166757d60387'] = 'EDL-1.0' + + # https://raw.githubusercontent.com/jquery/esprima/3.1.3/LICENSE.BSD + crunched_md5sums['80fa7b56a28e8c902e6af194003220a5'] = 'BSD-2-Clause' + # https://raw.githubusercontent.com/npm/npm-install-checks/master/LICENSE + crunched_md5sums['e659f77bfd9002659e112d0d3d59b2c1'] = 'BSD-2-Clause' + # https://raw.githubusercontent.com/silverwind/default-gateway/4.2.0/LICENSE + crunched_md5sums['4c641f2d995c47f5cb08bdb4b5b6ea05'] = 'BSD-2-Clause' + # https://raw.githubusercontent.com/tad-lispy/node-damerau-levenshtein/v1.0.5/LICENSE + crunched_md5sums['2b8c039b2b9a25f0feb4410c4542d346'] = 'BSD-2-Clause' + # https://raw.githubusercontent.com/terser/terser/v3.17.0/LICENSE + crunched_md5sums['8bd23871802951c9ad63855151204c2c'] = 'BSD-2-Clause' + # https://raw.githubusercontent.com/alexei/sprintf.js/1.0.3/LICENSE + crunched_md5sums['008c22318c8ea65928bf730ddd0273e3'] = 'BSD-3-Clause' + # https://raw.githubusercontent.com/Caligatio/jsSHA/v3.2.0/LICENSE + crunched_md5sums['0e46634a01bfef056892949acaea85b1'] = 'BSD-3-Clause' + # https://raw.githubusercontent.com/d3/d3-path/v1.0.9/LICENSE + crunched_md5sums['b5f72aef53d3b2b432702c30b0215666'] = 'BSD-3-Clause' + # https://raw.githubusercontent.com/feross/ieee754/v1.1.13/LICENSE + crunched_md5sums['a39327c997c20da0937955192d86232d'] = 'BSD-3-Clause' + # https://raw.githubusercontent.com/joyent/node-extsprintf/v1.3.0/LICENSE + crunched_md5sums['721f23a96ff4161ca3a5f071bbe18108'] = 'MIT' + # https://raw.githubusercontent.com/pvorb/clone/v0.2.0/LICENSE + crunched_md5sums['b376d29a53c9573006b9970709231431'] = 'MIT' + # https://raw.githubusercontent.com/andris9/encoding/v0.1.12/LICENSE + crunched_md5sums['85d8a977ee9d7c5ab4ac03c9b95431c4'] = 'MIT-0' + # https://raw.githubusercontent.com/faye/websocket-driver-node/0.7.3/LICENSE.md + crunched_md5sums['b66384e7137e41a9b1904ef4d39703b6'] = 'Apache-2.0' + # https://raw.githubusercontent.com/less/less.js/v4.1.1/LICENSE + crunched_md5sums['b27575459e02221ccef97ec0bfd457ae'] = 'Apache-2.0' + # https://raw.githubusercontent.com/microsoft/TypeScript/v3.5.3/LICENSE.txt + crunched_md5sums['a54a1a6a39e7f9dbb4a23a42f5c7fd1c'] = 'Apache-2.0' + # https://raw.githubusercontent.com/request/request/v2.87.0/LICENSE + crunched_md5sums['1034431802e57486b393d00c5d262b8a'] = 'Apache-2.0' + # https://raw.githubusercontent.com/dchest/tweetnacl-js/v0.14.5/LICENSE + crunched_md5sums['75605e6bdd564791ab698fca65c94a4f'] = 'Unlicense' + # https://raw.githubusercontent.com/stackgl/gl-mat3/v2.0.0/LICENSE.md + crunched_md5sums['75512892d6f59dddb6d1c7e191957e9c'] = 'Zlib' + + commonlicdir = d.getVar('COMMON_LICENSE_DIR') + for fn in sorted(os.listdir(commonlicdir)): + md5value, lictext = crunch_license(os.path.join(commonlicdir, fn)) + if md5value not in crunched_md5sums: + crunched_md5sums[md5value] = fn + elif fn != crunched_md5sums[md5value]: + bb.debug(2, "crunched_md5sums['%s'] is already set to '%s' rather than '%s'" % (md5value, crunched_md5sums[md5value], fn)) + else: + bb.debug(2, "crunched_md5sums['%s'] is already set to '%s'" % (md5value, crunched_md5sums[md5value])) + + return crunched_md5sums + +def crunch_license(licfile): + ''' + Remove non-material text from a license file and then calculate its + md5sum. This works well for licenses that contain a copyright statement, + but is also a useful way to handle people's insistence upon reformatting + the license text slightly (with no material difference to the text of the + license). + ''' + + import oe.utils + + # Note: these are carefully constructed! + license_title_re = re.compile(r'^#*\(? *(This is )?([Tt]he )?.{0,15} ?[Ll]icen[sc]e( \(.{1,10}\))?\)?[:\.]? ?#*$') + license_statement_re = re.compile(r'^((This (project|software)|.{1,10}) is( free software)? (released|licen[sc]ed)|(Released|Licen[cs]ed)) under the .{1,10} [Ll]icen[sc]e:?$') + copyright_re = re.compile(r'^ *[#\*]* *(Modified work |MIT LICENSED )?Copyright ?(\([cC]\))? .*$') + disclaimer_re = re.compile(r'^ *\*? ?All [Rr]ights [Rr]eserved\.$') + email_re = re.compile(r'^.*<[\w\.-]*@[\w\.\-]*>$') + header_re = re.compile(r'^(\/\**!?)? ?[\-=\*]* ?(\*\/)?$') + tag_re = re.compile(r'^ *@?\(?([Ll]icense|MIT)\)?$') + url_re = re.compile(r'^ *[#\*]* *https?:\/\/[\w\.\/\-]+$') + lictext = [] with open(licfile, 'r', errors='surrogateescape') as f: for line in f: # Drop opening statements if copyright_re.match(line): continue + elif disclaimer_re.match(line): + continue + elif email_re.match(line): + continue + elif header_re.match(line): + continue + elif tag_re.match(line): + continue + elif url_re.match(line): + continue elif license_title_re.match(line): continue elif license_statement_re.match(line): continue - # Squash spaces, and replace smart quotes, double quotes - # and backticks with single quotes + # Strip comment symbols + line = line.replace('*', '') \ + .replace('#', '') + # Unify spelling + line = line.replace('sub-license', 'sublicense') + # Squash spaces line = oe.utils.squashspaces(line.strip()) + # Replace smart quotes, double quotes and backticks with single quotes line = line.replace(u"\u2018", "'").replace(u"\u2019", "'").replace(u"\u201c","'").replace(u"\u201d", "'").replace('"', '\'').replace('`', '\'') + # Unify brackets + line = line.replace("{", "[").replace("}", "]") if line: lictext.append(line) @@ -1124,31 +1214,40 @@ def crunch_license(licfile): except UnicodeEncodeError: md5val = None lictext = '' - license = crunched_md5sums.get(md5val, None) - return license, md5val, lictext + return md5val, lictext def guess_license(srctree, d): import bb md5sums = get_license_md5sums(d) + crunched_md5sums = crunch_known_licenses(d) + licenses = [] licspecs = ['*LICEN[CS]E*', 'COPYING*', '*[Ll]icense*', 'LEGAL*', '[Ll]egal*', '*GPL*', 'README.lic*', 'COPYRIGHT*', '[Cc]opyright*', 'e[dp]l-v10'] + skip_extensions = (".html", ".js", ".json", ".svg", ".ts", ".go") licfiles = [] for root, dirs, files in os.walk(srctree): for fn in files: + if fn.endswith(skip_extensions): + continue for spec in licspecs: if fnmatch.fnmatch(fn, spec): fullpath = os.path.join(root, fn) if not fullpath in licfiles: licfiles.append(fullpath) - for licfile in licfiles: + for licfile in sorted(licfiles): md5value = bb.utils.md5_file(licfile) license = md5sums.get(md5value, None) if not license: - license, crunched_md5, lictext = crunch_license(licfile) - if not license: + crunched_md5, lictext = crunch_license(licfile) + license = crunched_md5sums.get(crunched_md5, None) + if lictext and not license: license = 'Unknown' - licenses.append((license, os.path.relpath(licfile, srctree), md5value)) + logger.info("Please add the following line for '%s' to a 'lib/recipetool/licenses.csv' " \ + "and replace `Unknown` with the license:\n" \ + "%s,Unknown" % (os.path.relpath(licfile, srctree), md5value)) + if license: + licenses.append((license, os.path.relpath(licfile, srctree), md5value)) # FIXME should we grab at least one source file with a license header and add that too? @@ -1162,6 +1261,7 @@ def split_pkg_licenses(licvalues, packages, outlines, fallback_licenses=None, pn """ pkglicenses = {pn: []} for license, licpath, _ in licvalues: + license = fixup_license(license) for pkgname, pkgpath in packages.items(): if licpath.startswith(pkgpath + '/'): if pkgname in pkglicenses: @@ -1174,11 +1274,14 @@ def split_pkg_licenses(licvalues, packages, outlines, fallback_licenses=None, pn pkglicenses[pn].append(license) outlicenses = {} for pkgname in packages: - license = ' '.join(list(set(pkglicenses.get(pkgname, ['Unknown'])))) or 'Unknown' - if license == 'Unknown' and pkgname in fallback_licenses: + # Assume AND operator between license files + license = ' & '.join(list(set(pkglicenses.get(pkgname, ['Unknown'])))) or 'Unknown' + if license == 'Unknown' and fallback_licenses and pkgname in fallback_licenses: license = fallback_licenses[pkgname] - outlines.append('LICENSE_%s = "%s"' % (pkgname, license)) - outlicenses[pkgname] = license.split() + licenses = tidy_licenses(license) + license = ' & '.join(licenses) + outlines.append('LICENSE:%s = "%s"' % (pkgname, license)) + outlicenses[pkgname] = licenses return outlicenses def read_pkgconfig_provides(d): @@ -1311,6 +1414,7 @@ def register_commands(subparsers): parser_create.add_argument('-B', '--srcbranch', help='Branch in source repository if fetching from an SCM such as git (default master)') parser_create.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') parser_create.add_argument('--npm-dev', action="store_true", help='For npm, also fetch devDependencies') + parser_create.add_argument('--no-pypi', action="store_true", help='Do not inherit pypi class') parser_create.add_argument('--devtool', action="store_true", help=argparse.SUPPRESS) parser_create.add_argument('--mirrors', action="store_true", help='Enable PREMIRRORS and MIRRORS for source tree fetching (disabled by default).') parser_create.set_defaults(func=create_recipe) diff --git a/scripts/lib/recipetool/create_buildsys.py b/scripts/lib/recipetool/create_buildsys.py index 35a97c9345..ec9d510e23 100644 --- a/scripts/lib/recipetool/create_buildsys.py +++ b/scripts/lib/recipetool/create_buildsys.py @@ -5,9 +5,9 @@ # SPDX-License-Identifier: GPL-2.0-only # +import os import re import logging -import glob from recipetool.create import RecipeHandler, validate_pv logger = logging.getLogger('recipetool') @@ -137,15 +137,15 @@ class CmakeRecipeHandler(RecipeHandler): deps = [] unmappedpkgs = [] - proj_re = re.compile('project\s*\(([^)]*)\)', re.IGNORECASE) - pkgcm_re = re.compile('pkg_check_modules\s*\(\s*[a-zA-Z0-9-_]+\s*(REQUIRED)?\s+([^)\s]+)\s*\)', re.IGNORECASE) - pkgsm_re = re.compile('pkg_search_module\s*\(\s*[a-zA-Z0-9-_]+\s*(REQUIRED)?((\s+[^)\s]+)+)\s*\)', re.IGNORECASE) - findpackage_re = re.compile('find_package\s*\(\s*([a-zA-Z0-9-_]+)\s*.*', re.IGNORECASE) - findlibrary_re = re.compile('find_library\s*\(\s*[a-zA-Z0-9-_]+\s*(NAMES\s+)?([a-zA-Z0-9-_ ]+)\s*.*') - checklib_re = re.compile('check_library_exists\s*\(\s*([^\s)]+)\s*.*', re.IGNORECASE) - include_re = re.compile('include\s*\(\s*([^)\s]*)\s*\)', re.IGNORECASE) - subdir_re = re.compile('add_subdirectory\s*\(\s*([^)\s]*)\s*([^)\s]*)\s*\)', re.IGNORECASE) - dep_re = re.compile('([^ ><=]+)( *[<>=]+ *[^ ><=]+)?') + proj_re = re.compile(r'project\s*\(([^)]*)\)', re.IGNORECASE) + pkgcm_re = re.compile(r'pkg_check_modules\s*\(\s*[a-zA-Z0-9-_]+\s*(REQUIRED)?\s+([^)\s]+)\s*\)', re.IGNORECASE) + pkgsm_re = re.compile(r'pkg_search_module\s*\(\s*[a-zA-Z0-9-_]+\s*(REQUIRED)?((\s+[^)\s]+)+)\s*\)', re.IGNORECASE) + findpackage_re = re.compile(r'find_package\s*\(\s*([a-zA-Z0-9-_]+)\s*.*', re.IGNORECASE) + findlibrary_re = re.compile(r'find_library\s*\(\s*[a-zA-Z0-9-_]+\s*(NAMES\s+)?([a-zA-Z0-9-_ ]+)\s*.*') + checklib_re = re.compile(r'check_library_exists\s*\(\s*([^\s)]+)\s*.*', re.IGNORECASE) + include_re = re.compile(r'include\s*\(\s*([^)\s]*)\s*\)', re.IGNORECASE) + subdir_re = re.compile(r'add_subdirectory\s*\(\s*([^)\s]*)\s*([^)\s]*)\s*\)', re.IGNORECASE) + dep_re = re.compile(r'([^ ><=]+)( *[<>=]+ *[^ ><=]+)?') def find_cmake_package(pkg): RecipeHandler.load_devel_filemap(tinfoil.config_data) @@ -423,16 +423,16 @@ class AutotoolsRecipeHandler(RecipeHandler): 'makeinfo': 'texinfo', } - pkg_re = re.compile('PKG_CHECK_MODULES\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*') - pkgce_re = re.compile('PKG_CHECK_EXISTS\(\s*\[?([^,\]]*)\]?[),].*') - lib_re = re.compile('AC_CHECK_LIB\(\s*\[?([^,\]]*)\]?,.*') - libx_re = re.compile('AX_CHECK_LIBRARY\(\s*\[?[^,\]]*\]?,\s*\[?([^,\]]*)\]?,\s*\[?([a-zA-Z0-9-]*)\]?,.*') - progs_re = re.compile('_PROGS?\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*') - dep_re = re.compile('([^ ><=]+)( [<>=]+ [^ ><=]+)?') - ac_init_re = re.compile('AC_INIT\(\s*([^,]+),\s*([^,]+)[,)].*') - am_init_re = re.compile('AM_INIT_AUTOMAKE\(\s*([^,]+),\s*([^,]+)[,)].*') - define_re = re.compile('\s*(m4_)?define\(\s*([^,]+),\s*([^,]+)\)') - version_re = re.compile('([0-9.]+)') + pkg_re = re.compile(r'PKG_CHECK_MODULES\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*') + pkgce_re = re.compile(r'PKG_CHECK_EXISTS\(\s*\[?([^,\]]*)\]?[),].*') + lib_re = re.compile(r'AC_CHECK_LIB\(\s*\[?([^,\]]*)\]?,.*') + libx_re = re.compile(r'AX_CHECK_LIBRARY\(\s*\[?[^,\]]*\]?,\s*\[?([^,\]]*)\]?,\s*\[?([a-zA-Z0-9-]*)\]?,.*') + progs_re = re.compile(r'_PROGS?\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*') + dep_re = re.compile(r'([^ ><=]+)( [<>=]+ [^ ><=]+)?') + ac_init_re = re.compile(r'AC_INIT\(\s*([^,]+),\s*([^,]+)[,)].*') + am_init_re = re.compile(r'AM_INIT_AUTOMAKE\(\s*([^,]+),\s*([^,]+)[,)].*') + define_re = re.compile(r'\s*(m4_)?define\(\s*([^,]+),\s*([^,]+)\)') + version_re = re.compile(r'([0-9.]+)') defines = {} def subst_defines(value): @@ -545,7 +545,7 @@ class AutotoolsRecipeHandler(RecipeHandler): deps.append('zlib') elif keyword in ('AX_CHECK_OPENSSL', 'AX_LIB_CRYPTO'): deps.append('openssl') - elif keyword == 'AX_LIB_CURL': + elif keyword in ('AX_LIB_CURL', 'LIBCURL_CHECK_CONFIG'): deps.append('curl') elif keyword == 'AX_LIB_BEECRYPT': deps.append('beecrypt') @@ -624,6 +624,7 @@ class AutotoolsRecipeHandler(RecipeHandler): 'AX_CHECK_OPENSSL', 'AX_LIB_CRYPTO', 'AX_LIB_CURL', + 'LIBCURL_CHECK_CONFIG', 'AX_LIB_BEECRYPT', 'AX_LIB_EXPAT', 'AX_LIB_GCRYPT', diff --git a/scripts/lib/recipetool/create_buildsys_python.py b/scripts/lib/recipetool/create_buildsys_python.py index adfa377956..a807dafae5 100644 --- a/scripts/lib/recipetool/create_buildsys_python.py +++ b/scripts/lib/recipetool/create_buildsys_python.py @@ -8,9 +8,9 @@ import ast import codecs import collections -import distutils.command.build_py +import setuptools.command.build_py import email -import imp +import importlib import glob import itertools import logging @@ -18,7 +18,11 @@ import os import re import sys import subprocess +import json +import urllib.request from recipetool.create import RecipeHandler +from urllib.parse import urldefrag +from recipetool.create import determine_from_url logger = logging.getLogger('recipetool') @@ -37,7 +41,334 @@ class PythonRecipeHandler(RecipeHandler): assume_provided = ['builtins', 'os.path'] # Assumes that the host python3 builtin_module_names is sane for target too assume_provided = assume_provided + list(sys.builtin_module_names) + excluded_fields = [] + + classifier_license_map = { + 'License :: OSI Approved :: Academic Free License (AFL)': 'AFL', + 'License :: OSI Approved :: Apache Software License': 'Apache', + 'License :: OSI Approved :: Apple Public Source License': 'APSL', + 'License :: OSI Approved :: Artistic License': 'Artistic', + 'License :: OSI Approved :: Attribution Assurance License': 'AAL', + 'License :: OSI Approved :: BSD License': 'BSD-3-Clause', + 'License :: OSI Approved :: Boost Software License 1.0 (BSL-1.0)': 'BSL-1.0', + 'License :: OSI Approved :: CEA CNRS Inria Logiciel Libre License, version 2.1 (CeCILL-2.1)': 'CECILL-2.1', + 'License :: OSI Approved :: Common Development and Distribution License 1.0 (CDDL-1.0)': 'CDDL-1.0', + 'License :: OSI Approved :: Common Public License': 'CPL', + 'License :: OSI Approved :: Eclipse Public License 1.0 (EPL-1.0)': 'EPL-1.0', + 'License :: OSI Approved :: Eclipse Public License 2.0 (EPL-2.0)': 'EPL-2.0', + 'License :: OSI Approved :: Eiffel Forum License': 'EFL', + 'License :: OSI Approved :: European Union Public Licence 1.0 (EUPL 1.0)': 'EUPL-1.0', + 'License :: OSI Approved :: European Union Public Licence 1.1 (EUPL 1.1)': 'EUPL-1.1', + 'License :: OSI Approved :: European Union Public Licence 1.2 (EUPL 1.2)': 'EUPL-1.2', + 'License :: OSI Approved :: GNU Affero General Public License v3': 'AGPL-3.0-only', + 'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)': 'AGPL-3.0-or-later', + 'License :: OSI Approved :: GNU Free Documentation License (FDL)': 'GFDL', + 'License :: OSI Approved :: GNU General Public License (GPL)': 'GPL', + 'License :: OSI Approved :: GNU General Public License v2 (GPLv2)': 'GPL-2.0-only', + 'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)': 'GPL-2.0-or-later', + 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)': 'GPL-3.0-only', + 'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)': 'GPL-3.0-or-later', + 'License :: OSI Approved :: GNU Lesser General Public License v2 (LGPLv2)': 'LGPL-2.0-only', + 'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)': 'LGPL-2.0-or-later', + 'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)': 'LGPL-3.0-only', + 'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)': 'LGPL-3.0-or-later', + 'License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)': 'LGPL', + 'License :: OSI Approved :: Historical Permission Notice and Disclaimer (HPND)': 'HPND', + 'License :: OSI Approved :: IBM Public License': 'IPL', + 'License :: OSI Approved :: ISC License (ISCL)': 'ISC', + 'License :: OSI Approved :: Intel Open Source License': 'Intel', + 'License :: OSI Approved :: Jabber Open Source License': 'Jabber', + 'License :: OSI Approved :: MIT License': 'MIT', + 'License :: OSI Approved :: MIT No Attribution License (MIT-0)': 'MIT-0', + 'License :: OSI Approved :: MITRE Collaborative Virtual Workspace License (CVW)': 'CVWL', + 'License :: OSI Approved :: MirOS License (MirOS)': 'MirOS', + 'License :: OSI Approved :: Motosoto License': 'Motosoto', + 'License :: OSI Approved :: Mozilla Public License 1.0 (MPL)': 'MPL-1.0', + 'License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)': 'MPL-1.1', + 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)': 'MPL-2.0', + 'License :: OSI Approved :: Nethack General Public License': 'NGPL', + 'License :: OSI Approved :: Nokia Open Source License': 'Nokia', + 'License :: OSI Approved :: Open Group Test Suite License': 'OGTSL', + 'License :: OSI Approved :: Open Software License 3.0 (OSL-3.0)': 'OSL-3.0', + 'License :: OSI Approved :: PostgreSQL License': 'PostgreSQL', + 'License :: OSI Approved :: Python License (CNRI Python License)': 'CNRI-Python', + 'License :: OSI Approved :: Python Software Foundation License': 'PSF-2.0', + 'License :: OSI Approved :: Qt Public License (QPL)': 'QPL', + 'License :: OSI Approved :: Ricoh Source Code Public License': 'RSCPL', + 'License :: OSI Approved :: SIL Open Font License 1.1 (OFL-1.1)': 'OFL-1.1', + 'License :: OSI Approved :: Sleepycat License': 'Sleepycat', + 'License :: OSI Approved :: Sun Industry Standards Source License (SISSL)': 'SISSL', + 'License :: OSI Approved :: Sun Public License': 'SPL', + 'License :: OSI Approved :: The Unlicense (Unlicense)': 'Unlicense', + 'License :: OSI Approved :: Universal Permissive License (UPL)': 'UPL-1.0', + 'License :: OSI Approved :: University of Illinois/NCSA Open Source License': 'NCSA', + 'License :: OSI Approved :: Vovida Software License 1.0': 'VSL-1.0', + 'License :: OSI Approved :: W3C License': 'W3C', + 'License :: OSI Approved :: X.Net License': 'Xnet', + 'License :: OSI Approved :: Zope Public License': 'ZPL', + 'License :: OSI Approved :: zlib/libpng License': 'Zlib', + 'License :: Other/Proprietary License': 'Proprietary', + 'License :: Public Domain': 'PD', + } + + def __init__(self): + pass + + def process_url(self, args, classes, handled, extravalues): + """ + Convert any pypi url https://pypi.org/project/<package>/<version> into https://files.pythonhosted.org/packages/source/... + which corresponds to the archive location, and add pypi class + """ + + if 'url' in handled: + return None + + fetch_uri = None + source = args.source + required_version = args.version if args.version else None + match = re.match(r'https?://pypi.org/project/([^/]+)(?:/([^/]+))?/?$', urldefrag(source)[0]) + if match: + package = match.group(1) + version = match.group(2) if match.group(2) else required_version + + json_url = f"https://pypi.org/pypi/%s/json" % package + response = urllib.request.urlopen(json_url) + if response.status == 200: + data = json.loads(response.read()) + if not version: + # grab latest version + version = data["info"]["version"] + pypi_package = data["info"]["name"] + for release in reversed(data["releases"][version]): + if release["packagetype"] == "sdist": + fetch_uri = release["url"] + break + else: + logger.warning("Cannot handle pypi url %s: cannot fetch package information using %s", source, json_url) + return None + else: + match = re.match(r'^https?://files.pythonhosted.org/packages.*/(.*)-.*$', source) + if match: + fetch_uri = source + pypi_package = match.group(1) + _, version = determine_from_url(fetch_uri) + + if match and not args.no_pypi: + if required_version and version != required_version: + raise Exception("Version specified using --version/-V (%s) and version specified in the url (%s) do not match" % (required_version, version)) + # This is optionnal if BPN looks like "python-<pypi_package>" or "python3-<pypi_package>" (see pypi.bbclass) + # but at this point we cannot know because because user can specify the output name of the recipe on the command line + extravalues["PYPI_PACKAGE"] = pypi_package + # If the tarball extension is not 'tar.gz' (default value in pypi.bblcass) whe should set PYPI_PACKAGE_EXT in the recipe + pypi_package_ext = re.match(r'.*%s-%s\.(.*)$' % (pypi_package, version), fetch_uri) + if pypi_package_ext: + pypi_package_ext = pypi_package_ext.group(1) + if pypi_package_ext != "tar.gz": + extravalues["PYPI_PACKAGE_EXT"] = pypi_package_ext + + # Pypi class will handle S and SRC_URI variables, so remove them + # TODO: allow oe.recipeutils.patch_recipe_lines() to accept regexp so we can simplify the following to: + # extravalues['SRC_URI(?:\[.*?\])?'] = None + extravalues['S'] = None + extravalues['SRC_URI'] = None + + classes.append('pypi') + + handled.append('url') + return fetch_uri + + def handle_classifier_license(self, classifiers, existing_licenses=""): + + licenses = [] + for classifier in classifiers: + if classifier in self.classifier_license_map: + license = self.classifier_license_map[classifier] + if license == 'Apache' and 'Apache-2.0' in existing_licenses: + license = 'Apache-2.0' + elif license == 'GPL': + if 'GPL-2.0' in existing_licenses or 'GPLv2' in existing_licenses: + license = 'GPL-2.0' + elif 'GPL-3.0' in existing_licenses or 'GPLv3' in existing_licenses: + license = 'GPL-3.0' + elif license == 'LGPL': + if 'LGPL-2.1' in existing_licenses or 'LGPLv2.1' in existing_licenses: + license = 'LGPL-2.1' + elif 'LGPL-2.0' in existing_licenses or 'LGPLv2' in existing_licenses: + license = 'LGPL-2.0' + elif 'LGPL-3.0' in existing_licenses or 'LGPLv3' in existing_licenses: + license = 'LGPL-3.0' + licenses.append(license) + + if licenses: + return ' & '.join(licenses) + + return None + + def map_info_to_bbvar(self, info, extravalues): + + # Map PKG-INFO & setup.py fields to bitbake variables + for field, values in info.items(): + if field in self.excluded_fields: + continue + + if field not in self.bbvar_map: + continue + + if isinstance(values, str): + value = values + else: + value = ' '.join(str(v) for v in values if v) + + bbvar = self.bbvar_map[field] + if bbvar == "PN": + # by convention python recipes start with "python3-" + if not value.startswith('python'): + value = 'python3-' + value + + if bbvar not in extravalues and value: + extravalues[bbvar] = value + + def apply_info_replacements(self, info): + if not self.replacements: + return + + for variable, search, replace in self.replacements: + if variable not in info: + continue + + def replace_value(search, replace, value): + if replace is None: + if re.search(search, value): + return None + else: + new_value = re.sub(search, replace, value) + if value != new_value: + return new_value + return value + + value = info[variable] + if isinstance(value, str): + new_value = replace_value(search, replace, value) + if new_value is None: + del info[variable] + elif new_value != value: + info[variable] = new_value + elif hasattr(value, 'items'): + for dkey, dvalue in list(value.items()): + new_list = [] + for pos, a_value in enumerate(dvalue): + new_value = replace_value(search, replace, a_value) + if new_value is not None and new_value != value: + new_list.append(new_value) + + if value != new_list: + value[dkey] = new_list + else: + new_list = [] + for pos, a_value in enumerate(value): + new_value = replace_value(search, replace, a_value) + if new_value is not None and new_value != value: + new_list.append(new_value) + + if value != new_list: + info[variable] = new_list + + + def scan_python_dependencies(self, paths): + deps = set() + try: + dep_output = self.run_command(['pythondeps', '-d'] + paths) + except (OSError, subprocess.CalledProcessError): + pass + else: + for line in dep_output.splitlines(): + line = line.rstrip() + dep, filename = line.split('\t', 1) + if filename.endswith('/setup.py'): + continue + deps.add(dep) + + try: + provides_output = self.run_command(['pythondeps', '-p'] + paths) + except (OSError, subprocess.CalledProcessError): + pass + else: + provides_lines = (l.rstrip() for l in provides_output.splitlines()) + provides = set(l for l in provides_lines if l and l != 'setup') + deps -= provides + + return deps + + def parse_pkgdata_for_python_packages(self): + pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR') + + ldata = tinfoil.config_data.createCopy() + bb.parse.handle('classes-recipe/python3-dir.bbclass', ldata, True) + python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR') + + dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload') + python_dirs = [python_sitedir + os.sep, + os.path.join(os.path.dirname(python_sitedir), 'dist-packages') + os.sep, + os.path.dirname(python_sitedir) + os.sep] + packages = {} + for pkgdatafile in glob.glob('{}/runtime/*'.format(pkgdata_dir)): + files_info = None + with open(pkgdatafile, 'r') as f: + for line in f.readlines(): + field, value = line.split(': ', 1) + if field.startswith('FILES_INFO'): + files_info = ast.literal_eval(value) + break + else: + continue + + for fn in files_info: + for suffix in importlib.machinery.all_suffixes(): + if fn.endswith(suffix): + break + else: + continue + + if fn.startswith(dynload_dir + os.sep): + if '/.debug/' in fn: + continue + base = os.path.basename(fn) + provided = base.split('.', 1)[0] + packages[provided] = os.path.basename(pkgdatafile) + continue + + for python_dir in python_dirs: + if fn.startswith(python_dir): + relpath = fn[len(python_dir):] + relstart, _, relremaining = relpath.partition(os.sep) + if relstart.endswith('.egg'): + relpath = relremaining + base, _ = os.path.splitext(relpath) + + if '/.debug/' in base: + continue + if os.path.basename(base) == '__init__': + base = os.path.dirname(base) + base = base.replace(os.sep + os.sep, os.sep) + provided = base.replace(os.sep, '.') + packages[provided] = os.path.basename(pkgdatafile) + return packages + + @classmethod + def run_command(cls, cmd, **popenargs): + if 'stderr' not in popenargs: + popenargs['stderr'] = subprocess.STDOUT + try: + return subprocess.check_output(cmd, **popenargs).decode('utf-8') + except OSError as exc: + logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc) + raise + except subprocess.CalledProcessError as exc: + logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc.output) + raise + +class PythonSetupPyRecipeHandler(PythonRecipeHandler): bbvar_map = { 'Name': 'PN', 'Version': 'PV', @@ -45,9 +376,9 @@ class PythonRecipeHandler(RecipeHandler): 'Summary': 'SUMMARY', 'Description': 'DESCRIPTION', 'License': 'LICENSE', - 'Requires': 'RDEPENDS_${PN}', - 'Provides': 'RPROVIDES_${PN}', - 'Obsoletes': 'RREPLACES_${PN}', + 'Requires': 'RDEPENDS:${PN}', + 'Provides': 'RPROVIDES:${PN}', + 'Obsoletes': 'RREPLACES:${PN}', } # PN/PV are already set by recipetool core & desc can be extremely long excluded_fields = [ @@ -75,6 +406,7 @@ class PythonRecipeHandler(RecipeHandler): 'Supported-Platform', ] setuparg_multi_line_values = ['Description'] + replacements = [ ('License', r' +$', ''), ('License', r'^ +', ''), @@ -95,71 +427,161 @@ class PythonRecipeHandler(RecipeHandler): ('Install-requires', r'\[[^\]]+\]$', ''), ] - classifier_license_map = { - 'License :: OSI Approved :: Academic Free License (AFL)': 'AFL', - 'License :: OSI Approved :: Apache Software License': 'Apache', - 'License :: OSI Approved :: Apple Public Source License': 'APSL', - 'License :: OSI Approved :: Artistic License': 'Artistic', - 'License :: OSI Approved :: Attribution Assurance License': 'AAL', - 'License :: OSI Approved :: BSD License': 'BSD', - 'License :: OSI Approved :: Common Public License': 'CPL', - 'License :: OSI Approved :: Eiffel Forum License': 'EFL', - 'License :: OSI Approved :: European Union Public Licence 1.0 (EUPL 1.0)': 'EUPL-1.0', - 'License :: OSI Approved :: European Union Public Licence 1.1 (EUPL 1.1)': 'EUPL-1.1', - 'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)': 'AGPL-3.0+', - 'License :: OSI Approved :: GNU Affero General Public License v3': 'AGPL-3.0', - 'License :: OSI Approved :: GNU Free Documentation License (FDL)': 'GFDL', - 'License :: OSI Approved :: GNU General Public License (GPL)': 'GPL', - 'License :: OSI Approved :: GNU General Public License v2 (GPLv2)': 'GPL-2.0', - 'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)': 'GPL-2.0+', - 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)': 'GPL-3.0', - 'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)': 'GPL-3.0+', - 'License :: OSI Approved :: GNU Lesser General Public License v2 (LGPLv2)': 'LGPL-2.0', - 'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)': 'LGPL-2.0+', - 'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)': 'LGPL-3.0', - 'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)': 'LGPL-3.0+', - 'License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)': 'LGPL', - 'License :: OSI Approved :: IBM Public License': 'IPL', - 'License :: OSI Approved :: ISC License (ISCL)': 'ISC', - 'License :: OSI Approved :: Intel Open Source License': 'Intel', - 'License :: OSI Approved :: Jabber Open Source License': 'Jabber', - 'License :: OSI Approved :: MIT License': 'MIT', - 'License :: OSI Approved :: MITRE Collaborative Virtual Workspace License (CVW)': 'CVWL', - 'License :: OSI Approved :: Motosoto License': 'Motosoto', - 'License :: OSI Approved :: Mozilla Public License 1.0 (MPL)': 'MPL-1.0', - 'License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)': 'MPL-1.1', - 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)': 'MPL-2.0', - 'License :: OSI Approved :: Nethack General Public License': 'NGPL', - 'License :: OSI Approved :: Nokia Open Source License': 'Nokia', - 'License :: OSI Approved :: Open Group Test Suite License': 'OGTSL', - 'License :: OSI Approved :: Python License (CNRI Python License)': 'CNRI-Python', - 'License :: OSI Approved :: Python Software Foundation License': 'PSF', - 'License :: OSI Approved :: Qt Public License (QPL)': 'QPL', - 'License :: OSI Approved :: Ricoh Source Code Public License': 'RSCPL', - 'License :: OSI Approved :: Sleepycat License': 'Sleepycat', - 'License :: OSI Approved :: Sun Industry Standards Source License (SISSL)': '-- Sun Industry Standards Source License (SISSL)', - 'License :: OSI Approved :: Sun Public License': 'SPL', - 'License :: OSI Approved :: University of Illinois/NCSA Open Source License': 'NCSA', - 'License :: OSI Approved :: Vovida Software License 1.0': 'VSL-1.0', - 'License :: OSI Approved :: W3C License': 'W3C', - 'License :: OSI Approved :: X.Net License': 'Xnet', - 'License :: OSI Approved :: Zope Public License': 'ZPL', - 'License :: OSI Approved :: zlib/libpng License': 'Zlib', - } - def __init__(self): pass + def parse_setup_py(self, setupscript='./setup.py'): + with codecs.open(setupscript) as f: + info, imported_modules, non_literals, extensions = gather_setup_info(f) + + def _map(key): + key = key.replace('_', '-') + key = key[0].upper() + key[1:] + if key in self.setup_parse_map: + key = self.setup_parse_map[key] + return key + + # Naive mapping of setup() arguments to PKG-INFO field names + for d in [info, non_literals]: + for key, value in list(d.items()): + if key is None: + continue + new_key = _map(key) + if new_key != key: + del d[key] + d[new_key] = value + + return info, 'setuptools' in imported_modules, non_literals, extensions + + def get_setup_args_info(self, setupscript='./setup.py'): + cmd = ['python3', setupscript] + info = {} + keys = set(self.bbvar_map.keys()) + keys |= set(self.setuparg_list_fields) + keys |= set(self.setuparg_multi_line_values) + grouped_keys = itertools.groupby(keys, lambda k: (k in self.setuparg_list_fields, k in self.setuparg_multi_line_values)) + for index, keys in grouped_keys: + if index == (True, False): + # Splitlines output for each arg as a list value + for key in keys: + arg = self.setuparg_map.get(key, key.lower()) + try: + arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript)) + except (OSError, subprocess.CalledProcessError): + pass + else: + info[key] = [l.rstrip() for l in arg_info.splitlines()] + elif index == (False, True): + # Entire output for each arg + for key in keys: + arg = self.setuparg_map.get(key, key.lower()) + try: + arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript)) + except (OSError, subprocess.CalledProcessError): + pass + else: + info[key] = arg_info + else: + info.update(self.get_setup_byline(list(keys), setupscript)) + return info + + def get_setup_byline(self, fields, setupscript='./setup.py'): + info = {} + + cmd = ['python3', setupscript] + cmd.extend('--' + self.setuparg_map.get(f, f.lower()) for f in fields) + try: + info_lines = self.run_command(cmd, cwd=os.path.dirname(setupscript)).splitlines() + except (OSError, subprocess.CalledProcessError): + pass + else: + if len(fields) != len(info_lines): + logger.error('Mismatch between setup.py output lines and number of fields') + sys.exit(1) + + for lineno, line in enumerate(info_lines): + line = line.rstrip() + info[fields[lineno]] = line + return info + + def get_pkginfo(self, pkginfo_fn): + msg = email.message_from_file(open(pkginfo_fn, 'r')) + msginfo = {} + for field in msg.keys(): + values = msg.get_all(field) + if len(values) == 1: + msginfo[field] = values[0] + else: + msginfo[field] = values + return msginfo + + def scan_setup_python_deps(self, srctree, setup_info, setup_non_literals): + if 'Package-dir' in setup_info: + package_dir = setup_info['Package-dir'] + else: + package_dir = {} + + dist = setuptools.Distribution() + + class PackageDir(setuptools.command.build_py.build_py): + def __init__(self, package_dir): + self.package_dir = package_dir + self.dist = dist + super().__init__(self.dist) + + pd = PackageDir(package_dir) + to_scan = [] + if not any(v in setup_non_literals for v in ['Py-modules', 'Scripts', 'Packages']): + if 'Py-modules' in setup_info: + for module in setup_info['Py-modules']: + try: + package, module = module.rsplit('.', 1) + except ValueError: + package, module = '.', module + module_path = os.path.join(pd.get_package_dir(package), module + '.py') + to_scan.append(module_path) + + if 'Packages' in setup_info: + for package in setup_info['Packages']: + to_scan.append(pd.get_package_dir(package)) + + if 'Scripts' in setup_info: + to_scan.extend(setup_info['Scripts']) + else: + logger.info("Scanning the entire source tree, as one or more of the following setup keywords are non-literal: py_modules, scripts, packages.") + + if not to_scan: + to_scan = ['.'] + + logger.info("Scanning paths for packages & dependencies: %s", ', '.join(to_scan)) + + provided_packages = self.parse_pkgdata_for_python_packages() + scanned_deps = self.scan_python_dependencies([os.path.join(srctree, p) for p in to_scan]) + mapped_deps, unmapped_deps = set(self.base_pkgdeps), set() + for dep in scanned_deps: + mapped = provided_packages.get(dep) + if mapped: + logger.debug('Mapped %s to %s' % (dep, mapped)) + mapped_deps.add(mapped) + else: + logger.debug('Could not map %s' % dep) + unmapped_deps.add(dep) + return mapped_deps, unmapped_deps + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): + if 'buildsystem' in handled: return False + logger.debug("Trying setup.py parser") + # Check for non-zero size setup.py files setupfiles = RecipeHandler.checkfiles(srctree, ['setup.py']) for fn in setupfiles: if os.path.getsize(fn): break else: + logger.debug("No setup.py found") return False # setup.py is always parsed to get at certain required information, such as @@ -193,6 +615,18 @@ class PythonRecipeHandler(RecipeHandler): continue if line.startswith('['): + # PACKAGECONFIG must not contain expressions or whitespace + line = line.replace(" ", "") + line = line.replace(':', "") + line = line.replace('.', "-dot-") + line = line.replace('"', "") + line = line.replace('<', "-smaller-") + line = line.replace('>', "-bigger-") + line = line.replace('_', "-") + line = line.replace('(', "") + line = line.replace(')', "") + line = line.replace('!', "-not-") + line = line.replace('=', "-equals-") current_feature = line[1:-1] elif current_feature: extras_req[current_feature].append(line) @@ -226,51 +660,16 @@ class PythonRecipeHandler(RecipeHandler): if license_str: for i, line in enumerate(lines_before): - if line.startswith('LICENSE = '): + if line.startswith('##LICENSE_PLACEHOLDER##'): lines_before.insert(i, '# NOTE: License in setup.py/PKGINFO is: %s' % license_str) break if 'Classifier' in info: - existing_licenses = info.get('License', '') - licenses = [] - for classifier in info['Classifier']: - if classifier in self.classifier_license_map: - license = self.classifier_license_map[classifier] - if license == 'Apache' and 'Apache-2.0' in existing_licenses: - license = 'Apache-2.0' - elif license == 'GPL': - if 'GPL-2.0' in existing_licenses or 'GPLv2' in existing_licenses: - license = 'GPL-2.0' - elif 'GPL-3.0' in existing_licenses or 'GPLv3' in existing_licenses: - license = 'GPL-3.0' - elif license == 'LGPL': - if 'LGPL-2.1' in existing_licenses or 'LGPLv2.1' in existing_licenses: - license = 'LGPL-2.1' - elif 'LGPL-2.0' in existing_licenses or 'LGPLv2' in existing_licenses: - license = 'LGPL-2.0' - elif 'LGPL-3.0' in existing_licenses or 'LGPLv3' in existing_licenses: - license = 'LGPL-3.0' - licenses.append(license) - - if licenses: - info['License'] = ' & '.join(licenses) + license = self.handle_classifier_license(info['Classifier'], info.get('License', '')) + if license: + info['License'] = license - # Map PKG-INFO & setup.py fields to bitbake variables - for field, values in info.items(): - if field in self.excluded_fields: - continue - - if field not in self.bbvar_map: - continue - - if isinstance(values, str): - value = values - else: - value = ' '.join(str(v) for v in values if v) - - bbvar = self.bbvar_map[field] - if bbvar not in extravalues and value: - extravalues[bbvar] = value + self.map_info_to_bbvar(info, extravalues) mapped_deps, unmapped_deps = self.scan_setup_python_deps(srctree, setup_info, setup_non_literals) @@ -281,6 +680,7 @@ class PythonRecipeHandler(RecipeHandler): lines_after.append('# The following configs & dependencies are from setuptools extras_require.') lines_after.append('# These dependencies are optional, hence can be controlled via PACKAGECONFIG.') lines_after.append('# The upstream names may not correspond exactly to bitbake package names.') + lines_after.append('# The configs are might not correct, since PACKAGECONFIG does not support expressions as may used in requires.txt - they are just replaced by text.') lines_after.append('#') lines_after.append('# Uncomment this line to enable all the optional features.') lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req))) @@ -301,7 +701,7 @@ class PythonRecipeHandler(RecipeHandler): inst_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(inst_reqs)) lines_after.append('# WARNING: the following rdepends are from setuptools install_requires. These') lines_after.append('# upstream names may not correspond exactly to bitbake package names.') - lines_after.append('RDEPENDS_${{PN}} += "{}"'.format(' '.join(inst_req_deps))) + lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(inst_req_deps))) if mapped_deps: name = info.get('Name') @@ -313,7 +713,7 @@ class PythonRecipeHandler(RecipeHandler): lines_after.append('') lines_after.append('# WARNING: the following rdepends are determined through basic analysis of the') lines_after.append('# python sources, and might not be 100% accurate.') - lines_after.append('RDEPENDS_${{PN}} += "{}"'.format(' '.join(sorted(mapped_deps)))) + lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(sorted(mapped_deps)))) unmapped_deps -= set(extensions) unmapped_deps -= set(self.assume_provided) @@ -326,275 +726,283 @@ class PythonRecipeHandler(RecipeHandler): handled.append('buildsystem') - def get_pkginfo(self, pkginfo_fn): - msg = email.message_from_file(open(pkginfo_fn, 'r')) - msginfo = {} - for field in msg.keys(): - values = msg.get_all(field) - if len(values) == 1: - msginfo[field] = values[0] - else: - msginfo[field] = values - return msginfo +class PythonPyprojectTomlRecipeHandler(PythonRecipeHandler): + """Base class to support PEP517 and PEP518 + + PEP517 https://peps.python.org/pep-0517/#source-trees + PEP518 https://peps.python.org/pep-0518/#build-system-table + """ + # bitbake currently supports the 4 following backends + build_backend_map = { + "setuptools.build_meta": "python_setuptools_build_meta", + "poetry.core.masonry.api": "python_poetry_core", + "flit_core.buildapi": "python_flit_core", + "hatchling.build": "python_hatchling", + "maturin": "python_maturin", + "mesonpy": "python_mesonpy", + } - def parse_setup_py(self, setupscript='./setup.py'): - with codecs.open(setupscript) as f: - info, imported_modules, non_literals, extensions = gather_setup_info(f) + # setuptools.build_meta and flit declare project metadata into the "project" section of pyproject.toml + # according to PEP-621: https://packaging.python.org/en/latest/specifications/declaring-project-metadata/#declaring-project-metadata + # while poetry uses the "tool.poetry" section according to its official documentation: https://python-poetry.org/docs/pyproject/ + # keys from "project" and "tool.poetry" sections are almost the same except for the HOMEPAGE which is "homepage" for tool.poetry + # and "Homepage" for "project" section. So keep both + bbvar_map = { + "name": "PN", + "version": "PV", + "Homepage": "HOMEPAGE", + "homepage": "HOMEPAGE", + "description": "SUMMARY", + "license": "LICENSE", + "dependencies": "RDEPENDS:${PN}", + "requires": "DEPENDS", + } - def _map(key): - key = key.replace('_', '-') - key = key[0].upper() + key[1:] - if key in self.setup_parse_map: - key = self.setup_parse_map[key] - return key + replacements = [ + ("license", r" +$", ""), + ("license", r"^ +", ""), + ("license", r" ", "-"), + ("license", r"^GNU-", ""), + ("license", r"-[Ll]icen[cs]e(,?-[Vv]ersion)?", ""), + ("license", r"^UNKNOWN$", ""), + # Remove currently unhandled version numbers from these variables + ("requires", r"\[[^\]]+\]$", ""), + ("requires", r"^([^><= ]+).*", r"\1"), + ("dependencies", r"\[[^\]]+\]$", ""), + ("dependencies", r"^([^><= ]+).*", r"\1"), + ] - # Naive mapping of setup() arguments to PKG-INFO field names - for d in [info, non_literals]: - for key, value in list(d.items()): - if key is None: - continue - new_key = _map(key) - if new_key != key: - del d[key] - d[new_key] = value + excluded_native_pkgdeps = [ + # already provided by python_setuptools_build_meta.bbclass + "python3-setuptools-native", + "python3-wheel-native", + # already provided by python_poetry_core.bbclass + "python3-poetry-core-native", + # already provided by python_flit_core.bbclass + "python3-flit-core-native", + # already provided by python_mesonpy + "python3-meson-python-native", + ] - return info, 'setuptools' in imported_modules, non_literals, extensions + # add here a list of known and often used packages and the corresponding bitbake package + known_deps_map = { + "setuptools": "python3-setuptools", + "wheel": "python3-wheel", + "poetry-core": "python3-poetry-core", + "flit_core": "python3-flit-core", + "setuptools-scm": "python3-setuptools-scm", + "hatchling": "python3-hatchling", + "hatch-vcs": "python3-hatch-vcs", + "meson-python" : "python3-meson-python", + } - def get_setup_args_info(self, setupscript='./setup.py'): - cmd = ['python3', setupscript] - info = {} - keys = set(self.bbvar_map.keys()) - keys |= set(self.setuparg_list_fields) - keys |= set(self.setuparg_multi_line_values) - grouped_keys = itertools.groupby(keys, lambda k: (k in self.setuparg_list_fields, k in self.setuparg_multi_line_values)) - for index, keys in grouped_keys: - if index == (True, False): - # Splitlines output for each arg as a list value - for key in keys: - arg = self.setuparg_map.get(key, key.lower()) - try: - arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript)) - except (OSError, subprocess.CalledProcessError): - pass - else: - info[key] = [l.rstrip() for l in arg_info.splitlines()] - elif index == (False, True): - # Entire output for each arg - for key in keys: - arg = self.setuparg_map.get(key, key.lower()) - try: - arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript)) - except (OSError, subprocess.CalledProcessError): - pass - else: - info[key] = arg_info - else: - info.update(self.get_setup_byline(list(keys), setupscript)) - return info + def __init__(self): + pass - def get_setup_byline(self, fields, setupscript='./setup.py'): + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): info = {} + metadata = {} - cmd = ['python3', setupscript] - cmd.extend('--' + self.setuparg_map.get(f, f.lower()) for f in fields) - try: - info_lines = self.run_command(cmd, cwd=os.path.dirname(setupscript)).splitlines() - except (OSError, subprocess.CalledProcessError): - pass - else: - if len(fields) != len(info_lines): - logger.error('Mismatch between setup.py output lines and number of fields') - sys.exit(1) - - for lineno, line in enumerate(info_lines): - line = line.rstrip() - info[fields[lineno]] = line - return info - - def apply_info_replacements(self, info): - for variable, search, replace in self.replacements: - if variable not in info: - continue - - def replace_value(search, replace, value): - if replace is None: - if re.search(search, value): - return None - else: - new_value = re.sub(search, replace, value) - if value != new_value: - return new_value - return value - - value = info[variable] - if isinstance(value, str): - new_value = replace_value(search, replace, value) - if new_value is None: - del info[variable] - elif new_value != value: - info[variable] = new_value - elif hasattr(value, 'items'): - for dkey, dvalue in list(value.items()): - new_list = [] - for pos, a_value in enumerate(dvalue): - new_value = replace_value(search, replace, a_value) - if new_value is not None and new_value != value: - new_list.append(new_value) - - if value != new_list: - value[dkey] = new_list - else: - new_list = [] - for pos, a_value in enumerate(value): - new_value = replace_value(search, replace, a_value) - if new_value is not None and new_value != value: - new_list.append(new_value) - - if value != new_list: - info[variable] = new_list - - def scan_setup_python_deps(self, srctree, setup_info, setup_non_literals): - if 'Package-dir' in setup_info: - package_dir = setup_info['Package-dir'] - else: - package_dir = {} - - class PackageDir(distutils.command.build_py.build_py): - def __init__(self, package_dir): - self.package_dir = package_dir - - pd = PackageDir(package_dir) - to_scan = [] - if not any(v in setup_non_literals for v in ['Py-modules', 'Scripts', 'Packages']): - if 'Py-modules' in setup_info: - for module in setup_info['Py-modules']: - try: - package, module = module.rsplit('.', 1) - except ValueError: - package, module = '.', module - module_path = os.path.join(pd.get_package_dir(package), module + '.py') - to_scan.append(module_path) + if 'buildsystem' in handled: + return False - if 'Packages' in setup_info: - for package in setup_info['Packages']: - to_scan.append(pd.get_package_dir(package)) + logger.debug("Trying pyproject.toml parser") - if 'Scripts' in setup_info: - to_scan.extend(setup_info['Scripts']) + # Check for non-zero size setup.py files + setupfiles = RecipeHandler.checkfiles(srctree, ["pyproject.toml"]) + for fn in setupfiles: + if os.path.getsize(fn): + break else: - logger.info("Scanning the entire source tree, as one or more of the following setup keywords are non-literal: py_modules, scripts, packages.") - - if not to_scan: - to_scan = ['.'] - - logger.info("Scanning paths for packages & dependencies: %s", ', '.join(to_scan)) + logger.debug("No pyproject.toml found") + return False - provided_packages = self.parse_pkgdata_for_python_packages() - scanned_deps = self.scan_python_dependencies([os.path.join(srctree, p) for p in to_scan]) - mapped_deps, unmapped_deps = set(self.base_pkgdeps), set() - for dep in scanned_deps: - mapped = provided_packages.get(dep) - if mapped: - logger.debug('Mapped %s to %s' % (dep, mapped)) - mapped_deps.add(mapped) - else: - logger.debug('Could not map %s' % dep) - unmapped_deps.add(dep) - return mapped_deps, unmapped_deps + setupscript = os.path.join(srctree, "pyproject.toml") - def scan_python_dependencies(self, paths): - deps = set() try: - dep_output = self.run_command(['pythondeps', '-d'] + paths) - except (OSError, subprocess.CalledProcessError): - pass - else: - for line in dep_output.splitlines(): - line = line.rstrip() - dep, filename = line.split('\t', 1) - if filename.endswith('/setup.py'): - continue - deps.add(dep) + try: + import tomllib + except ImportError: + try: + import tomli as tomllib + except ImportError: + logger.error("Neither 'tomllib' nor 'tomli' could be imported, cannot scan pyproject.toml.") + return False + + try: + with open(setupscript, "rb") as f: + config = tomllib.load(f) + except Exception: + logger.exception("Failed to parse pyproject.toml") + return False + + build_backend = config["build-system"]["build-backend"] + if build_backend in self.build_backend_map: + classes.append(self.build_backend_map[build_backend]) + else: + logger.error( + "Unsupported build-backend: %s, cannot use pyproject.toml. Will try to use legacy setup.py" + % build_backend + ) + return False - try: - provides_output = self.run_command(['pythondeps', '-p'] + paths) - except (OSError, subprocess.CalledProcessError): - pass - else: - provides_lines = (l.rstrip() for l in provides_output.splitlines()) - provides = set(l for l in provides_lines if l and l != 'setup') - deps -= provides + licfile = "" - return deps + if build_backend == "poetry.core.masonry.api": + if "tool" in config and "poetry" in config["tool"]: + metadata = config["tool"]["poetry"] + else: + if "project" in config: + metadata = config["project"] + + if metadata: + for field, values in metadata.items(): + if field == "license": + # For setuptools.build_meta and flit, licence is a table + # but for poetry licence is a string + # for hatchling, both table (jsonschema) and string (iniconfig) have been used + if build_backend == "poetry.core.masonry.api": + value = values + else: + value = values.get("text", "") + if not value: + licfile = values.get("file", "") + continue + elif field == "dependencies" and build_backend == "poetry.core.masonry.api": + # For poetry backend, "dependencies" section looks like: + # [tool.poetry.dependencies] + # requests = "^2.13.0" + # requests = { version = "^2.13.0", source = "private" } + # See https://python-poetry.org/docs/master/pyproject/#dependencies-and-dependency-groups for more details + # This class doesn't handle versions anyway, so we just get the dependencies name here and construct a list + value = [] + for k in values.keys(): + value.append(k) + elif isinstance(values, dict): + for k, v in values.items(): + info[k] = v + continue + else: + value = values - def parse_pkgdata_for_python_packages(self): - suffixes = [t[0] for t in imp.get_suffixes()] - pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR') + info[field] = value - ldata = tinfoil.config_data.createCopy() - bb.parse.handle('classes/python3-dir.bbclass', ldata, True) - python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR') + # Grab the license value before applying replacements + license_str = info.get("license", "").strip() - dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload') - python_dirs = [python_sitedir + os.sep, - os.path.join(os.path.dirname(python_sitedir), 'dist-packages') + os.sep, - os.path.dirname(python_sitedir) + os.sep] - packages = {} - for pkgdatafile in glob.glob('{}/runtime/*'.format(pkgdata_dir)): - files_info = None - with open(pkgdatafile, 'r') as f: - for line in f.readlines(): - field, value = line.split(': ', 1) - if field == 'FILES_INFO': - files_info = ast.literal_eval(value) + if license_str: + for i, line in enumerate(lines_before): + if line.startswith("##LICENSE_PLACEHOLDER##"): + lines_before.insert( + i, "# NOTE: License in pyproject.toml is: %s" % license_str + ) break - else: - continue - for fn in files_info: - for suffix in suffixes: - if fn.endswith(suffix): - break - else: - continue + info["requires"] = config["build-system"]["requires"] + + self.apply_info_replacements(info) + + if "classifiers" in info: + license = self.handle_classifier_license( + info["classifiers"], info.get("license", "") + ) + if license: + if licfile: + lines = [] + md5value = bb.utils.md5_file(os.path.join(srctree, licfile)) + lines.append('LICENSE = "%s"' % license) + lines.append( + 'LIC_FILES_CHKSUM = "file://%s;md5=%s"' + % (licfile, md5value) + ) + lines.append("") + + # Replace the placeholder so we get the values in the right place in the recipe file + try: + pos = lines_before.index("##LICENSE_PLACEHOLDER##") + except ValueError: + pos = -1 + if pos == -1: + lines_before.extend(lines) + else: + lines_before[pos : pos + 1] = lines - if fn.startswith(dynload_dir + os.sep): - if '/.debug/' in fn: - continue - base = os.path.basename(fn) - provided = base.split('.', 1)[0] - packages[provided] = os.path.basename(pkgdatafile) - continue + handled.append(("license", [license, licfile, md5value])) + else: + info["license"] = license - for python_dir in python_dirs: - if fn.startswith(python_dir): - relpath = fn[len(python_dir):] - relstart, _, relremaining = relpath.partition(os.sep) - if relstart.endswith('.egg'): - relpath = relremaining - base, _ = os.path.splitext(relpath) + provided_packages = self.parse_pkgdata_for_python_packages() + provided_packages.update(self.known_deps_map) + native_mapped_deps, native_unmapped_deps = set(), set() + mapped_deps, unmapped_deps = set(), set() - if '/.debug/' in base: - continue - if os.path.basename(base) == '__init__': - base = os.path.dirname(base) - base = base.replace(os.sep + os.sep, os.sep) - provided = base.replace(os.sep, '.') - packages[provided] = os.path.basename(pkgdatafile) - return packages + if "requires" in info: + for require in info["requires"]: + mapped = provided_packages.get(require) - @classmethod - def run_command(cls, cmd, **popenargs): - if 'stderr' not in popenargs: - popenargs['stderr'] = subprocess.STDOUT - try: - return subprocess.check_output(cmd, **popenargs).decode('utf-8') - except OSError as exc: - logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc) - raise - except subprocess.CalledProcessError as exc: - logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc.output) - raise + if mapped: + logger.debug("Mapped %s to %s" % (require, mapped)) + native_mapped_deps.add(mapped) + else: + logger.debug("Could not map %s" % require) + native_unmapped_deps.add(require) + + info.pop("requires") + + if native_mapped_deps != set(): + native_mapped_deps = { + item + "-native" for item in native_mapped_deps + } + native_mapped_deps -= set(self.excluded_native_pkgdeps) + if native_mapped_deps != set(): + info["requires"] = " ".join(sorted(native_mapped_deps)) + + if native_unmapped_deps: + lines_after.append("") + lines_after.append( + "# WARNING: We were unable to map the following python package/module" + ) + lines_after.append( + "# dependencies to the bitbake packages which include them:" + ) + lines_after.extend( + "# {}".format(d) for d in sorted(native_unmapped_deps) + ) + + if "dependencies" in info: + for dependency in info["dependencies"]: + mapped = provided_packages.get(dependency) + if mapped: + logger.debug("Mapped %s to %s" % (dependency, mapped)) + mapped_deps.add(mapped) + else: + logger.debug("Could not map %s" % dependency) + unmapped_deps.add(dependency) + + info.pop("dependencies") + + if mapped_deps != set(): + if mapped_deps != set(): + info["dependencies"] = " ".join(sorted(mapped_deps)) + + if unmapped_deps: + lines_after.append("") + lines_after.append( + "# WARNING: We were unable to map the following python package/module" + ) + lines_after.append( + "# runtime dependencies to the bitbake packages which include them:" + ) + lines_after.extend( + "# {}".format(d) for d in sorted(unmapped_deps) + ) + + self.map_info_to_bbvar(info, extravalues) + + handled.append("buildsystem") + except Exception: + logger.exception("Failed to correctly handle pyproject.toml, falling back to another method") + return False def gather_setup_info(fileobj): @@ -710,5 +1118,7 @@ def has_non_literals(value): def register_recipe_handlers(handlers): - # We need to make sure this is ahead of the makefile fallback handler - handlers.append((PythonRecipeHandler(), 70)) + # We need to make sure these are ahead of the makefile fallback handler + # and the pyproject.toml handler ahead of the setup.py handler + handlers.append((PythonPyprojectTomlRecipeHandler(), 75)) + handlers.append((PythonSetupPyRecipeHandler(), 70)) diff --git a/scripts/lib/recipetool/create_go.py b/scripts/lib/recipetool/create_go.py new file mode 100644 index 0000000000..a85a2f2786 --- /dev/null +++ b/scripts/lib/recipetool/create_go.py @@ -0,0 +1,777 @@ +# Recipe creation tool - go support plugin +# +# The code is based on golang internals. See the afftected +# methods for further reference and information. +# +# Copyright (C) 2023 Weidmueller GmbH & Co KG +# Author: Lukas Funke <lukas.funke@weidmueller.com> +# +# SPDX-License-Identifier: GPL-2.0-only +# + + +from collections import namedtuple +from enum import Enum +from html.parser import HTMLParser +from recipetool.create import RecipeHandler, handle_license_vars +from recipetool.create import guess_license, tidy_licenses, fixup_license +from recipetool.create import determine_from_url +from urllib.error import URLError, HTTPError + +import bb.utils +import json +import logging +import os +import re +import subprocess +import sys +import shutil +import tempfile +import urllib.parse +import urllib.request + + +GoImport = namedtuple('GoImport', 'root vcs url suffix') +logger = logging.getLogger('recipetool') +CodeRepo = namedtuple( + 'CodeRepo', 'path codeRoot codeDir pathMajor pathPrefix pseudoMajor') + +tinfoil = None + +# Regular expression to parse pseudo semantic version +# see https://go.dev/ref/mod#pseudo-versions +re_pseudo_semver = re.compile( + r"^v[0-9]+\.(0\.0-|\d+\.\d+-([^+]*\.)?0\.)(?P<utc>\d{14})-(?P<commithash>[A-Za-z0-9]+)(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$") +# Regular expression to parse semantic version +re_semver = re.compile( + r"^v(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$") + + +def tinfoil_init(instance): + global tinfoil + tinfoil = instance + + +class GoRecipeHandler(RecipeHandler): + """Class to handle the go recipe creation""" + + @staticmethod + def __ensure_go(): + """Check if the 'go' command is available in the recipes""" + recipe = "go-native" + if not tinfoil.recipes_parsed: + tinfoil.parse_recipes() + try: + rd = tinfoil.parse_recipe(recipe) + except bb.providers.NoProvider: + bb.error( + "Nothing provides '%s' which is required for the build" % (recipe)) + bb.note( + "You will likely need to add a layer that provides '%s'" % (recipe)) + return None + + bindir = rd.getVar('STAGING_BINDIR_NATIVE') + gopath = os.path.join(bindir, 'go') + + if not os.path.exists(gopath): + tinfoil.build_targets(recipe, 'addto_recipe_sysroot') + + if not os.path.exists(gopath): + logger.error( + '%s required to process specified source, but %s did not seem to populate it' % 'go', recipe) + return None + + return bindir + + def __resolve_repository_static(self, modulepath): + """Resolve the repository in a static manner + + The method is based on the go implementation of + `repoRootFromVCSPaths` in + https://github.com/golang/go/blob/master/src/cmd/go/internal/vcs/vcs.go + """ + + url = urllib.parse.urlparse("https://" + modulepath) + req = urllib.request.Request(url.geturl()) + + try: + resp = urllib.request.urlopen(req) + # Some modulepath are just redirects to github (or some other vcs + # hoster). Therefore, we check if this modulepath redirects to + # somewhere else + if resp.geturl() != url.geturl(): + bb.debug(1, "%s is redirectred to %s" % + (url.geturl(), resp.geturl())) + url = urllib.parse.urlparse(resp.geturl()) + modulepath = url.netloc + url.path + + except URLError as url_err: + # This is probably because the module path + # contains the subdir and major path. Thus, + # we ignore this error for now + logger.debug( + 1, "Failed to fetch page from [%s]: %s" % (url, str(url_err))) + + host, _, _ = modulepath.partition('/') + + class vcs(Enum): + pathprefix = "pathprefix" + regexp = "regexp" + type = "type" + repo = "repo" + check = "check" + schemelessRepo = "schemelessRepo" + + # GitHub + vcsGitHub = {} + vcsGitHub[vcs.pathprefix] = "github.com" + vcsGitHub[vcs.regexp] = re.compile( + r'^(?P<root>github\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$') + vcsGitHub[vcs.type] = "git" + vcsGitHub[vcs.repo] = "https://\\g<root>" + + # Bitbucket + vcsBitbucket = {} + vcsBitbucket[vcs.pathprefix] = "bitbucket.org" + vcsBitbucket[vcs.regexp] = re.compile( + r'^(?P<root>bitbucket\.org/(?P<bitname>[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/(?P<suffix>[A-Za-z0-9_.\-]+))*$') + vcsBitbucket[vcs.type] = "git" + vcsBitbucket[vcs.repo] = "https://\\g<root>" + + # IBM DevOps Services (JazzHub) + vcsIBMDevOps = {} + vcsIBMDevOps[vcs.pathprefix] = "hub.jazz.net/git" + vcsIBMDevOps[vcs.regexp] = re.compile( + r'^(?P<root>hub\.jazz\.net/git/[a-z0-9]+/[A-Za-z0-9_.\-]+)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$') + vcsIBMDevOps[vcs.type] = "git" + vcsIBMDevOps[vcs.repo] = "https://\\g<root>" + + # Git at Apache + vcsApacheGit = {} + vcsApacheGit[vcs.pathprefix] = "git.apache.org" + vcsApacheGit[vcs.regexp] = re.compile( + r'^(?P<root>git\.apache\.org/[a-z0-9_.\-]+\.git)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$') + vcsApacheGit[vcs.type] = "git" + vcsApacheGit[vcs.repo] = "https://\\g<root>" + + # Git at OpenStack + vcsOpenStackGit = {} + vcsOpenStackGit[vcs.pathprefix] = "git.openstack.org" + vcsOpenStackGit[vcs.regexp] = re.compile( + r'^(?P<root>git\.openstack\.org/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(\.git)?(/(?P<suffix>[A-Za-z0-9_.\-]+))*$') + vcsOpenStackGit[vcs.type] = "git" + vcsOpenStackGit[vcs.repo] = "https://\\g<root>" + + # chiselapp.com for fossil + vcsChiselapp = {} + vcsChiselapp[vcs.pathprefix] = "chiselapp.com" + vcsChiselapp[vcs.regexp] = re.compile( + r'^(?P<root>chiselapp\.com/user/[A-Za-z0-9]+/repository/[A-Za-z0-9_.\-]+)$') + vcsChiselapp[vcs.type] = "fossil" + vcsChiselapp[vcs.repo] = "https://\\g<root>" + + # General syntax for any server. + # Must be last. + vcsGeneralServer = {} + vcsGeneralServer[vcs.regexp] = re.compile( + "(?P<root>(?P<repo>([a-z0-9.\\-]+\\.)+[a-z0-9.\\-]+(:[0-9]+)?(/~?[A-Za-z0-9_.\\-]+)+?)\\.(?P<vcs>bzr|fossil|git|hg|svn))(/~?(?P<suffix>[A-Za-z0-9_.\\-]+))*$") + vcsGeneralServer[vcs.schemelessRepo] = True + + vcsPaths = [vcsGitHub, vcsBitbucket, vcsIBMDevOps, + vcsApacheGit, vcsOpenStackGit, vcsChiselapp, + vcsGeneralServer] + + if modulepath.startswith("example.net") or modulepath == "rsc.io": + logger.warning("Suspicious module path %s" % modulepath) + return None + if modulepath.startswith("http:") or modulepath.startswith("https:"): + logger.warning("Import path should not start with %s %s" % + ("http", "https")) + return None + + rootpath = None + vcstype = None + repourl = None + suffix = None + + for srv in vcsPaths: + m = srv[vcs.regexp].match(modulepath) + if vcs.pathprefix in srv: + if host == srv[vcs.pathprefix]: + rootpath = m.group('root') + vcstype = srv[vcs.type] + repourl = m.expand(srv[vcs.repo]) + suffix = m.group('suffix') + break + elif m and srv[vcs.schemelessRepo]: + rootpath = m.group('root') + vcstype = m[vcs.type] + repourl = m[vcs.repo] + suffix = m.group('suffix') + break + + return GoImport(rootpath, vcstype, repourl, suffix) + + def __resolve_repository_dynamic(self, modulepath): + """Resolve the repository root in a dynamic manner. + + The method is based on the go implementation of + `repoRootForImportDynamic` in + https://github.com/golang/go/blob/master/src/cmd/go/internal/vcs/vcs.go + """ + url = urllib.parse.urlparse("https://" + modulepath) + + class GoImportHTMLParser(HTMLParser): + + def __init__(self): + super().__init__() + self.__srv = {} + + def handle_starttag(self, tag, attrs): + if tag == 'meta' and list( + filter(lambda a: (a[0] == 'name' and a[1] == 'go-import'), attrs)): + content = list( + filter(lambda a: (a[0] == 'content'), attrs)) + if content: + srv = content[0][1].split() + self.__srv[srv[0]] = srv + + def go_import(self, modulepath): + if modulepath in self.__srv: + srv = self.__srv[modulepath] + return GoImport(srv[0], srv[1], srv[2], None) + return None + + url = url.geturl() + "?go-get=1" + req = urllib.request.Request(url) + + try: + body = urllib.request.urlopen(req).read() + except HTTPError as http_err: + logger.warning( + "Unclean status when fetching page from [%s]: %s", url, str(http_err)) + body = http_err.fp.read() + except URLError as url_err: + logger.warning( + "Failed to fetch page from [%s]: %s", url, str(url_err)) + return None + + parser = GoImportHTMLParser() + parser.feed(body.decode('utf-8')) + parser.close() + + return parser.go_import(modulepath) + + def __resolve_from_golang_proxy(self, modulepath, version): + """ + Resolves repository data from golang proxy + """ + url = urllib.parse.urlparse("https://proxy.golang.org/" + + modulepath + + "/@v/" + + version + + ".info") + + # Transform url to lower case, golang proxy doesn't like mixed case + req = urllib.request.Request(url.geturl().lower()) + + try: + resp = urllib.request.urlopen(req) + except URLError as url_err: + logger.warning( + "Failed to fetch page from [%s]: %s", url, str(url_err)) + return None + + golang_proxy_res = resp.read().decode('utf-8') + modinfo = json.loads(golang_proxy_res) + + if modinfo and 'Origin' in modinfo: + origin = modinfo['Origin'] + _root_url = urllib.parse.urlparse(origin['URL']) + + # We normalize the repo URL since we don't want the scheme in it + _subdir = origin['Subdir'] if 'Subdir' in origin else None + _root, _, _ = self.__split_path_version(modulepath) + if _subdir: + _root = _root[:-len(_subdir)].strip('/') + + _commit = origin['Hash'] + _vcs = origin['VCS'] + return (GoImport(_root, _vcs, _root_url.geturl(), None), _commit) + + return None + + def __resolve_repository(self, modulepath): + """ + Resolves src uri from go module-path + """ + repodata = self.__resolve_repository_static(modulepath) + if not repodata or not repodata.url: + repodata = self.__resolve_repository_dynamic(modulepath) + if not repodata or not repodata.url: + logger.error( + "Could not resolve repository for module path '%s'" % modulepath) + # There is no way to recover from this + sys.exit(14) + if repodata: + logger.debug(1, "Resolved download path for import '%s' => %s" % ( + modulepath, repodata.url)) + return repodata + + def __split_path_version(self, path): + i = len(path) + dot = False + for j in range(i, 0, -1): + if path[j - 1] < '0' or path[j - 1] > '9': + break + if path[j - 1] == '.': + dot = True + break + i = j - 1 + + if i <= 1 or i == len( + path) or path[i - 1] != 'v' or path[i - 2] != '/': + return path, "", True + + prefix, pathMajor = path[:i - 2], path[i - 2:] + if dot or len( + pathMajor) <= 2 or pathMajor[2] == '0' or pathMajor == "/v1": + return path, "", False + + return prefix, pathMajor, True + + def __get_path_major(self, pathMajor): + if not pathMajor: + return "" + + if pathMajor[0] != '/' and pathMajor[0] != '.': + logger.error( + "pathMajor suffix %s passed to PathMajorPrefix lacks separator", pathMajor) + + if pathMajor.startswith(".v") and pathMajor.endswith("-unstable"): + pathMajor = pathMajor[:len("-unstable") - 2] + + return pathMajor[1:] + + def __build_coderepo(self, repo, path): + codedir = "" + pathprefix, pathMajor, _ = self.__split_path_version(path) + if repo.root == path: + pathprefix = path + elif path.startswith(repo.root): + codedir = pathprefix[len(repo.root):].strip('/') + + pseudoMajor = self.__get_path_major(pathMajor) + + logger.debug("root='%s', codedir='%s', prefix='%s', pathMajor='%s', pseudoMajor='%s'", + repo.root, codedir, pathprefix, pathMajor, pseudoMajor) + + return CodeRepo(path, repo.root, codedir, + pathMajor, pathprefix, pseudoMajor) + + def __resolve_version(self, repo, path, version): + hash = None + coderoot = self.__build_coderepo(repo, path) + + def vcs_fetch_all(): + tmpdir = tempfile.mkdtemp() + clone_cmd = "%s clone --bare %s %s" % ('git', repo.url, tmpdir) + bb.process.run(clone_cmd) + log_cmd = "git log --all --pretty='%H %d' --decorate=short" + output, _ = bb.process.run( + log_cmd, shell=True, stderr=subprocess.PIPE, cwd=tmpdir) + bb.utils.prunedir(tmpdir) + return output.strip().split('\n') + + def vcs_fetch_remote(tag): + # add * to grab ^{} + refs = {} + ls_remote_cmd = "git ls-remote -q --tags {} {}*".format( + repo.url, tag) + output, _ = bb.process.run(ls_remote_cmd) + output = output.strip().split('\n') + for line in output: + f = line.split(maxsplit=1) + if len(f) != 2: + continue + + for prefix in ["HEAD", "refs/heads/", "refs/tags/"]: + if f[1].startswith(prefix): + refs[f[1][len(prefix):]] = f[0] + + for key, hash in refs.items(): + if key.endswith(r"^{}"): + refs[key.strip(r"^{}")] = hash + + return refs[tag] + + m_pseudo_semver = re_pseudo_semver.match(version) + + if m_pseudo_semver: + remote_refs = vcs_fetch_all() + short_commit = m_pseudo_semver.group('commithash') + for l in remote_refs: + r = l.split(maxsplit=1) + sha1 = r[0] if len(r) else None + if not sha1: + logger.error( + "Ups: could not resolve abbref commit for %s" % short_commit) + + elif sha1.startswith(short_commit): + hash = sha1 + break + else: + m_semver = re_semver.match(version) + if m_semver: + + def get_sha1_remote(re): + rsha1 = None + for line in remote_refs: + # Split lines of the following format: + # 22e90d9b964610628c10f673ca5f85b8c2a2ca9a (tag: sometag) + lineparts = line.split(maxsplit=1) + sha1 = lineparts[0] if len(lineparts) else None + refstring = lineparts[1] if len( + lineparts) == 2 else None + if refstring: + # Normalize tag string and split in case of multiple + # regs e.g. (tag: speech/v1.10.0, tag: orchestration/v1.5.0 ...) + refs = refstring.strip('(), ').split(',') + for ref in refs: + if re.match(ref.strip()): + rsha1 = sha1 + return rsha1 + + semver = "v" + m_semver.group('major') + "."\ + + m_semver.group('minor') + "."\ + + m_semver.group('patch') \ + + (("-" + m_semver.group('prerelease')) + if m_semver.group('prerelease') else "") + + tag = os.path.join( + coderoot.codeDir, semver) if coderoot.codeDir else semver + + # probe tag using 'ls-remote', which is faster than fetching + # complete history + hash = vcs_fetch_remote(tag) + if not hash: + # backup: fetch complete history + remote_refs = vcs_fetch_all() + hash = get_sha1_remote( + re.compile(fr"(tag:|HEAD ->) ({tag})")) + + logger.debug( + "Resolving commit for tag '%s' -> '%s'", tag, hash) + return hash + + def __generate_srcuri_inline_fcn(self, path, version, replaces=None): + """Generate SRC_URI functions for go imports""" + + logger.info("Resolving repository for module %s", path) + # First try to resolve repo and commit from golang proxy + # Most info is already there and we don't have to go through the + # repository or even perform the version resolve magic + golang_proxy_info = self.__resolve_from_golang_proxy(path, version) + if golang_proxy_info: + repo = golang_proxy_info[0] + commit = golang_proxy_info[1] + else: + # Fallback + # Resolve repository by 'hand' + repo = self.__resolve_repository(path) + commit = self.__resolve_version(repo, path, version) + + url = urllib.parse.urlparse(repo.url) + repo_url = url.netloc + url.path + + coderoot = self.__build_coderepo(repo, path) + + inline_fcn = "${@go_src_uri(" + inline_fcn += f"'{repo_url}','{version}'" + if repo_url != path: + inline_fcn += f",path='{path}'" + if coderoot.codeDir: + inline_fcn += f",subdir='{coderoot.codeDir}'" + if repo.vcs != 'git': + inline_fcn += f",vcs='{repo.vcs}'" + if replaces: + inline_fcn += f",replaces='{replaces}'" + if coderoot.pathMajor: + inline_fcn += f",pathmajor='{coderoot.pathMajor}'" + inline_fcn += ")}" + + return inline_fcn, commit + + def __go_handle_dependencies(self, go_mod, srctree, localfilesdir, extravalues, d): + + import re + src_uris = [] + src_revs = [] + + def generate_src_rev(path, version, commithash): + src_rev = f"# {path}@{version} => {commithash}\n" + # Ups...maybe someone manipulated the source repository and the + # version or commit could not be resolved. This is a sign of + # a) the supply chain was manipulated (bad) + # b) the implementation for the version resolving didn't work + # anymore (less bad) + if not commithash: + src_rev += f"#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n" + src_rev += f"#!!! Could not resolve version !!!\n" + src_rev += f"#!!! Possible supply chain attack !!!\n" + src_rev += f"#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n" + src_rev += f"SRCREV_{path.replace('/', '.')} = \"{commithash}\"" + + return src_rev + + # we first go over replacement list, because we are essentialy + # interested only in the replaced path + if go_mod['Replace']: + for replacement in go_mod['Replace']: + oldpath = replacement['Old']['Path'] + path = replacement['New']['Path'] + version = '' + if 'Version' in replacement['New']: + version = replacement['New']['Version'] + + if os.path.exists(os.path.join(srctree, path)): + # the module refers to the local path, remove it from requirement list + # because it's a local module + go_mod['Require'][:] = [v for v in go_mod['Require'] if v.get('Path') != oldpath] + else: + # Replace the path and the version, so we don't iterate replacement list anymore + for require in go_mod['Require']: + if require['Path'] == oldpath: + require.update({'Path': path, 'Version': version}) + break + + for require in go_mod['Require']: + path = require['Path'] + version = require['Version'] + + inline_fcn, commithash = self.__generate_srcuri_inline_fcn( + path, version) + src_uris.append(inline_fcn) + src_revs.append(generate_src_rev(path, version, commithash)) + + # strip version part from module URL /vXX + baseurl = re.sub(r'/v(\d+)$', '', go_mod['Module']['Path']) + pn, _ = determine_from_url(baseurl) + go_mods_basename = "%s-modules.inc" % pn + + go_mods_filename = os.path.join(localfilesdir, go_mods_basename) + with open(go_mods_filename, "w") as f: + # We introduce this indirection to make the tests a little easier + f.write("SRC_URI += \"${GO_DEPENDENCIES_SRC_URI}\"\n") + f.write("GO_DEPENDENCIES_SRC_URI = \"\\\n") + for uri in src_uris: + f.write(" " + uri + " \\\n") + f.write("\"\n\n") + for rev in src_revs: + f.write(rev + "\n") + + extravalues['extrafiles'][go_mods_basename] = go_mods_filename + + def __go_run_cmd(self, cmd, cwd, d): + return bb.process.run(cmd, env=dict(os.environ, PATH=d.getVar('PATH')), + shell=True, cwd=cwd) + + def __go_native_version(self, d): + stdout, _ = self.__go_run_cmd("go version", None, d) + m = re.match(r".*\sgo((\d+).(\d+).(\d+))\s([\w\/]*)", stdout) + major = int(m.group(2)) + minor = int(m.group(3)) + patch = int(m.group(4)) + + return major, minor, patch + + def __go_mod_patch(self, srctree, localfilesdir, extravalues, d): + + patchfilename = "go.mod.patch" + go_native_version_major, go_native_version_minor, _ = self.__go_native_version( + d) + self.__go_run_cmd("go mod tidy -go=%d.%d" % + (go_native_version_major, go_native_version_minor), srctree, d) + stdout, _ = self.__go_run_cmd("go mod edit -json", srctree, d) + + # Create patch in order to upgrade go version + self.__go_run_cmd("git diff go.mod > %s" % (patchfilename), srctree, d) + # Restore original state + self.__go_run_cmd("git checkout HEAD go.mod go.sum", srctree, d) + + go_mod = json.loads(stdout) + tmpfile = os.path.join(localfilesdir, patchfilename) + shutil.move(os.path.join(srctree, patchfilename), tmpfile) + + extravalues['extrafiles'][patchfilename] = tmpfile + + return go_mod, patchfilename + + def __go_mod_vendor(self, go_mod, srctree, localfilesdir, extravalues, d): + # Perform vendoring to retrieve the correct modules.txt + tmp_vendor_dir = tempfile.mkdtemp() + + # -v causes to go to print modules.txt to stderr + _, stderr = self.__go_run_cmd( + "go mod vendor -v -o %s" % (tmp_vendor_dir), srctree, d) + + modules_txt_basename = "modules.txt" + modules_txt_filename = os.path.join(localfilesdir, modules_txt_basename) + with open(modules_txt_filename, "w") as f: + f.write(stderr) + + extravalues['extrafiles'][modules_txt_basename] = modules_txt_filename + + licenses = [] + lic_files_chksum = [] + licvalues = guess_license(tmp_vendor_dir, d) + shutil.rmtree(tmp_vendor_dir) + + if licvalues: + for licvalue in licvalues: + license = licvalue[0] + lics = tidy_licenses(fixup_license(license)) + lics = [lic for lic in lics if lic not in licenses] + if len(lics): + licenses.extend(lics) + lic_files_chksum.append( + 'file://src/${GO_IMPORT}/vendor/%s;md5=%s' % (licvalue[1], licvalue[2])) + + # strip version part from module URL /vXX + baseurl = re.sub(r'/v(\d+)$', '', go_mod['Module']['Path']) + pn, _ = determine_from_url(baseurl) + licenses_basename = "%s-licenses.inc" % pn + + licenses_filename = os.path.join(localfilesdir, licenses_basename) + with open(licenses_filename, "w") as f: + f.write("GO_MOD_LICENSES = \"%s\"\n\n" % + ' & '.join(sorted(licenses, key=str.casefold))) + # We introduce this indirection to make the tests a little easier + f.write("LIC_FILES_CHKSUM += \"${VENDORED_LIC_FILES_CHKSUM}\"\n") + f.write("VENDORED_LIC_FILES_CHKSUM = \"\\\n") + for lic in lic_files_chksum: + f.write(" " + lic + " \\\n") + f.write("\"\n") + + extravalues['extrafiles'][licenses_basename] = licenses_filename + + def process(self, srctree, classes, lines_before, + lines_after, handled, extravalues): + + if 'buildsystem' in handled: + return False + + files = RecipeHandler.checkfiles(srctree, ['go.mod']) + if not files: + return False + + d = bb.data.createCopy(tinfoil.config_data) + go_bindir = self.__ensure_go() + if not go_bindir: + sys.exit(14) + + d.prependVar('PATH', '%s:' % go_bindir) + handled.append('buildsystem') + classes.append("go-vendor") + + stdout, _ = self.__go_run_cmd("go mod edit -json", srctree, d) + + go_mod = json.loads(stdout) + go_import = go_mod['Module']['Path'] + go_version_match = re.match("([0-9]+).([0-9]+)", go_mod['Go']) + go_version_major = int(go_version_match.group(1)) + go_version_minor = int(go_version_match.group(2)) + src_uris = [] + + localfilesdir = tempfile.mkdtemp(prefix='recipetool-go-') + extravalues.setdefault('extrafiles', {}) + + # Use an explicit name determined from the module name because it + # might differ from the actual URL for replaced modules + # strip version part from module URL /vXX + baseurl = re.sub(r'/v(\d+)$', '', go_mod['Module']['Path']) + pn, _ = determine_from_url(baseurl) + + # go.mod files with version < 1.17 may not include all indirect + # dependencies. Thus, we have to upgrade the go version. + if go_version_major == 1 and go_version_minor < 17: + logger.warning( + "go.mod files generated by Go < 1.17 might have incomplete indirect dependencies.") + go_mod, patchfilename = self.__go_mod_patch(srctree, localfilesdir, + extravalues, d) + src_uris.append( + "file://%s;patchdir=src/${GO_IMPORT}" % (patchfilename)) + + # Check whether the module is vendored. If so, we have nothing to do. + # Otherwise we gather all dependencies and add them to the recipe + if not os.path.exists(os.path.join(srctree, "vendor")): + + # Write additional $BPN-modules.inc file + self.__go_mod_vendor(go_mod, srctree, localfilesdir, extravalues, d) + lines_before.append("LICENSE += \" & ${GO_MOD_LICENSES}\"") + lines_before.append("require %s-licenses.inc" % (pn)) + + self.__rewrite_src_uri(lines_before, ["file://modules.txt"]) + + self.__go_handle_dependencies(go_mod, srctree, localfilesdir, extravalues, d) + lines_before.append("require %s-modules.inc" % (pn)) + + # Do generic license handling + handle_license_vars(srctree, lines_before, handled, extravalues, d) + self.__rewrite_lic_uri(lines_before) + + lines_before.append("GO_IMPORT = \"{}\"".format(baseurl)) + lines_before.append("SRCREV_FORMAT = \"${BPN}\"") + + def __update_lines_before(self, updated, newlines, lines_before): + if updated: + del lines_before[:] + for line in newlines: + # Hack to avoid newlines that edit_metadata inserts + if line.endswith('\n'): + line = line[:-1] + lines_before.append(line) + return updated + + def __rewrite_lic_uri(self, lines_before): + + def varfunc(varname, origvalue, op, newlines): + if varname == 'LIC_FILES_CHKSUM': + new_licenses = [] + licenses = origvalue.split('\\') + for license in licenses: + if not license: + logger.warning("No license file was detected for the main module!") + # the license list of the main recipe must be empty + # this can happen for example in case of CLOSED license + # Fall through to complete recipe generation + continue + license = license.strip() + uri, chksum = license.split(';', 1) + url = urllib.parse.urlparse(uri) + new_uri = os.path.join( + url.scheme + "://", "src", "${GO_IMPORT}", url.netloc + url.path) + ";" + chksum + new_licenses.append(new_uri) + + return new_licenses, None, -1, True + return origvalue, None, 0, True + + updated, newlines = bb.utils.edit_metadata( + lines_before, ['LIC_FILES_CHKSUM'], varfunc) + return self.__update_lines_before(updated, newlines, lines_before) + + def __rewrite_src_uri(self, lines_before, additional_uris = []): + + def varfunc(varname, origvalue, op, newlines): + if varname == 'SRC_URI': + src_uri = ["git://${GO_IMPORT};destsuffix=git/src/${GO_IMPORT};nobranch=1;name=${BPN};protocol=https"] + src_uri.extend(additional_uris) + return src_uri, None, -1, True + return origvalue, None, 0, True + + updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc) + return self.__update_lines_before(updated, newlines, lines_before) + + +def register_recipe_handlers(handlers): + handlers.append((GoRecipeHandler(), 60)) diff --git a/scripts/lib/recipetool/create_kmod.py b/scripts/lib/recipetool/create_kmod.py index 85b5c48e53..cc00106961 100644 --- a/scripts/lib/recipetool/create_kmod.py +++ b/scripts/lib/recipetool/create_kmod.py @@ -113,7 +113,7 @@ class KernelModuleRecipeHandler(RecipeHandler): kdirpath, _ = check_target(compile_lines, install=False) if manual_install or not install_lines: - lines_after.append('EXTRA_OEMAKE_append_task-install = " -C ${STAGING_KERNEL_DIR} M=${S}"') + lines_after.append('EXTRA_OEMAKE:append:task-install = " -C ${STAGING_KERNEL_DIR} M=${S}"') elif install_target and install_target != 'modules_install': lines_after.append('MODULES_INSTALL_TARGET = "install"') diff --git a/scripts/lib/recipetool/create_npm.py b/scripts/lib/recipetool/create_npm.py index 2bcae91dfa..113a89f6a6 100644 --- a/scripts/lib/recipetool/create_npm.py +++ b/scripts/lib/recipetool/create_npm.py @@ -6,16 +6,20 @@ """Recipe creation tool - npm module support plugin""" import json +import logging import os import re import sys import tempfile import bb from bb.fetch2.npm import NpmEnvironment +from bb.fetch2.npm import npm_package from bb.fetch2.npmsw import foreach_dependencies from recipetool.create import RecipeHandler +from recipetool.create import get_license_md5sums from recipetool.create import guess_license from recipetool.create import split_pkg_licenses +logger = logging.getLogger('recipetool') TINFOIL = None @@ -28,15 +32,6 @@ class NpmRecipeHandler(RecipeHandler): """Class to handle the npm recipe creation""" @staticmethod - def _npm_name(name): - """Generate a Yocto friendly npm name""" - name = re.sub("/", "-", name) - name = name.lower() - name = re.sub(r"[^\-a-z0-9]", "", name) - name = name.strip("-") - return name - - @staticmethod def _get_registry(lines): """Get the registry value from the 'npm://registry' url""" registry = None @@ -118,23 +113,32 @@ class NpmRecipeHandler(RecipeHandler): licfiles = [] packages = {} - def _licfiles_append(licfile): - """Append 'licfile' to the license files list""" - licfilepath = os.path.join(srctree, licfile) - licmd5 = bb.utils.md5_file(licfilepath) - licfiles.append("file://%s;md5=%s" % (licfile, licmd5)) - # Handle the parent package - _licfiles_append("package.json") packages["${PN}"] = "" + def _licfiles_append_fallback_readme_files(destdir): + """Append README files as fallback to license files if a license files is missing""" + + fallback = True + readmes = [] + basedir = os.path.join(srctree, destdir) + for fn in os.listdir(basedir): + upper = fn.upper() + if upper.startswith("README"): + fullpath = os.path.join(basedir, fn) + readmes.append(fullpath) + if upper.startswith("COPYING") or "LICENCE" in upper or "LICENSE" in upper: + fallback = False + if fallback: + for readme in readmes: + licfiles.append(os.path.relpath(readme, srctree)) + # Handle the dependencies - def _handle_dependency(name, params, deptree): - suffix = "-".join([self._npm_name(dep) for dep in deptree]) - destdirs = [os.path.join("node_modules", dep) for dep in deptree] - destdir = os.path.join(*destdirs) - _licfiles_append(os.path.join(destdir, "package.json")) - packages["${PN}-" + suffix] = destdir + def _handle_dependency(name, params, destdir): + deptree = destdir.split('node_modules/') + suffix = "-".join([npm_package(dep) for dep in deptree]) + packages["${PN}" + suffix] = destdir + _licfiles_append_fallback_readme_files(destdir) with open(shrinkwrap_file, "r") as f: shrinkwrap = json.load(f) @@ -142,6 +146,23 @@ class NpmRecipeHandler(RecipeHandler): foreach_dependencies(shrinkwrap, _handle_dependency, dev) return licfiles, packages + + # Handle the peer dependencies + def _handle_peer_dependency(self, shrinkwrap_file): + """Check if package has peer dependencies and show warning if it is the case""" + with open(shrinkwrap_file, "r") as f: + shrinkwrap = json.load(f) + + packages = shrinkwrap.get("packages", {}) + peer_deps = packages.get("", {}).get("peerDependencies", {}) + + for peer_dep in peer_deps: + peer_dep_yocto_name = npm_package(peer_dep) + bb.warn(peer_dep + " is a peer dependencie of the actual package. " + + "Please add this peer dependencie to the RDEPENDS variable as %s and generate its recipe with devtool" + % peer_dep_yocto_name) + + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): """Handle the npm recipe creation""" @@ -160,7 +181,7 @@ class NpmRecipeHandler(RecipeHandler): if "name" not in data or "version" not in data: return False - extravalues["PN"] = self._npm_name(data["name"]) + extravalues["PN"] = npm_package(data["name"]) extravalues["PV"] = data["version"] if "description" in data: @@ -229,7 +250,7 @@ class NpmRecipeHandler(RecipeHandler): value = origvalue.replace("version=" + data["version"], "version=${PV}") value = value.replace("version=latest", "version=${PV}") values = [line.strip() for line in value.strip('\n').splitlines()] - if "dependencies" in shrinkwrap: + if "dependencies" in shrinkwrap.get("packages", {}).get("", {}): values.append(url_recipe) return values, None, 4, False @@ -246,12 +267,42 @@ class NpmRecipeHandler(RecipeHandler): bb.note("Handling licences ...") (licfiles, packages) = self._handle_licenses(srctree, shrinkwrap_file, dev) - extravalues["LIC_FILES_CHKSUM"] = licfiles - split_pkg_licenses(guess_license(srctree, d), packages, lines_after, []) + + def _guess_odd_license(licfiles): + import bb + + md5sums = get_license_md5sums(d, linenumbers=True) + + chksums = [] + licenses = [] + for licfile in licfiles: + f = os.path.join(srctree, licfile) + md5value = bb.utils.md5_file(f) + (license, beginline, endline, md5) = md5sums.get(md5value, + (None, "", "", "")) + if not license: + license = "Unknown" + logger.info("Please add the following line for '%s' to a " + "'lib/recipetool/licenses.csv' and replace `Unknown`, " + "`X`, `Y` and `MD5` with the license, begin line, " + "end line and partial MD5 checksum:\n" \ + "%s,Unknown,X,Y,MD5" % (licfile, md5value)) + chksums.append("file://%s%s%s;md5=%s" % (licfile, + ";beginline=%s" % (beginline) if beginline else "", + ";endline=%s" % (endline) if endline else "", + md5 if md5 else md5value)) + licenses.append((license, licfile, md5value)) + return (licenses, chksums) + + (licenses, extravalues["LIC_FILES_CHKSUM"]) = _guess_odd_license(licfiles) + split_pkg_licenses([*licenses, *guess_license(srctree, d)], packages, lines_after) classes.append("npm") handled.append("buildsystem") + # Check if package has peer dependencies and inform the user + self._handle_peer_dependency(shrinkwrap_file) + return True def register_recipe_handlers(handlers): diff --git a/scripts/lib/recipetool/licenses.csv b/scripts/lib/recipetool/licenses.csv new file mode 100644 index 0000000000..80851111b3 --- /dev/null +++ b/scripts/lib/recipetool/licenses.csv @@ -0,0 +1,37 @@ +0636e73ff0215e8d672dc4c32c317bb3,GPL-2.0-only +12f884d2ae1ff87c09e5b7ccc2c4ca7e,GPL-2.0-only +18810669f13b87348459e611d31ab760,GPL-2.0-only +252890d9eee26aab7b432e8b8a616475,LGPL-2.0-only +2d5025d4aa3495befef8f17206a5b0a1,LGPL-2.1-only +3214f080875748938ba060314b4f727d,LGPL-2.0-only +385c55653886acac3821999a3ccd17b3,Artistic-1.0 | GPL-2.0-only +393a5ca445f6965873eca0259a17f833,GPL-2.0-only +3b83ef96387f14655fc854ddc3c6bd57,Apache-2.0 +3bf50002aefd002f49e7bb854063f7e7,LGPL-2.0-only +4325afd396febcb659c36b49533135d4,GPL-2.0-only +4fbd65380cdd255951079008b364516c,LGPL-2.1-only +54c7042be62e169199200bc6477f04d1,BSD-3-Clause +55ca817ccb7d5b5b66355690e9abc605,LGPL-2.0-only +59530bdf33659b29e73d4adb9f9f6552,GPL-2.0-only +5f30f0716dfdd0d91eb439ebec522ec2,LGPL-2.0-only +6a6a8e020838b23406c81b19c1d46df6,LGPL-3.0-only +751419260aa954499f7abaabaa882bbe,GPL-2.0-only +7fbc338309ac38fefcd64b04bb903e34,LGPL-2.1-only +8ca43cbc842c2336e835926c2166c28b,GPL-2.0-only +94d55d512a9ba36caa9b7df079bae19f,GPL-2.0-only +9ac2e7cff1ddaf48b6eab6028f23ef88,GPL-2.0-only +9f604d8a4f8e74f4f5140845a21b6674,LGPL-2.0-only +a6f89e2100d9b6cdffcea4f398e37343,LGPL-2.1-only +b234ee4d69f5fce4486a80fdaf4a4263,GPL-2.0-only +bbb461211a33b134d42ed5ee802b37ff,LGPL-2.1-only +bfe1f75d606912a4111c90743d6c7325,MPL-1.1-only +c93c0550bd3173f4504b2cbd8991e50b,GPL-2.0-only +d32239bcb673463ab874e80d47fae504,GPL-3.0-only +d7810fab7487fb0aad327b76f1be7cd7,GPL-2.0-only +d8045f3b8f929c1cb29a1e3fd737b499,LGPL-2.1-only +db979804f025cf55aabec7129cb671ed,LGPL-2.0-only +eb723b61539feef013de476e68b5c50a,GPL-2.0-only +ebb5c50ab7cab4baeffba14977030c07,GPL-2.0-only +f27defe1e96c2e1ecd4e0c9be8967949,GPL-3.0-only +fad9b3332be894bab9bc501572864b29,LGPL-2.1-only +fbc093901857fcd118f065f900982c24,LGPL-2.1-only diff --git a/scripts/lib/recipetool/setvar.py b/scripts/lib/recipetool/setvar.py index f8e2ee75fb..b5ad335cae 100644 --- a/scripts/lib/recipetool/setvar.py +++ b/scripts/lib/recipetool/setvar.py @@ -49,6 +49,7 @@ def setvar(args): for patch in patches: for line in patch: sys.stdout.write(line) + tinfoil.modified_files() return 0 diff --git a/scripts/lib/resulttool/log.py b/scripts/lib/resulttool/log.py index eb3927ec82..15148ca288 100644 --- a/scripts/lib/resulttool/log.py +++ b/scripts/lib/resulttool/log.py @@ -28,12 +28,10 @@ def show_reproducible(result, reproducible, logger): def log(args, logger): results = resultutils.load_resultsdata(args.source) - ptest_count = sum(1 for _, _, _, r in resultutils.test_run_results(results) if 'ptestresult.sections' in r) - if ptest_count > 1 and not args.prepend_run: - print("%i ptest sections found. '--prepend-run' is required" % ptest_count) - return 1 - for _, run_name, _, r in resultutils.test_run_results(results): + if args.list_ptest: + print('\n'.join(sorted(r['ptestresult.sections'].keys()))) + if args.dump_ptest: for sectname in ['ptestresult.sections', 'ltpposixresult.sections', 'ltpresult.sections']: if sectname in r: @@ -48,6 +46,9 @@ def log(args, logger): os.makedirs(dest_dir, exist_ok=True) dest = os.path.join(dest_dir, '%s.log' % name) + if os.path.exists(dest): + print("Overlapping ptest logs found, skipping %s. The '--prepend-run' option would avoid this" % name) + continue print(dest) with open(dest, 'w') as f: f.write(logdata) @@ -86,6 +87,8 @@ def register_commands(subparsers): parser.set_defaults(func=log) parser.add_argument('source', help='the results file/directory/URL to import') + parser.add_argument('--list-ptest', action='store_true', + help='list the ptest test names') parser.add_argument('--ptest', action='append', default=[], help='show logs for a ptest') parser.add_argument('--dump-ptest', metavar='DIR', diff --git a/scripts/lib/resulttool/regression.py b/scripts/lib/resulttool/regression.py index 9f952951b3..10e7d13841 100644 --- a/scripts/lib/resulttool/regression.py +++ b/scripts/lib/resulttool/regression.py @@ -7,15 +7,209 @@ # import resulttool.resultutils as resultutils -import json from oeqa.utils.git import GitRepo import oeqa.utils.gitarchive as gitarchive -def compare_result(logger, base_name, target_name, base_result, target_result): +METADATA_MATCH_TABLE = { + "oeselftest": "OESELFTEST_METADATA" +} + +OESELFTEST_METADATA_GUESS_TABLE={ + "trigger-build-posttrigger": { + "run_all_tests": False, + "run_tests":["buildoptions.SourceMirroring.test_yocto_source_mirror"], + "skips": None, + "machine": None, + "select_tags":None, + "exclude_tags": None + }, + "reproducible": { + "run_all_tests": False, + "run_tests":["reproducible"], + "skips": None, + "machine": None, + "select_tags":None, + "exclude_tags": None + }, + "arch-qemu-quick": { + "run_all_tests": True, + "run_tests":None, + "skips": None, + "machine": None, + "select_tags":["machine"], + "exclude_tags": None + }, + "arch-qemu-full-x86-or-x86_64": { + "run_all_tests": True, + "run_tests":None, + "skips": None, + "machine": None, + "select_tags":["machine", "toolchain-system"], + "exclude_tags": None + }, + "arch-qemu-full-others": { + "run_all_tests": True, + "run_tests":None, + "skips": None, + "machine": None, + "select_tags":["machine", "toolchain-user"], + "exclude_tags": None + }, + "selftest": { + "run_all_tests": True, + "run_tests":None, + "skips": ["distrodata.Distrodata.test_checkpkg", "buildoptions.SourceMirroring.test_yocto_source_mirror", "reproducible"], + "machine": None, + "select_tags":None, + "exclude_tags": ["machine", "toolchain-system", "toolchain-user"] + }, + "bringup": { + "run_all_tests": True, + "run_tests":None, + "skips": ["distrodata.Distrodata.test_checkpkg", "buildoptions.SourceMirroring.test_yocto_source_mirror"], + "machine": None, + "select_tags":None, + "exclude_tags": ["machine", "toolchain-system", "toolchain-user"] + } +} + +STATUS_STRINGS = { + "None": "No matching test result" +} + +REGRESSIONS_DISPLAY_LIMIT=50 + +MISSING_TESTS_BANNER = "-------------------------- Missing tests --------------------------" +ADDITIONAL_DATA_BANNER = "--------------------- Matches and improvements --------------------" + +def test_has_at_least_one_matching_tag(test, tag_list): + return "oetags" in test and any(oetag in tag_list for oetag in test["oetags"]) + +def all_tests_have_at_least_one_matching_tag(results, tag_list): + return all(test_has_at_least_one_matching_tag(test_result, tag_list) or test_name.startswith("ptestresult") for (test_name, test_result) in results.items()) + +def any_test_have_any_matching_tag(results, tag_list): + return any(test_has_at_least_one_matching_tag(test, tag_list) for test in results.values()) + +def have_skipped_test(result, test_prefix): + return all( result[test]['status'] == "SKIPPED" for test in result if test.startswith(test_prefix)) + +def have_all_tests_skipped(result, test_prefixes_list): + return all(have_skipped_test(result, test_prefix) for test_prefix in test_prefixes_list) + +def guess_oeselftest_metadata(results): + """ + When an oeselftest test result is lacking OESELFTEST_METADATA, we can try to guess it based on results content. + Check results for specific values (absence/presence of oetags, number and name of executed tests...), + and if it matches one of known configuration from autobuilder configuration, apply guessed OSELFTEST_METADATA + to it to allow proper test filtering. + This guessing process is tightly coupled to config.json in autobuilder. It should trigger less and less, + as new tests will have OESELFTEST_METADATA properly appended at test reporting time + """ + + if len(results) == 1 and "buildoptions.SourceMirroring.test_yocto_source_mirror" in results: + return OESELFTEST_METADATA_GUESS_TABLE['trigger-build-posttrigger'] + elif all(result.startswith("reproducible") for result in results): + return OESELFTEST_METADATA_GUESS_TABLE['reproducible'] + elif all_tests_have_at_least_one_matching_tag(results, ["machine"]): + return OESELFTEST_METADATA_GUESS_TABLE['arch-qemu-quick'] + elif all_tests_have_at_least_one_matching_tag(results, ["machine", "toolchain-system"]): + return OESELFTEST_METADATA_GUESS_TABLE['arch-qemu-full-x86-or-x86_64'] + elif all_tests_have_at_least_one_matching_tag(results, ["machine", "toolchain-user"]): + return OESELFTEST_METADATA_GUESS_TABLE['arch-qemu-full-others'] + elif not any_test_have_any_matching_tag(results, ["machine", "toolchain-user", "toolchain-system"]): + if have_all_tests_skipped(results, ["distrodata.Distrodata.test_checkpkg", "buildoptions.SourceMirroring.test_yocto_source_mirror", "reproducible"]): + return OESELFTEST_METADATA_GUESS_TABLE['selftest'] + elif have_all_tests_skipped(results, ["distrodata.Distrodata.test_checkpkg", "buildoptions.SourceMirroring.test_yocto_source_mirror"]): + return OESELFTEST_METADATA_GUESS_TABLE['bringup'] + + return None + + +def metadata_matches(base_configuration, target_configuration): + """ + For passed base and target, check test type. If test type matches one of + properties described in METADATA_MATCH_TABLE, compare metadata if it is + present in base. Return true if metadata matches, or if base lacks some + data (either TEST_TYPE or the corresponding metadata) + """ + test_type = base_configuration.get('TEST_TYPE') + if test_type not in METADATA_MATCH_TABLE: + return True + + metadata_key = METADATA_MATCH_TABLE.get(test_type) + if target_configuration.get(metadata_key) != base_configuration.get(metadata_key): + return False + + return True + + +def machine_matches(base_configuration, target_configuration): + return base_configuration.get('MACHINE') == target_configuration.get('MACHINE') + + +def can_be_compared(logger, base, target): + """ + Some tests are not relevant to be compared, for example some oeselftest + run with different tests sets or parameters. Return true if tests can be + compared + """ + ret = True + base_configuration = base['configuration'] + target_configuration = target['configuration'] + + # Older test results lack proper OESELFTEST_METADATA: if not present, try to guess it based on tests results. + if base_configuration.get('TEST_TYPE') == 'oeselftest' and 'OESELFTEST_METADATA' not in base_configuration: + guess = guess_oeselftest_metadata(base['result']) + if guess is None: + logger.error(f"ERROR: did not manage to guess oeselftest metadata for {base_configuration['STARTTIME']}") + else: + logger.debug(f"Enriching {base_configuration['STARTTIME']} with {guess}") + base_configuration['OESELFTEST_METADATA'] = guess + if target_configuration.get('TEST_TYPE') == 'oeselftest' and 'OESELFTEST_METADATA' not in target_configuration: + guess = guess_oeselftest_metadata(target['result']) + if guess is None: + logger.error(f"ERROR: did not manage to guess oeselftest metadata for {target_configuration['STARTTIME']}") + else: + logger.debug(f"Enriching {target_configuration['STARTTIME']} with {guess}") + target_configuration['OESELFTEST_METADATA'] = guess + + # Test runs with LTP results in should only be compared with other runs with LTP tests in them + if base_configuration.get('TEST_TYPE') == 'runtime' and any(result.startswith("ltpresult") for result in base['result']): + ret = target_configuration.get('TEST_TYPE') == 'runtime' and any(result.startswith("ltpresult") for result in target['result']) + + return ret and metadata_matches(base_configuration, target_configuration) \ + and machine_matches(base_configuration, target_configuration) + +def get_status_str(raw_status): + raw_status_lower = raw_status.lower() if raw_status else "None" + return STATUS_STRINGS.get(raw_status_lower, raw_status) + +def get_additional_info_line(new_pass_count, new_tests): + result=[] + if new_tests: + result.append(f'+{new_tests} test(s) present') + if new_pass_count: + result.append(f'+{new_pass_count} test(s) now passing') + + if not result: + return "" + + return ' -> ' + ', '.join(result) + '\n' + +def compare_result(logger, base_name, target_name, base_result, target_result, display_limit=None): base_result = base_result.get('result') target_result = target_result.get('result') result = {} + new_tests = 0 + regressions = {} + resultstring = "" + new_tests = 0 + new_pass_count = 0 + + display_limit = int(display_limit) if display_limit else REGRESSIONS_DISPLAY_LIMIT + if base_result and target_result: for k in base_result: base_testcase = base_result[k] @@ -27,12 +221,47 @@ def compare_result(logger, base_name, target_name, base_result, target_result): result[k] = {'base': base_status, 'target': target_status} else: logger.error('Failed to retrieved base test case status: %s' % k) + + # Also count new tests that were not present in base results: it + # could be newly added tests, but it could also highlights some tests + # renames or fixed faulty ptests + for k in target_result: + if k not in base_result: + new_tests += 1 if result: - resultstring = "Regression: %s\n %s\n" % (base_name, target_name) - for k in sorted(result): - resultstring += ' %s: %s -> %s\n' % (k, result[k]['base'], result[k]['target']) + new_pass_count = sum(test['target'] is not None and test['target'].startswith("PASS") for test in result.values()) + # Print a regression report only if at least one test has a regression status (FAIL, SKIPPED, absent...) + if new_pass_count < len(result): + resultstring = "Regression: %s\n %s\n" % (base_name, target_name) + for k in sorted(result): + if not result[k]['target'] or not result[k]['target'].startswith("PASS"): + # Differentiate each ptest kind when listing regressions + key_parts = k.split('.') + key = '.'.join(key_parts[:2]) if k.startswith('ptest') else key_parts[0] + # Append new regression to corresponding test family + regressions[key] = regressions.setdefault(key, []) + [' %s: %s -> %s\n' % (k, get_status_str(result[k]['base']), get_status_str(result[k]['target']))] + resultstring += f" Total: {sum([len(regressions[r]) for r in regressions])} new regression(s):\n" + for k in regressions: + resultstring += f" {len(regressions[k])} regression(s) for {k}\n" + count_to_print=min([display_limit, len(regressions[k])]) if display_limit > 0 else len(regressions[k]) + resultstring += ''.join(regressions[k][:count_to_print]) + if count_to_print < len(regressions[k]): + resultstring+=' [...]\n' + if new_pass_count > 0: + resultstring += f' Additionally, {new_pass_count} previously failing test(s) is/are now passing\n' + if new_tests > 0: + resultstring += f' Additionally, {new_tests} new test(s) is/are present\n' + else: + resultstring = "%s\n%s\n" % (base_name, target_name) + result = None else: - resultstring = "Match: %s\n %s" % (base_name, target_name) + resultstring = "%s\n%s\n" % (base_name, target_name) + + if not result: + additional_info = get_additional_info_line(new_pass_count, new_tests) + if additional_info: + resultstring += additional_info + return result, resultstring def get_results(logger, source): @@ -44,12 +273,38 @@ def regression(args, logger): regression_common(args, logger, base_results, target_results) +# Some test case naming is poor and contains random strings, particularly lttng/babeltrace. +# Truncating the test names works since they contain file and line number identifiers +# which allows us to match them without the random components. +def fixup_ptest_names(results, logger): + for r in results: + for i in results[r]: + tests = list(results[r][i]['result'].keys()) + for test in tests: + new = None + if test.startswith(("ptestresult.lttng-tools.", "ptestresult.babeltrace.", "ptestresult.babeltrace2")) and "_-_" in test: + new = test.split("_-_")[0] + elif test.startswith(("ptestresult.curl.")) and "__" in test: + new = test.split("__")[0] + elif test.startswith(("ptestresult.dbus.")) and "__" in test: + new = test.split("__")[0] + elif test.startswith("ptestresult.binutils") and "build-st-" in test: + new = test.split(" ")[0] + elif test.startswith("ptestresult.gcc") and "/tmp/runtest." in test: + new = ".".join(test.split(".")[:2]) + if new: + results[r][i]['result'][new] = results[r][i]['result'][test] + del results[r][i]['result'][test] + def regression_common(args, logger, base_results, target_results): if args.base_result_id: base_results = resultutils.filter_resultsdata(base_results, args.base_result_id) if args.target_result_id: target_results = resultutils.filter_resultsdata(target_results, args.target_result_id) + fixup_ptest_names(base_results, logger) + fixup_ptest_names(target_results, logger) + matches = [] regressions = [] notfound = [] @@ -62,7 +317,9 @@ def regression_common(args, logger, base_results, target_results): # removing any pairs which match for c in base.copy(): for b in target.copy(): - res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b]) + if not can_be_compared(logger, base_results[a][c], target_results[a][b]): + continue + res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b], args.limit) if not res: matches.append(resstr) base.remove(c) @@ -71,15 +328,18 @@ def regression_common(args, logger, base_results, target_results): # Should only now see regressions, we may not be able to match multiple pairs directly for c in base: for b in target: - res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b]) + if not can_be_compared(logger, base_results[a][c], target_results[a][b]): + continue + res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b], args.limit) if res: regressions.append(resstr) else: notfound.append("%s not found in target" % a) - print("\n".join(sorted(matches))) print("\n".join(sorted(regressions))) + print("\n" + MISSING_TESTS_BANNER + "\n") print("\n".join(sorted(notfound))) - + print("\n" + ADDITIONAL_DATA_BANNER + "\n") + print("\n".join(sorted(matches))) return 0 def regression_git(args, logger): @@ -183,4 +443,5 @@ def register_commands(subparsers): parser_build.add_argument('--commit-number', help="Revision number to search for, redundant if --commit is specified") parser_build.add_argument('--commit2', help="Revision to compare with") parser_build.add_argument('--commit-number2', help="Revision number to compare with, redundant if --commit2 is specified") + parser_build.add_argument('-l', '--limit', default=REGRESSIONS_DISPLAY_LIMIT, help="Maximum number of changes to display per test. Can be set to 0 to print all changes") diff --git a/scripts/lib/resulttool/report.py b/scripts/lib/resulttool/report.py index f0ca50ebe2..a349510ab8 100644 --- a/scripts/lib/resulttool/report.py +++ b/scripts/lib/resulttool/report.py @@ -176,7 +176,10 @@ class ResultsTextReport(object): vals['sort'] = line['testseries'] + "_" + line['result_id'] vals['failed_testcases'] = line['failed_testcases'] for k in cols: - vals[k] = "%d (%s%%)" % (line[k], format(line[k] / total_tested * 100, '.0f')) + if total_tested: + vals[k] = "%d (%s%%)" % (line[k], format(line[k] / total_tested * 100, '.0f')) + else: + vals[k] = "0 (0%)" for k in maxlen: if k in vals and len(vals[k]) > maxlen[k]: maxlen[k] = len(vals[k]) diff --git a/scripts/lib/resulttool/resultutils.py b/scripts/lib/resulttool/resultutils.py index 8917022d36..c5521d81bd 100644 --- a/scripts/lib/resulttool/resultutils.py +++ b/scripts/lib/resulttool/resultutils.py @@ -58,7 +58,11 @@ def append_resultsdata(results, f, configmap=store_map, configvars=extra_configv testseries = posixpath.basename(posixpath.dirname(url.path)) else: with open(f, "r") as filedata: - data = json.load(filedata) + try: + data = json.load(filedata) + except json.decoder.JSONDecodeError: + print("Cannot decode {}. Possible corruption. Skipping.".format(f)) + data = "" testseries = os.path.basename(os.path.dirname(f)) else: data = f @@ -142,7 +146,7 @@ def generic_get_log(sectionname, results, section): return decode_log(ptest['log']) def ptestresult_get_log(results, section): - return generic_get_log('ptestresuls.sections', results, section) + return generic_get_log('ptestresult.sections', results, section) def generic_get_rawlogs(sectname, results): if sectname not in results: diff --git a/scripts/lib/scriptutils.py b/scripts/lib/scriptutils.py index f92255d8dc..f23e53cba9 100644 --- a/scripts/lib/scriptutils.py +++ b/scripts/lib/scriptutils.py @@ -5,7 +5,6 @@ # SPDX-License-Identifier: GPL-2.0-only # -import argparse import glob import logging import os @@ -18,13 +17,14 @@ import sys import tempfile import threading import importlib -from importlib import machinery +import importlib.machinery +import importlib.util class KeepAliveStreamHandler(logging.StreamHandler): def __init__(self, keepalive=True, **kwargs): super().__init__(**kwargs) if keepalive is True: - keepalive = 5000 # default timeout + keepalive = 5000 # default timeout self._timeout = threading.Condition() self._stop = False @@ -35,9 +35,9 @@ class KeepAliveStreamHandler(logging.StreamHandler): with self._timeout: if not self._timeout.wait(keepalive): self.emit(logging.LogRecord("keepalive", logging.INFO, - None, None, "Keepalive message", None, None)) + None, None, "Keepalive message", None, None)) - self._thread = threading.Thread(target = thread, daemon = True) + self._thread = threading.Thread(target=thread, daemon=True) self._thread.start() def close(self): @@ -71,18 +71,19 @@ def logger_setup_color(logger, color='auto'): for handler in logger.handlers: if (isinstance(handler, logging.StreamHandler) and - isinstance(handler.formatter, BBLogFormatter)): + isinstance(handler.formatter, BBLogFormatter)): if color == 'always' or (color == 'auto' and handler.stream.isatty()): handler.formatter.enable_color() def load_plugins(logger, plugins, pluginpath): - def load_plugin(name): logger.debug('Loading plugin %s' % name) - spec = importlib.machinery.PathFinder.find_spec(name, path=[pluginpath] ) + spec = importlib.machinery.PathFinder.find_spec(name, path=[pluginpath]) if spec: - return spec.loader.load_module() + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod def plugin_name(filename): return os.path.splitext(os.path.basename(filename))[0] @@ -176,6 +177,7 @@ def fetch_url(tinfoil, srcuri, srcrev, destdir, logger, preserve_tmp=False, mirr f.write('BB_STRICT_CHECKSUM = "ignore"\n') f.write('SRC_URI = "%s"\n' % srcuri) f.write('SRCREV = "%s"\n' % srcrev) + f.write('PV = "0.0+"\n') f.write('WORKDIR = "%s"\n' % tmpworkdir) # Set S out of the way so it doesn't get created under the workdir f.write('S = "%s"\n' % os.path.join(tmpdir, 'emptysrc')) @@ -215,7 +217,8 @@ def fetch_url(tinfoil, srcuri, srcrev, destdir, logger, preserve_tmp=False, mirr pathvars = ['T', 'RECIPE_SYSROOT', 'RECIPE_SYSROOT_NATIVE'] for pathvar in pathvars: path = rd.getVar(pathvar) - shutil.rmtree(path) + if os.path.exists(path): + shutil.rmtree(path) finally: if fetchrecipe: try: @@ -274,6 +277,6 @@ def filter_src_subdirs(pth): Used by devtool and recipetool. """ dirlist = os.listdir(pth) - filterout = ['git.indirectionsymlink', 'source-date-epoch'] + filterout = ['git.indirectionsymlink', 'source-date-epoch', 'sstate-install-recipe_qa'] dirlist = [x for x in dirlist if x not in filterout] return dirlist diff --git a/scripts/lib/wic/canned-wks/efi-bootdisk.wks.in b/scripts/lib/wic/canned-wks/efi-bootdisk.wks.in index 7300e65e32..2fd286ff98 100644 --- a/scripts/lib/wic/canned-wks/efi-bootdisk.wks.in +++ b/scripts/lib/wic/canned-wks/efi-bootdisk.wks.in @@ -1,3 +1,3 @@ bootloader --ptable gpt -part /boot --source rootfs --rootfs-dir=${IMAGE_ROOTFS}/boot --fstype=vfat --label boot --active --align 1024 --use-uuid --overhead-factor 1.0 +part /boot --source rootfs --rootfs-dir=${IMAGE_ROOTFS}/boot --fstype=vfat --label boot --active --align 1024 --use-uuid --overhead-factor 1.1 part / --source rootfs --fstype=ext4 --label root --align 1024 --exclude-path boot/ diff --git a/scripts/lib/wic/canned-wks/qemuloongarch.wks b/scripts/lib/wic/canned-wks/qemuloongarch.wks new file mode 100644 index 0000000000..8465c7a8c0 --- /dev/null +++ b/scripts/lib/wic/canned-wks/qemuloongarch.wks @@ -0,0 +1,3 @@ +# short-description: Create qcow2 image for LoongArch QEMU machines + +part / --source rootfs --fstype=ext4 --label root --align 4096 --size 5G diff --git a/scripts/lib/wic/canned-wks/qemux86-directdisk.wks b/scripts/lib/wic/canned-wks/qemux86-directdisk.wks index 22b45217f1..808997611a 100644 --- a/scripts/lib/wic/canned-wks/qemux86-directdisk.wks +++ b/scripts/lib/wic/canned-wks/qemux86-directdisk.wks @@ -4,5 +4,5 @@ include common.wks.inc -bootloader --timeout=0 --append="rw oprofile.timer=1 rootfstype=ext4 " +bootloader --timeout=0 --append="rw oprofile.timer=1 rootfstype=ext4 console=tty console=ttyS0 " diff --git a/scripts/lib/wic/engine.py b/scripts/lib/wic/engine.py index 018815b966..674ccfc244 100644 --- a/scripts/lib/wic/engine.py +++ b/scripts/lib/wic/engine.py @@ -19,10 +19,10 @@ import os import tempfile import json import subprocess +import shutil import re from collections import namedtuple, OrderedDict -from distutils.spawn import find_executable from wic import WicError from wic.filemap import sparse_copy @@ -245,7 +245,7 @@ class Disk: for path in pathlist.split(':'): self.paths = "%s%s:%s" % (native_sysroot, path, self.paths) - self.parted = find_executable("parted", self.paths) + self.parted = shutil.which("parted", path=self.paths) if not self.parted: raise WicError("Can't find executable parted") @@ -283,7 +283,7 @@ class Disk: "resize2fs", "mkswap", "mkdosfs", "debugfs","blkid"): aname = "_%s" % name if aname not in self.__dict__: - setattr(self, aname, find_executable(name, self.paths)) + setattr(self, aname, shutil.which(name, path=self.paths)) if aname not in self.__dict__ or self.__dict__[aname] is None: raise WicError("Can't find executable '{}'".format(name)) return self.__dict__[aname] diff --git a/scripts/lib/wic/filemap.py b/scripts/lib/wic/filemap.py index 4d9da28172..85b39d5d74 100644 --- a/scripts/lib/wic/filemap.py +++ b/scripts/lib/wic/filemap.py @@ -46,6 +46,13 @@ def get_block_size(file_obj): bsize = stat.st_blksize else: raise IOError("Unable to determine block size") + + # The logic in this script only supports a maximum of a 4KB + # block size + max_block_size = 4 * 1024 + if bsize > max_block_size: + bsize = max_block_size + return bsize class ErrorNotSupp(Exception): diff --git a/scripts/lib/wic/help.py b/scripts/lib/wic/help.py index bd3a2b97df..163535e431 100644 --- a/scripts/lib/wic/help.py +++ b/scripts/lib/wic/help.py @@ -637,7 +637,7 @@ DESCRIPTION oe-core: directdisk.bbclass and mkefidisk.sh. The difference between wic and those examples is that with wic the functionality of those scripts is implemented by a general-purpose partitioning - 'language' based on Redhat kickstart syntax). + 'language' based on Red Hat kickstart syntax). The initial motivation and design considerations that lead to the current tool are described exhaustively in Yocto Bug #3847 @@ -840,8 +840,8 @@ DESCRIPTION meanings. The commands are based on the Fedora kickstart documentation but with modifications to reflect wic capabilities. - http://fedoraproject.org/wiki/Anaconda/Kickstart#part_or_partition - http://fedoraproject.org/wiki/Anaconda/Kickstart#bootloader + https://pykickstart.readthedocs.io/en/latest/kickstart-docs.html#part-or-partition + https://pykickstart.readthedocs.io/en/latest/kickstart-docs.html#bootloader Commands @@ -930,6 +930,7 @@ DESCRIPTION ext4 btrfs squashfs + erofs swap --fsoptions: Specifies a free-form string of options to be @@ -939,6 +940,12 @@ DESCRIPTION quotes. If not specified, the default string is "defaults". + --fspassno: Specifies the order in which filesystem checks are done + at boot time by fsck. See fs_passno parameter of + fstab(5). This parameter will be copied into the + /etc/fstab file of the installed system. If not + specified the default value of "0" will be used. + --label label: Specifies the label to give to the filesystem to be made on the partition. If the given label is already in use by another filesystem, @@ -990,6 +997,9 @@ DESCRIPTION multiple partitions and we want to keep the right permissions and usernames in all the partitions. + --no-fstab-update: This option is specific to wic. It does not update the + '/etc/fstab' stock file for the given partition. + --extra-space: This option is specific to wic. It adds extra space after the space filled by the content of the partition. The final size can go @@ -1108,7 +1118,7 @@ COMMAND: TOPIC: overview - Presents an overall overview of Wic plugins - Presents an overview and API for Wic plugins - kickstart - Presents a Wic kicstart file reference + kickstart - Presents a Wic kickstart file reference Examples: diff --git a/scripts/lib/wic/ksparser.py b/scripts/lib/wic/ksparser.py index 3eb669da39..7ef3dc83dd 100644 --- a/scripts/lib/wic/ksparser.py +++ b/scripts/lib/wic/ksparser.py @@ -155,9 +155,11 @@ class KickStart(): part.add_argument('--change-directory') part.add_argument("--extra-space", type=sizetype("M")) part.add_argument('--fsoptions', dest='fsopts') + part.add_argument('--fspassno', dest='fspassno') part.add_argument('--fstype', default='vfat', choices=('ext2', 'ext3', 'ext4', 'btrfs', - 'squashfs', 'vfat', 'msdos', 'swap')) + 'squashfs', 'vfat', 'msdos', 'erofs', + 'swap', 'none')) part.add_argument('--mkfs-extraopts', default='') part.add_argument('--label') part.add_argument('--use-label', action='store_true') @@ -169,6 +171,7 @@ class KickStart(): part.add_argument('--rootfs-dir') part.add_argument('--type', default='primary', choices = ('primary', 'logical')) + part.add_argument('--hidden', action='store_true') # --size and --fixed-size cannot be specified together; options # ----extra-space and --overhead-factor should also raise a parser @@ -184,11 +187,13 @@ class KickStart(): part.add_argument('--use-uuid', action='store_true') part.add_argument('--uuid') part.add_argument('--fsuuid') + part.add_argument('--no-fstab-update', action='store_true') + part.add_argument('--mbr', action='store_true') bootloader = subparsers.add_parser('bootloader') bootloader.add_argument('--append') bootloader.add_argument('--configfile') - bootloader.add_argument('--ptable', choices=('msdos', 'gpt'), + bootloader.add_argument('--ptable', choices=('msdos', 'gpt', 'gpt-hybrid'), default='msdos') bootloader.add_argument('--timeout', type=int) bootloader.add_argument('--source') @@ -229,6 +234,10 @@ class KickStart(): err = "%s:%d: SquashFS does not support LABEL" \ % (confpath, lineno) raise KickStartError(err) + # erofs does not support filesystem labels + if parsed.fstype == 'erofs' and parsed.label: + err = "%s:%d: erofs does not support LABEL" % (confpath, lineno) + raise KickStartError(err) if parsed.fstype == 'msdos' or parsed.fstype == 'vfat': if parsed.fsuuid: if parsed.fsuuid.upper().startswith('0X'): diff --git a/scripts/lib/wic/misc.py b/scripts/lib/wic/misc.py index 75b219cd3f..1a7c140fa6 100644 --- a/scripts/lib/wic/misc.py +++ b/scripts/lib/wic/misc.py @@ -16,16 +16,17 @@ import logging import os import re import subprocess +import shutil from collections import defaultdict -from distutils import spawn from wic import WicError logger = logging.getLogger('wic') # executable -> recipe pairs for exec_native_cmd -NATIVE_RECIPES = {"bmaptool": "bmap-tools", +NATIVE_RECIPES = {"bmaptool": "bmaptool", + "dumpe2fs": "e2fsprogs", "grub-mkimage": "grub-efi", "isohybrid": "syslinux", "mcopy": "mtools", @@ -35,6 +36,7 @@ NATIVE_RECIPES = {"bmaptool": "bmap-tools", "mkdosfs": "dosfstools", "mkisofs": "cdrtools", "mkfs.btrfs": "btrfs-tools", + "mkfs.erofs": "erofs-utils", "mkfs.ext2": "e2fsprogs", "mkfs.ext3": "e2fsprogs", "mkfs.ext4": "e2fsprogs", @@ -121,7 +123,7 @@ def find_executable(cmd, paths): if provided and "%s-native" % recipe in provided: return True - return spawn.find_executable(cmd, paths) + return shutil.which(cmd, path=paths) def exec_native_cmd(cmd_and_args, native_sysroot, pseudo=""): """ @@ -139,11 +141,12 @@ def exec_native_cmd(cmd_and_args, native_sysroot, pseudo=""): cmd_and_args = pseudo + cmd_and_args hosttools_dir = get_bitbake_var("HOSTTOOLS_DIR") + target_sys = get_bitbake_var("TARGET_SYS") - native_paths = "%s/sbin:%s/usr/sbin:%s/usr/bin:%s/bin:%s" % \ + native_paths = "%s/sbin:%s/usr/sbin:%s/usr/bin:%s/usr/bin/%s:%s/bin:%s" % \ (native_sysroot, native_sysroot, - native_sysroot, native_sysroot, - hosttools_dir) + native_sysroot, native_sysroot, target_sys, + native_sysroot, hosttools_dir) native_cmd_and_args = "export PATH=%s:$PATH;%s" % \ (native_paths, cmd_and_args) diff --git a/scripts/lib/wic/partition.py b/scripts/lib/wic/partition.py index e574f40c47..795707ec5d 100644 --- a/scripts/lib/wic/partition.py +++ b/scripts/lib/wic/partition.py @@ -33,6 +33,7 @@ class Partition(): self.include_path = args.include_path self.change_directory = args.change_directory self.fsopts = args.fsopts + self.fspassno = args.fspassno self.fstype = args.fstype self.label = args.label self.use_label = args.use_label @@ -54,9 +55,12 @@ class Partition(): self.uuid = args.uuid self.fsuuid = args.fsuuid self.type = args.type + self.no_fstab_update = args.no_fstab_update self.updated_fstab_path = None self.has_fstab = False self.update_fstab_in_rootfs = False + self.hidden = args.hidden + self.mbr = args.mbr self.lineno = lineno self.source_file = "" @@ -104,7 +108,7 @@ class Partition(): extra_blocks = self.extra_space rootfs_size = actual_rootfs_size + extra_blocks - rootfs_size *= self.overhead_factor + rootfs_size = int(rootfs_size * self.overhead_factor) logger.debug("Added %d extra blocks to %s to get to %d total blocks", extra_blocks, self.mountpoint, rootfs_size) @@ -131,6 +135,8 @@ class Partition(): self.update_fstab_in_rootfs = True if not self.source: + if self.fstype == "none" or self.no_table: + return if not self.size and not self.fixed_size: raise WicError("The %s partition has a size of zero. Please " "specify a non-zero --size/--fixed-size for that " @@ -141,9 +147,9 @@ class Partition(): native_sysroot) self.source_file = "%s/fs.%s" % (cr_workdir, self.fstype) else: - if self.fstype == 'squashfs': - raise WicError("It's not possible to create empty squashfs " - "partition '%s'" % (self.mountpoint)) + if self.fstype in ('squashfs', 'erofs'): + raise WicError("It's not possible to create empty %s " + "partition '%s'" % (self.fstype, self.mountpoint)) rootfs = "%s/fs_%s.%s.%s" % (cr_workdir, self.label, self.lineno, self.fstype) @@ -170,7 +176,7 @@ class Partition(): # Split sourceparams string of the form key1=val1[,key2=val2,...] # into a dict. Also accepts valueless keys i.e. without = splitted = self.sourceparams.split(',') - srcparams_dict = dict(par.split('=', 1) for par in splitted if par) + srcparams_dict = dict((par.split('=', 1) + [None])[:2] for par in splitted if par) plugin = PluginMgr.get_plugins('source')[self.source] plugin.do_configure_partition(self, srcparams_dict, creator, @@ -278,6 +284,20 @@ class Partition(): extraopts = self.mkfs_extraopts or "-F -i 8192" + if os.getenv('SOURCE_DATE_EPOCH'): + sde_time = int(os.getenv('SOURCE_DATE_EPOCH')) + if pseudo: + pseudo = "export E2FSPROGS_FAKE_TIME=%s;%s " % (sde_time, pseudo) + else: + pseudo = "export E2FSPROGS_FAKE_TIME=%s; " % sde_time + + # Set hash_seed to generate deterministic directory indexes + namespace = uuid.UUID("e7429877-e7b3-4a68-a5c9-2f2fdf33d460") + if self.fsuuid: + namespace = uuid.UUID(self.fsuuid) + hash_seed = str(uuid.uuid5(namespace, str(sde_time))) + extraopts += " -E hash_seed=%s" % hash_seed + label_str = "" if self.label: label_str = "-L %s" % self.label @@ -286,7 +306,7 @@ class Partition(): (self.fstype, extraopts, rootfs, label_str, self.fsuuid, rootfs_dir) exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo) - if self.updated_fstab_path and self.has_fstab: + if self.updated_fstab_path and self.has_fstab and not self.no_fstab_update: debugfs_script_path = os.path.join(cr_workdir, "debugfs_script") with open(debugfs_script_path, "w") as f: f.write("cd etc\n") @@ -298,6 +318,32 @@ class Partition(): mkfs_cmd = "fsck.%s -pvfD %s" % (self.fstype, rootfs) exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo) + if os.getenv('SOURCE_DATE_EPOCH'): + sde_time = hex(int(os.getenv('SOURCE_DATE_EPOCH'))) + debugfs_script_path = os.path.join(cr_workdir, "debugfs_script") + files = [] + for root, dirs, others in os.walk(rootfs_dir): + base = root.replace(rootfs_dir, "").rstrip(os.sep) + files += [ "/" if base == "" else base ] + files += [ base + "/" + n for n in dirs + others ] + with open(debugfs_script_path, "w") as f: + f.write("set_current_time %s\n" % (sde_time)) + if self.updated_fstab_path and self.has_fstab and not self.no_fstab_update: + f.write("set_inode_field /etc/fstab mtime %s\n" % (sde_time)) + f.write("set_inode_field /etc/fstab mtime_extra 0\n") + for file in set(files): + for time in ["atime", "ctime", "crtime"]: + f.write("set_inode_field \"%s\" %s %s\n" % (file, time, sde_time)) + f.write("set_inode_field \"%s\" %s_extra 0\n" % (file, time)) + for time in ["wtime", "mkfs_time", "lastcheck"]: + f.write("set_super_value %s %s\n" % (time, sde_time)) + for time in ["mtime", "first_error_time", "last_error_time"]: + f.write("set_super_value %s 0\n" % (time)) + debugfs_cmd = "debugfs -w -f %s %s" % (debugfs_script_path, rootfs) + exec_native_cmd(debugfs_cmd, native_sysroot) + + self.check_for_Y2038_problem(rootfs, native_sysroot) + def prepare_rootfs_btrfs(self, rootfs, cr_workdir, oe_builddir, rootfs_dir, native_sysroot, pseudo): """ @@ -337,8 +383,6 @@ class Partition(): label_str = "-n %s" % self.label size_str = "" - if self.fstype == 'msdos': - size_str = "-F 16" # FAT 16 extraopts = self.mkfs_extraopts or '-S 512' @@ -350,8 +394,8 @@ class Partition(): mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (rootfs, rootfs_dir) exec_native_cmd(mcopy_cmd, native_sysroot) - if self.updated_fstab_path and self.has_fstab: - mcopy_cmd = "mcopy -i %s %s ::/etc/fstab" % (rootfs, self.updated_fstab_path) + if self.updated_fstab_path and self.has_fstab and not self.no_fstab_update: + mcopy_cmd = "mcopy -m -i %s %s ::/etc/fstab" % (rootfs, self.updated_fstab_path) exec_native_cmd(mcopy_cmd, native_sysroot) chmod_cmd = "chmod 644 %s" % rootfs @@ -369,6 +413,19 @@ class Partition(): (rootfs_dir, rootfs, extraopts) exec_native_cmd(squashfs_cmd, native_sysroot, pseudo=pseudo) + def prepare_rootfs_erofs(self, rootfs, cr_workdir, oe_builddir, rootfs_dir, + native_sysroot, pseudo): + """ + Prepare content for a erofs rootfs partition. + """ + extraopts = self.mkfs_extraopts or '' + erofs_cmd = "mkfs.erofs %s -U %s %s %s" % \ + (extraopts, self.fsuuid, rootfs, rootfs_dir) + exec_native_cmd(erofs_cmd, native_sysroot, pseudo=pseudo) + + def prepare_empty_partition_none(self, rootfs, oe_builddir, native_sysroot): + pass + def prepare_empty_partition_ext(self, rootfs, oe_builddir, native_sysroot): """ @@ -388,6 +445,8 @@ class Partition(): (self.fstype, extraopts, label_str, self.fsuuid, rootfs) exec_native_cmd(mkfs_cmd, native_sysroot) + self.check_for_Y2038_problem(rootfs, native_sysroot) + def prepare_empty_partition_btrfs(self, rootfs, oe_builddir, native_sysroot): """ @@ -418,8 +477,6 @@ class Partition(): label_str = "-n %s" % self.label size_str = "" - if self.fstype == 'msdos': - size_str = "-F 16" # FAT 16 extraopts = self.mkfs_extraopts or '-S 512' @@ -449,3 +506,37 @@ class Partition(): mkswap_cmd = "mkswap %s -U %s %s" % (label_str, self.fsuuid, path) exec_native_cmd(mkswap_cmd, native_sysroot) + + def check_for_Y2038_problem(self, rootfs, native_sysroot): + """ + Check if the filesystem is affected by the Y2038 problem + (Y2038 problem = 32 bit time_t overflow in January 2038) + """ + def get_err_str(part): + err = "The {} filesystem {} has no Y2038 support." + if part.mountpoint: + args = [part.fstype, "mounted at %s" % part.mountpoint] + elif part.label: + args = [part.fstype, "labeled '%s'" % part.label] + elif part.part_name: + args = [part.fstype, "in partition '%s'" % part.part_name] + else: + args = [part.fstype, "in partition %s" % part.num] + return err.format(*args) + + # ext2 and ext3 are always affected by the Y2038 problem + if self.fstype in ["ext2", "ext3"]: + logger.warn(get_err_str(self)) + return + + ret, out = exec_native_cmd("dumpe2fs %s" % rootfs, native_sysroot) + + # if ext4 is affected by the Y2038 problem depends on the inode size + for line in out.splitlines(): + if line.startswith("Inode size:"): + size = int(line.split(":")[1].strip()) + if size < 256: + logger.warn("%s Inodes (of size %d) are too small." % + (get_err_str(self), size)) + break + diff --git a/scripts/lib/wic/pluginbase.py b/scripts/lib/wic/pluginbase.py index d9b4e57747..b64568339b 100644 --- a/scripts/lib/wic/pluginbase.py +++ b/scripts/lib/wic/pluginbase.py @@ -9,9 +9,11 @@ __all__ = ['ImagerPlugin', 'SourcePlugin'] import os import logging +import types from collections import defaultdict -from importlib.machinery import SourceFileLoader +import importlib +import importlib.util from wic import WicError from wic.misc import get_bitbake_var @@ -54,7 +56,9 @@ class PluginMgr: mname = fname[:-3] mpath = os.path.join(ppath, fname) logger.debug("loading plugin module %s", mpath) - SourceFileLoader(mname, mpath).load_module() + spec = importlib.util.spec_from_file_location(mname, mpath) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) return PLUGINS.get(ptype) diff --git a/scripts/lib/wic/plugins/imager/direct.py b/scripts/lib/wic/plugins/imager/direct.py index 7e1c1c03ab..a1d152659b 100644 --- a/scripts/lib/wic/plugins/imager/direct.py +++ b/scripts/lib/wic/plugins/imager/direct.py @@ -54,6 +54,7 @@ class DirectPlugin(ImagerPlugin): self.native_sysroot = native_sysroot self.oe_builddir = oe_builddir + self.debug = options.debug self.outdir = options.outdir self.compressor = options.compressor self.bmap = options.bmap @@ -76,7 +77,8 @@ class DirectPlugin(ImagerPlugin): image_path = self._full_path(self.workdir, self.parts[0].disk, "direct") self._image = PartitionedImage(image_path, self.ptable_format, - self.parts, self.native_sysroot) + self.parts, self.native_sysroot, + options.extra_space) def setup_workdir(self, workdir): if workdir: @@ -115,7 +117,7 @@ class DirectPlugin(ImagerPlugin): updated = False for part in self.parts: if not part.realnum or not part.mountpoint \ - or part.mountpoint == "/": + or part.mountpoint == "/" or not (part.mountpoint.startswith('/') or part.mountpoint == "swap"): continue if part.use_uuid: @@ -136,8 +138,9 @@ class DirectPlugin(ImagerPlugin): device_name = "/dev/%s%s%d" % (part.disk, prefix, part.realnum) opts = part.fsopts if part.fsopts else "defaults" + passno = part.fspassno if part.fspassno else "0" line = "\t".join([device_name, part.mountpoint, part.fstype, - opts, "0", "0"]) + "\n" + opts, "0", passno]) + "\n" fstab_lines.append(line) updated = True @@ -146,6 +149,9 @@ class DirectPlugin(ImagerPlugin): self.updated_fstab_path = os.path.join(self.workdir, "fstab") with open(self.updated_fstab_path, "w") as f: f.writelines(fstab_lines) + if os.getenv('SOURCE_DATE_EPOCH'): + fstab_time = int(os.getenv('SOURCE_DATE_EPOCH')) + os.utime(self.updated_fstab_path, (fstab_time, fstab_time)) def _full_path(self, path, name, extention): """ Construct full file path to a file we generate. """ @@ -257,6 +263,8 @@ class DirectPlugin(ImagerPlugin): if part.mountpoint == "/": if part.uuid: return "PARTUUID=%s" % part.uuid + elif part.label and self.ptable_format != 'msdos': + return "PARTLABEL=%s" % part.label else: suffix = 'p' if part.disk.startswith('mmcblk') else '' return "/dev/%s%s%-d" % (part.disk, suffix, part.realnum) @@ -274,8 +282,9 @@ class DirectPlugin(ImagerPlugin): if os.path.isfile(path): shutil.move(path, os.path.join(self.outdir, fname)) - # remove work directory - shutil.rmtree(self.workdir, ignore_errors=True) + # remove work directory when it is not in debugging mode + if not self.debug: + shutil.rmtree(self.workdir, ignore_errors=True) # Overhead of the MBR partitioning scheme (just one sector) MBR_OVERHEAD = 1 @@ -291,7 +300,7 @@ class PartitionedImage(): Partitioned image in a file. """ - def __init__(self, path, ptable_format, partitions, native_sysroot=None): + def __init__(self, path, ptable_format, partitions, native_sysroot=None, extra_space=0): self.path = path # Path to the image file self.numpart = 0 # Number of allocated partitions self.realpart = 0 # Number of partitions in the partition table @@ -304,7 +313,10 @@ class PartitionedImage(): # all partitions (in bytes) self.ptable_format = ptable_format # Partition table format # Disk system identifier - self.identifier = random.SystemRandom().randint(1, 0xffffffff) + if os.getenv('SOURCE_DATE_EPOCH'): + self.identifier = random.Random(int(os.getenv('SOURCE_DATE_EPOCH'))).randint(1, 0xffffffff) + else: + self.identifier = random.SystemRandom().randint(1, 0xffffffff) self.partitions = partitions self.partimages = [] @@ -312,6 +324,7 @@ class PartitionedImage(): self.sector_size = SECTOR_SIZE self.native_sysroot = native_sysroot num_real_partitions = len([p for p in self.partitions if not p.no_table]) + self.extra_space = extra_space # calculate the real partition number, accounting for partitions not # in the partition table and logical partitions @@ -329,7 +342,7 @@ class PartitionedImage(): # generate parition and filesystem UUIDs for part in self.partitions: if not part.uuid and part.use_uuid: - if self.ptable_format == 'gpt': + if self.ptable_format in ('gpt', 'gpt-hybrid'): part.uuid = str(uuid.uuid4()) else: # msdos partition table part.uuid = '%08x-%02d' % (self.identifier, part.realnum) @@ -385,6 +398,10 @@ class PartitionedImage(): raise WicError("setting custom partition type is not " \ "implemented for msdos partitions") + if part.mbr and self.ptable_format != 'gpt-hybrid': + raise WicError("Partition may only be included in MBR with " \ + "a gpt-hybrid partition table") + # Get the disk where the partition is located self.numpart += 1 if not part.no_table: @@ -393,7 +410,7 @@ class PartitionedImage(): if self.numpart == 1: if self.ptable_format == "msdos": overhead = MBR_OVERHEAD - elif self.ptable_format == "gpt": + elif self.ptable_format in ("gpt", "gpt-hybrid"): overhead = GPT_OVERHEAD # Skip one sector required for the partitioning scheme overhead @@ -477,10 +494,11 @@ class PartitionedImage(): # Once all the partitions have been layed out, we can calculate the # minumim disk size self.min_size = self.offset - if self.ptable_format == "gpt": + if self.ptable_format in ("gpt", "gpt-hybrid"): self.min_size += GPT_OVERHEAD self.min_size *= self.sector_size + self.min_size += self.extra_space def _create_partition(self, device, parttype, fstype, start, size): """ Create a partition on an image described by the 'device' object. """ @@ -497,22 +515,49 @@ class PartitionedImage(): return exec_native_cmd(cmd, self.native_sysroot) + def _write_identifier(self, device, identifier): + logger.debug("Set disk identifier %x", identifier) + with open(device, 'r+b') as img: + img.seek(0x1B8) + img.write(identifier.to_bytes(4, 'little')) + + def _make_disk(self, device, ptable_format, min_size): + logger.debug("Creating sparse file %s", device) + with open(device, 'w') as sparse: + os.ftruncate(sparse.fileno(), min_size) + + logger.debug("Initializing partition table for %s", device) + exec_native_cmd("parted -s %s mklabel %s" % (device, ptable_format), + self.native_sysroot) + + def _write_disk_guid(self): + if self.ptable_format in ('gpt', 'gpt-hybrid'): + if os.getenv('SOURCE_DATE_EPOCH'): + self.disk_guid = uuid.UUID(int=int(os.getenv('SOURCE_DATE_EPOCH'))) + else: + self.disk_guid = uuid.uuid4() + + logger.debug("Set disk guid %s", self.disk_guid) + sfdisk_cmd = "sfdisk --disk-id %s %s" % (self.path, self.disk_guid) + exec_native_cmd(sfdisk_cmd, self.native_sysroot) + def create(self): - logger.debug("Creating sparse file %s", self.path) - with open(self.path, 'w') as sparse: - os.ftruncate(sparse.fileno(), self.min_size) + self._make_disk(self.path, + "gpt" if self.ptable_format == "gpt-hybrid" else self.ptable_format, + self.min_size) - logger.debug("Initializing partition table for %s", self.path) - exec_native_cmd("parted -s %s mklabel %s" % - (self.path, self.ptable_format), self.native_sysroot) + self._write_identifier(self.path, self.identifier) + self._write_disk_guid() - logger.debug("Set disk identifier %x", self.identifier) - with open(self.path, 'r+b') as img: - img.seek(0x1B8) - img.write(self.identifier.to_bytes(4, 'little')) + if self.ptable_format == "gpt-hybrid": + mbr_path = self.path + ".mbr" + self._make_disk(mbr_path, "msdos", self.min_size) + self._write_identifier(mbr_path, self.identifier) logger.debug("Creating partitions") + hybrid_mbr_part_num = 0 + for part in self.partitions: if part.num == 0: continue @@ -557,11 +602,19 @@ class PartitionedImage(): self._create_partition(self.path, part.type, parted_fs_type, part.start, part.size_sec) - if part.part_name: + if self.ptable_format == "gpt-hybrid" and part.mbr: + hybrid_mbr_part_num += 1 + if hybrid_mbr_part_num > 4: + raise WicError("Extended MBR partitions are not supported in hybrid MBR") + self._create_partition(mbr_path, "primary", + parted_fs_type, part.start, part.size_sec) + + if self.ptable_format in ("gpt", "gpt-hybrid") and (part.part_name or part.label): + partition_label = part.part_name if part.part_name else part.label logger.debug("partition %d: set name to %s", - part.num, part.part_name) + part.num, partition_label) exec_native_cmd("sgdisk --change-name=%d:%s %s" % \ - (part.num, part.part_name, + (part.num, partition_label, self.path), self.native_sysroot) if part.part_type: @@ -571,32 +624,55 @@ class PartitionedImage(): (part.num, part.part_type, self.path), self.native_sysroot) - if part.uuid and self.ptable_format == "gpt": + if part.uuid and self.ptable_format in ("gpt", "gpt-hybrid"): logger.debug("partition %d: set UUID to %s", part.num, part.uuid) exec_native_cmd("sgdisk --partition-guid=%d:%s %s" % \ (part.num, part.uuid, self.path), self.native_sysroot) - if part.label and self.ptable_format == "gpt": - logger.debug("partition %d: set name to %s", - part.num, part.label) - exec_native_cmd("parted -s %s name %d %s" % \ - (self.path, part.num, part.label), - self.native_sysroot) - if part.active: - flag_name = "legacy_boot" if self.ptable_format == 'gpt' else "boot" + flag_name = "legacy_boot" if self.ptable_format in ('gpt', 'gpt-hybrid') else "boot" logger.debug("Set '%s' flag for partition '%s' on disk '%s'", flag_name, part.num, self.path) exec_native_cmd("parted -s %s set %d %s on" % \ (self.path, part.num, flag_name), self.native_sysroot) + if self.ptable_format == 'gpt-hybrid' and part.mbr: + exec_native_cmd("parted -s %s set %d %s on" % \ + (mbr_path, hybrid_mbr_part_num, "boot"), + self.native_sysroot) if part.system_id: exec_native_cmd("sfdisk --part-type %s %s %s" % \ (self.path, part.num, part.system_id), self.native_sysroot) + if part.hidden and self.ptable_format == "gpt": + logger.debug("Set hidden attribute for partition '%s' on disk '%s'", + part.num, self.path) + exec_native_cmd("sfdisk --part-attrs %s %s RequiredPartition" % \ + (self.path, part.num), + self.native_sysroot) + + if self.ptable_format == "gpt-hybrid": + # Write a protective GPT partition + hybrid_mbr_part_num += 1 + if hybrid_mbr_part_num > 4: + raise WicError("Extended MBR partitions are not supported in hybrid MBR") + + # parted cannot directly create a protective GPT partition, so + # create with an arbitrary type, then change it to the correct type + # with sfdisk + self._create_partition(mbr_path, "primary", "fat32", 1, GPT_OVERHEAD) + exec_native_cmd("sfdisk --part-type %s %d 0xee" % (mbr_path, hybrid_mbr_part_num), + self.native_sysroot) + + # Copy hybrid MBR + with open(mbr_path, "rb") as mbr_file: + with open(self.path, "r+b") as image_file: + mbr = mbr_file.read(512) + image_file.write(mbr) + def cleanup(self): pass diff --git a/scripts/lib/wic/plugins/source/bootimg-efi.py b/scripts/lib/wic/plugins/source/bootimg-efi.py index cdc72543c2..13a9cddf4e 100644 --- a/scripts/lib/wic/plugins/source/bootimg-efi.py +++ b/scripts/lib/wic/plugins/source/bootimg-efi.py @@ -12,6 +12,7 @@ import logging import os +import tempfile import shutil import re @@ -34,6 +35,26 @@ class BootimgEFIPlugin(SourcePlugin): name = 'bootimg-efi' @classmethod + def _copy_additional_files(cls, hdddir, initrd, dtb): + bootimg_dir = get_bitbake_var("DEPLOY_DIR_IMAGE") + if not bootimg_dir: + raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting") + + if initrd: + initrds = initrd.split(';') + for rd in initrds: + cp_cmd = "cp %s/%s %s" % (bootimg_dir, rd, hdddir) + exec_cmd(cp_cmd, True) + else: + logger.debug("Ignoring missing initrd") + + if dtb: + if ';' in dtb: + raise WicError("Only one DTB supported, exiting") + cp_cmd = "cp %s/%s %s" % (bootimg_dir, dtb, hdddir) + exec_cmd(cp_cmd, True) + + @classmethod def do_configure_grubefi(cls, hdddir, creator, cr_workdir, source_params): """ Create loader-specific (grub-efi) config @@ -52,18 +73,9 @@ class BootimgEFIPlugin(SourcePlugin): "get it from %s." % configfile) initrd = source_params.get('initrd') + dtb = source_params.get('dtb') - if initrd: - bootimg_dir = get_bitbake_var("DEPLOY_DIR_IMAGE") - if not bootimg_dir: - raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting") - - initrds = initrd.split(';') - for rd in initrds: - cp_cmd = "cp %s/%s %s" % (bootimg_dir, rd, hdddir) - exec_cmd(cp_cmd, True) - else: - logger.debug("Ignoring missing initrd") + cls._copy_additional_files(hdddir, initrd, dtb) if not custom_cfg: # Create grub configuration using parameters from wks file @@ -97,6 +109,9 @@ class BootimgEFIPlugin(SourcePlugin): grubefi_conf += " /%s" % rd grubefi_conf += "\n" + if dtb: + grubefi_conf += "devicetree /%s\n" % dtb + grubefi_conf += "}\n" logger.debug("Writing grubefi config %s/hdd/boot/EFI/BOOT/grub.cfg", @@ -118,24 +133,18 @@ class BootimgEFIPlugin(SourcePlugin): bootloader = creator.ks.bootloader + unified_image = source_params.get('create-unified-kernel-image') == "true" + loader_conf = "" - loader_conf += "default boot\n" + if not unified_image: + loader_conf += "default boot\n" loader_conf += "timeout %d\n" % bootloader.timeout initrd = source_params.get('initrd') + dtb = source_params.get('dtb') - if initrd: - # obviously we need to have a common common deploy var - bootimg_dir = get_bitbake_var("DEPLOY_DIR_IMAGE") - if not bootimg_dir: - raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting") - - initrds = initrd.split(';') - for rd in initrds: - cp_cmd = "cp %s/%s %s" % (bootimg_dir, rd, hdddir) - exec_cmd(cp_cmd, True) - else: - logger.debug("Ignoring missing initrd") + if not unified_image: + cls._copy_additional_files(hdddir, initrd, dtb) logger.debug("Writing systemd-boot config " "%s/hdd/boot/loader/loader.conf", cr_workdir) @@ -183,11 +192,15 @@ class BootimgEFIPlugin(SourcePlugin): for rd in initrds: boot_conf += "initrd /%s\n" % rd - logger.debug("Writing systemd-boot config " - "%s/hdd/boot/loader/entries/boot.conf", cr_workdir) - cfg = open("%s/hdd/boot/loader/entries/boot.conf" % cr_workdir, "w") - cfg.write(boot_conf) - cfg.close() + if dtb: + boot_conf += "devicetree /%s\n" % dtb + + if not unified_image: + logger.debug("Writing systemd-boot config " + "%s/hdd/boot/loader/entries/boot.conf", cr_workdir) + cfg = open("%s/hdd/boot/loader/entries/boot.conf" % cr_workdir, "w") + cfg.write(boot_conf) + cfg.close() @classmethod @@ -207,6 +220,8 @@ class BootimgEFIPlugin(SourcePlugin): cls.do_configure_grubefi(hdddir, creator, cr_workdir, source_params) elif source_params['loader'] == 'systemd-boot': cls.do_configure_systemdboot(hdddir, creator, cr_workdir, source_params) + elif source_params['loader'] == 'uefi-kernel': + pass else: raise WicError("unrecognized bootimg-efi loader: %s" % source_params['loader']) except KeyError: @@ -288,9 +303,107 @@ class BootimgEFIPlugin(SourcePlugin): kernel = "%s-%s.bin" % \ (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME")) - install_cmd = "install -m 0644 %s/%s %s/%s" % \ - (staging_kernel_dir, kernel, hdddir, kernel) - exec_cmd(install_cmd) + if source_params.get('create-unified-kernel-image') == "true": + initrd = source_params.get('initrd') + if not initrd: + raise WicError("initrd= must be specified when create-unified-kernel-image=true, exiting") + + deploy_dir = get_bitbake_var("DEPLOY_DIR_IMAGE") + efi_stub = glob("%s/%s" % (deploy_dir, "linux*.efi.stub")) + if len(efi_stub) == 0: + raise WicError("Unified Kernel Image EFI stub not found, exiting") + efi_stub = efi_stub[0] + + with tempfile.TemporaryDirectory() as tmp_dir: + label = source_params.get('label') + label_conf = "root=%s" % creator.rootdev + if label: + label_conf = "LABEL=%s" % label + + bootloader = creator.ks.bootloader + cmdline = open("%s/cmdline" % tmp_dir, "w") + cmdline.write("%s %s" % (label_conf, bootloader.append)) + cmdline.close() + + initrds = initrd.split(';') + initrd = open("%s/initrd" % tmp_dir, "wb") + for f in initrds: + with open("%s/%s" % (deploy_dir, f), 'rb') as in_file: + shutil.copyfileobj(in_file, initrd) + initrd.close() + + # Searched by systemd-boot: + # https://systemd.io/BOOT_LOADER_SPECIFICATION/#type-2-efi-unified-kernel-images + install_cmd = "install -d %s/EFI/Linux" % hdddir + exec_cmd(install_cmd) + + staging_dir_host = get_bitbake_var("STAGING_DIR_HOST") + target_sys = get_bitbake_var("TARGET_SYS") + + objdump_cmd = "%s-objdump" % target_sys + objdump_cmd += " -p %s" % efi_stub + objdump_cmd += " | awk '{ if ($1 == \"SectionAlignment\"){print $2} }'" + + ret, align_str = exec_native_cmd(objdump_cmd, native_sysroot) + align = int(align_str, 16) + + objdump_cmd = "%s-objdump" % target_sys + objdump_cmd += " -h %s | tail -2" % efi_stub + ret, output = exec_native_cmd(objdump_cmd, native_sysroot) + + offset = int(output.split()[2], 16) + int(output.split()[3], 16) + + osrel_off = offset + align - offset % align + osrel_path = "%s/usr/lib/os-release" % staging_dir_host + osrel_sz = os.stat(osrel_path).st_size + + cmdline_off = osrel_off + osrel_sz + cmdline_off = cmdline_off + align - cmdline_off % align + cmdline_sz = os.stat(cmdline.name).st_size + + dtb_off = cmdline_off + cmdline_sz + dtb_off = dtb_off + align - dtb_off % align + + dtb = source_params.get('dtb') + if dtb: + if ';' in dtb: + raise WicError("Only one DTB supported, exiting") + dtb_path = "%s/%s" % (deploy_dir, dtb) + dtb_params = '--add-section .dtb=%s --change-section-vma .dtb=0x%x' % \ + (dtb_path, dtb_off) + linux_off = dtb_off + os.stat(dtb_path).st_size + linux_off = linux_off + align - linux_off % align + else: + dtb_params = '' + linux_off = dtb_off + + linux_path = "%s/%s" % (staging_kernel_dir, kernel) + linux_sz = os.stat(linux_path).st_size + + initrd_off = linux_off + linux_sz + initrd_off = initrd_off + align - initrd_off % align + + # https://www.freedesktop.org/software/systemd/man/systemd-stub.html + objcopy_cmd = "%s-objcopy" % target_sys + objcopy_cmd += " --enable-deterministic-archives" + objcopy_cmd += " --preserve-dates" + objcopy_cmd += " --add-section .osrel=%s" % osrel_path + objcopy_cmd += " --change-section-vma .osrel=0x%x" % osrel_off + objcopy_cmd += " --add-section .cmdline=%s" % cmdline.name + objcopy_cmd += " --change-section-vma .cmdline=0x%x" % cmdline_off + objcopy_cmd += dtb_params + objcopy_cmd += " --add-section .linux=%s" % linux_path + objcopy_cmd += " --change-section-vma .linux=0x%x" % linux_off + objcopy_cmd += " --add-section .initrd=%s" % initrd.name + objcopy_cmd += " --change-section-vma .initrd=0x%x" % initrd_off + objcopy_cmd += " %s %s/EFI/Linux/linux.efi" % (efi_stub, hdddir) + + exec_native_cmd(objcopy_cmd, native_sysroot) + else: + if source_params.get('install-kernel-into-boot-dir') != 'false': + install_cmd = "install -m 0644 %s/%s %s/%s" % \ + (staging_kernel_dir, kernel, hdddir, kernel) + exec_cmd(install_cmd) if get_bitbake_var("IMAGE_EFI_BOOT_FILES"): for src_path, dst_path in cls.install_task: @@ -312,6 +425,28 @@ class BootimgEFIPlugin(SourcePlugin): for mod in [x for x in os.listdir(kernel_dir) if x.startswith("systemd-")]: cp_cmd = "cp %s/%s %s/EFI/BOOT/%s" % (kernel_dir, mod, hdddir, mod[8:]) exec_cmd(cp_cmd, True) + elif source_params['loader'] == 'uefi-kernel': + kernel = get_bitbake_var("KERNEL_IMAGETYPE") + if not kernel: + raise WicError("Empty KERNEL_IMAGETYPE %s\n" % target) + target = get_bitbake_var("TARGET_SYS") + if not target: + raise WicError("Unknown arch (TARGET_SYS) %s\n" % target) + + if re.match("x86_64", target): + kernel_efi_image = "bootx64.efi" + elif re.match('i.86', target): + kernel_efi_image = "bootia32.efi" + elif re.match('aarch64', target): + kernel_efi_image = "bootaa64.efi" + elif re.match('arm', target): + kernel_efi_image = "bootarm.efi" + else: + raise WicError("UEFI stub kernel is incompatible with target %s" % target) + + for mod in [x for x in os.listdir(kernel_dir) if x.startswith(kernel)]: + cp_cmd = "cp %s/%s %s/EFI/BOOT/%s" % (kernel_dir, mod, hdddir, kernel_efi_image) + exec_cmd(cp_cmd, True) else: raise WicError("unrecognized bootimg-efi loader: %s" % source_params['loader']) @@ -323,6 +458,11 @@ class BootimgEFIPlugin(SourcePlugin): cp_cmd = "cp %s %s/" % (startup, hdddir) exec_cmd(cp_cmd, True) + for paths in part.include_path or []: + for path in paths: + cp_cmd = "cp -r %s %s/" % (path, hdddir) + exec_cmd(cp_cmd, True) + du_cmd = "du -bks %s" % hdddir out = exec_cmd(du_cmd) blocks = int(out.split()[0]) @@ -337,6 +477,13 @@ class BootimgEFIPlugin(SourcePlugin): logger.debug("Added %d extra blocks to %s to get to %d total blocks", extra_blocks, part.mountpoint, blocks) + # required for compatibility with certain devices expecting file system + # block count to be equal to partition block count + if blocks < part.fixed_size: + blocks = part.fixed_size + logger.debug("Overriding %s to %d total blocks for compatibility", + part.mountpoint, blocks) + # dosfs image, created by mkdosfs bootimg = "%s/boot.img" % cr_workdir diff --git a/scripts/lib/wic/plugins/source/bootimg-partition.py b/scripts/lib/wic/plugins/source/bootimg-partition.py index 5dbe2558d2..1071d1af3f 100644 --- a/scripts/lib/wic/plugins/source/bootimg-partition.py +++ b/scripts/lib/wic/plugins/source/bootimg-partition.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # # DESCRIPTION @@ -30,6 +32,7 @@ class BootimgPartitionPlugin(SourcePlugin): """ name = 'bootimg-partition' + image_boot_files_var_name = 'IMAGE_BOOT_FILES' @classmethod def do_configure_partition(cls, part, source_params, cr, cr_workdir, @@ -54,12 +57,12 @@ class BootimgPartitionPlugin(SourcePlugin): else: var = "" - boot_files = get_bitbake_var("IMAGE_BOOT_FILES" + var) + boot_files = get_bitbake_var(cls.image_boot_files_var_name + var) if boot_files is not None: break if boot_files is None: - raise WicError('No boot files defined, IMAGE_BOOT_FILES unset for entry #%d' % part.lineno) + raise WicError('No boot files defined, %s unset for entry #%d' % (cls.image_boot_files_var_name, part.lineno)) logger.debug('Boot files: %s', boot_files) @@ -110,7 +113,7 @@ class BootimgPartitionPlugin(SourcePlugin): # Use a custom configuration for extlinux.conf extlinux_conf = custom_cfg logger.debug("Using custom configuration file " - "%s for extlinux.cfg", configfile) + "%s for extlinux.conf", configfile) else: raise WicError("configfile is specified but failed to " "get it from %s." % configfile) diff --git a/scripts/lib/wic/plugins/source/bootimg-pcbios.py b/scripts/lib/wic/plugins/source/bootimg-pcbios.py index f2639e7004..a207a83530 100644 --- a/scripts/lib/wic/plugins/source/bootimg-pcbios.py +++ b/scripts/lib/wic/plugins/source/bootimg-pcbios.py @@ -122,7 +122,7 @@ class BootimgPcbiosPlugin(SourcePlugin): syslinux_conf += "DEFAULT boot\n" syslinux_conf += "LABEL boot\n" - kernel = "/vmlinuz" + kernel = "/" + get_bitbake_var("KERNEL_IMAGETYPE") syslinux_conf += "KERNEL " + kernel + "\n" syslinux_conf += "APPEND label=boot root=%s %s\n" % \ @@ -155,8 +155,8 @@ class BootimgPcbiosPlugin(SourcePlugin): kernel = "%s-%s.bin" % \ (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME")) - cmds = ("install -m 0644 %s/%s %s/vmlinuz" % - (staging_kernel_dir, kernel, hdddir), + cmds = ("install -m 0644 %s/%s %s/%s" % + (staging_kernel_dir, kernel, hdddir, get_bitbake_var("KERNEL_IMAGETYPE")), "install -m 444 %s/syslinux/ldlinux.sys %s/ldlinux.sys" % (bootimg_dir, hdddir), "install -m 0644 %s/syslinux/vesamenu.c32 %s/vesamenu.c32" % @@ -186,8 +186,10 @@ class BootimgPcbiosPlugin(SourcePlugin): # dosfs image, created by mkdosfs bootimg = "%s/boot%s.img" % (cr_workdir, part.lineno) - dosfs_cmd = "mkdosfs -n boot -i %s -S 512 -C %s %d" % \ - (part.fsuuid, bootimg, blocks) + label = part.label if part.label else "boot" + + dosfs_cmd = "mkdosfs -n %s -i %s -S 512 -C %s %d" % \ + (label, part.fsuuid, bootimg, blocks) exec_native_cmd(dosfs_cmd, native_sysroot) mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (bootimg, hdddir) diff --git a/scripts/lib/wic/plugins/source/empty.py b/scripts/lib/wic/plugins/source/empty.py index 041617d648..4178912377 100644 --- a/scripts/lib/wic/plugins/source/empty.py +++ b/scripts/lib/wic/plugins/source/empty.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: MIT # @@ -7,9 +9,19 @@ # To use it you must pass "empty" as argument for the "--source" parameter in # the wks file. For example: # part foo --source empty --ondisk sda --size="1024" --align 1024 +# +# The plugin supports writing zeros to the start of the +# partition. This is useful to overwrite old content like +# filesystem signatures which may be re-recognized otherwise. +# This feature can be enabled with +# '--sourceparams="[fill|size=<N>[S|s|K|k|M|G]][,][bs=<N>[S|s|K|k|M|G]]"' +# Conflicting or missing options throw errors. import logging +import os +from wic import WicError +from wic.ksparser import sizetype from wic.pluginbase import SourcePlugin logger = logging.getLogger('wic') @@ -17,6 +29,16 @@ logger = logging.getLogger('wic') class EmptyPartitionPlugin(SourcePlugin): """ Populate unformatted empty partition. + + The following sourceparams are supported: + - fill + Fill the entire partition with zeros. Requires '--fixed-size' option + to be set. + - size=<N>[S|s|K|k|M|G] + Set the first N bytes of the partition to zero. Default unit is 'K'. + - bs=<N>[S|s|K|k|M|G] + Write at most N bytes at a time during source file creation. + Defaults to '1M'. Default unit is 'K'. """ name = 'empty' @@ -29,4 +51,39 @@ class EmptyPartitionPlugin(SourcePlugin): Called to do the actual content population for a partition i.e. it 'prepares' the partition to be incorporated into the image. """ - return + get_byte_count = sizetype('K', True) + size = 0 + + if 'fill' in source_params and 'size' in source_params: + raise WicError("Conflicting source parameters 'fill' and 'size' specified, exiting.") + + # Set the size of the zeros to be written to the partition + if 'fill' in source_params: + if part.fixed_size == 0: + raise WicError("Source parameter 'fill' only works with the '--fixed-size' option, exiting.") + size = get_byte_count(part.fixed_size) + elif 'size' in source_params: + size = get_byte_count(source_params['size']) + + if size == 0: + # Nothing to do, create empty partition + return + + if 'bs' in source_params: + bs = get_byte_count(source_params['bs']) + else: + bs = get_byte_count('1M') + + # Create a binary file of the requested size filled with zeros + source_file = os.path.join(cr_workdir, 'empty-plugin-zeros%s.bin' % part.lineno) + if not os.path.exists(os.path.dirname(source_file)): + os.makedirs(os.path.dirname(source_file)) + + quotient, remainder = divmod(size, bs) + with open(source_file, 'wb') as file: + for _ in range(quotient): + file.write(bytearray(bs)) + file.write(bytearray(remainder)) + + part.size = (size + 1024 - 1) // 1024 # size in KB rounded up + part.source_file = source_file diff --git a/scripts/lib/wic/plugins/source/isoimage-isohybrid.py b/scripts/lib/wic/plugins/source/isoimage-isohybrid.py index afc9ea0f8f..607356ad13 100644 --- a/scripts/lib/wic/plugins/source/isoimage-isohybrid.py +++ b/scripts/lib/wic/plugins/source/isoimage-isohybrid.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # # DESCRIPTION diff --git a/scripts/lib/wic/plugins/source/rawcopy.py b/scripts/lib/wic/plugins/source/rawcopy.py index 3c4997d8ba..21903c2f23 100644 --- a/scripts/lib/wic/plugins/source/rawcopy.py +++ b/scripts/lib/wic/plugins/source/rawcopy.py @@ -1,9 +1,13 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # import logging import os +import signal +import subprocess from wic import WicError from wic.pluginbase import SourcePlugin @@ -21,6 +25,10 @@ class RawCopyPlugin(SourcePlugin): @staticmethod def do_image_label(fstype, dst, label): + # don't create label when fstype is none + if fstype == 'none': + return + if fstype.startswith('ext'): cmd = 'tune2fs -L %s %s' % (label, dst) elif fstype in ('msdos', 'vfat'): @@ -29,15 +37,35 @@ class RawCopyPlugin(SourcePlugin): cmd = 'btrfs filesystem label %s %s' % (dst, label) elif fstype == 'swap': cmd = 'mkswap -L %s %s' % (label, dst) - elif fstype == 'squashfs': - raise WicError("It's not possible to update a squashfs " - "filesystem label '%s'" % (label)) + elif fstype in ('squashfs', 'erofs'): + raise WicError("It's not possible to update a %s " + "filesystem label '%s'" % (fstype, label)) else: raise WicError("Cannot update filesystem label: " "Unknown fstype: '%s'" % (fstype)) exec_cmd(cmd) + @staticmethod + def do_image_uncompression(src, dst, workdir): + def subprocess_setup(): + # Python installs a SIGPIPE handler by default. This is usually not what + # non-Python subprocesses expect. + # SIGPIPE errors are known issues with gzip/bash + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + + extension = os.path.splitext(src)[1] + decompressor = { + ".bz2": "bzip2", + ".gz": "gzip", + ".xz": "xz", + ".zst": "zstd -f", + }.get(extension) + if not decompressor: + raise WicError("Not supported compressor filename extension: %s" % extension) + cmd = "%s -dc %s > %s" % (decompressor, src, dst) + subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True, cwd=workdir) + @classmethod def do_prepare_partition(cls, part, source_params, cr, cr_workdir, oe_builddir, bootimg_dir, kernel_dir, @@ -56,7 +84,13 @@ class RawCopyPlugin(SourcePlugin): if 'file' not in source_params: raise WicError("No file specified") - src = os.path.join(kernel_dir, source_params['file']) + if 'unpack' in source_params: + img = os.path.join(kernel_dir, source_params['file']) + src = os.path.join(cr_workdir, os.path.splitext(source_params['file'])[0]) + RawCopyPlugin.do_image_uncompression(img, src, cr_workdir) + else: + src = os.path.join(kernel_dir, source_params['file']) + dst = os.path.join(cr_workdir, "%s.%s" % (os.path.basename(source_params['file']), part.lineno)) if not os.path.exists(os.path.dirname(dst)): diff --git a/scripts/lib/wic/plugins/source/rootfs.py b/scripts/lib/wic/plugins/source/rootfs.py index 96d940a91d..c990143c0d 100644 --- a/scripts/lib/wic/plugins/source/rootfs.py +++ b/scripts/lib/wic/plugins/source/rootfs.py @@ -35,7 +35,7 @@ class RootfsPlugin(SourcePlugin): @staticmethod def __validate_path(cmd, rootfs_dir, path): if os.path.isabs(path): - logger.error("%s: Must be relative: %s" % (cmd, orig_path)) + logger.error("%s: Must be relative: %s" % (cmd, path)) sys.exit(1) # Disallow climbing outside of parent directory using '..', @@ -43,14 +43,14 @@ class RootfsPlugin(SourcePlugin): # directory, or modify a directory outside OpenEmbedded). full_path = os.path.realpath(os.path.join(rootfs_dir, path)) if not full_path.startswith(os.path.realpath(rootfs_dir)): - logger.error("%s: Must point inside the rootfs:" % (cmd, path)) + logger.error("%s: Must point inside the rootfs: %s" % (cmd, path)) sys.exit(1) return full_path @staticmethod def __get_rootfs_dir(rootfs_dir): - if os.path.isdir(rootfs_dir): + if rootfs_dir and os.path.isdir(rootfs_dir): return os.path.realpath(rootfs_dir) image_rootfs_dir = get_bitbake_var("IMAGE_ROOTFS", rootfs_dir) @@ -97,6 +97,9 @@ class RootfsPlugin(SourcePlugin): part.has_fstab = os.path.exists(os.path.join(part.rootfs_dir, "etc/fstab")) pseudo_dir = os.path.join(part.rootfs_dir, "../pseudo") if not os.path.lexists(pseudo_dir): + pseudo_dir = os.path.join(cls.__get_rootfs_dir(None), '../pseudo') + + if not os.path.lexists(pseudo_dir): logger.warn("%s folder does not exist. " "Usernames and permissions will be invalid " % pseudo_dir) pseudo_dir = None @@ -218,10 +221,10 @@ class RootfsPlugin(SourcePlugin): # Update part.has_fstab here as fstab may have been added or # removed by the above modifications. part.has_fstab = os.path.exists(os.path.join(new_rootfs, "etc/fstab")) - if part.update_fstab_in_rootfs and part.has_fstab: + if part.update_fstab_in_rootfs and part.has_fstab and not part.no_fstab_update: fstab_path = os.path.join(new_rootfs, "etc/fstab") # Assume that fstab should always be owned by root with fixed permissions - install_cmd = "install -m 0644 %s %s" % (part.updated_fstab_path, fstab_path) + install_cmd = "install -m 0644 -p %s %s" % (part.updated_fstab_path, fstab_path) if new_pseudo: pseudo = cls.__get_pseudo(native_sysroot, new_rootfs, new_pseudo) else: |