diff options
Diffstat (limited to 'meta/lib/oe/packagedata.py')
| -rw-r--r-- | meta/lib/oe/packagedata.py | 276 |
1 files changed, 272 insertions, 4 deletions
diff --git a/meta/lib/oe/packagedata.py b/meta/lib/oe/packagedata.py index 22261d271e..2d1d6ddeb7 100644 --- a/meta/lib/oe/packagedata.py +++ b/meta/lib/oe/packagedata.py @@ -1,9 +1,16 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # import codecs import os +import json +import bb.compress.zstd +import oe.path + +from glob import glob def packaged(pkg, d): return os.access(get_subpkgedata_fn(pkg, d) + '.packaged', os.R_OK) @@ -19,7 +26,7 @@ def read_pkgdatafile(fn): import re with open(fn, 'r') as f: lines = f.readlines() - r = re.compile("(^.+?):\s+(.*)") + r = re.compile(r"(^.+?):\s+(.*)") for l in lines: m = r.match(l) if m: @@ -45,18 +52,30 @@ def read_pkgdata(pn, d): return read_pkgdatafile(fn) # -# Collapse FOO_pkg variables into FOO +# Collapse FOO:pkg variables into FOO # def read_subpkgdata_dict(pkg, d): ret = {} subd = read_pkgdatafile(get_subpkgedata_fn(pkg, d)) for var in subd: - newvar = var.replace("_" + pkg, "") - if newvar == var and var + "_" + pkg in subd: + newvar = var.replace(":" + pkg, "") + if newvar == var and var + ":" + pkg in subd: continue ret[newvar] = subd[var] return ret +def read_subpkgdata_extended(pkg, d): + import json + import bb.compress.zstd + + fn = d.expand("${PKGDATA_DIR}/extended/%s.json.zstd" % pkg) + try: + num_threads = int(d.getVar("BB_NUMBER_THREADS")) + with bb.compress.zstd.open(fn, "rt", encoding="utf-8", num_threads=num_threads) as f: + return json.load(f) + except FileNotFoundError: + return None + def _pkgmap(d): """Return a dictionary mapping package to recipe name.""" @@ -96,3 +115,252 @@ def recipename(pkg, d): """Return the recipe name for the given binary package name.""" return pkgmap(d).get(pkg) + +def foreach_runtime_provider_pkgdata(d, rdep, include_rdep=False): + pkgdata_dir = d.getVar("PKGDATA_DIR") + possibles = set() + try: + possibles |= set(os.listdir("%s/runtime-rprovides/%s/" % (pkgdata_dir, rdep))) + except OSError: + pass + + if include_rdep: + possibles.add(rdep) + + for p in sorted(list(possibles)): + rdep_data = read_subpkgdata(p, d) + yield p, rdep_data + +def get_package_mapping(pkg, basepkg, d, depversions=None): + import oe.packagedata + + data = oe.packagedata.read_subpkgdata(pkg, d) + key = "PKG:%s" % pkg + + if key in data: + if bb.data.inherits_class('allarch', d) and bb.data.inherits_class('packagegroup', d) and pkg != data[key]: + bb.error("An allarch packagegroup shouldn't depend on packages which are dynamically renamed (%s to %s)" % (pkg, data[key])) + # Have to avoid undoing the write_extra_pkgs(global_variants...) + if bb.data.inherits_class('allarch', d) and not d.getVar('MULTILIB_VARIANTS') \ + and data[key] == basepkg: + return pkg + if depversions == []: + # Avoid returning a mapping if the renamed package rprovides its original name + rprovkey = "RPROVIDES:%s" % pkg + if rprovkey in data: + if pkg in bb.utils.explode_dep_versions2(data[rprovkey]): + bb.note("%s rprovides %s, not replacing the latter" % (data[key], pkg)) + return pkg + # Do map to rewritten package name + return data[key] + + return pkg + +def get_package_additional_metadata(pkg_type, d): + base_key = "PACKAGE_ADD_METADATA" + for key in ("%s_%s" % (base_key, pkg_type.upper()), base_key): + if d.getVar(key, False) is None: + continue + d.setVarFlag(key, "type", "list") + if d.getVarFlag(key, "separator") is None: + d.setVarFlag(key, "separator", "\\n") + metadata_fields = [field.strip() for field in oe.data.typed_value(key, d)] + return "\n".join(metadata_fields).strip() + +def runtime_mapping_rename(varname, pkg, d): + #bb.note("%s before: %s" % (varname, d.getVar(varname))) + + new_depends = {} + deps = bb.utils.explode_dep_versions2(d.getVar(varname) or "") + for depend, depversions in deps.items(): + new_depend = get_package_mapping(depend, pkg, d, depversions) + if depend != new_depend: + bb.note("package name mapping done: %s -> %s" % (depend, new_depend)) + new_depends[new_depend] = deps[depend] + + d.setVar(varname, bb.utils.join_deps(new_depends, commasep=False)) + + #bb.note("%s after: %s" % (varname, d.getVar(varname))) + +def emit_pkgdata(pkgfiles, d): + def process_postinst_on_target(pkg, mlprefix): + pkgval = d.getVar('PKG:%s' % pkg) + if pkgval is None: + pkgval = pkg + + defer_fragment = """ +if [ -n "$D" ]; then + $INTERCEPT_DIR/postinst_intercept delay_to_first_boot %s mlprefix=%s + exit 0 +fi +""" % (pkgval, mlprefix) + + postinst = d.getVar('pkg_postinst:%s' % pkg) + postinst_ontarget = d.getVar('pkg_postinst_ontarget:%s' % pkg) + + if postinst_ontarget: + bb.debug(1, 'adding deferred pkg_postinst_ontarget() to pkg_postinst() for %s' % pkg) + if not postinst: + postinst = '#!/bin/sh\n' + postinst += defer_fragment + postinst += postinst_ontarget + d.setVar('pkg_postinst:%s' % pkg, postinst) + + def add_set_e_to_scriptlets(pkg): + for scriptlet_name in ('pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm'): + scriptlet = d.getVar('%s:%s' % (scriptlet_name, pkg)) + if scriptlet: + scriptlet_split = scriptlet.split('\n') + if scriptlet_split[0].startswith("#!"): + scriptlet = scriptlet_split[0] + "\nset -e\n" + "\n".join(scriptlet_split[1:]) + else: + scriptlet = "set -e\n" + "\n".join(scriptlet_split[0:]) + d.setVar('%s:%s' % (scriptlet_name, pkg), scriptlet) + + def write_if_exists(f, pkg, var): + def encode(str): + import codecs + c = codecs.getencoder("unicode_escape") + return c(str)[0].decode("latin1") + + val = d.getVar('%s:%s' % (var, pkg)) + if val: + f.write('%s:%s: %s\n' % (var, pkg, encode(val))) + return val + val = d.getVar('%s' % (var)) + if val: + f.write('%s: %s\n' % (var, encode(val))) + return val + + def write_extra_pkgs(variants, pn, packages, pkgdatadir): + for variant in variants: + with open("%s/%s-%s" % (pkgdatadir, variant, pn), 'w') as fd: + fd.write("PACKAGES: %s\n" % ' '.join( + map(lambda pkg: '%s-%s' % (variant, pkg), packages.split()))) + + def write_extra_runtime_pkgs(variants, packages, pkgdatadir): + for variant in variants: + for pkg in packages.split(): + ml_pkg = "%s-%s" % (variant, pkg) + subdata_file = "%s/runtime/%s" % (pkgdatadir, ml_pkg) + with open(subdata_file, 'w') as fd: + fd.write("PKG:%s: %s" % (ml_pkg, pkg)) + + packages = d.getVar('PACKAGES') + pkgdest = d.getVar('PKGDEST') + pkgdatadir = d.getVar('PKGDESTWORK') + + data_file = pkgdatadir + d.expand("/${PN}") + with open(data_file, 'w') as fd: + fd.write("PACKAGES: %s\n" % packages) + + pkgdebugsource = d.getVar("PKGDEBUGSOURCES") or [] + + pn = d.getVar('PN') + global_variants = (d.getVar('MULTILIB_GLOBAL_VARIANTS') or "").split() + variants = (d.getVar('MULTILIB_VARIANTS') or "").split() + + if bb.data.inherits_class('kernel', d) or bb.data.inherits_class('module-base', d): + write_extra_pkgs(variants, pn, packages, pkgdatadir) + + if bb.data.inherits_class('allarch', d) and not variants \ + and not bb.data.inherits_class('packagegroup', d): + write_extra_pkgs(global_variants, pn, packages, pkgdatadir) + + workdir = d.getVar('WORKDIR') + + for pkg in packages.split(): + pkgval = d.getVar('PKG:%s' % pkg) + if pkgval is None: + pkgval = pkg + d.setVar('PKG:%s' % pkg, pkg) + + extended_data = { + "files_info": {} + } + + pkgdestpkg = os.path.join(pkgdest, pkg) + files = {} + files_extra = {} + total_size = 0 + seen = set() + for f in pkgfiles[pkg]: + fpath = os.sep + os.path.relpath(f, pkgdestpkg) + + fstat = os.lstat(f) + files[fpath] = fstat.st_size + + extended_data["files_info"].setdefault(fpath, {}) + extended_data["files_info"][fpath]['size'] = fstat.st_size + + if fstat.st_ino not in seen: + seen.add(fstat.st_ino) + total_size += fstat.st_size + + if fpath in pkgdebugsource: + extended_data["files_info"][fpath]['debugsrc'] = pkgdebugsource[fpath] + del pkgdebugsource[fpath] + + d.setVar('FILES_INFO:' + pkg , json.dumps(files, sort_keys=True)) + + process_postinst_on_target(pkg, d.getVar("MLPREFIX")) + add_set_e_to_scriptlets(pkg) + + subdata_file = pkgdatadir + "/runtime/%s" % pkg + with open(subdata_file, 'w') as sf: + for var in (d.getVar('PKGDATA_VARS') or "").split(): + val = write_if_exists(sf, pkg, var) + + write_if_exists(sf, pkg, 'FILERPROVIDESFLIST') + for dfile in sorted((d.getVar('FILERPROVIDESFLIST:' + pkg) or "").split()): + write_if_exists(sf, pkg, 'FILERPROVIDES:' + dfile) + + write_if_exists(sf, pkg, 'FILERDEPENDSFLIST') + for dfile in sorted((d.getVar('FILERDEPENDSFLIST:' + pkg) or "").split()): + write_if_exists(sf, pkg, 'FILERDEPENDS:' + dfile) + + sf.write('%s:%s: %d\n' % ('PKGSIZE', pkg, total_size)) + + subdata_extended_file = pkgdatadir + "/extended/%s.json.zstd" % pkg + num_threads = int(d.getVar("BB_NUMBER_THREADS")) + with bb.compress.zstd.open(subdata_extended_file, "wt", encoding="utf-8", num_threads=num_threads) as f: + json.dump(extended_data, f, sort_keys=True, separators=(",", ":")) + + # Symlinks needed for rprovides lookup + rprov = d.getVar('RPROVIDES:%s' % pkg) or d.getVar('RPROVIDES') + if rprov: + for p in bb.utils.explode_deps(rprov): + subdata_sym = pkgdatadir + "/runtime-rprovides/%s/%s" % (p, pkg) + bb.utils.mkdirhier(os.path.dirname(subdata_sym)) + oe.path.relsymlink(subdata_file, subdata_sym, True) + + allow_empty = d.getVar('ALLOW_EMPTY:%s' % pkg) + if not allow_empty: + allow_empty = d.getVar('ALLOW_EMPTY') + root = "%s/%s" % (pkgdest, pkg) + os.chdir(root) + g = glob('*') + if g or allow_empty == "1": + # Symlinks needed for reverse lookups (from the final package name) + subdata_sym = pkgdatadir + "/runtime-reverse/%s" % pkgval + oe.path.relsymlink(subdata_file, subdata_sym, True) + + packagedfile = pkgdatadir + '/runtime/%s.packaged' % pkg + open(packagedfile, 'w').close() + + if bb.data.inherits_class('kernel', d) or bb.data.inherits_class('module-base', d): + write_extra_runtime_pkgs(variants, packages, pkgdatadir) + + if bb.data.inherits_class('allarch', d) and not variants \ + and not bb.data.inherits_class('packagegroup', d): + write_extra_runtime_pkgs(global_variants, packages, pkgdatadir) + +def mapping_rename_hook(d): + """ + Rewrite variables to account for package renaming in things + like debian.bbclass or manual PKG variable name changes + """ + pkg = d.getVar("PKG") + oe.packagedata.runtime_mapping_rename("RDEPENDS", pkg, d) + oe.packagedata.runtime_mapping_rename("RRECOMMENDS", pkg, d) + oe.packagedata.runtime_mapping_rename("RSUGGESTS", pkg, d) |