diff options
Diffstat (limited to 'meta/classes-global')
21 files changed, 9787 insertions, 0 deletions
diff --git a/meta/classes-global/base.bbclass b/meta/classes-global/base.bbclass new file mode 100644 index 0000000000..0999b42daa --- /dev/null +++ b/meta/classes-global/base.bbclass @@ -0,0 +1,749 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +BB_DEFAULT_TASK ?= "build" +CLASSOVERRIDE ?= "class-target" + +inherit patch +inherit staging + +inherit mirrors +inherit utils +inherit utility-tasks +inherit logging + +PACKAGECONFIG_CONFARGS ??= "" + +inherit metadata_scm + +def lsb_distro_identifier(d): + adjust = d.getVar('LSB_DISTRO_ADJUST') + adjust_func = None + if adjust: + try: + adjust_func = globals()[adjust] + except KeyError: + pass + return oe.lsb.distro_identifier(adjust_func) + +die() { + bbfatal_log "$*" +} + +oe_runmake_call() { + bbnote ${MAKE} ${EXTRA_OEMAKE} "$@" + ${MAKE} ${EXTRA_OEMAKE} "$@" +} + +oe_runmake() { + oe_runmake_call "$@" || die "oe_runmake failed" +} + + +def get_base_dep(d): + if d.getVar('INHIBIT_DEFAULT_DEPS', False): + return "" + return "${BASE_DEFAULT_DEPS}" + +BASE_DEFAULT_DEPS = "virtual/${HOST_PREFIX}gcc virtual/${HOST_PREFIX}compilerlibs virtual/libc" + +BASEDEPENDS = "" +BASEDEPENDS:class-target = "${@get_base_dep(d)}" +BASEDEPENDS:class-nativesdk = "${@get_base_dep(d)}" + +DEPENDS:prepend="${BASEDEPENDS} " + +FILESPATH = "${@base_set_filespath(["${FILE_DIRNAME}/${BP}", "${FILE_DIRNAME}/${BPN}", "${FILE_DIRNAME}/files"], d)}" +# THISDIR only works properly with imediate expansion as it has to run +# in the context of the location its used (:=) +THISDIR = "${@os.path.dirname(d.getVar('FILE'))}" + +def extra_path_elements(d): + path = "" + elements = (d.getVar('EXTRANATIVEPATH') or "").split() + for e in elements: + path = path + "${STAGING_BINDIR_NATIVE}/" + e + ":" + return path + +PATH:prepend = "${@extra_path_elements(d)}" + +def get_lic_checksum_file_list(d): + filelist = [] + lic_files = d.getVar("LIC_FILES_CHKSUM") or '' + tmpdir = d.getVar("TMPDIR") + s = d.getVar("S") + b = d.getVar("B") + workdir = d.getVar("WORKDIR") + + urls = lic_files.split() + for url in urls: + # We only care about items that are absolute paths since + # any others should be covered by SRC_URI. + try: + (method, host, path, user, pswd, parm) = bb.fetch.decodeurl(url) + if method != "file" or not path: + raise bb.fetch.MalformedUrl(url) + + if path[0] == '/': + if path.startswith((tmpdir, s, b, workdir)): + continue + filelist.append(path + ":" + str(os.path.exists(path))) + except bb.fetch.MalformedUrl: + bb.fatal(d.getVar('PN') + ": LIC_FILES_CHKSUM contains an invalid URL: " + url) + return " ".join(filelist) + +def setup_hosttools_dir(dest, toolsvar, d, fatal=True): + tools = d.getVar(toolsvar).split() + origbbenv = d.getVar("BB_ORIGENV", False) + path = origbbenv.getVar("PATH") + # Need to ignore our own scripts directories to avoid circular links + for p in path.split(":"): + if p.endswith("/scripts"): + path = path.replace(p, "/ignoreme") + bb.utils.mkdirhier(dest) + notfound = [] + for tool in tools: + desttool = os.path.join(dest, tool) + if not os.path.exists(desttool): + # clean up dead symlink + if os.path.islink(desttool): + os.unlink(desttool) + srctool = bb.utils.which(path, tool, executable=True) + # gcc/g++ may link to ccache on some hosts, e.g., + # /usr/local/bin/ccache/gcc -> /usr/bin/ccache, then which(gcc) + # would return /usr/local/bin/ccache/gcc, but what we need is + # /usr/bin/gcc, this code can check and fix that. + if os.path.islink(srctool) and os.path.basename(os.readlink(srctool)) == 'ccache': + srctool = bb.utils.which(path, tool, executable=True, direction=1) + if srctool: + os.symlink(srctool, desttool) + else: + notfound.append(tool) + + if notfound and fatal: + bb.fatal("The following required tools (as specified by HOSTTOOLS) appear to be unavailable in PATH, please install them in order to proceed:\n %s" % " ".join(notfound)) + +# We can't use vardepvalue against do_fetch directly since that would overwrite +# the other task dependencies so we use an indirect function. +python fetcher_hashes_dummyfunc() { + return +} +fetcher_hashes_dummyfunc[vardepvalue] = "${@bb.fetch.get_hashvalue(d)}" + +addtask fetch +do_fetch[dirs] = "${DL_DIR}" +do_fetch[file-checksums] = "${@bb.fetch.get_checksum_file_list(d)}" +do_fetch[file-checksums] += " ${@get_lic_checksum_file_list(d)}" +do_fetch[prefuncs] += "fetcher_hashes_dummyfunc" +do_fetch[network] = "1" +python base_do_fetch() { + + src_uri = (d.getVar('SRC_URI') or "").split() + if not src_uri: + return + + try: + fetcher = bb.fetch2.Fetch(src_uri, d) + fetcher.download() + except bb.fetch2.BBFetchException as e: + bb.fatal("Bitbake Fetcher Error: " + repr(e)) +} + +addtask unpack after do_fetch +do_unpack[dirs] = "${WORKDIR}" + +do_unpack[cleandirs] = "${@d.getVar('S') if os.path.normpath(d.getVar('S')) != os.path.normpath(d.getVar('WORKDIR')) else os.path.join('${S}', 'patches')}" + +python base_do_unpack() { + src_uri = (d.getVar('SRC_URI') or "").split() + if not src_uri: + return + + try: + fetcher = bb.fetch2.Fetch(src_uri, d) + fetcher.unpack(d.getVar('WORKDIR')) + except bb.fetch2.BBFetchException as e: + bb.fatal("Bitbake Fetcher Error: " + repr(e)) +} + +SSTATETASKS += "do_deploy_source_date_epoch" + +do_deploy_source_date_epoch () { + mkdir -p ${SDE_DEPLOYDIR} + if [ -e ${SDE_FILE} ]; then + echo "Deploying SDE from ${SDE_FILE} -> ${SDE_DEPLOYDIR}." + cp -p ${SDE_FILE} ${SDE_DEPLOYDIR}/__source_date_epoch.txt + else + echo "${SDE_FILE} not found!" + fi +} + +python do_deploy_source_date_epoch_setscene () { + sstate_setscene(d) + bb.utils.mkdirhier(d.getVar('SDE_DIR')) + sde_file = os.path.join(d.getVar('SDE_DEPLOYDIR'), '__source_date_epoch.txt') + if os.path.exists(sde_file): + target = d.getVar('SDE_FILE') + bb.debug(1, "Moving setscene SDE file %s -> %s" % (sde_file, target)) + bb.utils.rename(sde_file, target) + else: + bb.debug(1, "%s not found!" % sde_file) +} + +do_deploy_source_date_epoch[dirs] = "${SDE_DEPLOYDIR}" +do_deploy_source_date_epoch[sstate-plaindirs] = "${SDE_DEPLOYDIR}" +addtask do_deploy_source_date_epoch_setscene +addtask do_deploy_source_date_epoch before do_configure after do_patch + +python create_source_date_epoch_stamp() { + # Version: 1 + source_date_epoch = oe.reproducible.get_source_date_epoch(d, d.getVar('S')) + oe.reproducible.epochfile_write(source_date_epoch, d.getVar('SDE_FILE'), d) +} +do_unpack[postfuncs] += "create_source_date_epoch_stamp" + +def get_source_date_epoch_value(d): + return oe.reproducible.epochfile_read(d.getVar('SDE_FILE'), d) + +def get_layers_branch_rev(d): + revisions = oe.buildcfg.get_layer_revisions(d) + layers_branch_rev = ["%-20s = \"%s:%s\"" % (r[1], r[2], r[3]) for r in revisions] + i = len(layers_branch_rev)-1 + p1 = layers_branch_rev[i].find("=") + s1 = layers_branch_rev[i][p1:] + while i > 0: + p2 = layers_branch_rev[i-1].find("=") + s2= layers_branch_rev[i-1][p2:] + if s1 == s2: + layers_branch_rev[i-1] = layers_branch_rev[i-1][0:p2] + i -= 1 + else: + i -= 1 + p1 = layers_branch_rev[i].find("=") + s1= layers_branch_rev[i][p1:] + return layers_branch_rev + + +BUILDCFG_FUNCS ??= "buildcfg_vars get_layers_branch_rev buildcfg_neededvars" +BUILDCFG_FUNCS[type] = "list" + +def buildcfg_vars(d): + statusvars = oe.data.typed_value('BUILDCFG_VARS', d) + for var in statusvars: + value = d.getVar(var) + if value is not None: + yield '%-20s = "%s"' % (var, value) + +def buildcfg_neededvars(d): + needed_vars = oe.data.typed_value("BUILDCFG_NEEDEDVARS", d) + pesteruser = [] + for v in needed_vars: + val = d.getVar(v) + if not val or val == 'INVALID': + pesteruser.append(v) + + if pesteruser: + bb.fatal('The following variable(s) were not set: %s\nPlease set them directly, or choose a MACHINE or DISTRO that sets them.' % ', '.join(pesteruser)) + +addhandler base_eventhandler +base_eventhandler[eventmask] = "bb.event.ConfigParsed bb.event.MultiConfigParsed bb.event.BuildStarted bb.event.RecipePreFinalise bb.event.RecipeParsed" +python base_eventhandler() { + import bb.runqueue + + if isinstance(e, bb.event.ConfigParsed): + if not d.getVar("NATIVELSBSTRING", False): + d.setVar("NATIVELSBSTRING", lsb_distro_identifier(d)) + d.setVar("ORIGNATIVELSBSTRING", d.getVar("NATIVELSBSTRING", False)) + d.setVar('BB_VERSION', bb.__version__) + + # There might be no bb.event.ConfigParsed event if bitbake server is + # running, so check bb.event.BuildStarted too to make sure ${HOSTTOOLS_DIR} + # exists. + if isinstance(e, bb.event.ConfigParsed) or \ + (isinstance(e, bb.event.BuildStarted) and not os.path.exists(d.getVar('HOSTTOOLS_DIR'))): + # Works with the line in layer.conf which changes PATH to point here + setup_hosttools_dir(d.getVar('HOSTTOOLS_DIR'), 'HOSTTOOLS', d) + setup_hosttools_dir(d.getVar('HOSTTOOLS_DIR'), 'HOSTTOOLS_NONFATAL', d, fatal=False) + + if isinstance(e, bb.event.MultiConfigParsed): + # We need to expand SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS in each of the multiconfig data stores + # own contexts so the variables get expanded correctly for that arch, then inject back into + # the main data store. + deps = [] + for config in e.mcdata: + deps.append(e.mcdata[config].getVar("SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS")) + deps = " ".join(deps) + e.mcdata[''].setVar("SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS", deps) + + if isinstance(e, bb.event.BuildStarted): + localdata = bb.data.createCopy(d) + statuslines = [] + for func in oe.data.typed_value('BUILDCFG_FUNCS', localdata): + g = globals() + if func not in g: + bb.warn("Build configuration function '%s' does not exist" % func) + else: + flines = g[func](localdata) + if flines: + statuslines.extend(flines) + + statusheader = d.getVar('BUILDCFG_HEADER') + if statusheader: + bb.plain('\n%s\n%s\n' % (statusheader, '\n'.join(statuslines))) + + # This code is to silence warnings where the SDK variables overwrite the + # target ones and we'd see duplicate key names overwriting each other + # for various PREFERRED_PROVIDERS + if isinstance(e, bb.event.RecipePreFinalise): + if d.getVar("TARGET_PREFIX") == d.getVar("SDK_PREFIX"): + d.delVar("PREFERRED_PROVIDER_virtual/${TARGET_PREFIX}binutils") + d.delVar("PREFERRED_PROVIDER_virtual/${TARGET_PREFIX}gcc") + d.delVar("PREFERRED_PROVIDER_virtual/${TARGET_PREFIX}g++") + d.delVar("PREFERRED_PROVIDER_virtual/${TARGET_PREFIX}compilerlibs") + + if isinstance(e, bb.event.RecipeParsed): + # + # If we have multiple providers of virtual/X and a PREFERRED_PROVIDER_virtual/X is set + # skip parsing for all the other providers which will mean they get uninstalled from the + # sysroot since they're now "unreachable". This makes switching virtual/kernel work in + # particular. + # + pn = d.getVar('PN') + source_mirror_fetch = d.getVar('SOURCE_MIRROR_FETCH', False) + if not source_mirror_fetch: + provs = (d.getVar("PROVIDES") or "").split() + multiprovidersallowed = (d.getVar("BB_MULTI_PROVIDER_ALLOWED") or "").split() + for p in provs: + if p.startswith("virtual/") and p not in multiprovidersallowed: + profprov = d.getVar("PREFERRED_PROVIDER_" + p) + if profprov and pn != profprov: + raise bb.parse.SkipRecipe("PREFERRED_PROVIDER_%s set to %s, not %s" % (p, profprov, pn)) +} + +CONFIGURESTAMPFILE = "${WORKDIR}/configure.sstate" +CLEANBROKEN = "0" + +addtask configure after do_patch +do_configure[dirs] = "${B}" +base_do_configure() { + if [ -n "${CONFIGURESTAMPFILE}" -a -e "${CONFIGURESTAMPFILE}" ]; then + if [ "`cat ${CONFIGURESTAMPFILE}`" != "${BB_TASKHASH}" ]; then + cd ${B} + if [ "${CLEANBROKEN}" != "1" -a \( -e Makefile -o -e makefile -o -e GNUmakefile \) ]; then + oe_runmake clean + fi + # -ignore_readdir_race does not work correctly with -delete; + # use xargs to avoid spurious build failures + find ${B} -ignore_readdir_race -name \*.la -type f -print0 | xargs -0 rm -f + fi + fi + if [ -n "${CONFIGURESTAMPFILE}" ]; then + mkdir -p `dirname ${CONFIGURESTAMPFILE}` + echo ${BB_TASKHASH} > ${CONFIGURESTAMPFILE} + fi +} + +addtask compile after do_configure +do_compile[dirs] = "${B}" +base_do_compile() { + if [ -e Makefile -o -e makefile -o -e GNUmakefile ]; then + oe_runmake || die "make failed" + else + bbnote "nothing to compile" + fi +} + +addtask install after do_compile +do_install[dirs] = "${B}" +# Remove and re-create ${D} so that it is guaranteed to be empty +do_install[cleandirs] = "${D}" + +base_do_install() { + : +} + +addtask build after do_populate_sysroot +do_build[noexec] = "1" +do_build[recrdeptask] += "do_deploy" +do_build () { + : +} + +def set_packagetriplet(d): + archs = [] + tos = [] + tvs = [] + + archs.append(d.getVar("PACKAGE_ARCHS").split()) + tos.append(d.getVar("TARGET_OS")) + tvs.append(d.getVar("TARGET_VENDOR")) + + def settriplet(d, varname, archs, tos, tvs): + triplets = [] + for i in range(len(archs)): + for arch in archs[i]: + triplets.append(arch + tvs[i] + "-" + tos[i]) + triplets.reverse() + d.setVar(varname, " ".join(triplets)) + + settriplet(d, "PKGTRIPLETS", archs, tos, tvs) + + variants = d.getVar("MULTILIB_VARIANTS") or "" + for item in variants.split(): + localdata = bb.data.createCopy(d) + overrides = localdata.getVar("OVERRIDES", False) + ":virtclass-multilib-" + item + localdata.setVar("OVERRIDES", overrides) + + archs.append(localdata.getVar("PACKAGE_ARCHS").split()) + tos.append(localdata.getVar("TARGET_OS")) + tvs.append(localdata.getVar("TARGET_VENDOR")) + + settriplet(d, "PKGMLTRIPLETS", archs, tos, tvs) + +python () { + import string, re + + # Handle backfilling + oe.utils.features_backfill("DISTRO_FEATURES", d) + oe.utils.features_backfill("MACHINE_FEATURES", d) + + if d.getVar("S")[-1] == '/': + bb.warn("Recipe %s sets S variable with trailing slash '%s', remove it" % (d.getVar("PN"), d.getVar("S"))) + if d.getVar("B")[-1] == '/': + bb.warn("Recipe %s sets B variable with trailing slash '%s', remove it" % (d.getVar("PN"), d.getVar("B"))) + + if os.path.normpath(d.getVar("WORKDIR")) != os.path.normpath(d.getVar("S")): + d.appendVar("PSEUDO_IGNORE_PATHS", ",${S}") + if os.path.normpath(d.getVar("WORKDIR")) != os.path.normpath(d.getVar("B")): + d.appendVar("PSEUDO_IGNORE_PATHS", ",${B}") + + # To add a recipe to the skip list , set: + # SKIP_RECIPE[pn] = "message" + pn = d.getVar('PN') + skip_msg = d.getVarFlag('SKIP_RECIPE', pn) + if skip_msg: + bb.debug(1, "Skipping %s %s" % (pn, skip_msg)) + raise bb.parse.SkipRecipe("Recipe will be skipped because: %s" % (skip_msg)) + + # Handle PACKAGECONFIG + # + # These take the form: + # + # PACKAGECONFIG ??= "<default options>" + # PACKAGECONFIG[foo] = "--enable-foo,--disable-foo,foo_depends,foo_runtime_depends,foo_runtime_recommends,foo_conflict_packageconfig" + pkgconfigflags = d.getVarFlags("PACKAGECONFIG") or {} + if pkgconfigflags: + pkgconfig = (d.getVar('PACKAGECONFIG') or "").split() + pn = d.getVar("PN") + + mlprefix = d.getVar("MLPREFIX") + + def expandFilter(appends, extension, prefix): + appends = bb.utils.explode_deps(d.expand(" ".join(appends))) + newappends = [] + for a in appends: + if a.endswith("-native") or ("-cross-" in a): + newappends.append(a) + elif a.startswith("virtual/"): + subs = a.split("/", 1)[1] + if subs.startswith(prefix): + newappends.append(a + extension) + else: + newappends.append("virtual/" + prefix + subs + extension) + else: + if a.startswith(prefix): + newappends.append(a + extension) + else: + newappends.append(prefix + a + extension) + return newappends + + def appendVar(varname, appends): + if not appends: + return + if varname.find("DEPENDS") != -1: + if bb.data.inherits_class('nativesdk', d) or bb.data.inherits_class('cross-canadian', d) : + appends = expandFilter(appends, "", "nativesdk-") + elif bb.data.inherits_class('native', d): + appends = expandFilter(appends, "-native", "") + elif mlprefix: + appends = expandFilter(appends, "", mlprefix) + varname = d.expand(varname) + d.appendVar(varname, " " + " ".join(appends)) + + extradeps = [] + extrardeps = [] + extrarrecs = [] + extraconf = [] + for flag, flagval in sorted(pkgconfigflags.items()): + items = flagval.split(",") + num = len(items) + if num > 6: + bb.error("%s: PACKAGECONFIG[%s] Only enable,disable,depend,rdepend,rrecommend,conflict_packageconfig can be specified!" + % (d.getVar('PN'), flag)) + + if flag in pkgconfig: + if num >= 3 and items[2]: + extradeps.append(items[2]) + if num >= 4 and items[3]: + extrardeps.append(items[3]) + if num >= 5 and items[4]: + extrarrecs.append(items[4]) + if num >= 1 and items[0]: + extraconf.append(items[0]) + elif num >= 2 and items[1]: + extraconf.append(items[1]) + + if num >= 6 and items[5]: + conflicts = set(items[5].split()) + invalid = conflicts.difference(set(pkgconfigflags.keys())) + if invalid: + bb.error("%s: PACKAGECONFIG[%s] Invalid conflict package config%s '%s' specified." + % (d.getVar('PN'), flag, 's' if len(invalid) > 1 else '', ' '.join(invalid))) + + if flag in pkgconfig: + intersec = conflicts.intersection(set(pkgconfig)) + if intersec: + bb.fatal("%s: PACKAGECONFIG[%s] Conflict package config%s '%s' set in PACKAGECONFIG." + % (d.getVar('PN'), flag, 's' if len(intersec) > 1 else '', ' '.join(intersec))) + + appendVar('DEPENDS', extradeps) + appendVar('RDEPENDS:${PN}', extrardeps) + appendVar('RRECOMMENDS:${PN}', extrarrecs) + appendVar('PACKAGECONFIG_CONFARGS', extraconf) + + pn = d.getVar('PN') + license = d.getVar('LICENSE') + if license == "INVALID" and pn != "defaultpkgname": + bb.fatal('This recipe does not have the LICENSE field set (%s)' % pn) + + if bb.data.inherits_class('license', d): + check_license_format(d) + unmatched_license_flags = check_license_flags(d) + if unmatched_license_flags: + for unmatched in unmatched_license_flags: + message = "Has a restricted license '%s' which is not listed in your LICENSE_FLAGS_ACCEPTED." % unmatched + details = d.getVarFlag("LICENSE_FLAGS_DETAILS", unmatched) + if details: + message += "\n" + details + bb.debug(1, "Skipping %s: %s" % (pn, message)) + raise bb.parse.SkipRecipe(message) + + # If we're building a target package we need to use fakeroot (pseudo) + # in order to capture permissions, owners, groups and special files + if not bb.data.inherits_class('native', d) and not bb.data.inherits_class('cross', d): + d.appendVarFlag('do_prepare_recipe_sysroot', 'depends', ' virtual/fakeroot-native:do_populate_sysroot') + d.appendVarFlag('do_install', 'depends', ' virtual/fakeroot-native:do_populate_sysroot') + d.setVarFlag('do_install', 'fakeroot', '1') + d.appendVarFlag('do_package', 'depends', ' virtual/fakeroot-native:do_populate_sysroot') + d.setVarFlag('do_package', 'fakeroot', '1') + d.setVarFlag('do_package_setscene', 'fakeroot', '1') + d.appendVarFlag('do_package_setscene', 'depends', ' virtual/fakeroot-native:do_populate_sysroot') + d.setVarFlag('do_devshell', 'fakeroot', '1') + d.appendVarFlag('do_devshell', 'depends', ' virtual/fakeroot-native:do_populate_sysroot') + + need_machine = d.getVar('COMPATIBLE_MACHINE') + if need_machine and not d.getVar('PARSE_ALL_RECIPES', False): + import re + compat_machines = (d.getVar('MACHINEOVERRIDES') or "").split(":") + for m in compat_machines: + if re.match(need_machine, m): + break + else: + raise bb.parse.SkipRecipe("incompatible with machine %s (not in COMPATIBLE_MACHINE)" % d.getVar('MACHINE')) + + source_mirror_fetch = d.getVar('SOURCE_MIRROR_FETCH', False) or d.getVar('PARSE_ALL_RECIPES', False) + if not source_mirror_fetch: + need_host = d.getVar('COMPATIBLE_HOST') + if need_host: + import re + this_host = d.getVar('HOST_SYS') + if not re.match(need_host, this_host): + raise bb.parse.SkipRecipe("incompatible with host %s (not in COMPATIBLE_HOST)" % this_host) + + bad_licenses = (d.getVar('INCOMPATIBLE_LICENSE') or "").split() + + check_license = False if pn.startswith("nativesdk-") else True + for t in ["-native", "-cross-${TARGET_ARCH}", "-cross-initial-${TARGET_ARCH}", + "-crosssdk-${SDK_SYS}", "-crosssdk-initial-${SDK_SYS}", + "-cross-canadian-${TRANSLATED_TARGET_ARCH}"]: + if pn.endswith(d.expand(t)): + check_license = False + if pn.startswith("gcc-source-"): + check_license = False + + if check_license and bad_licenses: + bad_licenses = expand_wildcard_licenses(d, bad_licenses) + + exceptions = (d.getVar("INCOMPATIBLE_LICENSE_EXCEPTIONS") or "").split() + + for lic_exception in exceptions: + if ":" in lic_exception: + lic_exception = lic_exception.split(":")[1] + if lic_exception in oe.license.obsolete_license_list(): + bb.fatal("Obsolete license %s used in INCOMPATIBLE_LICENSE_EXCEPTIONS" % lic_exception) + + pkgs = d.getVar('PACKAGES').split() + skipped_pkgs = {} + unskipped_pkgs = [] + for pkg in pkgs: + remaining_bad_licenses = oe.license.apply_pkg_license_exception(pkg, bad_licenses, exceptions) + + incompatible_lic = incompatible_license(d, remaining_bad_licenses, pkg) + if incompatible_lic: + skipped_pkgs[pkg] = incompatible_lic + else: + unskipped_pkgs.append(pkg) + + if unskipped_pkgs: + for pkg in skipped_pkgs: + bb.debug(1, "Skipping the package %s at do_rootfs because of incompatible license(s): %s" % (pkg, ' '.join(skipped_pkgs[pkg]))) + d.setVar('_exclude_incompatible-' + pkg, ' '.join(skipped_pkgs[pkg])) + for pkg in unskipped_pkgs: + bb.debug(1, "Including the package %s" % pkg) + else: + incompatible_lic = incompatible_license(d, bad_licenses) + for pkg in skipped_pkgs: + incompatible_lic += skipped_pkgs[pkg] + incompatible_lic = sorted(list(set(incompatible_lic))) + + if incompatible_lic: + bb.debug(1, "Skipping recipe %s because of incompatible license(s): %s" % (pn, ' '.join(incompatible_lic))) + raise bb.parse.SkipRecipe("it has incompatible license(s): %s" % ' '.join(incompatible_lic)) + + srcuri = d.getVar('SRC_URI') + for uri_string in srcuri.split(): + uri = bb.fetch.URI(uri_string) + # Also check downloadfilename as the URL path might not be useful for sniffing + path = uri.params.get("downloadfilename", uri.path) + + # HTTP/FTP use the wget fetcher + if uri.scheme in ("http", "https", "ftp"): + d.appendVarFlag('do_fetch', 'depends', ' wget-native:do_populate_sysroot') + + # Svn packages should DEPEND on subversion-native + if uri.scheme == "svn": + d.appendVarFlag('do_fetch', 'depends', ' subversion-native:do_populate_sysroot') + + # Git packages should DEPEND on git-native + elif uri.scheme in ("git", "gitsm"): + d.appendVarFlag('do_fetch', 'depends', ' git-native:do_populate_sysroot') + + # Mercurial packages should DEPEND on mercurial-native + elif uri.scheme == "hg": + d.appendVar("EXTRANATIVEPATH", ' python3-native ') + d.appendVarFlag('do_fetch', 'depends', ' mercurial-native:do_populate_sysroot ca-certificates-native:do_populate_sysroot') + + # OSC packages should DEPEND on osc-native + elif uri.scheme == "osc": + d.appendVarFlag('do_fetch', 'depends', ' osc-native:do_populate_sysroot') + + elif uri.scheme == "npm": + d.appendVarFlag('do_fetch', 'depends', ' nodejs-native:do_populate_sysroot') + + elif uri.scheme == "repo": + d.appendVarFlag('do_fetch', 'depends', ' repo-native:do_populate_sysroot') + + # *.lz4 should DEPEND on lz4-native for unpacking + if path.endswith('.lz4'): + d.appendVarFlag('do_unpack', 'depends', ' lz4-native:do_populate_sysroot') + + # *.zst should DEPEND on zstd-native for unpacking + elif path.endswith('.zst'): + d.appendVarFlag('do_unpack', 'depends', ' zstd-native:do_populate_sysroot') + + # *.lz should DEPEND on lzip-native for unpacking + elif path.endswith('.lz'): + d.appendVarFlag('do_unpack', 'depends', ' lzip-native:do_populate_sysroot') + + # *.xz should DEPEND on xz-native for unpacking + elif path.endswith('.xz') or path.endswith('.txz'): + d.appendVarFlag('do_unpack', 'depends', ' xz-native:do_populate_sysroot') + + # .zip should DEPEND on unzip-native for unpacking + elif path.endswith('.zip') or path.endswith('.jar'): + d.appendVarFlag('do_unpack', 'depends', ' unzip-native:do_populate_sysroot') + + # Some rpm files may be compressed internally using xz (for example, rpms from Fedora) + elif path.endswith('.rpm'): + d.appendVarFlag('do_unpack', 'depends', ' xz-native:do_populate_sysroot') + + # *.deb should DEPEND on xz-native for unpacking + elif path.endswith('.deb'): + d.appendVarFlag('do_unpack', 'depends', ' xz-native:do_populate_sysroot') + + # *.7z should DEPEND on p7zip-native for unpacking + elif path.endswith('.7z'): + d.appendVarFlag('do_unpack', 'depends', ' p7zip-native:do_populate_sysroot') + + set_packagetriplet(d) + + # 'multimachine' handling + mach_arch = d.getVar('MACHINE_ARCH') + pkg_arch = d.getVar('PACKAGE_ARCH') + + if (pkg_arch == mach_arch): + # Already machine specific - nothing further to do + return + + # + # We always try to scan SRC_URI for urls with machine overrides + # unless the package sets SRC_URI_OVERRIDES_PACKAGE_ARCH=0 + # + override = d.getVar('SRC_URI_OVERRIDES_PACKAGE_ARCH') + if override != '0': + paths = [] + fpaths = (d.getVar('FILESPATH') or '').split(':') + machine = d.getVar('MACHINE') + for p in fpaths: + if os.path.basename(p) == machine and os.path.isdir(p): + paths.append(p) + + if paths: + for s in srcuri.split(): + if not s.startswith("file://"): + continue + fetcher = bb.fetch2.Fetch([s], d) + local = fetcher.localpath(s) + for mp in paths: + if local.startswith(mp): + #bb.note("overriding PACKAGE_ARCH from %s to %s for %s" % (pkg_arch, mach_arch, pn)) + d.setVar('PACKAGE_ARCH', "${MACHINE_ARCH}") + return + + packages = d.getVar('PACKAGES').split() + for pkg in packages: + pkgarch = d.getVar("PACKAGE_ARCH_%s" % pkg) + + # We could look for != PACKAGE_ARCH here but how to choose + # if multiple differences are present? + # Look through PACKAGE_ARCHS for the priority order? + if pkgarch and pkgarch == mach_arch: + d.setVar('PACKAGE_ARCH', "${MACHINE_ARCH}") + bb.warn("Recipe %s is marked as only being architecture specific but seems to have machine specific packages?! The recipe may as well mark itself as machine specific directly." % d.getVar("PN")) +} + +addtask cleansstate after do_clean +python do_cleansstate() { + sstate_clean_cachefiles(d) +} +addtask cleanall after do_cleansstate +do_cleansstate[nostamp] = "1" + +python do_cleanall() { + src_uri = (d.getVar('SRC_URI') or "").split() + if not src_uri: + return + + try: + fetcher = bb.fetch2.Fetch(src_uri, d) + fetcher.clean() + except bb.fetch2.BBFetchException as e: + bb.fatal(str(e)) +} +do_cleanall[nostamp] = "1" + + +EXPORT_FUNCTIONS do_fetch do_unpack do_configure do_compile do_install diff --git a/meta/classes-global/buildstats.bbclass b/meta/classes-global/buildstats.bbclass new file mode 100644 index 0000000000..f49a67aa4f --- /dev/null +++ b/meta/classes-global/buildstats.bbclass @@ -0,0 +1,302 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +BUILDSTATS_BASE = "${TMPDIR}/buildstats/" + +################################################################################ +# Build statistics gathering. +# +# The CPU and Time gathering/tracking functions and bbevent inspiration +# were written by Christopher Larson. +# +################################################################################ + +def get_buildprocess_cputime(pid): + with open("/proc/%d/stat" % pid, "r") as f: + fields = f.readline().rstrip().split() + # 13: utime, 14: stime, 15: cutime, 16: cstime + return sum(int(field) for field in fields[13:16]) + +def get_process_cputime(pid): + import resource + with open("/proc/%d/stat" % pid, "r") as f: + fields = f.readline().rstrip().split() + stats = { + 'utime' : fields[13], + 'stime' : fields[14], + 'cutime' : fields[15], + 'cstime' : fields[16], + } + iostats = {} + if os.path.isfile("/proc/%d/io" % pid): + with open("/proc/%d/io" % pid, "r") as f: + while True: + i = f.readline().strip() + if not i: + break + if not ":" in i: + # one more extra line is appended (empty or containing "0") + # most probably due to race condition in kernel while + # updating IO stats + break + i = i.split(": ") + iostats[i[0]] = i[1] + resources = resource.getrusage(resource.RUSAGE_SELF) + childres = resource.getrusage(resource.RUSAGE_CHILDREN) + return stats, iostats, resources, childres + +def get_cputime(): + with open("/proc/stat", "r") as f: + fields = f.readline().rstrip().split()[1:] + return sum(int(field) for field in fields) + +def set_timedata(var, d, server_time): + d.setVar(var, server_time) + +def get_timedata(var, d, end_time): + oldtime = d.getVar(var, False) + if oldtime is None: + return + return end_time - oldtime + +def set_buildtimedata(var, d): + import time + time = time.time() + cputime = get_cputime() + proctime = get_buildprocess_cputime(os.getpid()) + d.setVar(var, (time, cputime, proctime)) + +def get_buildtimedata(var, d): + import time + timedata = d.getVar(var, False) + if timedata is None: + return + oldtime, oldcpu, oldproc = timedata + procdiff = get_buildprocess_cputime(os.getpid()) - oldproc + cpudiff = get_cputime() - oldcpu + end_time = time.time() + timediff = end_time - oldtime + if cpudiff > 0: + cpuperc = float(procdiff) * 100 / cpudiff + else: + cpuperc = None + return timediff, cpuperc + +def write_task_data(status, logfile, e, d): + with open(os.path.join(logfile), "a") as f: + elapsedtime = get_timedata("__timedata_task", d, e.time) + if elapsedtime: + f.write(d.expand("${PF}: %s\n" % e.task)) + f.write(d.expand("Elapsed time: %0.2f seconds\n" % elapsedtime)) + cpu, iostats, resources, childres = get_process_cputime(os.getpid()) + if cpu: + f.write("utime: %s\n" % cpu['utime']) + f.write("stime: %s\n" % cpu['stime']) + f.write("cutime: %s\n" % cpu['cutime']) + f.write("cstime: %s\n" % cpu['cstime']) + for i in iostats: + f.write("IO %s: %s\n" % (i, iostats[i])) + rusages = ["ru_utime", "ru_stime", "ru_maxrss", "ru_minflt", "ru_majflt", "ru_inblock", "ru_oublock", "ru_nvcsw", "ru_nivcsw"] + for i in rusages: + f.write("rusage %s: %s\n" % (i, getattr(resources, i))) + for i in rusages: + f.write("Child rusage %s: %s\n" % (i, getattr(childres, i))) + if status == "passed": + f.write("Status: PASSED \n") + else: + f.write("Status: FAILED \n") + f.write("Ended: %0.2f \n" % e.time) + +def write_host_data(logfile, e, d, type): + import subprocess, os, datetime + # minimum time allowed for each command to run, in seconds + time_threshold = 0.5 + limit = 10 + # the total number of commands + num_cmds = 0 + msg = "" + if type == "interval": + # interval at which data will be logged + interval = d.getVar("BB_HEARTBEAT_EVENT", False) + if interval is None: + bb.warn("buildstats: Collecting host data at intervals failed. Set BB_HEARTBEAT_EVENT=\"<interval>\" in conf/local.conf for the interval at which host data will be logged.") + d.setVar("BB_LOG_HOST_STAT_ON_INTERVAL", "0") + return + interval = int(interval) + cmds = d.getVar('BB_LOG_HOST_STAT_CMDS_INTERVAL') + msg = "Host Stats: Collecting data at %d second intervals.\n" % interval + if cmds is None: + d.setVar("BB_LOG_HOST_STAT_ON_INTERVAL", "0") + bb.warn("buildstats: Collecting host data at intervals failed. Set BB_LOG_HOST_STAT_CMDS_INTERVAL=\"command1 ; command2 ; ... \" in conf/local.conf.") + return + if type == "failure": + cmds = d.getVar('BB_LOG_HOST_STAT_CMDS_FAILURE') + msg = "Host Stats: Collecting data on failure.\n" + msg += "Failed at task: " + e.task + "\n" + if cmds is None: + d.setVar("BB_LOG_HOST_STAT_ON_FAILURE", "0") + bb.warn("buildstats: Collecting host data on failure failed. Set BB_LOG_HOST_STAT_CMDS_FAILURE=\"command1 ; command2 ; ... \" in conf/local.conf.") + return + c_san = [] + for cmd in cmds.split(";"): + if len(cmd) == 0: + continue + num_cmds += 1 + c_san.append(cmd) + if num_cmds == 0: + if type == "interval": + d.setVar("BB_LOG_HOST_STAT_ON_INTERVAL", "0") + if type == "failure": + d.setVar("BB_LOG_HOST_STAT_ON_FAILURE", "0") + return + + # return if the interval is not enough to run all commands within the specified BB_HEARTBEAT_EVENT interval + if type == "interval": + limit = interval / num_cmds + if limit <= time_threshold: + d.setVar("BB_LOG_HOST_STAT_ON_INTERVAL", "0") + bb.warn("buildstats: Collecting host data failed. BB_HEARTBEAT_EVENT interval not enough to run the specified commands. Increase value of BB_HEARTBEAT_EVENT in conf/local.conf.") + return + + # set the environment variables + path = d.getVar("PATH") + opath = d.getVar("BB_ORIGENV", False).getVar("PATH") + ospath = os.environ['PATH'] + os.environ['PATH'] = path + ":" + opath + ":" + ospath + with open(logfile, "a") as f: + f.write("Event Time: %f\nDate: %s\n" % (e.time, datetime.datetime.now())) + f.write("%s" % msg) + for c in c_san: + try: + output = subprocess.check_output(c.split(), stderr=subprocess.STDOUT, timeout=limit).decode('utf-8') + except (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError) as err: + output = "Error running command: %s\n%s\n" % (c, err) + f.write("%s\n%s\n" % (c, output)) + # reset the environment + os.environ['PATH'] = ospath + +python run_buildstats () { + import bb.build + import bb.event + import time, subprocess, platform + + bn = d.getVar('BUILDNAME') + ######################################################################## + # bitbake fires HeartbeatEvent even before a build has been + # triggered, causing BUILDNAME to be None + ######################################################################## + if bn is not None: + bsdir = os.path.join(d.getVar('BUILDSTATS_BASE'), bn) + taskdir = os.path.join(bsdir, d.getVar('PF')) + if isinstance(e, bb.event.HeartbeatEvent) and bb.utils.to_boolean(d.getVar("BB_LOG_HOST_STAT_ON_INTERVAL")): + bb.utils.mkdirhier(bsdir) + write_host_data(os.path.join(bsdir, "host_stats_interval"), e, d, "interval") + + if isinstance(e, bb.event.BuildStarted): + ######################################################################## + # If the kernel was not configured to provide I/O statistics, issue + # a one time warning. + ######################################################################## + if not os.path.isfile("/proc/%d/io" % os.getpid()): + bb.warn("The Linux kernel on your build host was not configured to provide process I/O statistics. (CONFIG_TASK_IO_ACCOUNTING is not set)") + + ######################################################################## + # at first pass make the buildstats hierarchy and then + # set the buildname + ######################################################################## + bb.utils.mkdirhier(bsdir) + set_buildtimedata("__timedata_build", d) + build_time = os.path.join(bsdir, "build_stats") + # write start of build into build_time + with open(build_time, "a") as f: + host_info = platform.uname() + f.write("Host Info: ") + for x in host_info: + if x: + f.write(x + " ") + f.write("\n") + f.write("Build Started: %0.2f \n" % d.getVar('__timedata_build', False)[0]) + + elif isinstance(e, bb.event.BuildCompleted): + build_time = os.path.join(bsdir, "build_stats") + with open(build_time, "a") as f: + ######################################################################## + # Write build statistics for the build + ######################################################################## + timedata = get_buildtimedata("__timedata_build", d) + if timedata: + time, cpu = timedata + # write end of build and cpu used into build_time + f.write("Elapsed time: %0.2f seconds \n" % (time)) + if cpu: + f.write("CPU usage: %0.1f%% \n" % cpu) + + if isinstance(e, bb.build.TaskStarted): + set_timedata("__timedata_task", d, e.time) + bb.utils.mkdirhier(taskdir) + # write into the task event file the name and start time + with open(os.path.join(taskdir, e.task), "a") as f: + f.write("Event: %s \n" % bb.event.getName(e)) + f.write("Started: %0.2f \n" % e.time) + + elif isinstance(e, bb.build.TaskSucceeded): + write_task_data("passed", os.path.join(taskdir, e.task), e, d) + if e.task == "do_rootfs": + bs = os.path.join(bsdir, "build_stats") + with open(bs, "a") as f: + rootfs = d.getVar('IMAGE_ROOTFS') + if os.path.isdir(rootfs): + try: + rootfs_size = subprocess.check_output(["du", "-sh", rootfs], + stderr=subprocess.STDOUT).decode('utf-8') + f.write("Uncompressed Rootfs size: %s" % rootfs_size) + except subprocess.CalledProcessError as err: + bb.warn("Failed to get rootfs size: %s" % err.output.decode('utf-8')) + + elif isinstance(e, bb.build.TaskFailed): + # Can have a failure before TaskStarted so need to mkdir here too + bb.utils.mkdirhier(taskdir) + write_task_data("failed", os.path.join(taskdir, e.task), e, d) + ######################################################################## + # Lets make things easier and tell people where the build failed in + # build_status. We do this here because BuildCompleted triggers no + # matter what the status of the build actually is + ######################################################################## + build_status = os.path.join(bsdir, "build_stats") + with open(build_status, "a") as f: + f.write(d.expand("Failed at: ${PF} at task: %s \n" % e.task)) + if bb.utils.to_boolean(d.getVar("BB_LOG_HOST_STAT_ON_FAILURE")): + write_host_data(os.path.join(bsdir, "host_stats_%s_failure" % e.task), e, d, "failure") +} + +addhandler run_buildstats +run_buildstats[eventmask] = "bb.event.BuildStarted bb.event.BuildCompleted bb.event.HeartbeatEvent bb.build.TaskStarted bb.build.TaskSucceeded bb.build.TaskFailed" + +python runqueue_stats () { + import buildstats + from bb import event, runqueue + # We should not record any samples before the first task has started, + # because that's the first activity shown in the process chart. + # Besides, at that point we are sure that the build variables + # are available that we need to find the output directory. + # The persistent SystemStats is stored in the datastore and + # closed when the build is done. + system_stats = d.getVar('_buildstats_system_stats', False) + if not system_stats and isinstance(e, (bb.runqueue.sceneQueueTaskStarted, bb.runqueue.runQueueTaskStarted)): + system_stats = buildstats.SystemStats(d) + d.setVar('_buildstats_system_stats', system_stats) + if system_stats: + # Ensure that we sample at important events. + done = isinstance(e, bb.event.BuildCompleted) + if system_stats.sample(e, force=done): + d.setVar('_buildstats_system_stats', system_stats) + if done: + system_stats.close() + d.delVar('_buildstats_system_stats') +} + +addhandler runqueue_stats +runqueue_stats[eventmask] = "bb.runqueue.sceneQueueTaskStarted bb.runqueue.runQueueTaskStarted bb.event.HeartbeatEvent bb.event.BuildCompleted bb.event.MonitorDiskEvent" diff --git a/meta/classes-global/debian.bbclass b/meta/classes-global/debian.bbclass new file mode 100644 index 0000000000..e2a129d028 --- /dev/null +++ b/meta/classes-global/debian.bbclass @@ -0,0 +1,141 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +# Debian package renaming only occurs when a package is built +# We therefore have to make sure we build all runtime packages +# before building the current package to make the packages runtime +# depends are correct +# +# Custom library package names can be defined setting +# DEBIANNAME: + pkgname to the desired name. +# +# Better expressed as ensure all RDEPENDS package before we package +# This means we can't have circular RDEPENDS/RRECOMMENDS + +AUTO_LIBNAME_PKGS = "${PACKAGES}" + +inherit package + +python debian_package_name_hook () { + import glob, copy, stat, errno, re, pathlib, subprocess + + pkgdest = d.getVar("PKGDEST") + packages = d.getVar('PACKAGES') + so_re = re.compile(r"lib.*\.so") + + def socrunch(s): + s = s.lower().replace('_', '-') + m = re.match(r"^(.*)(.)\.so\.(.*)$", s) + if m is None: + return None + if m.group(2) in '0123456789': + bin = '%s%s-%s' % (m.group(1), m.group(2), m.group(3)) + else: + bin = m.group(1) + m.group(2) + m.group(3) + dev = m.group(1) + m.group(2) + return (bin, dev) + + def isexec(path): + try: + s = os.stat(path) + except (os.error, AttributeError): + return 0 + return (s[stat.ST_MODE] & stat.S_IEXEC) + + def add_rprovides(pkg, d): + newpkg = d.getVar('PKG:' + pkg) + if newpkg and newpkg != pkg: + provs = (d.getVar('RPROVIDES:' + pkg) or "").split() + if pkg not in provs: + d.appendVar('RPROVIDES:' + pkg, " " + pkg + " (=" + d.getVar("PKGV") + ")") + + def auto_libname(packages, orig_pkg): + p = lambda var: pathlib.PurePath(d.getVar(var)) + libdirs = (p("base_libdir"), p("libdir")) + bindirs = (p("base_bindir"), p("base_sbindir"), p("bindir"), p("sbindir")) + + sonames = [] + has_bins = 0 + has_libs = 0 + for f in pkgfiles[orig_pkg]: + # This is .../packages-split/orig_pkg/ + pkgpath = pathlib.PurePath(pkgdest, orig_pkg) + # Strip pkgpath off the full path to a file in the package, re-root + # so it is absolute, and then get the parent directory of the file. + path = pathlib.PurePath("/") / (pathlib.PurePath(f).relative_to(pkgpath).parent) + if path in bindirs: + has_bins = 1 + if path in libdirs: + has_libs = 1 + if so_re.match(os.path.basename(f)): + try: + cmd = [d.expand("${TARGET_PREFIX}objdump"), "-p", f] + output = subprocess.check_output(cmd).decode("utf-8") + for m in re.finditer(r"\s+SONAME\s+([^\s]+)", output): + if m.group(1) not in sonames: + sonames.append(m.group(1)) + except subprocess.CalledProcessError: + pass + bb.debug(1, 'LIBNAMES: pkg %s libs %d bins %d sonames %s' % (orig_pkg, has_libs, has_bins, sonames)) + soname = None + if len(sonames) == 1: + soname = sonames[0] + elif len(sonames) > 1: + lead = d.getVar('LEAD_SONAME') + if lead: + r = re.compile(lead) + filtered = [] + for s in sonames: + if r.match(s): + filtered.append(s) + if len(filtered) == 1: + soname = filtered[0] + elif len(filtered) > 1: + bb.note("Multiple matches (%s) for LEAD_SONAME '%s'" % (", ".join(filtered), lead)) + else: + bb.note("Multiple libraries (%s) found, but LEAD_SONAME '%s' doesn't match any of them" % (", ".join(sonames), lead)) + else: + bb.note("Multiple libraries (%s) found and LEAD_SONAME not defined" % ", ".join(sonames)) + + if has_libs and not has_bins and soname: + soname_result = socrunch(soname) + if soname_result: + (pkgname, devname) = soname_result + for pkg in packages.split(): + if (d.getVar('PKG:' + pkg, False) or d.getVar('DEBIAN_NOAUTONAME:' + pkg, False)): + add_rprovides(pkg, d) + continue + debian_pn = d.getVar('DEBIANNAME:' + pkg, False) + if debian_pn: + newpkg = debian_pn + elif pkg == orig_pkg: + newpkg = pkgname + else: + newpkg = pkg.replace(orig_pkg, devname, 1) + mlpre=d.getVar('MLPREFIX') + if mlpre: + if not newpkg.find(mlpre) == 0: + newpkg = mlpre + newpkg + if newpkg != pkg: + bb.note("debian: renaming %s to %s" % (pkg, newpkg)) + d.setVar('PKG:' + pkg, newpkg) + add_rprovides(pkg, d) + else: + add_rprovides(orig_pkg, d) + + # reversed sort is needed when some package is substring of another + # ie in ncurses we get without reverse sort: + # DEBUG: LIBNAMES: pkgname libtic5 devname libtic pkg ncurses-libtic orig_pkg ncurses-libtic debian_pn None newpkg libtic5 + # and later + # DEBUG: LIBNAMES: pkgname libtic5 devname libtic pkg ncurses-libticw orig_pkg ncurses-libtic debian_pn None newpkg libticw + # so we need to handle ncurses-libticw->libticw5 before ncurses-libtic->libtic5 + for pkg in sorted((d.getVar('AUTO_LIBNAME_PKGS') or "").split(), reverse=True): + auto_libname(packages, pkg) +} + +EXPORT_FUNCTIONS package_name_hook + +DEBIAN_NAMES = "1" diff --git a/meta/classes-global/devshell.bbclass b/meta/classes-global/devshell.bbclass new file mode 100644 index 0000000000..4c23049cf0 --- /dev/null +++ b/meta/classes-global/devshell.bbclass @@ -0,0 +1,164 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +inherit terminal + +DEVSHELL = "${SHELL}" + +python do_devshell () { + if d.getVarFlag("do_devshell", "manualfakeroot"): + d.prependVar("DEVSHELL", "pseudo ") + fakeenv = d.getVar("FAKEROOTENV").split() + for f in fakeenv: + k = f.split("=") + d.setVar(k[0], k[1]) + d.appendVar("OE_TERMINAL_EXPORTS", " " + k[0]) + d.delVarFlag("do_devshell", "fakeroot") + + oe_terminal(d.getVar('DEVSHELL'), 'OpenEmbedded Developer Shell', d) +} + +addtask devshell after do_patch do_prepare_recipe_sysroot + +# The directory that the terminal starts in +DEVSHELL_STARTDIR ?= "${S}" +do_devshell[dirs] = "${DEVSHELL_STARTDIR}" +do_devshell[nostamp] = "1" +do_devshell[network] = "1" + +# devshell and fakeroot/pseudo need careful handling since only the final +# command should run under fakeroot emulation, any X connection should +# be done as the normal user. We therfore carefully construct the envionment +# manually +python () { + if d.getVarFlag("do_devshell", "fakeroot"): + # We need to signal our code that we want fakeroot however we + # can't manipulate the environment and variables here yet (see YOCTO #4795) + d.setVarFlag("do_devshell", "manualfakeroot", "1") + d.delVarFlag("do_devshell", "fakeroot") +} + +def pydevshell(d): + + import code + import select + import signal + import termios + + m, s = os.openpty() + sname = os.ttyname(s) + + def noechoicanon(fd): + old = termios.tcgetattr(fd) + old[3] = old[3] &~ termios.ECHO &~ termios.ICANON + # &~ termios.ISIG + termios.tcsetattr(fd, termios.TCSADRAIN, old) + + # No echo or buffering over the pty + noechoicanon(s) + + pid = os.fork() + if pid: + os.close(m) + oe_terminal("oepydevshell-internal.py %s %d" % (sname, pid), 'OpenEmbedded Developer PyShell', d) + os._exit(0) + else: + os.close(s) + + os.dup2(m, sys.stdin.fileno()) + os.dup2(m, sys.stdout.fileno()) + os.dup2(m, sys.stderr.fileno()) + + bb.utils.nonblockingfd(sys.stdout) + bb.utils.nonblockingfd(sys.stderr) + bb.utils.nonblockingfd(sys.stdin) + + _context = { + "os": os, + "bb": bb, + "time": time, + "d": d, + } + + ps1 = "pydevshell> " + ps2 = "... " + buf = [] + more = False + + i = code.InteractiveInterpreter(locals=_context) + print("OE PyShell (PN = %s)\n" % d.getVar("PN")) + + def prompt(more): + if more: + prompt = ps2 + else: + prompt = ps1 + sys.stdout.write(prompt) + sys.stdout.flush() + + # Restore Ctrl+C since bitbake masks this + def signal_handler(signal, frame): + raise KeyboardInterrupt + signal.signal(signal.SIGINT, signal_handler) + + child = None + + prompt(more) + while True: + try: + try: + (r, _, _) = select.select([sys.stdin], [], [], 1) + if not r: + continue + line = sys.stdin.readline().strip() + if not line: + prompt(more) + continue + except EOFError as e: + sys.stdout.write("\n") + sys.stdout.flush() + except (OSError, IOError) as e: + if e.errno == 11: + continue + if e.errno == 5: + return + raise + else: + if not child: + child = int(line) + continue + buf.append(line) + source = "\n".join(buf) + more = i.runsource(source, "<pyshell>") + if not more: + buf = [] + sys.stderr.flush() + prompt(more) + except KeyboardInterrupt: + i.write("\nKeyboardInterrupt\n") + buf = [] + more = False + prompt(more) + except SystemExit: + # Easiest way to ensure everything exits + os.kill(child, signal.SIGTERM) + break + +python do_pydevshell() { + import signal + + try: + pydevshell(d) + except SystemExit: + # Stop the SIGTERM above causing an error exit code + return + finally: + return +} +addtask pydevshell after do_patch + +do_pydevshell[nostamp] = "1" +do_pydevshell[network] = "1" diff --git a/meta/classes-global/insane.bbclass b/meta/classes-global/insane.bbclass new file mode 100644 index 0000000000..e963001d09 --- /dev/null +++ b/meta/classes-global/insane.bbclass @@ -0,0 +1,1664 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +# BB Class inspired by ebuild.sh +# +# This class will test files after installation for certain +# security issues and other kind of issues. +# +# Checks we do: +# -Check the ownership and permissions +# -Check the RUNTIME path for the $TMPDIR +# -Check if .la files wrongly point to workdir +# -Check if .pc files wrongly point to workdir +# -Check if packages contains .debug directories or .so files +# where they should be in -dev or -dbg +# -Check if config.log contains traces to broken autoconf tests +# -Check invalid characters (non-utf8) on some package metadata +# -Ensure that binaries in base_[bindir|sbindir|libdir] do not link +# into exec_prefix +# -Check that scripts in base_[bindir|sbindir|libdir] do not reference +# files under exec_prefix +# -Check if the package name is upper case + +# Elect whether a given type of error is a warning or error, they may +# have been set by other files. +WARN_QA ?= " libdir xorg-driver-abi buildpaths \ + textrel incompatible-license files-invalid \ + infodir build-deps src-uri-bad symlink-to-sysroot multilib \ + invalid-packageconfig host-user-contaminated uppercase-pn \ + mime mime-xdg unlisted-pkg-lics unhandled-features-check \ + missing-update-alternatives native-last missing-ptest \ + license-exists license-no-generic license-syntax license-format \ + license-incompatible license-file-missing obsolete-license \ + 32bit-time virtual-slash \ + " +ERROR_QA ?= "dev-so debug-deps dev-deps debug-files arch pkgconfig la \ + perms dep-cmp pkgvarcheck perm-config perm-line perm-link \ + split-strip packages-list pkgv-undefined var-undefined \ + version-going-backwards expanded-d invalid-chars \ + license-checksum dev-elf file-rdeps configure-unsafe \ + configure-gettext perllocalpod shebang-size \ + already-stripped installed-vs-shipped ldflags compile-host-path \ + install-host-path pn-overrides unknown-configure-option \ + useless-rpaths rpaths staticdev empty-dirs \ + patch-fuzz \ + " +# Add usrmerge QA check based on distro feature +ERROR_QA:append = "${@bb.utils.contains('DISTRO_FEATURES', 'usrmerge', ' usrmerge', '', d)}" +ERROR_QA:append:layer-core = " patch-status" +WARN_QA:append:layer-core = " missing-metadata missing-maintainer" + +FAKEROOT_QA = "host-user-contaminated" +FAKEROOT_QA[doc] = "QA tests which need to run under fakeroot. If any \ +enabled tests are listed here, the do_package_qa task will run under fakeroot." + +ALL_QA = "${WARN_QA} ${ERROR_QA}" + +UNKNOWN_CONFIGURE_OPT_IGNORE ?= "--enable-nls --disable-nls --disable-silent-rules --disable-dependency-tracking --with-libtool-sysroot --disable-static" + +# This is a list of directories that are expected to be empty. +QA_EMPTY_DIRS ?= " \ + /dev/pts \ + /media \ + /proc \ + /run \ + /tmp \ + ${localstatedir}/run \ + ${localstatedir}/volatile \ +" +# It is possible to specify why a directory is expected to be empty by defining +# QA_EMPTY_DIRS_RECOMMENDATION:<path>, which will then be included in the error +# message if the directory is not empty. If it is not specified for a directory, +# then "but it is expected to be empty" will be used. + +def package_qa_clean_path(path, d, pkg=None): + """ + Remove redundant paths from the path for display. If pkg isn't set then + TMPDIR is stripped, otherwise PKGDEST/pkg is stripped. + """ + if pkg: + path = path.replace(os.path.join(d.getVar("PKGDEST"), pkg), "/") + return path.replace(d.getVar("TMPDIR"), "/").replace("//", "/") + +QAPATHTEST[shebang-size] = "package_qa_check_shebang_size" +def package_qa_check_shebang_size(path, name, d, elf, messages): + import stat + if os.path.islink(path) or stat.S_ISFIFO(os.stat(path).st_mode) or elf: + return + + try: + with open(path, 'rb') as f: + stanza = f.readline(130) + except IOError: + return + + if stanza.startswith(b'#!'): + try: + stanza.decode("utf-8") + except UnicodeDecodeError: + #If it is not a text file, it is not a script + return + + if len(stanza) > 129: + oe.qa.add_message(messages, "shebang-size", "%s: %s maximum shebang size exceeded, the maximum size is 128." % (name, package_qa_clean_path(path, d, name))) + return + +QAPATHTEST[libexec] = "package_qa_check_libexec" +def package_qa_check_libexec(path,name, d, elf, messages): + + # Skip the case where the default is explicitly /usr/libexec + libexec = d.getVar('libexecdir') + if libexec == "/usr/libexec": + return True + + if 'libexec' in path.split(os.path.sep): + oe.qa.add_message(messages, "libexec", "%s: %s is using libexec please relocate to %s" % (name, package_qa_clean_path(path, d, name), libexec)) + return False + + return True + +QAPATHTEST[rpaths] = "package_qa_check_rpath" +def package_qa_check_rpath(file,name, d, elf, messages): + """ + Check for dangerous RPATHs + """ + if not elf: + return + + if os.path.islink(file): + return + + bad_dirs = [d.getVar('BASE_WORKDIR'), d.getVar('STAGING_DIR_TARGET')] + + phdrs = elf.run_objdump("-p", d) + + import re + rpath_re = re.compile(r"\s+RPATH\s+(.*)") + for line in phdrs.split("\n"): + m = rpath_re.match(line) + if m: + rpath = m.group(1) + for dir in bad_dirs: + if dir in rpath: + oe.qa.add_message(messages, "rpaths", "package %s contains bad RPATH %s in file %s" % (name, rpath, file)) + +QAPATHTEST[useless-rpaths] = "package_qa_check_useless_rpaths" +def package_qa_check_useless_rpaths(file, name, d, elf, messages): + """ + Check for RPATHs that are useless but not dangerous + """ + def rpath_eq(a, b): + return os.path.normpath(a) == os.path.normpath(b) + + if not elf: + return + + if os.path.islink(file): + return + + libdir = d.getVar("libdir") + base_libdir = d.getVar("base_libdir") + + phdrs = elf.run_objdump("-p", d) + + import re + rpath_re = re.compile(r"\s+RPATH\s+(.*)") + for line in phdrs.split("\n"): + m = rpath_re.match(line) + if m: + rpath = m.group(1) + if rpath_eq(rpath, libdir) or rpath_eq(rpath, base_libdir): + # The dynamic linker searches both these places anyway. There is no point in + # looking there again. + oe.qa.add_message(messages, "useless-rpaths", "%s: %s contains probably-redundant RPATH %s" % (name, package_qa_clean_path(file, d, name), rpath)) + +QAPATHTEST[dev-so] = "package_qa_check_dev" +def package_qa_check_dev(path, name, d, elf, messages): + """ + Check for ".so" library symlinks in non-dev packages + """ + + if not name.endswith("-dev") and not name.endswith("-dbg") and not name.endswith("-ptest") and not name.startswith("nativesdk-") and path.endswith(".so") and os.path.islink(path): + oe.qa.add_message(messages, "dev-so", "non -dev/-dbg/nativesdk- package %s contains symlink .so '%s'" % \ + (name, package_qa_clean_path(path, d, name))) + +QAPATHTEST[dev-elf] = "package_qa_check_dev_elf" +def package_qa_check_dev_elf(path, name, d, elf, messages): + """ + Check that -dev doesn't contain real shared libraries. The test has to + check that the file is not a link and is an ELF object as some recipes + install link-time .so files that are linker scripts. + """ + if name.endswith("-dev") and path.endswith(".so") and not os.path.islink(path) and elf: + oe.qa.add_message(messages, "dev-elf", "-dev package %s contains non-symlink .so '%s'" % \ + (name, package_qa_clean_path(path, d, name))) + +QAPATHTEST[staticdev] = "package_qa_check_staticdev" +def package_qa_check_staticdev(path, name, d, elf, messages): + """ + Check for ".a" library in non-staticdev packages + There are a number of exceptions to this rule, -pic packages can contain + static libraries, the _nonshared.a belong with their -dev packages and + libgcc.a, libgcov.a will be skipped in their packages + """ + + if not name.endswith("-pic") and not name.endswith("-staticdev") and not name.endswith("-ptest") and path.endswith(".a") and not path.endswith("_nonshared.a") and not '/usr/lib/debug-static/' in path and not '/.debug-static/' in path: + oe.qa.add_message(messages, "staticdev", "non -staticdev package contains static .a library: %s path '%s'" % \ + (name, package_qa_clean_path(path, d, name))) + +QAPATHTEST[mime] = "package_qa_check_mime" +def package_qa_check_mime(path, name, d, elf, messages): + """ + Check if package installs mime types to /usr/share/mime/packages + while no inheriting mime.bbclass + """ + + if d.getVar("datadir") + "/mime/packages" in path and path.endswith('.xml') and not bb.data.inherits_class("mime", d): + oe.qa.add_message(messages, "mime", "package contains mime types but does not inherit mime: %s path '%s'" % \ + (name, package_qa_clean_path(path, d, name))) + +QAPATHTEST[mime-xdg] = "package_qa_check_mime_xdg" +def package_qa_check_mime_xdg(path, name, d, elf, messages): + """ + Check if package installs desktop file containing MimeType and requires + mime-types.bbclass to create /usr/share/applications/mimeinfo.cache + """ + + if d.getVar("datadir") + "/applications" in path and path.endswith('.desktop') and not bb.data.inherits_class("mime-xdg", d): + mime_type_found = False + try: + with open(path, 'r') as f: + for line in f.read().split('\n'): + if 'MimeType' in line: + mime_type_found = True + break; + except: + # At least libreoffice installs symlinks with absolute paths that are dangling here. + # We could implement some magic but for few (one) recipes it is not worth the effort so just warn: + wstr = "%s cannot open %s - is it a symlink with absolute path?\n" % (name, package_qa_clean_path(path, d, name)) + wstr += "Please check if (linked) file contains key 'MimeType'.\n" + pkgname = name + if name == d.getVar('PN'): + pkgname = '${PN}' + wstr += "If yes: add \'inhert mime-xdg\' and \'MIME_XDG_PACKAGES += \"%s\"\' / if no add \'INSANE_SKIP:%s += \"mime-xdg\"\' to recipe." % (pkgname, pkgname) + oe.qa.add_message(messages, "mime-xdg", wstr) + if mime_type_found: + oe.qa.add_message(messages, "mime-xdg", "%s: contains desktop file with key 'MimeType' but does not inhert mime-xdg: %s" % \ + (name, package_qa_clean_path(path, d, name))) + +def package_qa_check_libdir(d): + """ + Check for wrong library installation paths. For instance, catch + recipes installing /lib/bar.so when ${base_libdir}="lib32" or + installing in /usr/lib64 when ${libdir}="/usr/lib" + """ + import re + + pkgdest = d.getVar('PKGDEST') + base_libdir = d.getVar("base_libdir") + os.sep + libdir = d.getVar("libdir") + os.sep + libexecdir = d.getVar("libexecdir") + os.sep + exec_prefix = d.getVar("exec_prefix") + os.sep + + messages = [] + + # The re's are purposely fuzzy, as some there are some .so.x.y.z files + # that don't follow the standard naming convention. It checks later + # that they are actual ELF files + lib_re = re.compile(r"^/lib.+\.so(\..+)?$") + exec_re = re.compile(r"^%s.*/lib.+\.so(\..+)?$" % exec_prefix) + + for root, dirs, files in os.walk(pkgdest): + if root == pkgdest: + # Skip subdirectories for any packages with libdir in INSANE_SKIP + skippackages = [] + for package in dirs: + if 'libdir' in (d.getVar('INSANE_SKIP:' + package) or "").split(): + bb.note("Package %s skipping libdir QA test" % (package)) + skippackages.append(package) + elif d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-file-directory' and package.endswith("-dbg"): + bb.note("Package %s skipping libdir QA test for PACKAGE_DEBUG_SPLIT_STYLE equals debug-file-directory" % (package)) + skippackages.append(package) + for package in skippackages: + dirs.remove(package) + for file in files: + full_path = os.path.join(root, file) + rel_path = os.path.relpath(full_path, pkgdest) + if os.sep in rel_path: + package, rel_path = rel_path.split(os.sep, 1) + rel_path = os.sep + rel_path + if lib_re.match(rel_path): + if base_libdir not in rel_path: + # make sure it's an actual ELF file + elf = oe.qa.ELFFile(full_path) + try: + elf.open() + messages.append("%s: found library in wrong location: %s" % (package, rel_path)) + except (oe.qa.NotELFFileError): + pass + if exec_re.match(rel_path): + if libdir not in rel_path and libexecdir not in rel_path: + # make sure it's an actual ELF file + elf = oe.qa.ELFFile(full_path) + try: + elf.open() + messages.append("%s: found library in wrong location: %s" % (package, rel_path)) + except (oe.qa.NotELFFileError): + pass + + if messages: + oe.qa.handle_error("libdir", "\n".join(messages), d) + +QAPATHTEST[debug-files] = "package_qa_check_dbg" +def package_qa_check_dbg(path, name, d, elf, messages): + """ + Check for ".debug" files or directories outside of the dbg package + """ + + if not "-dbg" in name and not "-ptest" in name: + if '.debug' in path.split(os.path.sep): + oe.qa.add_message(messages, "debug-files", "%s: non debug package contains .debug directory %s" % \ + (name, package_qa_clean_path(path, d, name))) + +QAPATHTEST[arch] = "package_qa_check_arch" +def package_qa_check_arch(path,name,d, elf, messages): + """ + Check if archs are compatible + """ + import re, oe.elf + + if not elf: + return + + target_os = d.getVar('HOST_OS') + target_arch = d.getVar('HOST_ARCH') + provides = d.getVar('PROVIDES') + bpn = d.getVar('BPN') + + if target_arch == "allarch": + pn = d.getVar('PN') + oe.qa.add_message(messages, "arch", pn + ": Recipe inherits the allarch class, but has packaged architecture-specific binaries") + return + + # FIXME: Cross package confuse this check, so just skip them + for s in ['cross', 'nativesdk', 'cross-canadian']: + if bb.data.inherits_class(s, d): + return + + # avoid following links to /usr/bin (e.g. on udev builds) + # we will check the files pointed to anyway... + if os.path.islink(path): + return + + #if this will throw an exception, then fix the dict above + (machine, osabi, abiversion, littleendian, bits) \ + = oe.elf.machine_dict(d)[target_os][target_arch] + + # Check the architecture and endiannes of the binary + is_32 = (("virtual/kernel" in provides) or bb.data.inherits_class("module", d)) and \ + (target_os == "linux-gnux32" or target_os == "linux-muslx32" or \ + target_os == "linux-gnu_ilp32" or re.match(r'mips64.*32', d.getVar('DEFAULTTUNE'))) + is_bpf = (oe.qa.elf_machine_to_string(elf.machine()) == "BPF") + if not ((machine == elf.machine()) or is_32 or is_bpf): + oe.qa.add_message(messages, "arch", "Architecture did not match (%s, expected %s) in %s" % \ + (oe.qa.elf_machine_to_string(elf.machine()), oe.qa.elf_machine_to_string(machine), package_qa_clean_path(path, d, name))) + elif not ((bits == elf.abiSize()) or is_32 or is_bpf): + oe.qa.add_message(messages, "arch", "Bit size did not match (%d, expected %d) in %s" % \ + (elf.abiSize(), bits, package_qa_clean_path(path, d, name))) + elif not ((littleendian == elf.isLittleEndian()) or is_bpf): + oe.qa.add_message(messages, "arch", "Endiannes did not match (%d, expected %d) in %s" % \ + (elf.isLittleEndian(), littleendian, package_qa_clean_path(path, d, name))) + +QAPATHTEST[desktop] = "package_qa_check_desktop" +def package_qa_check_desktop(path, name, d, elf, messages): + """ + Run all desktop files through desktop-file-validate. + """ + if path.endswith(".desktop"): + desktop_file_validate = os.path.join(d.getVar('STAGING_BINDIR_NATIVE'),'desktop-file-validate') + output = os.popen("%s %s" % (desktop_file_validate, path)) + # This only produces output on errors + for l in output: + oe.qa.add_message(messages, "desktop", "Desktop file issue: " + l.strip()) + +QAPATHTEST[textrel] = "package_qa_textrel" +def package_qa_textrel(path, name, d, elf, messages): + """ + Check if the binary contains relocations in .text + """ + + if not elf: + return + + if os.path.islink(path): + return + + phdrs = elf.run_objdump("-p", d) + sane = True + + import re + textrel_re = re.compile(r"\s+TEXTREL\s+") + for line in phdrs.split("\n"): + if textrel_re.match(line): + sane = False + break + + if not sane: + path = package_qa_clean_path(path, d, name) + oe.qa.add_message(messages, "textrel", "%s: ELF binary %s has relocations in .text" % (name, path)) + +QAPATHTEST[ldflags] = "package_qa_hash_style" +def package_qa_hash_style(path, name, d, elf, messages): + """ + Check if the binary has the right hash style... + """ + + if not elf: + return + + if os.path.islink(path): + return + + gnu_hash = "--hash-style=gnu" in d.getVar('LDFLAGS') + if not gnu_hash: + gnu_hash = "--hash-style=both" in d.getVar('LDFLAGS') + if not gnu_hash: + return + + sane = False + has_syms = False + + phdrs = elf.run_objdump("-p", d) + + # If this binary has symbols, we expect it to have GNU_HASH too. + for line in phdrs.split("\n"): + if "SYMTAB" in line: + has_syms = True + if "GNU_HASH" in line or "MIPS_XHASH" in line: + sane = True + if ("[mips32]" in line or "[mips64]" in line) and d.getVar('TCLIBC') == "musl": + sane = True + if has_syms and not sane: + path = package_qa_clean_path(path, d, name) + oe.qa.add_message(messages, "ldflags", "File %s in package %s doesn't have GNU_HASH (didn't pass LDFLAGS?)" % (path, name)) + + +QAPATHTEST[buildpaths] = "package_qa_check_buildpaths" +def package_qa_check_buildpaths(path, name, d, elf, messages): + """ + Check for build paths inside target files and error if paths are not + explicitly ignored. + """ + import stat + + # Ignore symlinks/devs/fifos + mode = os.lstat(path).st_mode + if stat.S_ISLNK(mode) or stat.S_ISBLK(mode) or stat.S_ISFIFO(mode) or stat.S_ISCHR(mode) or stat.S_ISSOCK(mode): + return + + tmpdir = bytes(d.getVar('TMPDIR'), encoding="utf-8") + with open(path, 'rb') as f: + file_content = f.read() + if tmpdir in file_content: + trimmed = path.replace(os.path.join (d.getVar("PKGDEST"), name), "") + oe.qa.add_message(messages, "buildpaths", "File %s in package %s contains reference to TMPDIR" % (trimmed, name)) + + +QAPATHTEST[xorg-driver-abi] = "package_qa_check_xorg_driver_abi" +def package_qa_check_xorg_driver_abi(path, name, d, elf, messages): + """ + Check that all packages containing Xorg drivers have ABI dependencies + """ + + # Skip dev, dbg or nativesdk packages + if name.endswith("-dev") or name.endswith("-dbg") or name.startswith("nativesdk-"): + return + + driverdir = d.expand("${libdir}/xorg/modules/drivers/") + if driverdir in path and path.endswith(".so"): + mlprefix = d.getVar('MLPREFIX') or '' + for rdep in bb.utils.explode_deps(d.getVar('RDEPENDS:' + name) or ""): + if rdep.startswith("%sxorg-abi-" % mlprefix): + return + oe.qa.add_message(messages, "xorg-driver-abi", "Package %s contains Xorg driver (%s) but no xorg-abi- dependencies" % (name, os.path.basename(path))) + +QAPATHTEST[infodir] = "package_qa_check_infodir" +def package_qa_check_infodir(path, name, d, elf, messages): + """ + Check that /usr/share/info/dir isn't shipped in a particular package + """ + infodir = d.expand("${infodir}/dir") + + if infodir in path: + oe.qa.add_message(messages, "infodir", "The /usr/share/info/dir file is not meant to be shipped in a particular package.") + +QAPATHTEST[symlink-to-sysroot] = "package_qa_check_symlink_to_sysroot" +def package_qa_check_symlink_to_sysroot(path, name, d, elf, messages): + """ + Check that the package doesn't contain any absolute symlinks to the sysroot. + """ + if os.path.islink(path): + target = os.readlink(path) + if os.path.isabs(target): + tmpdir = d.getVar('TMPDIR') + if target.startswith(tmpdir): + trimmed = path.replace(os.path.join (d.getVar("PKGDEST"), name), "") + oe.qa.add_message(messages, "symlink-to-sysroot", "Symlink %s in %s points to TMPDIR" % (trimmed, name)) + +QAPATHTEST[32bit-time] = "check_32bit_symbols" +def check_32bit_symbols(path, packagename, d, elf, messages): + """ + Check that ELF files do not use any 32 bit time APIs from glibc. + """ + thirtytwo_bit_time_archs = {'arm','armeb','mipsarcho32','powerpc','x86'} + overrides = set(d.getVar('OVERRIDES').split(':')) + if not (thirtytwo_bit_time_archs & overrides): + return + + import re + # This list is manually constructed by searching the image folder of the + # glibc recipe for __USE_TIME_BITS64. There is no good way to do this + # automatically. + api32 = { + # /usr/include/time.h + "clock_getres", "clock_gettime", "clock_nanosleep", "clock_settime", + "ctime", "ctime_r", "difftime", "gmtime", "gmtime_r", "localtime", + "localtime_r", "mktime", "nanosleep", "time", "timegm", "timelocal", + "timer_gettime", "timer_settime", "timespec_get", "timespec_getres", + # /usr/include/bits/time.h + "clock_adjtime", + # /usr/include/signal.h + "sigtimedwait", + # /usr/include/sys/time.h + "adjtime", + "futimes", "futimesat", "getitimer", "gettimeofday", "lutimes", + "setitimer", "settimeofday", "utimes", + # /usr/include/sys/timex.h + "adjtimex", "ntp_adjtime", "ntp_gettime", "ntp_gettimex", + # /usr/include/sys/wait.h + "wait3", "wait4", + # /usr/include/sys/stat.h + "fstat", "fstat64", "fstatat", "fstatat64", "futimens", "lstat", + "lstat64", "stat", "stat64", "utimensat", + # /usr/include/sys/poll.h + "ppoll", + # /usr/include/sys/resource.h + "getrusage", + # /usr/include/sys/ioctl.h + "ioctl", + # /usr/include/sys/select.h + "select", "pselect", + # /usr/include/sys/prctl.h + "prctl", + # /usr/include/sys/epoll.h + "epoll_pwait2", + # /usr/include/sys/timerfd.h + "timerfd_gettime", "timerfd_settime", + # /usr/include/sys/socket.h + "getsockopt", "recvmmsg", "recvmsg", "sendmmsg", "sendmsg", + "setsockopt", + # /usr/include/sys/msg.h + "msgctl", + # /usr/include/sys/sem.h + "semctl", "semtimedop", + # /usr/include/sys/shm.h + "shmctl", + # /usr/include/pthread.h + "pthread_clockjoin_np", "pthread_cond_clockwait", + "pthread_cond_timedwait", "pthread_mutex_clocklock", + "pthread_mutex_timedlock", "pthread_rwlock_clockrdlock", + "pthread_rwlock_clockwrlock", "pthread_rwlock_timedrdlock", + "pthread_rwlock_timedwrlock", "pthread_timedjoin_np", + # /usr/include/semaphore.h + "sem_clockwait", "sem_timedwait", + # /usr/include/threads.h + "cnd_timedwait", "mtx_timedlock", "thrd_sleep", + # /usr/include/aio.h + "aio_cancel", "aio_error", "aio_read", "aio_return", "aio_suspend", + "aio_write", "lio_listio", + # /usr/include/mqueue.h + "mq_timedreceive", "mq_timedsend", + # /usr/include/glob.h + "glob", "glob64", "globfree", "globfree64", + # /usr/include/sched.h + "sched_rr_get_interval", + # /usr/include/fcntl.h + "fcntl", "fcntl64", + # /usr/include/utime.h + "utime", + # /usr/include/ftw.h + "ftw", "ftw64", "nftw", "nftw64", + # /usr/include/fts.h + "fts64_children", "fts64_close", "fts64_open", "fts64_read", + "fts64_set", "fts_children", "fts_close", "fts_open", "fts_read", + "fts_set", + # /usr/include/netdb.h + "gai_suspend", + } + + ptrn = re.compile( + r''' + (?P<value>[\da-fA-F]+) \s+ + (?P<flags>[lgu! ][w ][C ][W ][Ii ][dD ]F) \s+ + (?P<section>\*UND\*) \s+ + (?P<alignment>(?P<size>[\da-fA-F]+)) \s+ + (?P<symbol> + ''' + + r'(?P<notag>' + r'|'.join(sorted(api32)) + r')' + + r''' + (@+(?P<tag>GLIBC_\d+\.\d+\S*))) + ''', re.VERBOSE + ) + + # elf is a oe.qa.ELFFile object + if elf is not None: + phdrs = elf.run_objdump("-tw", d) + syms = re.finditer(ptrn, phdrs) + usedapis = {sym.group('notag') for sym in syms} + if usedapis: + elfpath = package_qa_clean_path(path, d, packagename) + # Remove any .debug dir, heuristic that probably works + # At this point, any symbol information is stripped into the debug + # package, so that is the only place we will find them. + elfpath = elfpath.replace('.debug/', '') + allowed = "32bit-time" in (d.getVar('INSANE_SKIP') or '').split() + if not allowed: + msgformat = elfpath + " uses 32-bit api '%s'" + for sym in usedapis: + oe.qa.add_message(messages, '32bit-time', msgformat % sym) + oe.qa.add_message( + messages, '32bit-time', + 'Suppress with INSANE_SKIP = "32bit-time"' + ) + +# Check license variables +do_populate_lic[postfuncs] += "populate_lic_qa_checksum" +python populate_lic_qa_checksum() { + """ + Check for changes in the license files. + """ + + lic_files = d.getVar('LIC_FILES_CHKSUM') or '' + lic = d.getVar('LICENSE') + pn = d.getVar('PN') + + if lic == "CLOSED": + return + + if not lic_files and d.getVar('SRC_URI'): + oe.qa.handle_error("license-checksum", pn + ": Recipe file fetches files and does not have license file information (LIC_FILES_CHKSUM)", d) + + srcdir = d.getVar('S') + corebase_licensefile = d.getVar('COREBASE') + "/LICENSE" + for url in lic_files.split(): + try: + (type, host, path, user, pswd, parm) = bb.fetch.decodeurl(url) + except bb.fetch.MalformedUrl: + oe.qa.handle_error("license-checksum", pn + ": LIC_FILES_CHKSUM contains an invalid URL: " + url, d) + continue + srclicfile = os.path.join(srcdir, path) + if not os.path.isfile(srclicfile): + oe.qa.handle_error("license-checksum", pn + ": LIC_FILES_CHKSUM points to an invalid file: " + srclicfile, d) + continue + + if (srclicfile == corebase_licensefile): + bb.warn("${COREBASE}/LICENSE is not a valid license file, please use '${COMMON_LICENSE_DIR}/MIT' for a MIT License file in LIC_FILES_CHKSUM. This will become an error in the future") + + recipemd5 = parm.get('md5', '') + beginline, endline = 0, 0 + if 'beginline' in parm: + beginline = int(parm['beginline']) + if 'endline' in parm: + endline = int(parm['endline']) + + if (not beginline) and (not endline): + md5chksum = bb.utils.md5_file(srclicfile) + with open(srclicfile, 'r', errors='replace') as f: + license = f.read().splitlines() + else: + with open(srclicfile, 'rb') as f: + import hashlib + lineno = 0 + license = [] + try: + m = hashlib.new('MD5', usedforsecurity=False) + except TypeError: + m = hashlib.new('MD5') + for line in f: + lineno += 1 + if (lineno >= beginline): + if ((lineno <= endline) or not endline): + m.update(line) + license.append(line.decode('utf-8', errors='replace').rstrip()) + else: + break + md5chksum = m.hexdigest() + if recipemd5 == md5chksum: + bb.note (pn + ": md5 checksum matched for ", url) + else: + if recipemd5: + msg = pn + ": The LIC_FILES_CHKSUM does not match for " + url + msg = msg + "\n" + pn + ": The new md5 checksum is " + md5chksum + max_lines = int(d.getVar('QA_MAX_LICENSE_LINES') or 20) + if not license or license[-1] != '': + # Ensure that our license text ends with a line break + # (will be added with join() below). + license.append('') + remove = len(license) - max_lines + if remove > 0: + start = max_lines // 2 + end = start + remove - 1 + del license[start:end] + license.insert(start, '...') + msg = msg + "\n" + pn + ": Here is the selected license text:" + \ + "\n" + \ + "{:v^70}".format(" beginline=%d " % beginline if beginline else "") + \ + "\n" + "\n".join(license) + \ + "{:^^70}".format(" endline=%d " % endline if endline else "") + if beginline: + if endline: + srcfiledesc = "%s (lines %d through to %d)" % (srclicfile, beginline, endline) + else: + srcfiledesc = "%s (beginning on line %d)" % (srclicfile, beginline) + elif endline: + srcfiledesc = "%s (ending on line %d)" % (srclicfile, endline) + else: + srcfiledesc = srclicfile + msg = msg + "\n" + pn + ": Check if the license information has changed in %s to verify that the LICENSE value \"%s\" remains valid" % (srcfiledesc, lic) + + else: + msg = pn + ": LIC_FILES_CHKSUM is not specified for " + url + msg = msg + "\n" + pn + ": The md5 checksum is " + md5chksum + oe.qa.handle_error("license-checksum", msg, d) + + oe.qa.exit_if_errors(d) +} + +def qa_check_staged(path,d): + """ + Check staged la and pc files for common problems like references to the work + directory. + + As this is run after every stage we should be able to find the one + responsible for the errors easily even if we look at every .pc and .la file. + """ + + tmpdir = d.getVar('TMPDIR') + workdir = os.path.join(tmpdir, "work") + recipesysroot = d.getVar("RECIPE_SYSROOT") + + if bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d): + pkgconfigcheck = workdir + else: + pkgconfigcheck = tmpdir + + skip = (d.getVar('INSANE_SKIP') or "").split() + skip_la = False + if 'la' in skip: + bb.note("Recipe %s skipping qa checking: la" % d.getVar('PN')) + skip_la = True + + skip_pkgconfig = False + if 'pkgconfig' in skip: + bb.note("Recipe %s skipping qa checking: pkgconfig" % d.getVar('PN')) + skip_pkgconfig = True + + skip_shebang_size = False + if 'shebang-size' in skip: + bb.note("Recipe %s skipping qa checkking: shebang-size" % d.getVar('PN')) + skip_shebang_size = True + + # find all .la and .pc files + # read the content + # and check for stuff that looks wrong + for root, dirs, files in os.walk(path): + for file in files: + path = os.path.join(root,file) + if file.endswith(".la") and not skip_la: + with open(path) as f: + file_content = f.read() + file_content = file_content.replace(recipesysroot, "") + if workdir in file_content: + error_msg = "%s failed sanity test (workdir) in path %s" % (file,root) + oe.qa.handle_error("la", error_msg, d) + elif file.endswith(".pc") and not skip_pkgconfig: + with open(path) as f: + file_content = f.read() + file_content = file_content.replace(recipesysroot, "") + if pkgconfigcheck in file_content: + error_msg = "%s failed sanity test (tmpdir) in path %s" % (file,root) + oe.qa.handle_error("pkgconfig", error_msg, d) + + if not skip_shebang_size: + errors = {} + package_qa_check_shebang_size(path, "", d, None, errors) + for e in errors: + oe.qa.handle_error(e, errors[e], d) + + +# Run all package-wide warnfuncs and errorfuncs +def package_qa_package(warnfuncs, errorfuncs, package, d): + warnings = {} + errors = {} + + for func in warnfuncs: + func(package, d, warnings) + for func in errorfuncs: + func(package, d, errors) + + for w in warnings: + oe.qa.handle_error(w, warnings[w], d) + for e in errors: + oe.qa.handle_error(e, errors[e], d) + + return len(errors) == 0 + +# Run all recipe-wide warnfuncs and errorfuncs +def package_qa_recipe(warnfuncs, errorfuncs, pn, d): + warnings = {} + errors = {} + + for func in warnfuncs: + func(pn, d, warnings) + for func in errorfuncs: + func(pn, d, errors) + + for w in warnings: + oe.qa.handle_error(w, warnings[w], d) + for e in errors: + oe.qa.handle_error(e, errors[e], d) + + return len(errors) == 0 + +def prepopulate_objdump_p(elf, d): + output = elf.run_objdump("-p", d) + return (elf.name, output) + +# Walk over all files in a directory and call func +def package_qa_walk(warnfuncs, errorfuncs, package, d): + #if this will throw an exception, then fix the dict above + target_os = d.getVar('HOST_OS') + target_arch = d.getVar('HOST_ARCH') + + warnings = {} + errors = {} + elves = {} + for path in pkgfiles[package]: + elf = None + if os.path.isfile(path): + elf = oe.qa.ELFFile(path) + try: + elf.open() + elf.close() + except oe.qa.NotELFFileError: + elf = None + if elf: + elves[path] = elf + + results = oe.utils.multiprocess_launch(prepopulate_objdump_p, elves.values(), d, extraargs=(d,)) + for item in results: + elves[item[0]].set_objdump("-p", item[1]) + + for path in pkgfiles[package]: + if path in elves: + elves[path].open() + for func in warnfuncs: + func(path, package, d, elves.get(path), warnings) + for func in errorfuncs: + func(path, package, d, elves.get(path), errors) + if path in elves: + elves[path].close() + + for w in warnings: + oe.qa.handle_error(w, warnings[w], d) + for e in errors: + oe.qa.handle_error(e, errors[e], d) + +def package_qa_check_rdepends(pkg, pkgdest, skip, taskdeps, packages, d): + # Don't do this check for kernel/module recipes, there aren't too many debug/development + # packages and you can get false positives e.g. on kernel-module-lirc-dev + if bb.data.inherits_class("kernel", d) or bb.data.inherits_class("module-base", d): + return + + if not "-dbg" in pkg and not "packagegroup-" in pkg and not "-image" in pkg: + localdata = bb.data.createCopy(d) + localdata.setVar('OVERRIDES', localdata.getVar('OVERRIDES') + ':' + pkg) + + # Now check the RDEPENDS + rdepends = bb.utils.explode_deps(localdata.getVar('RDEPENDS') or "") + + # Now do the sanity check!!! + if "build-deps" not in skip: + for rdepend in rdepends: + if "-dbg" in rdepend and "debug-deps" not in skip: + error_msg = "%s rdepends on %s" % (pkg,rdepend) + oe.qa.handle_error("debug-deps", error_msg, d) + if (not "-dev" in pkg and not "-staticdev" in pkg) and rdepend.endswith("-dev") and "dev-deps" not in skip: + error_msg = "%s rdepends on %s" % (pkg, rdepend) + oe.qa.handle_error("dev-deps", error_msg, d) + if rdepend not in packages: + rdep_data = oe.packagedata.read_subpkgdata(rdepend, d) + if rdep_data and 'PN' in rdep_data and rdep_data['PN'] in taskdeps: + continue + if not rdep_data or not 'PN' in rdep_data: + for _, rdep_data in oe.packagedata.foreach_runtime_provider_pkgdata(d, rdepend): + if rdep_data and 'PN' in rdep_data and rdep_data['PN'] in taskdeps: + break + if rdep_data and 'PN' in rdep_data and rdep_data['PN'] in taskdeps: + continue + if rdep_data and 'PN' in rdep_data: + error_msg = "%s rdepends on %s, but it isn't a build dependency, missing %s in DEPENDS or PACKAGECONFIG?" % (pkg, rdepend, rdep_data['PN']) + else: + error_msg = "%s rdepends on %s, but it isn't a build dependency?" % (pkg, rdepend) + oe.qa.handle_error("build-deps", error_msg, d) + + if "file-rdeps" not in skip: + ignored_file_rdeps = set(['/bin/sh', '/usr/bin/env', 'rtld(GNU_HASH)']) + if bb.utils.contains('DISTRO_FEATURES', 'usrmerge', True, False, d): + ignored_file_rdeps |= set(['/usr/bin/sh']) + if bb.data.inherits_class('nativesdk', d): + ignored_file_rdeps |= set(['/bin/bash', '/usr/bin/perl', 'perl']) + if bb.utils.contains('DISTRO_FEATURES', 'usrmerge', True, False, d): + ignored_file_rdeps |= set(['/usr/bin/bash']) + # For Saving the FILERDEPENDS + filerdepends = {} + rdep_data = oe.packagedata.read_subpkgdata(pkg, d) + for key in rdep_data: + if key.startswith("FILERDEPENDS:"): + for subkey in bb.utils.explode_deps(rdep_data[key]): + if subkey not in ignored_file_rdeps and \ + not subkey.startswith('perl('): + # We already know it starts with FILERDEPENDS_ + filerdepends[subkey] = key[13:] + + if filerdepends: + done = rdepends[:] + # Add the rprovides of itself + if pkg not in done: + done.insert(0, pkg) + + # The python is not a package, but python-core provides it, so + # skip checking /usr/bin/python if python is in the rdeps, in + # case there is a RDEPENDS:pkg = "python" in the recipe. + for py in [ d.getVar('MLPREFIX') + "python", "python" ]: + if py in done: + filerdepends.pop("/usr/bin/python",None) + done.remove(py) + for rdep in done: + # The file dependencies may contain package names, e.g., + # perl + filerdepends.pop(rdep,None) + + for _, rdep_data in oe.packagedata.foreach_runtime_provider_pkgdata(d, rdep, True): + for key in rdep_data: + if key.startswith("FILERPROVIDES:") or key.startswith("RPROVIDES:"): + for subkey in bb.utils.explode_deps(rdep_data[key]): + filerdepends.pop(subkey,None) + # Add the files list to the rprovides + if key.startswith("FILES_INFO:"): + # Use eval() to make it as a dict + for subkey in eval(rdep_data[key]): + filerdepends.pop(subkey,None) + + if not filerdepends: + # Break if all the file rdepends are met + break + if filerdepends: + for key in filerdepends: + error_msg = "%s contained in package %s requires %s, but no providers found in RDEPENDS:%s?" % \ + (filerdepends[key].replace(":%s" % pkg, "").replace("@underscore@", "_"), pkg, key, pkg) + oe.qa.handle_error("file-rdeps", error_msg, d) +package_qa_check_rdepends[vardepsexclude] = "OVERRIDES" + +def package_qa_check_deps(pkg, pkgdest, d): + + localdata = bb.data.createCopy(d) + localdata.setVar('OVERRIDES', pkg) + + def check_valid_deps(var): + try: + rvar = bb.utils.explode_dep_versions2(localdata.getVar(var) or "") + except ValueError as e: + bb.fatal("%s:%s: %s" % (var, pkg, e)) + for dep in rvar: + for v in rvar[dep]: + if v and not v.startswith(('< ', '= ', '> ', '<= ', '>=')): + error_msg = "%s:%s is invalid: %s (%s) only comparisons <, =, >, <=, and >= are allowed" % (var, pkg, dep, v) + oe.qa.handle_error("dep-cmp", error_msg, d) + + check_valid_deps('RDEPENDS') + check_valid_deps('RRECOMMENDS') + check_valid_deps('RSUGGESTS') + check_valid_deps('RPROVIDES') + check_valid_deps('RREPLACES') + check_valid_deps('RCONFLICTS') + +QAPKGTEST[usrmerge] = "package_qa_check_usrmerge" +def package_qa_check_usrmerge(pkg, d, messages): + + pkgdest = d.getVar('PKGDEST') + pkg_dir = pkgdest + os.sep + pkg + os.sep + merged_dirs = ['bin', 'sbin', 'lib'] + d.getVar('MULTILIB_VARIANTS').split() + for f in merged_dirs: + if os.path.exists(pkg_dir + f) and not os.path.islink(pkg_dir + f): + msg = "%s package is not obeying usrmerge distro feature. /%s should be relocated to /usr." % (pkg, f) + oe.qa.add_message(messages, "usrmerge", msg) + return False + return True + +QAPKGTEST[perllocalpod] = "package_qa_check_perllocalpod" +def package_qa_check_perllocalpod(pkg, d, messages): + """ + Check that the recipe didn't ship a perlocal.pod file, which shouldn't be + installed in a distribution package. cpan.bbclass sets NO_PERLLOCAL=1 to + handle this for most recipes. + """ + import glob + pkgd = oe.path.join(d.getVar('PKGDEST'), pkg) + podpath = oe.path.join(pkgd, d.getVar("libdir"), "perl*", "*", "*", "perllocal.pod") + + matches = glob.glob(podpath) + if matches: + matches = [package_qa_clean_path(path, d, pkg) for path in matches] + msg = "%s contains perllocal.pod (%s), should not be installed" % (pkg, " ".join(matches)) + oe.qa.add_message(messages, "perllocalpod", msg) + +QAPKGTEST[expanded-d] = "package_qa_check_expanded_d" +def package_qa_check_expanded_d(package, d, messages): + """ + Check for the expanded D (${D}) value in pkg_* and FILES + variables, warn the user to use it correctly. + """ + sane = True + expanded_d = d.getVar('D') + + for var in 'FILES','pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm': + bbvar = d.getVar(var + ":" + package) or "" + if expanded_d in bbvar: + if var == 'FILES': + oe.qa.add_message(messages, "expanded-d", "FILES in %s recipe should not contain the ${D} variable as it references the local build directory not the target filesystem, best solution is to remove the ${D} reference" % package) + sane = False + else: + oe.qa.add_message(messages, "expanded-d", "%s in %s recipe contains ${D}, it should be replaced by $D instead" % (var, package)) + sane = False + return sane + +QAPKGTEST[unlisted-pkg-lics] = "package_qa_check_unlisted_pkg_lics" +def package_qa_check_unlisted_pkg_lics(package, d, messages): + """ + Check that all licenses for a package are among the licenses for the recipe. + """ + pkg_lics = d.getVar('LICENSE:' + package) + if not pkg_lics: + return True + + recipe_lics_set = oe.license.list_licenses(d.getVar('LICENSE')) + package_lics = oe.license.list_licenses(pkg_lics) + unlisted = package_lics - recipe_lics_set + if unlisted: + oe.qa.add_message(messages, "unlisted-pkg-lics", + "LICENSE:%s includes licenses (%s) that are not " + "listed in LICENSE" % (package, ' '.join(unlisted))) + return False + obsolete = set(oe.license.obsolete_license_list()) & package_lics - recipe_lics_set + if obsolete: + oe.qa.add_message(messages, "obsolete-license", + "LICENSE:%s includes obsolete licenses %s" % (package, ' '.join(obsolete))) + return False + return True + +QAPKGTEST[empty-dirs] = "package_qa_check_empty_dirs" +def package_qa_check_empty_dirs(pkg, d, messages): + """ + Check for the existence of files in directories that are expected to be + empty. + """ + + pkgd = oe.path.join(d.getVar('PKGDEST'), pkg) + for dir in (d.getVar('QA_EMPTY_DIRS') or "").split(): + empty_dir = oe.path.join(pkgd, dir) + if os.path.exists(empty_dir) and os.listdir(empty_dir): + recommendation = (d.getVar('QA_EMPTY_DIRS_RECOMMENDATION:' + dir) or + "but it is expected to be empty") + msg = "%s installs files in %s, %s" % (pkg, dir, recommendation) + oe.qa.add_message(messages, "empty-dirs", msg) + +def package_qa_check_encoding(keys, encode, d): + def check_encoding(key, enc): + sane = True + value = d.getVar(key) + if value: + try: + s = value.encode(enc) + except UnicodeDecodeError as e: + error_msg = "%s has non %s characters" % (key,enc) + sane = False + oe.qa.handle_error("invalid-chars", error_msg, d) + return sane + + for key in keys: + sane = check_encoding(key, encode) + if not sane: + break + +HOST_USER_UID := "${@os.getuid()}" +HOST_USER_GID := "${@os.getgid()}" + +QAPATHTEST[host-user-contaminated] = "package_qa_check_host_user" +def package_qa_check_host_user(path, name, d, elf, messages): + """Check for paths outside of /home which are owned by the user running bitbake.""" + + if not os.path.lexists(path): + return + + dest = d.getVar('PKGDEST') + pn = d.getVar('PN') + home = os.path.join(dest, name, 'home') + if path == home or path.startswith(home + os.sep): + return + + try: + stat = os.lstat(path) + except OSError as exc: + import errno + if exc.errno != errno.ENOENT: + raise + else: + check_uid = int(d.getVar('HOST_USER_UID')) + if stat.st_uid == check_uid: + oe.qa.add_message(messages, "host-user-contaminated", "%s: %s is owned by uid %d, which is the same as the user running bitbake. This may be due to host contamination" % (pn, package_qa_clean_path(path, d, name), check_uid)) + return False + + check_gid = int(d.getVar('HOST_USER_GID')) + if stat.st_gid == check_gid: + oe.qa.add_message(messages, "host-user-contaminated", "%s: %s is owned by gid %d, which is the same as the user running bitbake. This may be due to host contamination" % (pn, package_qa_clean_path(path, d, name), check_gid)) + return False + return True + +QARECIPETEST[unhandled-features-check] = "package_qa_check_unhandled_features_check" +def package_qa_check_unhandled_features_check(pn, d, messages): + if not bb.data.inherits_class('features_check', d): + var_set = False + for kind in ['DISTRO', 'MACHINE', 'COMBINED']: + for var in ['ANY_OF_' + kind + '_FEATURES', 'REQUIRED_' + kind + '_FEATURES', 'CONFLICT_' + kind + '_FEATURES']: + if d.getVar(var) is not None or d.hasOverrides(var): + var_set = True + if var_set: + oe.qa.handle_error("unhandled-features-check", "%s: recipe doesn't inherit features_check" % pn, d) + +QARECIPETEST[missing-update-alternatives] = "package_qa_check_missing_update_alternatives" +def package_qa_check_missing_update_alternatives(pn, d, messages): + # Look at all packages and find out if any of those sets ALTERNATIVE variable + # without inheriting update-alternatives class + for pkg in (d.getVar('PACKAGES') or '').split(): + if d.getVar('ALTERNATIVE:%s' % pkg) and not bb.data.inherits_class('update-alternatives', d): + oe.qa.handle_error("missing-update-alternatives", "%s: recipe defines ALTERNATIVE:%s but doesn't inherit update-alternatives. This might fail during do_rootfs later!" % (pn, pkg), d) + +# The PACKAGE FUNC to scan each package +python do_package_qa () { + import subprocess + import oe.packagedata + + bb.note("DO PACKAGE QA") + + main_lic = d.getVar('LICENSE') + + # Check for obsolete license references in main LICENSE (packages are checked below for any changes) + main_licenses = oe.license.list_licenses(d.getVar('LICENSE')) + obsolete = set(oe.license.obsolete_license_list()) & main_licenses + if obsolete: + oe.qa.handle_error("obsolete-license", "Recipe LICENSE includes obsolete licenses %s" % ' '.join(obsolete), d) + + bb.build.exec_func("read_subpackage_metadata", d) + + # Check non UTF-8 characters on recipe's metadata + package_qa_check_encoding(['DESCRIPTION', 'SUMMARY', 'LICENSE', 'SECTION'], 'utf-8', d) + + logdir = d.getVar('T') + pn = d.getVar('PN') + + # Scan the packages... + pkgdest = d.getVar('PKGDEST') + packages = set((d.getVar('PACKAGES') or '').split()) + + global pkgfiles + pkgfiles = {} + for pkg in packages: + pkgfiles[pkg] = [] + pkgdir = os.path.join(pkgdest, pkg) + for walkroot, dirs, files in os.walk(pkgdir): + # Don't walk into top-level CONTROL or DEBIAN directories as these + # are temporary directories created by do_package. + if walkroot == pkgdir: + for control in ("CONTROL", "DEBIAN"): + if control in dirs: + dirs.remove(control) + for file in files: + pkgfiles[pkg].append(os.path.join(walkroot, file)) + + # no packages should be scanned + if not packages: + return + + import re + # The package name matches the [a-z0-9.+-]+ regular expression + pkgname_pattern = re.compile(r"^[a-z0-9.+-]+$") + + taskdepdata = d.getVar("BB_TASKDEPDATA", False) + taskdeps = set() + for dep in taskdepdata: + taskdeps.add(taskdepdata[dep][0]) + + def parse_test_matrix(matrix_name): + testmatrix = d.getVarFlags(matrix_name) or {} + g = globals() + warnchecks = [] + for w in (d.getVar("WARN_QA") or "").split(): + if w in skip: + continue + if w in testmatrix and testmatrix[w] in g: + warnchecks.append(g[testmatrix[w]]) + + errorchecks = [] + for e in (d.getVar("ERROR_QA") or "").split(): + if e in skip: + continue + if e in testmatrix and testmatrix[e] in g: + errorchecks.append(g[testmatrix[e]]) + return warnchecks, errorchecks + + for package in packages: + skip = set((d.getVar('INSANE_SKIP') or "").split() + + (d.getVar('INSANE_SKIP:' + package) or "").split()) + if skip: + bb.note("Package %s skipping QA tests: %s" % (package, str(skip))) + + bb.note("Checking Package: %s" % package) + # Check package name + if not pkgname_pattern.match(package): + oe.qa.handle_error("pkgname", + "%s doesn't match the [a-z0-9.+-]+ regex" % package, d) + + warn_checks, error_checks = parse_test_matrix("QAPATHTEST") + package_qa_walk(warn_checks, error_checks, package, d) + + warn_checks, error_checks = parse_test_matrix("QAPKGTEST") + package_qa_package(warn_checks, error_checks, package, d) + + package_qa_check_rdepends(package, pkgdest, skip, taskdeps, packages, d) + package_qa_check_deps(package, pkgdest, d) + + warn_checks, error_checks = parse_test_matrix("QARECIPETEST") + package_qa_recipe(warn_checks, error_checks, pn, d) + + if 'libdir' in d.getVar("ALL_QA").split(): + package_qa_check_libdir(d) + + oe.qa.exit_if_errors(d) +} + +# binutils is used for most checks, so need to set as dependency +# POPULATESYSROOTDEPS is defined in staging class. +do_package_qa[depends] += "${POPULATESYSROOTDEPS}" +do_package_qa[vardeps] = "${@bb.utils.contains('ERROR_QA', 'empty-dirs', 'QA_EMPTY_DIRS', '', d)}" +do_package_qa[vardepsexclude] = "BB_TASKDEPDATA" +do_package_qa[rdeptask] = "do_packagedata" +addtask do_package_qa after do_packagedata do_package before do_build + +# Add the package specific INSANE_SKIPs to the sstate dependencies +python() { + pkgs = (d.getVar('PACKAGES') or '').split() + for pkg in pkgs: + d.appendVarFlag("do_package_qa", "vardeps", " INSANE_SKIP:{}".format(pkg)) +} + +SSTATETASKS += "do_package_qa" +do_package_qa[sstate-inputdirs] = "" +do_package_qa[sstate-outputdirs] = "" +python do_package_qa_setscene () { + sstate_setscene(d) +} +addtask do_package_qa_setscene + +python do_qa_sysroot() { + bb.note("QA checking do_populate_sysroot") + sysroot_destdir = d.expand('${SYSROOT_DESTDIR}') + for sysroot_dir in d.expand('${SYSROOT_DIRS}').split(): + qa_check_staged(sysroot_destdir + sysroot_dir, d) + oe.qa.exit_with_message_if_errors("do_populate_sysroot for this recipe installed files with QA issues", d) +} +do_populate_sysroot[postfuncs] += "do_qa_sysroot" + +python do_qa_patch() { + import subprocess + + ########################################################################### + # Check patch.log for fuzz warnings + # + # Further information on why we check for patch fuzz warnings: + # http://lists.openembedded.org/pipermail/openembedded-core/2018-March/148675.html + # https://bugzilla.yoctoproject.org/show_bug.cgi?id=10450 + ########################################################################### + + logdir = d.getVar('T') + patchlog = os.path.join(logdir,"log.do_patch") + + if os.path.exists(patchlog): + fuzzheader = '--- Patch fuzz start ---' + fuzzfooter = '--- Patch fuzz end ---' + statement = "grep -e '%s' %s > /dev/null" % (fuzzheader, patchlog) + if subprocess.call(statement, shell=True) == 0: + msg = "Fuzz detected:\n\n" + fuzzmsg = "" + inFuzzInfo = False + f = open(patchlog, "r") + for line in f: + if fuzzheader in line: + inFuzzInfo = True + fuzzmsg = "" + elif fuzzfooter in line: + fuzzmsg = fuzzmsg.replace('\n\n', '\n') + msg += fuzzmsg + msg += "\n" + inFuzzInfo = False + elif inFuzzInfo and not 'Now at patch' in line: + fuzzmsg += line + f.close() + msg += "The context lines in the patches can be updated with devtool:\n" + msg += "\n" + msg += " devtool modify %s\n" % d.getVar('PN') + msg += " devtool finish --force-patch-refresh %s <layer_path>\n\n" % d.getVar('PN') + msg += "Don't forget to review changes done by devtool!\n" + msg += "\nPatch log indicates that patches do not apply cleanly." + oe.qa.handle_error("patch-fuzz", msg, d) + + # Check if the patch contains a correctly formatted and spelled Upstream-Status + import re + from oe import patch + + for url in patch.src_patches(d): + (_, _, fullpath, _, _, _) = bb.fetch.decodeurl(url) + + msg = oe.qa.check_upstream_status(fullpath) + if msg: + oe.qa.handle_error("patch-status", msg, d) + + ########################################################################### + # Check for missing ptests + ########################################################################### + def match_line_in_files(toplevel, filename_glob, line_regex): + import pathlib + try: + toppath = pathlib.Path(toplevel) + for entry in toppath.glob(filename_glob): + try: + with open(entry, 'r', encoding='utf-8', errors='ignore') as f: + for line in f.readlines(): + if re.match(line_regex, line): + return True + except FileNotFoundError: + # Broken symlink in source + pass + except FileNotFoundError: + # pathlib.Path.glob() might throw this when file/directory + # disappear while scanning. + bb.note("unimplemented-ptest: FileNotFoundError exception while scanning (disappearing file while scanning?). Check was ignored." % d.getVar('PN')) + pass + return False + + srcdir = d.getVar('S') + if not bb.utils.contains('DISTRO_FEATURES', 'ptest', True, False, d): + pass + elif bb.data.inherits_class('ptest', d): + bb.note("Package %s QA: skipping unimplemented-ptest: ptest implementation detected" % d.getVar('PN')) + elif srcdir == d.getVar('WORKDIR'): + bb.note("Package %s QA: skipping unimplemented-ptest: This check is not supported for recipe with \"S = \"${WORKDIR}\"" % d.getVar('PN')) + + # Detect perl Test:: based tests + elif os.path.exists(os.path.join(srcdir, "t")) and any(filename.endswith('.t') for filename in os.listdir(os.path.join(srcdir, 't'))): + oe.qa.handle_error("unimplemented-ptest", "%s: perl Test:: based tests detected" % d.getVar('PN'), d) + + # Detect pytest-based tests + elif match_line_in_files(srcdir, "**/*.py", r'\s*(?:import\s*pytest|from\s*pytest)'): + oe.qa.handle_error("unimplemented-ptest", "%s: pytest-based tests detected" % d.getVar('PN'), d) + + # Detect meson-based tests + elif os.path.exists(os.path.join(srcdir, "meson.build")) and match_line_in_files(srcdir, "**/meson.build", r'\s*test\s*\('): + oe.qa.handle_error("unimplemented-ptest", "%s: meson-based tests detected" % d.getVar('PN'), d) + + # Detect cmake-based tests + elif os.path.exists(os.path.join(srcdir, "CMakeLists.txt")) and match_line_in_files(srcdir, "**/CMakeLists.txt", r'\s*(?:add_test|enable_testing)\s*\('): + oe.qa.handle_error("unimplemented-ptest", "%s: cmake-based tests detected" % d.getVar('PN'), d) + + # Detect autotools-based·tests + elif os.path.exists(os.path.join(srcdir, "Makefile.in")) and (match_line_in_files(srcdir, "**/Makefile.in", r'\s*TESTS\s*\+?=') or match_line_in_files(srcdir,"**/*.at",r'.*AT_INIT')): + oe.qa.handle_error("unimplemented-ptest", "%s: autotools-based tests detected" % d.getVar('PN'), d) + + # Last resort, detect a test directory in sources + elif any(filename.lower() in ["test", "tests"] for filename in os.listdir(srcdir)): + oe.qa.handle_error("unimplemented-ptest", "%s: test subdirectory detected" % d.getVar('PN'), d) + + oe.qa.exit_if_errors(d) +} + +python do_qa_configure() { + import subprocess + + ########################################################################### + # Check config.log for cross compile issues + ########################################################################### + + configs = [] + workdir = d.getVar('WORKDIR') + + skip = (d.getVar('INSANE_SKIP') or "").split() + skip_configure_unsafe = False + if 'configure-unsafe' in skip: + bb.note("Recipe %s skipping qa checking: configure-unsafe" % d.getVar('PN')) + skip_configure_unsafe = True + + if bb.data.inherits_class('autotools', d) and not skip_configure_unsafe: + bb.note("Checking autotools environment for common misconfiguration") + for root, dirs, files in os.walk(workdir): + statement = "grep -q -F -e 'is unsafe for cross-compilation' %s" % \ + os.path.join(root,"config.log") + if "config.log" in files: + if subprocess.call(statement, shell=True) == 0: + error_msg = """This autoconf log indicates errors, it looked at host include and/or library paths while determining system capabilities. +Rerun configure task after fixing this.""" + oe.qa.handle_error("configure-unsafe", error_msg, d) + + if "configure.ac" in files: + configs.append(os.path.join(root,"configure.ac")) + if "configure.in" in files: + configs.append(os.path.join(root, "configure.in")) + + ########################################################################### + # Check gettext configuration and dependencies are correct + ########################################################################### + + skip_configure_gettext = False + if 'configure-gettext' in skip: + bb.note("Recipe %s skipping qa checking: configure-gettext" % d.getVar('PN')) + skip_configure_gettext = True + + cnf = d.getVar('EXTRA_OECONF') or "" + if not ("gettext" in d.getVar('P') or "gcc-runtime" in d.getVar('P') or \ + "--disable-nls" in cnf or skip_configure_gettext): + ml = d.getVar("MLPREFIX") or "" + if bb.data.inherits_class('cross-canadian', d): + gt = "nativesdk-gettext" + else: + gt = "gettext-native" + deps = bb.utils.explode_deps(d.getVar('DEPENDS') or "") + if gt not in deps: + for config in configs: + gnu = "grep \"^[[:space:]]*AM_GNU_GETTEXT\" %s >/dev/null" % config + if subprocess.call(gnu, shell=True) == 0: + error_msg = "AM_GNU_GETTEXT used but no inherit gettext" + oe.qa.handle_error("configure-gettext", error_msg, d) + + ########################################################################### + # Check unrecognised configure options (with a white list) + ########################################################################### + if bb.data.inherits_class("autotools", d): + bb.note("Checking configure output for unrecognised options") + try: + if bb.data.inherits_class("autotools", d): + flag = "WARNING: unrecognized options:" + log = os.path.join(d.getVar('B'), 'config.log') + output = subprocess.check_output(['grep', '-F', flag, log]).decode("utf-8").replace(', ', ' ').replace('"', '') + options = set() + for line in output.splitlines(): + options |= set(line.partition(flag)[2].split()) + ignore_opts = set(d.getVar("UNKNOWN_CONFIGURE_OPT_IGNORE").split()) + options -= ignore_opts + if options: + pn = d.getVar('PN') + error_msg = pn + ": configure was passed unrecognised options: " + " ".join(options) + oe.qa.handle_error("unknown-configure-option", error_msg, d) + except subprocess.CalledProcessError: + pass + + # Check invalid PACKAGECONFIG + pkgconfig = (d.getVar("PACKAGECONFIG") or "").split() + if pkgconfig: + pkgconfigflags = d.getVarFlags("PACKAGECONFIG") or {} + for pconfig in pkgconfig: + if pconfig not in pkgconfigflags: + pn = d.getVar('PN') + error_msg = "%s: invalid PACKAGECONFIG: %s" % (pn, pconfig) + oe.qa.handle_error("invalid-packageconfig", error_msg, d) + + oe.qa.exit_if_errors(d) +} + +python do_qa_unpack() { + src_uri = d.getVar('SRC_URI') + s_dir = d.getVar('S') + if src_uri and not os.path.exists(s_dir): + bb.warn('%s: the directory %s (%s) pointed to by the S variable doesn\'t exist - please set S within the recipe to point to where the source has been unpacked to' % (d.getVar('PN'), d.getVar('S', False), s_dir)) +} + +python do_recipe_qa() { + import re + + def test_missing_metadata(pn, d): + fn = d.getVar("FILE") + srcfile = d.getVar('SRC_URI').split() + # Check that SUMMARY is not the same as the default from bitbake.conf + if d.getVar('SUMMARY') == d.expand("${PN} version ${PV}-${PR}"): + oe.qa.handle_error("missing-metadata", "Recipe {} in {} does not contain a SUMMARY. Please add an entry.".format(pn, fn), d) + if not d.getVar('HOMEPAGE'): + if srcfile and srcfile[0].startswith('file') or not d.getVar('SRC_URI'): + # We are only interested in recipes SRC_URI fetched from external sources + pass + else: + oe.qa.handle_error("missing-metadata", "Recipe {} in {} does not contain a HOMEPAGE. Please add an entry.".format(pn, fn), d) + + def test_missing_maintainer(pn, d): + fn = d.getVar("FILE") + if pn.endswith("-native") or pn.startswith("nativesdk-") or "packagegroup-" in pn or "core-image-ptest-" in pn: + return + if not d.getVar('RECIPE_MAINTAINER'): + oe.qa.handle_error("missing-maintainer", "Recipe {} in {} does not have an assigned maintainer. Please add an entry into meta/conf/distro/include/maintainers.inc.".format(pn, fn), d) + + def test_srcuri(pn, d): + skip = (d.getVar('INSANE_SKIP') or "").split() + if 'src-uri-bad' in skip: + bb.note("Recipe %s skipping qa checking: src-uri-bad" % pn) + return + + if "${PN}" in d.getVar("SRC_URI", False): + oe.qa.handle_error("src-uri-bad", "%s: SRC_URI uses PN not BPN" % pn, d) + + for url in d.getVar("SRC_URI").split(): + # Search for github and gitlab URLs that pull unstable archives (comment for future greppers) + if re.search(r"git(hu|la)b\.com/.+/.+/archive/.+", url) or "//codeload.github.com/" in url: + oe.qa.handle_error("src-uri-bad", "%s: SRC_URI uses unstable GitHub/GitLab archives, convert recipe to use git protocol" % pn, d) + + pn = d.getVar('PN') + test_missing_metadata(pn, d) + test_missing_maintainer(pn, d) + test_srcuri(pn, d) + oe.qa.exit_if_errors(d) +} + +addtask do_recipe_qa before do_fetch do_package_qa do_build + +SSTATETASKS += "do_recipe_qa" +do_recipe_qa[sstate-inputdirs] = "" +do_recipe_qa[sstate-outputdirs] = "" +python do_recipe_qa_setscene () { + sstate_setscene(d) +} +addtask do_recipe_qa_setscene + +# Check for patch fuzz +do_patch[postfuncs] += "do_qa_patch " + +# Check broken config.log files, for packages requiring Gettext which +# don't have it in DEPENDS. +#addtask qa_configure after do_configure before do_compile +do_configure[postfuncs] += "do_qa_configure " + +# Check does S exist. +do_unpack[postfuncs] += "do_qa_unpack" + +python () { + import re + + tests = d.getVar('ALL_QA').split() + if "desktop" in tests: + d.appendVar("PACKAGE_DEPENDS", " desktop-file-utils-native") + + ########################################################################### + # Check various variables + ########################################################################### + + # Checking ${FILESEXTRAPATHS} + extrapaths = (d.getVar("FILESEXTRAPATHS") or "") + if '__default' not in extrapaths.split(":"): + msg = "FILESEXTRAPATHS-variable, must always use :prepend (or :append)\n" + msg += "type of assignment, and don't forget the colon.\n" + msg += "Please assign it with the format of:\n" + msg += " FILESEXTRAPATHS:append := \":${THISDIR}/Your_Files_Path\" or\n" + msg += " FILESEXTRAPATHS:prepend := \"${THISDIR}/Your_Files_Path:\"\n" + msg += "in your bbappend file\n\n" + msg += "Your incorrect assignment is:\n" + msg += "%s\n" % extrapaths + bb.warn(msg) + + overrides = d.getVar('OVERRIDES').split(':') + pn = d.getVar('PN') + if pn in overrides: + msg = 'Recipe %s has PN of "%s" which is in OVERRIDES, this can result in unexpected behaviour.' % (d.getVar("FILE"), pn) + oe.qa.handle_error("pn-overrides", msg, d) + prog = re.compile(r'[A-Z]') + if prog.search(pn): + oe.qa.handle_error("uppercase-pn", 'PN: %s is upper case, this can result in unexpected behavior.' % pn, d) + + # Some people mistakenly use DEPENDS:${PN} instead of DEPENDS and wonder + # why it doesn't work. + if (d.getVar(d.expand('DEPENDS:${PN}'))): + oe.qa.handle_error("pkgvarcheck", "recipe uses DEPENDS:${PN}, should use DEPENDS", d) + + # virtual/ is meaningless for these variables + if "virtual-slash" in (d.getVar("ALL_QA") or "").split(): + for k in ['RDEPENDS', 'RPROVIDES']: + for var in bb.utils.explode_deps(d.getVar(k + ':' + pn) or ""): + if var.startswith("virtual/"): + oe.qa.handle_error("virtual-slash", "%s is set to %s but the substring 'virtual/' holds no meaning in this context. It only works for build time dependencies, not runtime ones. It is suggested to use 'VIRTUAL-RUNTIME_' variables instead." % (k, var), d) + + issues = [] + if (d.getVar('PACKAGES') or "").split(): + for dep in (d.getVar('QADEPENDS') or "").split(): + d.appendVarFlag('do_package_qa', 'depends', " %s:do_populate_sysroot" % dep) + for var in 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RCONFLICTS', 'RPROVIDES', 'RREPLACES', 'FILES', 'pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm', 'ALLOW_EMPTY': + if d.getVar(var, False): + issues.append(var) + + fakeroot_tests = d.getVar('FAKEROOT_QA').split() + if set(tests) & set(fakeroot_tests): + d.setVarFlag('do_package_qa', 'fakeroot', '1') + d.appendVarFlag('do_package_qa', 'depends', ' virtual/fakeroot-native:do_populate_sysroot') + else: + d.setVarFlag('do_package_qa', 'rdeptask', '') + for i in issues: + oe.qa.handle_error("pkgvarcheck", "%s: Variable %s is set as not being package specific, please fix this." % (d.getVar("FILE"), i), d) + + if 'native-last' not in (d.getVar('INSANE_SKIP') or "").split(): + for native_class in ['native', 'nativesdk']: + if bb.data.inherits_class(native_class, d): + + inherited_classes = d.getVar('__inherit_cache', False) or [] + needle = "/" + native_class + + bbclassextend = (d.getVar('BBCLASSEXTEND') or '').split() + # BBCLASSEXTEND items are always added in the end + skip_classes = bbclassextend + if bb.data.inherits_class('native', d) or 'native' in bbclassextend: + # native also inherits nopackages and relocatable bbclasses + skip_classes.extend(['nopackages', 'relocatable']) + + broken_order = [] + for class_item in reversed(inherited_classes): + if needle not in class_item: + for extend_item in skip_classes: + if '/%s.bbclass' % extend_item in class_item: + break + else: + pn = d.getVar('PN') + broken_order.append(os.path.basename(class_item)) + else: + break + if broken_order: + oe.qa.handle_error("native-last", "%s: native/nativesdk class is not inherited last, this can result in unexpected behaviour. " + "Classes inherited after native/nativesdk: %s" % (pn, " ".join(broken_order)), d) + + oe.qa.exit_if_errors(d) +} diff --git a/meta/classes-global/license.bbclass b/meta/classes-global/license.bbclass new file mode 100644 index 0000000000..b2e0d3faba --- /dev/null +++ b/meta/classes-global/license.bbclass @@ -0,0 +1,426 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +# Populates LICENSE_DIRECTORY as set in distro config with the license files as set by +# LIC_FILES_CHKSUM. +# TODO: +# - There is a real issue revolving around license naming standards. + +LICENSE_DIRECTORY ??= "${DEPLOY_DIR}/licenses" +LICSSTATEDIR = "${WORKDIR}/license-destdir/" + +# Create extra package with license texts and add it to RRECOMMENDS:${PN} +LICENSE_CREATE_PACKAGE[type] = "boolean" +LICENSE_CREATE_PACKAGE ??= "0" +LICENSE_PACKAGE_SUFFIX ??= "-lic" +LICENSE_FILES_DIRECTORY ??= "${datadir}/licenses/" + +addtask populate_lic after do_patch before do_build +do_populate_lic[dirs] = "${LICSSTATEDIR}/${PN}" +do_populate_lic[cleandirs] = "${LICSSTATEDIR}" + +python do_populate_lic() { + """ + Populate LICENSE_DIRECTORY with licenses. + """ + lic_files_paths = find_license_files(d) + + # The base directory we wrangle licenses to + destdir = os.path.join(d.getVar('LICSSTATEDIR'), d.getVar('SSTATE_PKGARCH'), d.getVar('PN')) + copy_license_files(lic_files_paths, destdir) + info = get_recipe_info(d) + with open(os.path.join(destdir, "recipeinfo"), "w") as f: + for key in sorted(info.keys()): + f.write("%s: %s\n" % (key, info[key])) + oe.qa.exit_if_errors(d) +} + +PSEUDO_IGNORE_PATHS .= ",${@','.join(((d.getVar('COMMON_LICENSE_DIR') or '') + ' ' + (d.getVar('LICENSE_PATH') or '') + ' ' + d.getVar('COREBASE') + '/meta/COPYING').split())}" +# it would be better to copy them in do_install:append, but find_license_filesa is python +python perform_packagecopy:prepend () { + enabled = oe.data.typed_value('LICENSE_CREATE_PACKAGE', d) + if d.getVar('CLASSOVERRIDE') == 'class-target' and enabled: + lic_files_paths = find_license_files(d) + + # LICENSE_FILES_DIRECTORY starts with '/' so os.path.join cannot be used to join D and LICENSE_FILES_DIRECTORY + destdir = d.getVar('D') + os.path.join(d.getVar('LICENSE_FILES_DIRECTORY'), d.getVar('PN')) + copy_license_files(lic_files_paths, destdir) + add_package_and_files(d) +} +perform_packagecopy[vardeps] += "LICENSE_CREATE_PACKAGE" + +def get_recipe_info(d): + info = {} + info["PV"] = d.getVar("PV") + info["PR"] = d.getVar("PR") + info["LICENSE"] = d.getVar("LICENSE") + return info + +def add_package_and_files(d): + packages = d.getVar('PACKAGES') + files = d.getVar('LICENSE_FILES_DIRECTORY') + pn = d.getVar('PN') + pn_lic = "%s%s" % (pn, d.getVar('LICENSE_PACKAGE_SUFFIX', False)) + if pn_lic in packages.split(): + bb.warn("%s package already existed in %s." % (pn_lic, pn)) + else: + # first in PACKAGES to be sure that nothing else gets LICENSE_FILES_DIRECTORY + d.setVar('PACKAGES', "%s %s" % (pn_lic, packages)) + d.setVar('FILES:' + pn_lic, files) + +def copy_license_files(lic_files_paths, destdir): + import shutil + import errno + + bb.utils.mkdirhier(destdir) + for (basename, path, beginline, endline) in lic_files_paths: + try: + src = path + dst = os.path.join(destdir, basename) + if os.path.exists(dst): + os.remove(dst) + if os.path.islink(src): + src = os.path.realpath(src) + canlink = os.access(src, os.W_OK) and (os.stat(src).st_dev == os.stat(destdir).st_dev) and beginline is None and endline is None + if canlink: + try: + os.link(src, dst) + except OSError as err: + if err.errno == errno.EXDEV: + # Copy license files if hardlink is not possible even if st_dev is the + # same on source and destination (docker container with device-mapper?) + canlink = False + else: + raise + # Only chown if we did hardlink and we're running under pseudo + if canlink and os.environ.get('PSEUDO_DISABLED') == '0': + os.chown(dst,0,0) + if not canlink: + begin_idx = max(0, int(beginline) - 1) if beginline is not None else None + end_idx = max(0, int(endline)) if endline is not None else None + if begin_idx is None and end_idx is None: + shutil.copyfile(src, dst) + else: + with open(src, 'rb') as src_f: + with open(dst, 'wb') as dst_f: + dst_f.write(b''.join(src_f.readlines()[begin_idx:end_idx])) + + except Exception as e: + bb.warn("Could not copy license file %s to %s: %s" % (src, dst, e)) + +def find_license_files(d): + """ + Creates list of files used in LIC_FILES_CHKSUM and generic LICENSE files. + """ + import shutil + import oe.license + from collections import defaultdict, OrderedDict + + # All the license files for the package + lic_files = d.getVar('LIC_FILES_CHKSUM') or "" + pn = d.getVar('PN') + # The license files are located in S/LIC_FILE_CHECKSUM. + srcdir = d.getVar('S') + # Directory we store the generic licenses as set in the distro configuration + generic_directory = d.getVar('COMMON_LICENSE_DIR') + # List of basename, path tuples + lic_files_paths = [] + # hash for keep track generic lics mappings + non_generic_lics = {} + # Entries from LIC_FILES_CHKSUM + lic_chksums = {} + license_source_dirs = [] + license_source_dirs.append(generic_directory) + try: + additional_lic_dirs = d.getVar('LICENSE_PATH').split() + for lic_dir in additional_lic_dirs: + license_source_dirs.append(lic_dir) + except: + pass + + class FindVisitor(oe.license.LicenseVisitor): + def visit_Str(self, node): + # + # Until I figure out what to do with + # the two modifiers I support (or greater = + + # and "with exceptions" being * + # we'll just strip out the modifier and put + # the base license. + find_license(node.s.replace("+", "").replace("*", "")) + self.generic_visit(node) + + def visit_Constant(self, node): + find_license(node.value.replace("+", "").replace("*", "")) + self.generic_visit(node) + + def find_license(license_type): + try: + bb.utils.mkdirhier(gen_lic_dest) + except: + pass + spdx_generic = None + license_source = None + # If the generic does not exist we need to check to see if there is an SPDX mapping to it, + # unless NO_GENERIC_LICENSE is set. + for lic_dir in license_source_dirs: + if not os.path.isfile(os.path.join(lic_dir, license_type)): + if d.getVarFlag('SPDXLICENSEMAP', license_type) != None: + # Great, there is an SPDXLICENSEMAP. We can copy! + bb.debug(1, "We need to use a SPDXLICENSEMAP for %s" % (license_type)) + spdx_generic = d.getVarFlag('SPDXLICENSEMAP', license_type) + license_source = lic_dir + break + elif os.path.isfile(os.path.join(lic_dir, license_type)): + spdx_generic = license_type + license_source = lic_dir + break + + non_generic_lic = d.getVarFlag('NO_GENERIC_LICENSE', license_type) + if spdx_generic and license_source: + # we really should copy to generic_ + spdx_generic, however, that ends up messing the manifest + # audit up. This should be fixed in emit_pkgdata (or, we actually got and fix all the recipes) + + lic_files_paths.append(("generic_" + license_type, os.path.join(license_source, spdx_generic), + None, None)) + + # The user may attempt to use NO_GENERIC_LICENSE for a generic license which doesn't make sense + # and should not be allowed, warn the user in this case. + if d.getVarFlag('NO_GENERIC_LICENSE', license_type): + oe.qa.handle_error("license-no-generic", + "%s: %s is a generic license, please don't use NO_GENERIC_LICENSE for it." % (pn, license_type), d) + + elif non_generic_lic and non_generic_lic in lic_chksums: + # if NO_GENERIC_LICENSE is set, we copy the license files from the fetched source + # of the package rather than the license_source_dirs. + lic_files_paths.append(("generic_" + license_type, + os.path.join(srcdir, non_generic_lic), None, None)) + non_generic_lics[non_generic_lic] = license_type + else: + # Explicitly avoid the CLOSED license because this isn't generic + if license_type != 'CLOSED': + # And here is where we warn people that their licenses are lousy + oe.qa.handle_error("license-exists", + "%s: No generic license file exists for: %s in any provider" % (pn, license_type), d) + pass + + if not generic_directory: + bb.fatal("COMMON_LICENSE_DIR is unset. Please set this in your distro config") + + for url in lic_files.split(): + try: + (method, host, path, user, pswd, parm) = bb.fetch.decodeurl(url) + if method != "file" or not path: + raise bb.fetch.MalformedUrl() + except bb.fetch.MalformedUrl: + bb.fatal("%s: LIC_FILES_CHKSUM contains an invalid URL: %s" % (d.getVar('PF'), url)) + # We want the license filename and path + chksum = parm.get('md5', None) + beginline = parm.get('beginline') + endline = parm.get('endline') + lic_chksums[path] = (chksum, beginline, endline) + + v = FindVisitor() + try: + v.visit_string(d.getVar('LICENSE')) + except oe.license.InvalidLicense as exc: + bb.fatal('%s: %s' % (d.getVar('PF'), exc)) + except SyntaxError: + oe.qa.handle_error("license-syntax", + "%s: Failed to parse LICENSE: %s" % (d.getVar('PF'), d.getVar('LICENSE')), d) + # Add files from LIC_FILES_CHKSUM to list of license files + lic_chksum_paths = defaultdict(OrderedDict) + for path, data in sorted(lic_chksums.items()): + lic_chksum_paths[os.path.basename(path)][data] = (os.path.join(srcdir, path), data[1], data[2]) + for basename, files in lic_chksum_paths.items(): + if len(files) == 1: + # Don't copy again a LICENSE already handled as non-generic + if basename in non_generic_lics: + continue + data = list(files.values())[0] + lic_files_paths.append(tuple([basename] + list(data))) + else: + # If there are multiple different license files with identical + # basenames we rename them to <file>.0, <file>.1, ... + for i, data in enumerate(files.values()): + lic_files_paths.append(tuple(["%s.%d" % (basename, i)] + list(data))) + + return lic_files_paths + +def return_spdx(d, license): + """ + This function returns the spdx mapping of a license if it exists. + """ + return d.getVarFlag('SPDXLICENSEMAP', license) + +def canonical_license(d, license): + """ + Return the canonical (SPDX) form of the license if available (so GPLv3 + becomes GPL-3.0-only) or the passed license if there is no canonical form. + """ + return d.getVarFlag('SPDXLICENSEMAP', license) or license + +def expand_wildcard_licenses(d, wildcard_licenses): + """ + There are some common wildcard values users may want to use. Support them + here. + """ + licenses = set(wildcard_licenses) + mapping = { + "AGPL-3.0*" : ["AGPL-3.0-only", "AGPL-3.0-or-later"], + "GPL-3.0*" : ["GPL-3.0-only", "GPL-3.0-or-later"], + "LGPL-3.0*" : ["LGPL-3.0-only", "LGPL-3.0-or-later"], + } + for k in mapping: + if k in wildcard_licenses: + licenses.remove(k) + for item in mapping[k]: + licenses.add(item) + + for l in licenses: + if l in oe.license.obsolete_license_list(): + bb.fatal("Error, %s is an obsolete license, please use an SPDX reference in INCOMPATIBLE_LICENSE" % l) + if "*" in l: + bb.fatal("Error, %s is an invalid license wildcard entry" % l) + + return list(licenses) + +def incompatible_license_contains(license, truevalue, falsevalue, d): + license = canonical_license(d, license) + bad_licenses = (d.getVar('INCOMPATIBLE_LICENSE') or "").split() + bad_licenses = expand_wildcard_licenses(d, bad_licenses) + return truevalue if license in bad_licenses else falsevalue + +def incompatible_pkg_license(d, dont_want_licenses, license): + # Handles an "or" or two license sets provided by + # flattened_licenses(), pick one that works if possible. + def choose_lic_set(a, b): + return a if all(oe.license.license_ok(canonical_license(d, lic), + dont_want_licenses) for lic in a) else b + + try: + licenses = oe.license.flattened_licenses(license, choose_lic_set) + except oe.license.LicenseError as exc: + bb.fatal('%s: %s' % (d.getVar('P'), exc)) + + incompatible_lic = [] + for l in licenses: + license = canonical_license(d, l) + if not oe.license.license_ok(license, dont_want_licenses): + incompatible_lic.append(license) + + return sorted(incompatible_lic) + +def incompatible_license(d, dont_want_licenses, package=None): + """ + This function checks if a recipe has only incompatible licenses. It also + take into consideration 'or' operand. dont_want_licenses should be passed + as canonical (SPDX) names. + """ + import oe.license + license = d.getVar("LICENSE:%s" % package) if package else None + if not license: + license = d.getVar('LICENSE') + + return incompatible_pkg_license(d, dont_want_licenses, license) + +def check_license_flags(d): + """ + This function checks if a recipe has any LICENSE_FLAGS that + aren't acceptable. + + If it does, it returns the all LICENSE_FLAGS missing from the list + of acceptable license flags, or all of the LICENSE_FLAGS if there + is no list of acceptable flags. + + If everything is is acceptable, it returns None. + """ + + def license_flag_matches(flag, acceptlist, pn): + """ + Return True if flag matches something in acceptlist, None if not. + + Before we test a flag against the acceptlist, we append _${PN} + to it. We then try to match that string against the + acceptlist. This covers the normal case, where we expect + LICENSE_FLAGS to be a simple string like 'commercial', which + the user typically matches exactly in the acceptlist by + explicitly appending the package name e.g 'commercial_foo'. + If we fail the match however, we then split the flag across + '_' and append each fragment and test until we either match or + run out of fragments. + """ + flag_pn = ("%s_%s" % (flag, pn)) + for candidate in acceptlist: + if flag_pn == candidate: + return True + + flag_cur = "" + flagments = flag_pn.split("_") + flagments.pop() # we've already tested the full string + for flagment in flagments: + if flag_cur: + flag_cur += "_" + flag_cur += flagment + for candidate in acceptlist: + if flag_cur == candidate: + return True + return False + + def all_license_flags_match(license_flags, acceptlist): + """ Return all unmatched flags, None if all flags match """ + pn = d.getVar('PN') + split_acceptlist = acceptlist.split() + flags = [] + for flag in license_flags.split(): + if not license_flag_matches(flag, split_acceptlist, pn): + flags.append(flag) + return flags if flags else None + + license_flags = d.getVar('LICENSE_FLAGS') + if license_flags: + acceptlist = d.getVar('LICENSE_FLAGS_ACCEPTED') + if not acceptlist: + return license_flags.split() + unmatched_flags = all_license_flags_match(license_flags, acceptlist) + if unmatched_flags: + return unmatched_flags + return None + +def check_license_format(d): + """ + This function checks if LICENSE is well defined, + Validate operators in LICENSES. + No spaces are allowed between LICENSES. + """ + pn = d.getVar('PN') + licenses = d.getVar('LICENSE') + from oe.license import license_operator, license_operator_chars, license_pattern + + elements = list(filter(lambda x: x.strip(), license_operator.split(licenses))) + for pos, element in enumerate(elements): + if license_pattern.match(element): + if pos > 0 and license_pattern.match(elements[pos - 1]): + oe.qa.handle_error('license-format', + '%s: LICENSE value "%s" has an invalid format - license names ' \ + 'must be separated by the following characters to indicate ' \ + 'the license selection: %s' % + (pn, licenses, license_operator_chars), d) + elif not license_operator.match(element): + oe.qa.handle_error('license-format', + '%s: LICENSE value "%s" has an invalid separator "%s" that is not ' \ + 'in the valid list of separators (%s)' % + (pn, licenses, element, license_operator_chars), d) + +SSTATETASKS += "do_populate_lic" +do_populate_lic[sstate-inputdirs] = "${LICSSTATEDIR}" +do_populate_lic[sstate-outputdirs] = "${LICENSE_DIRECTORY}/" + +IMAGE_CLASSES:append = " license_image" + +python do_populate_lic_setscene () { + sstate_setscene(d) +} +addtask do_populate_lic_setscene diff --git a/meta/classes-global/logging.bbclass b/meta/classes-global/logging.bbclass new file mode 100644 index 0000000000..ce03abfe42 --- /dev/null +++ b/meta/classes-global/logging.bbclass @@ -0,0 +1,107 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +# The following logging mechanisms are to be used in bash functions of recipes. +# They are intended to map one to one in intention and output format with the +# python recipe logging functions of a similar naming convention: bb.plain(), +# bb.note(), etc. + +LOGFIFO = "${T}/fifo.${@os.getpid()}" + +# Print the output exactly as it is passed in. Typically used for output of +# tasks that should be seen on the console. Use sparingly. +# Output: logs console +bbplain() { + if [ -p ${LOGFIFO} ] ; then + printf "%b\0" "bbplain $*" > ${LOGFIFO} + else + echo "$*" + fi +} + +# Notify the user of a noteworthy condition. +# Output: logs +bbnote() { + if [ -p ${LOGFIFO} ] ; then + printf "%b\0" "bbnote $*" > ${LOGFIFO} + else + echo "NOTE: $*" + fi +} + +# Print a warning to the log. Warnings are non-fatal, and do not +# indicate a build failure. +# Output: logs console +bbwarn() { + if [ -p ${LOGFIFO} ] ; then + printf "%b\0" "bbwarn $*" > ${LOGFIFO} + else + echo "WARNING: $*" + fi +} + +# Print an error to the log. Errors are non-fatal in that the build can +# continue, but they do indicate a build failure. +# Output: logs console +bberror() { + if [ -p ${LOGFIFO} ] ; then + printf "%b\0" "bberror $*" > ${LOGFIFO} + else + echo "ERROR: $*" + fi +} + +# Print a fatal error to the log. Fatal errors indicate build failure +# and halt the build, exiting with an error code. +# Output: logs console +bbfatal() { + if [ -p ${LOGFIFO} ] ; then + printf "%b\0" "bbfatal $*" > ${LOGFIFO} + else + echo "ERROR: $*" + fi + exit 1 +} + +# Like bbfatal, except prevents the suppression of the error log by +# bitbake's UI. +# Output: logs console +bbfatal_log() { + if [ -p ${LOGFIFO} ] ; then + printf "%b\0" "bbfatal_log $*" > ${LOGFIFO} + else + echo "ERROR: $*" + fi + exit 1 +} + +# Print debug messages. These are appropriate for progress checkpoint +# messages to the logs. Depending on the debug log level, they may also +# go to the console. +# Output: logs console +# Usage: bbdebug 1 "first level debug message" +# bbdebug 2 "second level debug message" +bbdebug() { + USAGE='Usage: bbdebug [123] "message"' + if [ $# -lt 2 ]; then + bbfatal "$USAGE" + fi + + # Strip off the debug level and ensure it is an integer + DBGLVL=$1; shift + NONDIGITS=$(echo "$DBGLVL" | tr -d "[:digit:]") + if [ "$NONDIGITS" ]; then + bbfatal "$USAGE" + fi + + # All debug output is printed to the logs + if [ -p ${LOGFIFO} ] ; then + printf "%b\0" "bbdebug $DBGLVL $*" > ${LOGFIFO} + else + echo "DEBUG: $*" + fi +} + diff --git a/meta/classes-global/mirrors.bbclass b/meta/classes-global/mirrors.bbclass new file mode 100644 index 0000000000..862648eec5 --- /dev/null +++ b/meta/classes-global/mirrors.bbclass @@ -0,0 +1,104 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +MIRRORS += "\ +${DEBIAN_MIRROR} http://snapshot.debian.org/archive/debian/20180310T215105Z/pool \ +${DEBIAN_MIRROR} http://snapshot.debian.org/archive/debian-archive/20120328T092752Z/debian/pool \ +${DEBIAN_MIRROR} http://snapshot.debian.org/archive/debian-archive/20110127T084257Z/debian/pool \ +${DEBIAN_MIRROR} http://snapshot.debian.org/archive/debian-archive/20090802T004153Z/debian/pool \ +${DEBIAN_MIRROR} http://ftp.de.debian.org/debian/pool \ +${DEBIAN_MIRROR} http://ftp.au.debian.org/debian/pool \ +${DEBIAN_MIRROR} http://ftp.cl.debian.org/debian/pool \ +${DEBIAN_MIRROR} http://ftp.hr.debian.org/debian/pool \ +${DEBIAN_MIRROR} http://ftp.fi.debian.org/debian/pool \ +${DEBIAN_MIRROR} http://ftp.hk.debian.org/debian/pool \ +${DEBIAN_MIRROR} http://ftp.hu.debian.org/debian/pool \ +${DEBIAN_MIRROR} http://ftp.ie.debian.org/debian/pool \ +${DEBIAN_MIRROR} http://ftp.it.debian.org/debian/pool \ +${DEBIAN_MIRROR} http://ftp.jp.debian.org/debian/pool \ +${DEBIAN_MIRROR} http://ftp.no.debian.org/debian/pool \ +${DEBIAN_MIRROR} http://ftp.pl.debian.org/debian/pool \ +${DEBIAN_MIRROR} http://ftp.ro.debian.org/debian/pool \ +${DEBIAN_MIRROR} http://ftp.si.debian.org/debian/pool \ +${DEBIAN_MIRROR} http://ftp.es.debian.org/debian/pool \ +${DEBIAN_MIRROR} http://ftp.se.debian.org/debian/pool \ +${DEBIAN_MIRROR} http://ftp.tr.debian.org/debian/pool \ +${GNU_MIRROR} https://mirrors.kernel.org/gnu \ +${KERNELORG_MIRROR} http://www.kernel.org/pub \ +${GNUPG_MIRROR} ftp://ftp.gnupg.org/gcrypt \ +${GNUPG_MIRROR} ftp://ftp.franken.de/pub/crypt/mirror/ftp.gnupg.org/gcrypt \ +${GNUPG_MIRROR} ftp://mirrors.dotsrc.org/gcrypt \ +ftp://dante.ctan.org/tex-archive ftp://ftp.fu-berlin.de/tex/CTAN \ +ftp://dante.ctan.org/tex-archive http://sunsite.sut.ac.jp/pub/archives/ctan/ \ +ftp://dante.ctan.org/tex-archive http://ctan.unsw.edu.au/ \ +ftp://ftp.gnutls.org/gcrypt/gnutls ${GNUPG_MIRROR}/gnutls \ +http://ftp.info-zip.org/pub/infozip/src/ ftp://sunsite.icm.edu.pl/pub/unix/archiving/info-zip/src/ \ +http://www.mirrorservice.org/sites/lsof.itap.purdue.edu/pub/tools/unix/lsof/ http://www.mirrorservice.org/sites/lsof.itap.purdue.edu/pub/tools/unix/lsof/OLD/ \ +${APACHE_MIRROR} http://www.us.apache.org/dist \ +${APACHE_MIRROR} http://archive.apache.org/dist \ +http://downloads.sourceforge.net/watchdog/ http://fossies.org/linux/misc/ \ +${SAVANNAH_GNU_MIRROR} http://download-mirror.savannah.gnu.org/releases \ +${SAVANNAH_NONGNU_MIRROR} http://download-mirror.savannah.nongnu.org/releases \ +ftp://sourceware.org/pub http://mirrors.kernel.org/sourceware \ +ftp://sourceware.org/pub http://gd.tuwien.ac.at/gnu/sourceware \ +ftp://sourceware.org/pub http://ftp.gwdg.de/pub/linux/sources.redhat.com/sourceware \ +cvs://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \ +svn://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \ +git://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \ +gitsm://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \ +hg://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \ +bzr://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \ +p4://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \ +osc://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \ +https?://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \ +ftp://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \ +npm://.*/?.* http://downloads.yoctoproject.org/mirror/sources/ \ +cvs://.*/.* http://sources.openembedded.org/ \ +svn://.*/.* http://sources.openembedded.org/ \ +git://.*/.* http://sources.openembedded.org/ \ +gitsm://.*/.* http://sources.openembedded.org/ \ +hg://.*/.* http://sources.openembedded.org/ \ +bzr://.*/.* http://sources.openembedded.org/ \ +p4://.*/.* http://sources.openembedded.org/ \ +osc://.*/.* http://sources.openembedded.org/ \ +https?://.*/.* http://sources.openembedded.org/ \ +ftp://.*/.* http://sources.openembedded.org/ \ +npm://.*/?.* http://sources.openembedded.org/ \ +${CPAN_MIRROR} https://cpan.metacpan.org/ \ +https?://downloads.yoctoproject.org/releases/uninative/ https://mirrors.kernel.org/yocto/uninative/ \ +https?://downloads.yoctoproject.org/mirror/sources/ https://mirrors.kernel.org/yocto-sources/ \ +" + +# Use MIRRORS to provide git repo fallbacks using the https protocol, for cases +# where git native protocol fetches may fail due to local firewall rules, etc. + +MIRRORS += "\ +git://salsa.debian.org/.* git://salsa.debian.org/PATH;protocol=https \ +git://git.gnome.org/.* git://gitlab.gnome.org/GNOME/PATH;protocol=https \ +git://git.infradead.org/.* git://git.infraroot.at/PATH;protocol=https \ +git://.*/.* git://HOST/PATH;protocol=https \ +git://.*/.* git://HOST/git/PATH;protocol=https \ +" + +# Switch llvm, glibc and binutils recipes to use shallow clones as they're large and this +# improves user experience whilst allowing the flexibility of git urls in the recipes +BB_GIT_SHALLOW:pn-binutils = "1" +BB_GIT_SHALLOW:pn-binutils-cross-${TARGET_ARCH} = "1" +BB_GIT_SHALLOW:pn-binutils-cross-canadian-${TRANSLATED_TARGET_ARCH} = "1" +BB_GIT_SHALLOW:pn-binutils-cross-testsuite = "1" +BB_GIT_SHALLOW:pn-binutils-crosssdk-${SDK_SYS} = "1" +BB_GIT_SHALLOW:pn-binutils-native = "1" +BB_GIT_SHALLOW:pn-nativesdk-binutils = "1" + +BB_GIT_SHALLOW:pn-cross-localedef-native = "1" +BB_GIT_SHALLOW:pn-glibc = "1" +BB_GIT_SHALLOW:pn-glibc-tests = "1" +PREMIRRORS += "git://sourceware.org/git/glibc.git https://downloads.yoctoproject.org/mirror/sources/ \ + git://sourceware.org/git/binutils-gdb.git https://downloads.yoctoproject.org/mirror/sources/" + +BB_GIT_SHALLOW:pn-llvm = "1" +BB_GIT_SHALLOW:pn-llvm-native = "1" +BB_GIT_SHALLOW:pn-nativesdk-llvm = "1" diff --git a/meta/classes-global/package.bbclass b/meta/classes-global/package.bbclass new file mode 100644 index 0000000000..aa1eb5e901 --- /dev/null +++ b/meta/classes-global/package.bbclass @@ -0,0 +1,616 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +# +# Packaging process +# +# Executive summary: This class iterates over the functions listed in PACKAGEFUNCS +# Taking D and splitting it up into the packages listed in PACKAGES, placing the +# resulting output in PKGDEST. +# +# There are the following default steps but PACKAGEFUNCS can be extended: +# +# a) package_convert_pr_autoinc - convert AUTOINC in PKGV to ${PRSERV_PV_AUTOINC} +# +# b) perform_packagecopy - Copy D into PKGD +# +# c) package_do_split_locales - Split out the locale files, updates FILES and PACKAGES +# +# d) split_and_strip_files - split the files into runtime and debug and strip them. +# Debug files include debug info split, and associated sources that end up in -dbg packages +# +# e) fixup_perms - Fix up permissions in the package before we split it. +# +# f) populate_packages - Split the files in PKGD into separate packages in PKGDEST/<pkgname> +# Also triggers the binary stripping code to put files in -dbg packages. +# +# g) package_do_filedeps - Collect perfile run-time dependency metadata +# The data is stores in FILER{PROVIDES,DEPENDS}_file_pkg variables with +# a list of affected files in FILER{PROVIDES,DEPENDS}FLIST_pkg +# +# h) package_do_shlibs - Look at the shared libraries generated and autotmatically add any +# dependencies found. Also stores the package name so anyone else using this library +# knows which package to depend on. +# +# i) package_do_pkgconfig - Keep track of which packages need and provide which .pc files +# +# j) read_shlibdeps - Reads the stored shlibs information into the metadata +# +# k) package_depchains - Adds automatic dependencies to -dbg and -dev packages +# +# l) emit_pkgdata - saves the packaging data into PKGDATA_DIR for use in later +# packaging steps + +inherit packagedata +inherit chrpath +inherit package_pkgdata +inherit insane + +PKGD = "${WORKDIR}/package" +PKGDEST = "${WORKDIR}/packages-split" + +LOCALE_SECTION ?= '' + +ALL_MULTILIB_PACKAGE_ARCHS = "${@all_multilib_tune_values(d, 'PACKAGE_ARCHS')}" + +# rpm is used for the per-file dependency identification +# dwarfsrcfiles is used to determine the list of debug source files +PACKAGE_DEPENDS += "rpm-native dwarfsrcfiles-native" + +# If your postinstall can execute at rootfs creation time rather than on +# target but depends on a native/cross tool in order to execute, you need to +# list that tool in PACKAGE_WRITE_DEPS. Target package dependencies belong +# in the package dependencies as normal, this is just for native/cross support +# tools at rootfs build time. +PACKAGE_WRITE_DEPS ??= "" + +def legitimize_package_name(s): + return oe.package.legitimize_package_name(s) + +def do_split_packages(d, root, file_regex, output_pattern, description, postinst=None, recursive=False, hook=None, extra_depends=None, aux_files_pattern=None, postrm=None, allow_dirs=False, prepend=False, match_path=False, aux_files_pattern_verbatim=None, allow_links=False, summary=None): + """ + Used in .bb files to split up dynamically generated subpackages of a + given package, usually plugins or modules. + + Arguments: + root -- the path in which to search + file_regex -- regular expression to match searched files. Use + parentheses () to mark the part of this expression + that should be used to derive the module name (to be + substituted where %s is used in other function + arguments as noted below) + output_pattern -- pattern to use for the package names. Must include %s. + description -- description to set for each package. Must include %s. + postinst -- postinstall script to use for all packages (as a + string) + recursive -- True to perform a recursive search - default False + hook -- a hook function to be called for every match. The + function will be called with the following arguments + (in the order listed): + f: full path to the file/directory match + pkg: the package name + file_regex: as above + output_pattern: as above + modulename: the module name derived using file_regex + extra_depends -- extra runtime dependencies (RDEPENDS) to be set for + all packages. The default value of None causes a + dependency on the main package (${PN}) - if you do + not want this, pass '' for this parameter. + aux_files_pattern -- extra item(s) to be added to FILES for each + package. Can be a single string item or a list of + strings for multiple items. Must include %s. + postrm -- postrm script to use for all packages (as a string) + allow_dirs -- True allow directories to be matched - default False + prepend -- if True, prepend created packages to PACKAGES instead + of the default False which appends them + match_path -- match file_regex on the whole relative path to the + root rather than just the file name + aux_files_pattern_verbatim -- extra item(s) to be added to FILES for + each package, using the actual derived module name + rather than converting it to something legal for a + package name. Can be a single string item or a list + of strings for multiple items. Must include %s. + allow_links -- True to allow symlinks to be matched - default False + summary -- Summary to set for each package. Must include %s; + defaults to description if not set. + + """ + + dvar = d.getVar('PKGD') + root = d.expand(root) + output_pattern = d.expand(output_pattern) + extra_depends = d.expand(extra_depends) + + # If the root directory doesn't exist, don't error out later but silently do + # no splitting. + if not os.path.exists(dvar + root): + return [] + + ml = d.getVar("MLPREFIX") + if ml: + if not output_pattern.startswith(ml): + output_pattern = ml + output_pattern + + newdeps = [] + for dep in (extra_depends or "").split(): + if dep.startswith(ml): + newdeps.append(dep) + else: + newdeps.append(ml + dep) + if newdeps: + extra_depends = " ".join(newdeps) + + + packages = d.getVar('PACKAGES').split() + split_packages = set() + + if postinst: + postinst = '#!/bin/sh\n' + postinst + '\n' + if postrm: + postrm = '#!/bin/sh\n' + postrm + '\n' + if not recursive: + objs = os.listdir(dvar + root) + else: + objs = [] + for walkroot, dirs, files in os.walk(dvar + root): + for file in files: + relpath = os.path.join(walkroot, file).replace(dvar + root + '/', '', 1) + if relpath: + objs.append(relpath) + + if extra_depends == None: + extra_depends = d.getVar("PN") + + if not summary: + summary = description + + for o in sorted(objs): + import re, stat + if match_path: + m = re.match(file_regex, o) + else: + m = re.match(file_regex, os.path.basename(o)) + + if not m: + continue + f = os.path.join(dvar + root, o) + mode = os.lstat(f).st_mode + if not (stat.S_ISREG(mode) or (allow_links and stat.S_ISLNK(mode)) or (allow_dirs and stat.S_ISDIR(mode))): + continue + on = oe.package.legitimize_package_name(m.group(1)) + pkg = output_pattern % on + split_packages.add(pkg) + if not pkg in packages: + if prepend: + packages = [pkg] + packages + else: + packages.append(pkg) + oldfiles = d.getVar('FILES:' + pkg) + newfile = os.path.join(root, o) + # These names will be passed through glob() so if the filename actually + # contains * or ? (rare, but possible) we need to handle that specially + newfile = newfile.replace('*', '[*]') + newfile = newfile.replace('?', '[?]') + if not oldfiles: + the_files = [newfile] + if aux_files_pattern: + if type(aux_files_pattern) is list: + for fp in aux_files_pattern: + the_files.append(fp % on) + else: + the_files.append(aux_files_pattern % on) + if aux_files_pattern_verbatim: + if type(aux_files_pattern_verbatim) is list: + for fp in aux_files_pattern_verbatim: + the_files.append(fp % m.group(1)) + else: + the_files.append(aux_files_pattern_verbatim % m.group(1)) + d.setVar('FILES:' + pkg, " ".join(the_files)) + else: + d.setVar('FILES:' + pkg, oldfiles + " " + newfile) + if extra_depends != '': + d.appendVar('RDEPENDS:' + pkg, ' ' + extra_depends) + if not d.getVar('DESCRIPTION:' + pkg): + d.setVar('DESCRIPTION:' + pkg, description % on) + if not d.getVar('SUMMARY:' + pkg): + d.setVar('SUMMARY:' + pkg, summary % on) + if postinst: + d.setVar('pkg_postinst:' + pkg, postinst) + if postrm: + d.setVar('pkg_postrm:' + pkg, postrm) + if callable(hook): + hook(f, pkg, file_regex, output_pattern, m.group(1)) + + d.setVar('PACKAGES', ' '.join(packages)) + return list(split_packages) + +PACKAGE_DEPENDS += "file-native" + +python () { + if d.getVar('PACKAGES') != '': + deps = "" + for dep in (d.getVar('PACKAGE_DEPENDS') or "").split(): + deps += " %s:do_populate_sysroot" % dep + if bb.utils.contains('DISTRO_FEATURES', 'minidebuginfo', True, False, d): + deps += ' xz-native:do_populate_sysroot' + d.appendVarFlag('do_package', 'depends', deps) + + # shlibs requires any DEPENDS to have already packaged for the *.list files + d.appendVarFlag('do_package', 'deptask', " do_packagedata") +} + + +PRSERV_ACTIVE = "${@bool(d.getVar("PRSERV_HOST"))}" +PRSERV_ACTIVE[vardepvalue] = "${PRSERV_ACTIVE}" +package_get_auto_pr[vardepsexclude] = "BB_TASKDEPDATA" +package_get_auto_pr[vardeps] += "PRSERV_ACTIVE" +python package_get_auto_pr() { + import oe.prservice + + def get_do_package_hash(pn): + if d.getVar("BB_RUNTASK") != "do_package": + taskdepdata = d.getVar("BB_TASKDEPDATA", False) + for dep in taskdepdata: + if taskdepdata[dep][1] == "do_package" and taskdepdata[dep][0] == pn: + return taskdepdata[dep][6] + return None + + # Support per recipe PRSERV_HOST + pn = d.getVar('PN') + host = d.getVar("PRSERV_HOST_" + pn) + if not (host is None): + d.setVar("PRSERV_HOST", host) + + pkgv = d.getVar("PKGV") + + # PR Server not active, handle AUTOINC + if not d.getVar('PRSERV_HOST'): + d.setVar("PRSERV_PV_AUTOINC", "0") + return + + auto_pr = None + pv = d.getVar("PV") + version = d.getVar("PRAUTOINX") + pkgarch = d.getVar("PACKAGE_ARCH") + checksum = get_do_package_hash(pn) + + # If do_package isn't in the dependencies, we can't get the checksum... + if not checksum: + bb.warn('Task %s requested do_package unihash, but it was not available.' % d.getVar('BB_RUNTASK')) + #taskdepdata = d.getVar("BB_TASKDEPDATA", False) + #for dep in taskdepdata: + # bb.warn('%s:%s = %s' % (taskdepdata[dep][0], taskdepdata[dep][1], taskdepdata[dep][6])) + return + + if d.getVar('PRSERV_LOCKDOWN'): + auto_pr = d.getVar('PRAUTO_' + version + '_' + pkgarch) or d.getVar('PRAUTO_' + version) or None + if auto_pr is None: + bb.fatal("Can NOT get PRAUTO from lockdown exported file") + d.setVar('PRAUTO',str(auto_pr)) + return + + try: + conn = oe.prservice.prserv_make_conn(d) + if conn is not None: + if "AUTOINC" in pkgv: + srcpv = bb.fetch2.get_srcrev(d) + base_ver = "AUTOINC-%s" % version[:version.find(srcpv)] + value = conn.getPR(base_ver, pkgarch, srcpv) + d.setVar("PRSERV_PV_AUTOINC", str(value)) + + auto_pr = conn.getPR(version, pkgarch, checksum) + conn.close() + except Exception as e: + bb.fatal("Can NOT get PRAUTO, exception %s" % str(e)) + if auto_pr is None: + bb.fatal("Can NOT get PRAUTO from remote PR service") + d.setVar('PRAUTO',str(auto_pr)) +} + +# +# Package functions suitable for inclusion in PACKAGEFUNCS +# + +python package_setup_pkgv() { + pkgv = d.getVar("PKGV") + # Expand SRCPV into PKGV if not present + srcpv = bb.fetch.get_pkgv_string(d) + if srcpv and "+" in pkgv: + d.appendVar("PKGV", srcpv) + pkgv = d.getVar("PKGV") + + # Adjust pkgv as necessary... + if 'AUTOINC' in pkgv: + d.setVar("PKGV", pkgv.replace("AUTOINC", "${PRSERV_PV_AUTOINC}")) +} + + +python package_convert_pr_autoinc() { + # Change PRSERV_PV_AUTOINC and EXTENDPRAUTO usage to special values + d.setVar('PRSERV_PV_AUTOINC', '@PRSERV_PV_AUTOINC@') + d.setVar('EXTENDPRAUTO', '@EXTENDPRAUTO@') +} + +LOCALEBASEPN ??= "${PN}" +LOCALE_PATHS ?= "${datadir}/locale" + +python package_do_split_locales() { + oe.package.split_locales(d) +} + +python perform_packagecopy () { + import subprocess + import shutil + + dest = d.getVar('D') + dvar = d.getVar('PKGD') + + # Start by package population by taking a copy of the installed + # files to operate on + # Preserve sparse files and hard links + cmd = 'tar --exclude=./sysroot-only -cf - -C %s -p -S . | tar -xf - -C %s' % (dest, dvar) + subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) + + # replace RPATHs for the nativesdk binaries, to make them relocatable + if bb.data.inherits_class('nativesdk', d) or bb.data.inherits_class('cross-canadian', d): + rpath_replace (dvar, d) +} +perform_packagecopy[cleandirs] = "${PKGD}" +perform_packagecopy[dirs] = "${PKGD}" + +python populate_packages () { + oe.package.populate_packages(d) +} +populate_packages[dirs] = "${D}" + +python package_fixsymlinks () { + oe.package.process_fixsymlinks(pkgfiles, d) +} + +python package_package_name_hook() { + """ + A package_name_hook function can be used to rewrite the package names by + changing PKG. For an example, see debian.bbclass. + """ + pass +} + +EXPORT_FUNCTIONS package_name_hook + + +PKGDESTWORK = "${WORKDIR}/pkgdata" + +PKGDATA_VARS = "PN PE PV PR PKGE PKGV PKGR LICENSE DESCRIPTION SUMMARY RDEPENDS RPROVIDES RRECOMMENDS RSUGGESTS RREPLACES RCONFLICTS SECTION PKG ALLOW_EMPTY FILES CONFFILES FILES_INFO PACKAGE_ADD_METADATA pkg_postinst pkg_postrm pkg_preinst pkg_prerm" + +python emit_pkgdata() { + import oe.packagedata + oe.packagedata.emit_pkgdata(pkgfiles, d) +} +emit_pkgdata[dirs] = "${PKGDESTWORK}/runtime ${PKGDESTWORK}/runtime-reverse ${PKGDESTWORK}/runtime-rprovides ${PKGDESTWORK}/extended" + +ldconfig_postinst_fragment() { +if [ x"$D" = "x" ]; then + if [ -x /sbin/ldconfig ]; then /sbin/ldconfig ; fi +fi +} + +RPMDEPS = "${STAGING_LIBDIR_NATIVE}/rpm/rpmdeps --alldeps --define '__font_provides %{nil}'" + +python package_do_filedeps() { + oe.package.process_filedeps(pkgfiles, d) +} + +SHLIBSDIRS = "${WORKDIR_PKGDATA}/${MLPREFIX}shlibs2" +SHLIBSWORKDIR = "${PKGDESTWORK}/${MLPREFIX}shlibs2" + +python package_do_shlibs() { + oe.package.process_shlibs(pkgfiles, d) +} + +python package_do_pkgconfig () { + oe.package.process_pkgconfig(pkgfiles, d) +} + +python read_shlibdeps () { + pkglibdeps = oe.package.read_libdep_files(d) + + packages = d.getVar('PACKAGES').split() + for pkg in packages: + rdepends = bb.utils.explode_dep_versions2(d.getVar('RDEPENDS:' + pkg) or "") + for dep in sorted(pkglibdeps[pkg]): + # Add the dep if it's not already there, or if no comparison is set + if dep not in rdepends: + rdepends[dep] = [] + for v in pkglibdeps[pkg][dep]: + if v not in rdepends[dep]: + rdepends[dep].append(v) + d.setVar('RDEPENDS:' + pkg, bb.utils.join_deps(rdepends, commasep=False)) +} + +python package_depchains() { + oe.package.process_depchains(pkgfiles, d) +} + +# Since bitbake can't determine which variables are accessed during package +# iteration, we need to list them here: +PACKAGEVARS = "FILES RDEPENDS RRECOMMENDS SUMMARY DESCRIPTION RSUGGESTS RPROVIDES RCONFLICTS PKG ALLOW_EMPTY pkg_postinst pkg_postrm pkg_postinst_ontarget INITSCRIPT_NAME INITSCRIPT_PARAMS DEBIAN_NOAUTONAME ALTERNATIVE PKGE PKGV PKGR USERADD_PARAM GROUPADD_PARAM CONFFILES SYSTEMD_SERVICE LICENSE SECTION pkg_preinst pkg_prerm RREPLACES GROUPMEMS_PARAM SYSTEMD_AUTO_ENABLE SKIP_FILEDEPS PRIVATE_LIBS PACKAGE_ADD_METADATA" + +def gen_packagevar(d, pkgvars="PACKAGEVARS"): + ret = [] + pkgs = (d.getVar("PACKAGES") or "").split() + vars = (d.getVar(pkgvars) or "").split() + for v in vars: + ret.append(v) + for p in pkgs: + for v in vars: + ret.append(v + ":" + p) + + # Ensure that changes to INCOMPATIBLE_LICENSE re-run do_package for + # affected recipes. + ret.append('_exclude_incompatible-%s' % p) + return " ".join(ret) + + +# Functions for setting up PKGD +PACKAGE_PREPROCESS_FUNCS ?= "" +# Functions which split PKGD up into separate packages +PACKAGESPLITFUNCS ?= " \ + package_do_split_locales \ + populate_packages" +# Functions which process metadata based on split packages +PACKAGEFUNCS += " \ + package_fixsymlinks \ + package_name_hook \ + package_do_filedeps \ + package_do_shlibs \ + package_do_pkgconfig \ + read_shlibdeps \ + package_depchains \ + emit_pkgdata" + +python do_package () { + # Change the following version to cause sstate to invalidate the package + # cache. This is useful if an item this class depends on changes in a + # way that the output of this class changes. rpmdeps is a good example + # as any change to rpmdeps requires this to be rerun. + # PACKAGE_BBCLASS_VERSION = "5" + + # Init cachedpath + global cpath + cpath = oe.cachedpath.CachedPath() + + ########################################################################### + # Sanity test the setup + ########################################################################### + + packages = (d.getVar('PACKAGES') or "").split() + if len(packages) < 1: + bb.debug(1, "No packages to build, skipping do_package") + return + + workdir = d.getVar('WORKDIR') + outdir = d.getVar('DEPLOY_DIR') + dest = d.getVar('D') + dvar = d.getVar('PKGD') + pn = d.getVar('PN') + + if not workdir or not outdir or not dest or not dvar or not pn: + msg = "WORKDIR, DEPLOY_DIR, D, PN and PKGD all must be defined, unable to package" + oe.qa.handle_error("var-undefined", msg, d) + return + + bb.build.exec_func("package_setup_pkgv", d) + bb.build.exec_func("package_convert_pr_autoinc", d) + + # Check for conflict between renamed packages and existing ones + # for each package in PACKAGES, check if it will be renamed to an existing one + for p in packages: + rename = d.getVar('PKG:%s' % p) + if rename and rename in packages: + bb.fatal('package "%s" is renamed to "%s" using PKG:%s, but package name already exists' % (p, rename, p)) + + ########################################################################### + # Optimisations + ########################################################################### + + # Continually expanding complex expressions is inefficient, particularly + # when we write to the datastore and invalidate the expansion cache. This + # code pre-expands some frequently used variables + + def expandVar(x, d): + d.setVar(x, d.getVar(x)) + + for x in 'PN', 'PV', 'BPN', 'TARGET_SYS', 'EXTENDPRAUTO': + expandVar(x, d) + + ########################################################################### + # Setup PKGD (from D) + ########################################################################### + + bb.build.exec_func("package_prepare_pkgdata", d) + bb.build.exec_func("perform_packagecopy", d) + for f in (d.getVar('PACKAGE_PREPROCESS_FUNCS') or '').split(): + bb.build.exec_func(f, d) + oe.package.process_split_and_strip_files(d) + oe.package.fixup_perms(d) + + ########################################################################### + # Split up PKGD into PKGDEST + ########################################################################### + + cpath = oe.cachedpath.CachedPath() + + for f in (d.getVar('PACKAGESPLITFUNCS') or '').split(): + bb.build.exec_func(f, d) + + ########################################################################### + # Process PKGDEST + ########################################################################### + + # Build global list of files in each split package + global pkgfiles + pkgfiles = {} + packages = d.getVar('PACKAGES').split() + pkgdest = d.getVar('PKGDEST') + for pkg in packages: + pkgfiles[pkg] = [] + for walkroot, dirs, files in cpath.walk(pkgdest + "/" + pkg): + for file in files: + pkgfiles[pkg].append(walkroot + os.sep + file) + + for f in (d.getVar('PACKAGEFUNCS') or '').split(): + bb.build.exec_func(f, d) + + oe.qa.exit_if_errors(d) +} + +do_package[dirs] = "${SHLIBSWORKDIR} ${D}" +do_package[vardeps] += "${PACKAGE_PREPROCESS_FUNCS} ${PACKAGESPLITFUNCS} ${PACKAGEFUNCS} ${@gen_packagevar(d)}" +addtask package after do_install + +SSTATETASKS += "do_package" +do_package[cleandirs] = "${PKGDEST} ${PKGDESTWORK}" +do_package[sstate-plaindirs] = "${PKGD} ${PKGDEST} ${PKGDESTWORK}" +do_package_setscene[dirs] = "${STAGING_DIR}" + +python do_package_setscene () { + sstate_setscene(d) +} +addtask do_package_setscene + +# Copy from PKGDESTWORK to tempdirectory as tempdirectory can be cleaned at both +# do_package_setscene and do_packagedata_setscene leading to races +python do_packagedata () { + bb.build.exec_func("package_setup_pkgv", d) + bb.build.exec_func("package_get_auto_pr", d) + + src = d.expand("${PKGDESTWORK}") + dest = d.expand("${WORKDIR}/pkgdata-pdata-input") + oe.path.copyhardlinktree(src, dest) + + bb.build.exec_func("packagedata_translate_pr_autoinc", d) +} +do_packagedata[cleandirs] += "${WORKDIR}/pkgdata-pdata-input" + +# Translate the EXTENDPRAUTO and AUTOINC to the final values +packagedata_translate_pr_autoinc() { + find ${WORKDIR}/pkgdata-pdata-input -type f | xargs --no-run-if-empty \ + sed -e 's,@PRSERV_PV_AUTOINC@,${PRSERV_PV_AUTOINC},g' \ + -e 's,@EXTENDPRAUTO@,${EXTENDPRAUTO},g' -i +} + +addtask packagedata before do_build after do_package + +SSTATETASKS += "do_packagedata" +do_packagedata[sstate-inputdirs] = "${WORKDIR}/pkgdata-pdata-input" +do_packagedata[sstate-outputdirs] = "${PKGDATA_DIR}" +do_packagedata[stamp-extra-info] = "${MACHINE_ARCH}" + +python do_packagedata_setscene () { + sstate_setscene(d) +} +addtask do_packagedata_setscene + diff --git a/meta/classes-global/package_deb.bbclass b/meta/classes-global/package_deb.bbclass new file mode 100644 index 0000000000..1f10b15a00 --- /dev/null +++ b/meta/classes-global/package_deb.bbclass @@ -0,0 +1,333 @@ +# +# Copyright 2006-2008 OpenedHand Ltd. +# +# SPDX-License-Identifier: MIT +# + +inherit package + +IMAGE_PKGTYPE ?= "deb" + +DPKG_BUILDCMD ??= "dpkg-deb" + +DPKG_ARCH ?= "${@debian_arch_map(d.getVar('TARGET_ARCH'), d.getVar('TUNE_FEATURES'))}" +DPKG_ARCH[vardepvalue] = "${DPKG_ARCH}" + +PKGWRITEDIRDEB = "${WORKDIR}/deploy-debs" + +APTCONF_TARGET = "${WORKDIR}" + +APT_ARGS = "${@['', '--no-install-recommends'][d.getVar("NO_RECOMMENDATIONS") == "1"]}" + +def debian_arch_map(arch, tune): + tune_features = tune.split() + if arch == "allarch": + return "all" + if arch in ["i586", "i686"]: + return "i386" + if arch == "x86_64": + if "mx32" in tune_features: + return "x32" + return "amd64" + if arch.startswith("mips"): + endian = ["el", ""]["bigendian" in tune_features] + if "n64" in tune_features: + return "mips64" + endian + if "n32" in tune_features: + return "mipsn32" + endian + return "mips" + endian + if arch == "powerpc": + return arch + ["", "spe"]["spe" in tune_features] + if arch == "aarch64": + return "arm64" + if arch == "arm": + return arch + ["el", "hf"]["callconvention-hard" in tune_features] + return arch + +python do_package_deb () { + packages = d.getVar('PACKAGES') + if not packages: + bb.debug(1, "PACKAGES not defined, nothing to package") + return + + tmpdir = d.getVar('TMPDIR') + if os.access(os.path.join(tmpdir, "stamps", "DEB_PACKAGE_INDEX_CLEAN"),os.R_OK): + os.unlink(os.path.join(tmpdir, "stamps", "DEB_PACKAGE_INDEX_CLEAN")) + + oe.utils.multiprocess_launch(deb_write_pkg, packages.split(), d, extraargs=(d,)) +} +do_package_deb[vardeps] += "deb_write_pkg" +do_package_deb[vardepsexclude] = "BB_NUMBER_THREADS" + +def deb_write_pkg(pkg, d): + import re, copy + import textwrap + import subprocess + import collections + import codecs + + outdir = d.getVar('PKGWRITEDIRDEB') + pkgdest = d.getVar('PKGDEST') + + def cleanupcontrol(root): + for p in ['CONTROL', 'DEBIAN']: + p = os.path.join(root, p) + if os.path.exists(p): + bb.utils.prunedir(p) + + localdata = bb.data.createCopy(d) + root = "%s/%s" % (pkgdest, pkg) + + lf = bb.utils.lockfile(root + ".lock") + try: + + localdata.setVar('ROOT', '') + localdata.setVar('ROOT_%s' % pkg, root) + pkgname = localdata.getVar('PKG:%s' % pkg) + if not pkgname: + pkgname = pkg + localdata.setVar('PKG', pkgname) + + localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + pkg) + + basedir = os.path.join(os.path.dirname(root)) + + pkgoutdir = os.path.join(outdir, localdata.getVar('PACKAGE_ARCH')) + bb.utils.mkdirhier(pkgoutdir) + + os.chdir(root) + cleanupcontrol(root) + from glob import glob + g = glob('*') + if not g and localdata.getVar('ALLOW_EMPTY', False) != "1": + bb.note("Not creating empty archive for %s-%s-%s" % (pkg, localdata.getVar('PKGV'), localdata.getVar('PKGR'))) + return + + controldir = os.path.join(root, 'DEBIAN') + bb.utils.mkdirhier(controldir) + os.chmod(controldir, 0o755) + + ctrlfile = codecs.open(os.path.join(controldir, 'control'), 'w', 'utf-8') + + fields = [] + pe = d.getVar('PKGE') + if pe and int(pe) > 0: + fields.append(["Version: %s:%s-%s\n", ['PKGE', 'PKGV', 'PKGR']]) + else: + fields.append(["Version: %s-%s\n", ['PKGV', 'PKGR']]) + fields.append(["Description: %s\n", ['DESCRIPTION']]) + fields.append(["Section: %s\n", ['SECTION']]) + fields.append(["Priority: %s\n", ['PRIORITY']]) + fields.append(["Maintainer: %s\n", ['MAINTAINER']]) + fields.append(["Architecture: %s\n", ['DPKG_ARCH']]) + fields.append(["OE: %s\n", ['PN']]) + fields.append(["PackageArch: %s\n", ['PACKAGE_ARCH']]) + if d.getVar('HOMEPAGE'): + fields.append(["Homepage: %s\n", ['HOMEPAGE']]) + + # Package, Version, Maintainer, Description - mandatory + # Section, Priority, Essential, Architecture, Source, Depends, Pre-Depends, Recommends, Suggests, Conflicts, Replaces, Provides - Optional + + + def pullData(l, d): + l2 = [] + for i in l: + data = d.getVar(i) + if data is None: + raise KeyError(i) + if i == 'DPKG_ARCH' and d.getVar('PACKAGE_ARCH') == 'all': + data = 'all' + elif i == 'PACKAGE_ARCH' or i == 'DPKG_ARCH': + # The params in deb package control don't allow character + # `_', so change the arch's `_' to `-'. Such as `x86_64' + # -->`x86-64' + data = data.replace('_', '-') + l2.append(data) + return l2 + + ctrlfile.write("Package: %s\n" % pkgname) + if d.getVar('PACKAGE_ARCH') == "all": + ctrlfile.write("Multi-Arch: foreign\n") + # check for required fields + for (c, fs) in fields: + # Special behavior for description... + if 'DESCRIPTION' in fs: + summary = localdata.getVar('SUMMARY') or localdata.getVar('DESCRIPTION') or "." + ctrlfile.write('Description: %s\n' % summary) + description = localdata.getVar('DESCRIPTION') or "." + description = textwrap.dedent(description).strip() + if '\\n' in description: + # Manually indent + for t in description.split('\\n'): + ctrlfile.write(' %s\n' % (t.strip() or '.')) + else: + # Auto indent + ctrlfile.write('%s\n' % textwrap.fill(description.strip(), width=74, initial_indent=' ', subsequent_indent=' ')) + + else: + ctrlfile.write(c % tuple(pullData(fs, localdata))) + + # more fields + + custom_fields_chunk = oe.packagedata.get_package_additional_metadata("deb", localdata) + if custom_fields_chunk: + ctrlfile.write(custom_fields_chunk) + ctrlfile.write("\n") + + oe.packagedata.mapping_rename_hook(localdata) + + def debian_cmp_remap(var): + # dpkg does not allow for '(', ')' or ':' in a dependency name + # Replace any instances of them with '__' + # + # In debian '>' and '<' do not mean what it appears they mean + # '<' = less or equal + # '>' = greater or equal + # adjust these to the '<<' and '>>' equivalents + # Also, "=" specifiers only work if they have the PR in, so 1.2.3 != 1.2.3-r0 + # so to avoid issues, map this to ">= 1.2.3 << 1.2.3.0" + for dep in list(var.keys()): + if '(' in dep or '/' in dep: + newdep = re.sub(r'[(:)/]', '__', dep) + if newdep.startswith("__"): + newdep = "A" + newdep + if newdep != dep: + var[newdep] = var[dep] + del var[dep] + for dep in var: + for i, v in enumerate(var[dep]): + if (v or "").startswith("< "): + var[dep][i] = var[dep][i].replace("< ", "<< ") + elif (v or "").startswith("> "): + var[dep][i] = var[dep][i].replace("> ", ">> ") + elif (v or "").startswith("= ") and "-r" not in v: + ver = var[dep][i].replace("= ", "") + var[dep][i] = var[dep][i].replace("= ", ">= ") + var[dep].append("<< " + ver + ".0") + + rdepends = bb.utils.explode_dep_versions2(localdata.getVar("RDEPENDS") or "") + debian_cmp_remap(rdepends) + for dep in list(rdepends.keys()): + if dep == pkg: + del rdepends[dep] + continue + if '*' in dep: + del rdepends[dep] + rrecommends = bb.utils.explode_dep_versions2(localdata.getVar("RRECOMMENDS") or "") + debian_cmp_remap(rrecommends) + for dep in list(rrecommends.keys()): + if '*' in dep: + del rrecommends[dep] + rsuggests = bb.utils.explode_dep_versions2(localdata.getVar("RSUGGESTS") or "") + debian_cmp_remap(rsuggests) + # Deliberately drop version information here, not wanted/supported by deb + rprovides = dict.fromkeys(bb.utils.explode_dep_versions2(localdata.getVar("RPROVIDES") or ""), []) + # Remove file paths if any from rprovides, debian does not support custom providers + for key in list(rprovides.keys()): + if key.startswith('/'): + del rprovides[key] + rprovides = collections.OrderedDict(sorted(rprovides.items(), key=lambda x: x[0])) + debian_cmp_remap(rprovides) + rreplaces = bb.utils.explode_dep_versions2(localdata.getVar("RREPLACES") or "") + debian_cmp_remap(rreplaces) + rconflicts = bb.utils.explode_dep_versions2(localdata.getVar("RCONFLICTS") or "") + debian_cmp_remap(rconflicts) + if rdepends: + ctrlfile.write("Depends: %s\n" % bb.utils.join_deps(rdepends)) + if rsuggests: + ctrlfile.write("Suggests: %s\n" % bb.utils.join_deps(rsuggests)) + if rrecommends: + ctrlfile.write("Recommends: %s\n" % bb.utils.join_deps(rrecommends)) + if rprovides: + ctrlfile.write("Provides: %s\n" % bb.utils.join_deps(rprovides)) + if rreplaces: + ctrlfile.write("Replaces: %s\n" % bb.utils.join_deps(rreplaces)) + if rconflicts: + ctrlfile.write("Conflicts: %s\n" % bb.utils.join_deps(rconflicts)) + ctrlfile.close() + + for script in ["preinst", "postinst", "prerm", "postrm"]: + scriptvar = localdata.getVar('pkg_%s' % script) + if not scriptvar: + continue + scriptvar = scriptvar.strip() + scriptfile = open(os.path.join(controldir, script), 'w') + + if scriptvar.startswith("#!"): + pos = scriptvar.find("\n") + 1 + scriptfile.write(scriptvar[:pos]) + else: + pos = 0 + scriptfile.write("#!/bin/sh\n") + + # Prevent the prerm/postrm scripts from being run during an upgrade + if script in ('prerm', 'postrm'): + scriptfile.write('[ "$1" != "upgrade" ] || exit 0\n') + + scriptfile.write(scriptvar[pos:]) + scriptfile.write('\n') + scriptfile.close() + os.chmod(os.path.join(controldir, script), 0o755) + + conffiles_str = ' '.join(oe.package.get_conffiles(pkg, d)) + if conffiles_str: + conffiles = open(os.path.join(controldir, 'conffiles'), 'w') + for f in conffiles_str.split(): + if os.path.exists(oe.path.join(root, f)): + conffiles.write('%s\n' % f) + conffiles.close() + + os.chdir(basedir) + subprocess.check_output("PATH=\"%s\" %s -b %s %s" % (localdata.getVar("PATH"), localdata.getVar("DPKG_BUILDCMD"), + root, pkgoutdir), + stderr=subprocess.STDOUT, + shell=True) + + finally: + cleanupcontrol(root) + bb.utils.unlockfile(lf) + +# Otherwise allarch packages may change depending on override configuration +deb_write_pkg[vardepsexclude] = "OVERRIDES" + +# Have to list any variables referenced as X_<pkg> that aren't in pkgdata here +DEBEXTRAVARS = "PKGV PKGR PKGV DESCRIPTION SECTION PRIORITY MAINTAINER DPKG_ARCH PN HOMEPAGE PACKAGE_ADD_METADATA_DEB" +do_package_write_deb[vardeps] += "${@gen_packagevar(d, 'DEBEXTRAVARS')}" + +SSTATETASKS += "do_package_write_deb" +do_package_write_deb[sstate-inputdirs] = "${PKGWRITEDIRDEB}" +do_package_write_deb[sstate-outputdirs] = "${DEPLOY_DIR_DEB}" + +python do_package_write_deb_setscene () { + tmpdir = d.getVar('TMPDIR') + + if os.access(os.path.join(tmpdir, "stamps", "DEB_PACKAGE_INDEX_CLEAN"),os.R_OK): + os.unlink(os.path.join(tmpdir, "stamps", "DEB_PACKAGE_INDEX_CLEAN")) + + sstate_setscene(d) +} +addtask do_package_write_deb_setscene + +python () { + if d.getVar('PACKAGES') != '': + deps = ' dpkg-native:do_populate_sysroot virtual/fakeroot-native:do_populate_sysroot' + d.appendVarFlag('do_package_write_deb', 'depends', deps) + d.setVarFlag('do_package_write_deb', 'fakeroot', "1") + + # Needed to ensure PKG_xxx renaming of dependency packages works + d.setVarFlag('do_package_write_deb', 'deptask', "do_packagedata") + d.setVarFlag('do_package_write_deb', 'rdeptask', "do_packagedata") +} + +python do_package_write_deb () { + bb.build.exec_func("read_subpackage_metadata", d) + bb.build.exec_func("do_package_deb", d) +} +do_package_write_deb[dirs] = "${PKGWRITEDIRDEB}" +do_package_write_deb[cleandirs] = "${PKGWRITEDIRDEB}" +do_package_write_deb[depends] += "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'), 'do_populate_sysroot')}" +addtask package_write_deb after do_packagedata do_package do_deploy_source_date_epoch before do_build +do_build[rdeptask] += "do_package_write_deb" + +PACKAGEINDEXDEPS += "dpkg-native:do_populate_sysroot" +PACKAGEINDEXDEPS += "apt-native:do_populate_sysroot" diff --git a/meta/classes-global/package_ipk.bbclass b/meta/classes-global/package_ipk.bbclass new file mode 100644 index 0000000000..209568ae5f --- /dev/null +++ b/meta/classes-global/package_ipk.bbclass @@ -0,0 +1,300 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +inherit package + +IMAGE_PKGTYPE ?= "ipk" + +IPKGCONF_TARGET = "${WORKDIR}/opkg.conf" +IPKGCONF_SDK = "${WORKDIR}/opkg-sdk.conf" +IPKGCONF_SDK_TARGET = "${WORKDIR}/opkg-sdk-target.conf" + +PKGWRITEDIRIPK = "${WORKDIR}/deploy-ipks" + +# Program to be used to build opkg packages +OPKGBUILDCMD ??= 'opkg-build -Z zstd -a "${ZSTD_DEFAULTS}"' + +OPKG_ARGS += "--force_postinstall --prefer-arch-to-version" +OPKG_ARGS += "${@['', '--no-install-recommends'][d.getVar("NO_RECOMMENDATIONS") == "1"]}" +OPKG_ARGS += "${@['', '--add-exclude ' + ' --add-exclude '.join((d.getVar('PACKAGE_EXCLUDE') or "").split())][(d.getVar("PACKAGE_EXCLUDE") or "").strip() != ""]}" + +OPKGLIBDIR ??= "${localstatedir}/lib" + +python do_package_ipk () { + workdir = d.getVar('WORKDIR') + outdir = d.getVar('PKGWRITEDIRIPK') + tmpdir = d.getVar('TMPDIR') + pkgdest = d.getVar('PKGDEST') + if not workdir or not outdir or not tmpdir: + bb.error("Variables incorrectly set, unable to package") + return + + packages = d.getVar('PACKAGES') + if not packages or packages == '': + bb.debug(1, "No packages; nothing to do") + return + + # We're about to add new packages so the index needs to be checked + # so remove the appropriate stamp file. + if os.access(os.path.join(tmpdir, "stamps", "IPK_PACKAGE_INDEX_CLEAN"), os.R_OK): + os.unlink(os.path.join(tmpdir, "stamps", "IPK_PACKAGE_INDEX_CLEAN")) + + oe.utils.multiprocess_launch(ipk_write_pkg, packages.split(), d, extraargs=(d,)) +} +do_package_ipk[vardeps] += "ipk_write_pkg" +do_package_ipk[vardepsexclude] = "BB_NUMBER_THREADS" + +# FILE isn't included by default but we want the recipe to change if basename() changes +IPK_RECIPE_FILE = "${@os.path.basename(d.getVar('FILE'))}" +IPK_RECIPE_FILE[vardepvalue] = "${IPK_RECIPE_FILE}" + +def ipk_write_pkg(pkg, d): + import re, copy + import subprocess + import textwrap + import collections + import glob + + def cleanupcontrol(root): + for p in ['CONTROL', 'DEBIAN']: + p = os.path.join(root, p) + if os.path.exists(p): + bb.utils.prunedir(p) + + outdir = d.getVar('PKGWRITEDIRIPK') + pkgdest = d.getVar('PKGDEST') + recipesource = d.getVar('IPK_RECIPE_FILE') + + localdata = bb.data.createCopy(d) + root = "%s/%s" % (pkgdest, pkg) + + lf = bb.utils.lockfile(root + ".lock") + try: + localdata.setVar('ROOT', '') + localdata.setVar('ROOT_%s' % pkg, root) + pkgname = localdata.getVar('PKG:%s' % pkg) + if not pkgname: + pkgname = pkg + localdata.setVar('PKG', pkgname) + + localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + pkg) + + basedir = os.path.join(os.path.dirname(root)) + arch = localdata.getVar('PACKAGE_ARCH') + + if localdata.getVar('IPK_HIERARCHICAL_FEED', False) == "1": + # Spread packages across subdirectories so each isn't too crowded + if pkgname.startswith('lib'): + pkg_prefix = 'lib' + pkgname[3] + else: + pkg_prefix = pkgname[0] + + # Keep -dbg, -dev, -doc, -staticdev, -locale and -locale-* packages + # together. These package suffixes are taken from the definitions of + # PACKAGES and PACKAGES_DYNAMIC in meta/conf/bitbake.conf + if pkgname[-4:] in ('-dbg', '-dev', '-doc'): + pkg_subdir = pkgname[:-4] + elif pkgname.endswith('-staticdev'): + pkg_subdir = pkgname[:-10] + elif pkgname.endswith('-locale'): + pkg_subdir = pkgname[:-7] + elif '-locale-' in pkgname: + pkg_subdir = pkgname[:pkgname.find('-locale-')] + else: + pkg_subdir = pkgname + + pkgoutdir = "%s/%s/%s/%s" % (outdir, arch, pkg_prefix, pkg_subdir) + else: + pkgoutdir = "%s/%s" % (outdir, arch) + + bb.utils.mkdirhier(pkgoutdir) + os.chdir(root) + cleanupcontrol(root) + g = glob.glob('*') + if not g and localdata.getVar('ALLOW_EMPTY', False) != "1": + bb.note("Not creating empty archive for %s-%s-%s" % (pkg, localdata.getVar('PKGV'), localdata.getVar('PKGR'))) + return + + controldir = os.path.join(root, 'CONTROL') + bb.utils.mkdirhier(controldir) + ctrlfile = open(os.path.join(controldir, 'control'), 'w') + + fields = [] + pe = d.getVar('PKGE') + if pe and int(pe) > 0: + fields.append(["Version: %s:%s-%s\n", ['PKGE', 'PKGV', 'PKGR']]) + else: + fields.append(["Version: %s-%s\n", ['PKGV', 'PKGR']]) + fields.append(["Description: %s\n", ['DESCRIPTION']]) + fields.append(["Section: %s\n", ['SECTION']]) + fields.append(["Priority: %s\n", ['PRIORITY']]) + fields.append(["Maintainer: %s\n", ['MAINTAINER']]) + fields.append(["License: %s\n", ['LICENSE']]) + fields.append(["Architecture: %s\n", ['PACKAGE_ARCH']]) + fields.append(["OE: %s\n", ['PN']]) + if d.getVar('HOMEPAGE'): + fields.append(["Homepage: %s\n", ['HOMEPAGE']]) + + def pullData(l, d): + l2 = [] + for i in l: + l2.append(d.getVar(i)) + return l2 + + ctrlfile.write("Package: %s\n" % pkgname) + # check for required fields + for (c, fs) in fields: + for f in fs: + if localdata.getVar(f, False) is None: + raise KeyError(f) + # Special behavior for description... + if 'DESCRIPTION' in fs: + summary = localdata.getVar('SUMMARY') or localdata.getVar('DESCRIPTION') or "." + ctrlfile.write('Description: %s\n' % summary) + description = localdata.getVar('DESCRIPTION') or "." + description = textwrap.dedent(description).strip() + if '\\n' in description: + # Manually indent: multiline description includes a leading space + for t in description.split('\\n'): + ctrlfile.write(' %s\n' % (t.strip() or ' .')) + else: + # Auto indent + ctrlfile.write('%s\n' % textwrap.fill(description, width=74, initial_indent=' ', subsequent_indent=' ')) + else: + ctrlfile.write(c % tuple(pullData(fs, localdata))) + + custom_fields_chunk = oe.packagedata.get_package_additional_metadata("ipk", localdata) + if custom_fields_chunk is not None: + ctrlfile.write(custom_fields_chunk) + ctrlfile.write("\n") + + oe.packagedata.mapping_rename_hook(localdata) + + def debian_cmp_remap(var): + # In debian '>' and '<' do not mean what it appears they mean + # '<' = less or equal + # '>' = greater or equal + # adjust these to the '<<' and '>>' equivalents + # Also, "=" specifiers only work if they have the PR in, so 1.2.3 != 1.2.3-r0 + # so to avoid issues, map this to ">= 1.2.3 << 1.2.3.0" + for dep in var: + for i, v in enumerate(var[dep]): + if (v or "").startswith("< "): + var[dep][i] = var[dep][i].replace("< ", "<< ") + elif (v or "").startswith("> "): + var[dep][i] = var[dep][i].replace("> ", ">> ") + elif (v or "").startswith("= ") and "-r" not in v: + ver = var[dep][i].replace("= ", "") + var[dep][i] = var[dep][i].replace("= ", ">= ") + var[dep].append("<< " + ver + ".0") + + rdepends = bb.utils.explode_dep_versions2(localdata.getVar("RDEPENDS") or "") + debian_cmp_remap(rdepends) + rrecommends = bb.utils.explode_dep_versions2(localdata.getVar("RRECOMMENDS") or "") + debian_cmp_remap(rrecommends) + rsuggests = bb.utils.explode_dep_versions2(localdata.getVar("RSUGGESTS") or "") + debian_cmp_remap(rsuggests) + # Deliberately drop version information here, not wanted/supported by ipk + rprovides = dict.fromkeys(bb.utils.explode_dep_versions2(localdata.getVar("RPROVIDES") or ""), []) + rprovides = collections.OrderedDict(sorted(rprovides.items(), key=lambda x: x[0])) + debian_cmp_remap(rprovides) + rreplaces = bb.utils.explode_dep_versions2(localdata.getVar("RREPLACES") or "") + debian_cmp_remap(rreplaces) + rconflicts = bb.utils.explode_dep_versions2(localdata.getVar("RCONFLICTS") or "") + debian_cmp_remap(rconflicts) + + if rdepends: + ctrlfile.write("Depends: %s\n" % bb.utils.join_deps(rdepends)) + if rsuggests: + ctrlfile.write("Suggests: %s\n" % bb.utils.join_deps(rsuggests)) + if rrecommends: + ctrlfile.write("Recommends: %s\n" % bb.utils.join_deps(rrecommends)) + if rprovides: + ctrlfile.write("Provides: %s\n" % bb.utils.join_deps(rprovides)) + if rreplaces: + ctrlfile.write("Replaces: %s\n" % bb.utils.join_deps(rreplaces)) + if rconflicts: + ctrlfile.write("Conflicts: %s\n" % bb.utils.join_deps(rconflicts)) + ctrlfile.write("Source: %s\n" % recipesource) + ctrlfile.close() + + for script in ["preinst", "postinst", "prerm", "postrm"]: + scriptvar = localdata.getVar('pkg_%s' % script) + if not scriptvar: + continue + scriptfile = open(os.path.join(controldir, script), 'w') + scriptfile.write(scriptvar) + scriptfile.close() + os.chmod(os.path.join(controldir, script), 0o755) + + conffiles_str = ' '.join(oe.package.get_conffiles(pkg, d)) + if conffiles_str: + conffiles = open(os.path.join(controldir, 'conffiles'), 'w') + for f in conffiles_str.split(): + if os.path.exists(oe.path.join(root, f)): + conffiles.write('%s\n' % f) + conffiles.close() + + os.chdir(basedir) + subprocess.check_output("PATH=\"%s\" %s %s %s" % (localdata.getVar("PATH"), + d.getVar("OPKGBUILDCMD"), pkg, pkgoutdir), + stderr=subprocess.STDOUT, + shell=True) + + if d.getVar('IPK_SIGN_PACKAGES') == '1': + ipkver = "%s-%s" % (localdata.getVar('PKGV'), localdata.getVar('PKGR')) + ipk_to_sign = "%s/%s_%s_%s.ipk" % (pkgoutdir, pkgname, ipkver, localdata.getVar('PACKAGE_ARCH')) + sign_ipk(d, ipk_to_sign) + + finally: + cleanupcontrol(root) + bb.utils.unlockfile(lf) + +# Have to list any variables referenced as X_<pkg> that aren't in pkgdata here +IPKEXTRAVARS = "PRIORITY MAINTAINER PACKAGE_ARCH HOMEPAGE PACKAGE_ADD_METADATA_IPK" +ipk_write_pkg[vardeps] += "${@gen_packagevar(d, 'IPKEXTRAVARS')}" + +# Otherwise allarch packages may change depending on override configuration +ipk_write_pkg[vardepsexclude] = "OVERRIDES" + + +SSTATETASKS += "do_package_write_ipk" +do_package_write_ipk[sstate-inputdirs] = "${PKGWRITEDIRIPK}" +do_package_write_ipk[sstate-outputdirs] = "${DEPLOY_DIR_IPK}" + +python do_package_write_ipk_setscene () { + tmpdir = d.getVar('TMPDIR') + + if os.access(os.path.join(tmpdir, "stamps", "IPK_PACKAGE_INDEX_CLEAN"), os.R_OK): + os.unlink(os.path.join(tmpdir, "stamps", "IPK_PACKAGE_INDEX_CLEAN")) + + sstate_setscene(d) +} +addtask do_package_write_ipk_setscene + +python () { + if d.getVar('PACKAGES') != '': + deps = ' opkg-utils-native:do_populate_sysroot virtual/fakeroot-native:do_populate_sysroot zstd-native:do_populate_sysroot' + d.appendVarFlag('do_package_write_ipk', 'depends', deps) + d.setVarFlag('do_package_write_ipk', 'fakeroot', "1") + + # Needed to ensure PKG_xxx renaming of dependency packages works + d.setVarFlag('do_package_write_ipk', 'deptask', "do_packagedata") + d.setVarFlag('do_package_write_ipk', 'rdeptask', "do_packagedata") +} + +python do_package_write_ipk () { + bb.build.exec_func("read_subpackage_metadata", d) + bb.build.exec_func("do_package_ipk", d) +} +do_package_write_ipk[dirs] = "${PKGWRITEDIRIPK}" +do_package_write_ipk[cleandirs] = "${PKGWRITEDIRIPK}" +do_package_write_ipk[depends] += "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'), 'do_populate_sysroot')}" +addtask package_write_ipk after do_packagedata do_package do_deploy_source_date_epoch before do_build +do_build[rdeptask] += "do_package_write_ipk" + +PACKAGEINDEXDEPS += "opkg-utils-native:do_populate_sysroot" +PACKAGEINDEXDEPS += "opkg-native:do_populate_sysroot" diff --git a/meta/classes-global/package_pkgdata.bbclass b/meta/classes-global/package_pkgdata.bbclass new file mode 100644 index 0000000000..f653bd9240 --- /dev/null +++ b/meta/classes-global/package_pkgdata.bbclass @@ -0,0 +1,173 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +WORKDIR_PKGDATA = "${WORKDIR}/pkgdata-sysroot" + +def package_populate_pkgdata_dir(pkgdatadir, d): + import glob + + postinsts = [] + seendirs = set() + stagingdir = d.getVar("PKGDATA_DIR") + pkgarchs = ['${MACHINE_ARCH}'] + pkgarchs = pkgarchs + list(reversed(d.getVar("PACKAGE_EXTRA_ARCHS").split())) + pkgarchs.append('allarch') + + bb.utils.mkdirhier(pkgdatadir) + for pkgarch in pkgarchs: + for manifest in glob.glob(d.expand("${SSTATE_MANIFESTS}/manifest-%s-*.packagedata" % pkgarch)): + with open(manifest, "r") as f: + for l in f: + l = l.strip() + dest = l.replace(stagingdir, "") + if l.endswith("/"): + staging_copydir(l, pkgdatadir, dest, seendirs) + continue + try: + staging_copyfile(l, pkgdatadir, dest, postinsts, seendirs) + except FileExistsError: + continue + +python package_prepare_pkgdata() { + import copy + import glob + + taskdepdata = d.getVar("BB_TASKDEPDATA", False) + mytaskname = d.getVar("BB_RUNTASK") + if mytaskname.endswith("_setscene"): + mytaskname = mytaskname.replace("_setscene", "") + workdir = d.getVar("WORKDIR") + pn = d.getVar("PN") + stagingdir = d.getVar("PKGDATA_DIR") + pkgdatadir = d.getVar("WORKDIR_PKGDATA") + + # Detect bitbake -b usage + nodeps = d.getVar("BB_LIMITEDDEPS") or False + if nodeps: + staging_package_populate_pkgdata_dir(pkgdatadir, d) + return + + start = None + configuredeps = [] + for dep in taskdepdata: + data = taskdepdata[dep] + if data[1] == mytaskname and data[0] == pn: + start = dep + break + if start is None: + bb.fatal("Couldn't find ourself in BB_TASKDEPDATA?") + + # We need to figure out which sysroot files we need to expose to this task. + # This needs to match what would get restored from sstate, which is controlled + # ultimately by calls from bitbake to setscene_depvalid(). + # That function expects a setscene dependency tree. We build a dependency tree + # condensed to inter-sstate task dependencies, similar to that used by setscene + # tasks. We can then call into setscene_depvalid() and decide + # which dependencies we can "see" and should expose in the recipe specific sysroot. + setscenedeps = copy.deepcopy(taskdepdata) + + start = set([start]) + + sstatetasks = d.getVar("SSTATETASKS").split() + # Add recipe specific tasks referenced by setscene_depvalid() + sstatetasks.append("do_stash_locale") + + # If start is an sstate task (like do_package) we need to add in its direct dependencies + # else the code below won't recurse into them. + for dep in set(start): + for dep2 in setscenedeps[dep][3]: + start.add(dep2) + start.remove(dep) + + # Create collapsed do_populate_sysroot -> do_populate_sysroot tree + for dep in taskdepdata: + data = setscenedeps[dep] + if data[1] not in sstatetasks: + for dep2 in setscenedeps: + data2 = setscenedeps[dep2] + if dep in data2[3]: + data2[3].update(setscenedeps[dep][3]) + data2[3].remove(dep) + if dep in start: + start.update(setscenedeps[dep][3]) + start.remove(dep) + del setscenedeps[dep] + + # Remove circular references + for dep in setscenedeps: + if dep in setscenedeps[dep][3]: + setscenedeps[dep][3].remove(dep) + + # Direct dependencies should be present and can be depended upon + for dep in set(start): + if setscenedeps[dep][1] == "do_packagedata": + if dep not in configuredeps: + configuredeps.append(dep) + + msgbuf = [] + # Call into setscene_depvalid for each sub-dependency and only copy sysroot files + # for ones that would be restored from sstate. + done = list(start) + next = list(start) + while next: + new = [] + for dep in next: + data = setscenedeps[dep] + for datadep in data[3]: + if datadep in done: + continue + taskdeps = {} + taskdeps[dep] = setscenedeps[dep][:2] + taskdeps[datadep] = setscenedeps[datadep][:2] + retval = setscene_depvalid(datadep, taskdeps, [], d, msgbuf) + done.append(datadep) + new.append(datadep) + if retval: + msgbuf.append("Skipping setscene dependency %s" % datadep) + continue + if datadep not in configuredeps and setscenedeps[datadep][1] == "do_packagedata": + configuredeps.append(datadep) + msgbuf.append("Adding dependency on %s" % setscenedeps[datadep][0]) + else: + msgbuf.append("Following dependency on %s" % setscenedeps[datadep][0]) + next = new + + # This logging is too verbose for day to day use sadly + #bb.debug(2, "\n".join(msgbuf)) + + seendirs = set() + postinsts = [] + multilibs = {} + manifests = {} + + msg_adding = [] + + for dep in configuredeps: + c = setscenedeps[dep][0] + msg_adding.append(c) + + manifest, d2 = oe.sstatesig.find_sstate_manifest(c, setscenedeps[dep][2], "packagedata", d, multilibs) + destsysroot = pkgdatadir + + if manifest: + targetdir = destsysroot + with open(manifest, "r") as f: + manifests[dep] = manifest + for l in f: + l = l.strip() + dest = targetdir + l.replace(stagingdir, "") + if l.endswith("/"): + staging_copydir(l, targetdir, dest, seendirs) + continue + staging_copyfile(l, targetdir, dest, postinsts, seendirs) + + bb.note("Installed into pkgdata-sysroot: %s" % str(msg_adding)) + +} +package_prepare_pkgdata[cleandirs] = "${WORKDIR_PKGDATA}" +package_prepare_pkgdata[vardepsexclude] += "MACHINE_ARCH PACKAGE_EXTRA_ARCHS SDK_ARCH BUILD_ARCH SDK_OS BB_TASKDEPDATA SSTATETASKS" + + diff --git a/meta/classes-global/package_rpm.bbclass b/meta/classes-global/package_rpm.bbclass new file mode 100644 index 0000000000..790ccbfaf0 --- /dev/null +++ b/meta/classes-global/package_rpm.bbclass @@ -0,0 +1,771 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +inherit package + +IMAGE_PKGTYPE ?= "rpm" + +RPM = "rpm" +RPMBUILD = "rpmbuild" +RPMBUILD_COMPMODE ?= "${@'w19T%d.zstdio' % int(d.getVar('ZSTD_THREADS'))}" + +PKGWRITEDIRRPM = "${WORKDIR}/deploy-rpms" + +# Maintaining the perfile dependencies has significant overhead when writing the +# packages. When set, this value merges them for efficiency. +MERGEPERFILEDEPS = "1" + +# Filter dependencies based on a provided function. +def filter_deps(var, f): + import collections + + depends_dict = bb.utils.explode_dep_versions2(var) + newdeps_dict = collections.OrderedDict() + for dep in depends_dict: + if f(dep): + newdeps_dict[dep] = depends_dict[dep] + return bb.utils.join_deps(newdeps_dict, commasep=False) + +# Filter out absolute paths (typically /bin/sh and /usr/bin/env) and any perl +# dependencies for nativesdk packages. +def filter_nativesdk_deps(srcname, var): + if var and srcname.startswith("nativesdk-"): + var = filter_deps(var, lambda dep: not dep.startswith('/') and dep != 'perl' and not dep.startswith('perl(')) + return var + +# Construct per file dependencies file +def write_rpm_perfiledata(srcname, d): + workdir = d.getVar('WORKDIR') + packages = d.getVar('PACKAGES') + pkgd = d.getVar('PKGD') + + def dump_filerdeps(varname, outfile, d): + outfile.write("#!/usr/bin/env python3\n\n") + outfile.write("# Dependency table\n") + outfile.write('deps = {\n') + for pkg in packages.split(): + dependsflist_key = 'FILE' + varname + 'FLIST' + ":" + pkg + dependsflist = (d.getVar(dependsflist_key) or "") + for dfile in dependsflist.split(): + key = "FILE" + varname + ":" + dfile + ":" + pkg + deps = filter_nativesdk_deps(srcname, d.getVar(key) or "") + depends_dict = bb.utils.explode_dep_versions(deps) + file = dfile.replace("@underscore@", "_") + file = file.replace("@closebrace@", "]") + file = file.replace("@openbrace@", "[") + file = file.replace("@tab@", "\t") + file = file.replace("@space@", " ") + file = file.replace("@at@", "@") + outfile.write('"' + pkgd + file + '" : "') + for dep in depends_dict: + ver = depends_dict[dep] + if dep and ver: + ver = ver.replace("(", "") + ver = ver.replace(")", "") + outfile.write(dep + " " + ver + " ") + else: + outfile.write(dep + " ") + outfile.write('",\n') + outfile.write('}\n\n') + outfile.write("import sys\n") + outfile.write("while 1:\n") + outfile.write("\tline = sys.stdin.readline().strip()\n") + outfile.write("\tif not line:\n") + outfile.write("\t\tsys.exit(0)\n") + outfile.write("\tif line in deps:\n") + outfile.write("\t\tprint(deps[line] + '\\n')\n") + + # OE-core dependencies a.k.a. RPM requires + outdepends = workdir + "/" + srcname + ".requires" + + dependsfile = open(outdepends, 'w') + + dump_filerdeps('RDEPENDS', dependsfile, d) + + dependsfile.close() + os.chmod(outdepends, 0o755) + + # OE-core / RPM Provides + outprovides = workdir + "/" + srcname + ".provides" + + providesfile = open(outprovides, 'w') + + dump_filerdeps('RPROVIDES', providesfile, d) + + providesfile.close() + os.chmod(outprovides, 0o755) + + return (outdepends, outprovides) + + +python write_specfile () { + import oe.packagedata + import os,pwd,grp,stat + + # append information for logs and patches to %prep + def add_prep(d, spec_files_bottom): + if d.getVarFlag('ARCHIVER_MODE', 'srpm') == '1' and bb.data.inherits_class('archiver', d): + spec_files_bottom.append('%%prep -n %s' % d.getVar('PN')) + spec_files_bottom.append('%s' % "echo \"include logs and patches, Please check them in SOURCES\"") + spec_files_bottom.append('') + + # append the name of tarball to key word 'SOURCE' in xxx.spec. + def tail_source(d): + if d.getVarFlag('ARCHIVER_MODE', 'srpm') == '1' and bb.data.inherits_class('archiver', d): + ar_outdir = d.getVar('ARCHIVER_OUTDIR') + if not os.path.exists(ar_outdir): + return + source_list = os.listdir(ar_outdir) + source_number = 0 + for source in source_list: + # do_deploy_archives may have already run (from sstate) meaning a .src.rpm may already + # exist in ARCHIVER_OUTDIR so skip if present. + if source.endswith(".src.rpm"): + continue + # The rpmbuild doesn't need the root permission, but it needs + # to know the file's user and group name, the only user and + # group in fakeroot is "root" when working in fakeroot. + f = os.path.join(ar_outdir, source) + os.chown(f, 0, 0) + spec_preamble_top.append('Source%s: %s' % (source_number, source)) + source_number += 1 + + # In RPM, dependencies are of the format: pkg <>= Epoch:Version-Release + # This format is similar to OE, however there are restrictions on the + # characters that can be in a field. In the Version field, "-" + # characters are not allowed. "-" is allowed in the Release field. + # + # We translate the "-" in the version to a "+", by loading the PKGV + # from the dependent recipe, replacing the - with a +, and then using + # that value to do a replace inside of this recipe's dependencies. + # This preserves the "-" separator between the version and release, as + # well as any "-" characters inside of the release field. + # + # All of this has to happen BEFORE the mapping_rename_hook as + # after renaming we cannot look up the dependencies in the packagedata + # store. + def translate_vers(varname, d): + depends = d.getVar(varname) + if depends: + depends_dict = bb.utils.explode_dep_versions2(depends) + newdeps_dict = {} + for dep in depends_dict: + verlist = [] + for ver in depends_dict[dep]: + if '-' in ver: + subd = oe.packagedata.read_subpkgdata_dict(dep, d) + if 'PKGV' in subd: + pv = subd['PV'] + pkgv = subd['PKGV'] + reppv = pkgv.replace('-', '+') + if ver.startswith(pv): + ver = ver.replace(pv, reppv) + ver = ver.replace(pkgv, reppv) + if 'PKGR' in subd: + # Make sure PKGR rather than PR in ver + pr = '-' + subd['PR'] + pkgr = '-' + subd['PKGR'] + if pkgr not in ver: + ver = ver.replace(pr, pkgr) + verlist.append(ver) + else: + verlist.append(ver) + newdeps_dict[dep] = verlist + depends = bb.utils.join_deps(newdeps_dict) + d.setVar(varname, depends.strip()) + + # We need to change the style the dependency from BB to RPM + # This needs to happen AFTER the mapping_rename_hook + def print_deps(variable, tag, array, d): + depends = variable + if depends: + depends_dict = bb.utils.explode_dep_versions2(depends) + for dep in depends_dict: + for ver in depends_dict[dep]: + ver = ver.replace('(', '') + ver = ver.replace(')', '') + array.append("%s: %s %s" % (tag, dep, ver)) + if not len(depends_dict[dep]): + array.append("%s: %s" % (tag, dep)) + + def walk_files(walkpath, target, conffiles): + # We can race against the ipk/deb backends which create CONTROL or DEBIAN directories + # when packaging. We just ignore these files which are created in + # packages-split/ and not package/ + # We have the odd situation where the CONTROL/DEBIAN directory can be removed in the middle of + # of the walk, the isdir() test would then fail and the walk code would assume its a file + # hence we check for the names in files too. + for rootpath, dirs, files in os.walk(walkpath): + def get_attr(path): + stat_f = os.stat(rootpath + "/" + path, follow_symlinks=False) + mode = stat.S_IMODE(stat_f.st_mode) + try: + owner = pwd.getpwuid(stat_f.st_uid).pw_name + except Exception as e: + bb.error("Content of /etc/passwd in sysroot:\n{}".format( + open(d.getVar("RECIPE_SYSROOT") +"/etc/passwd").read())) + raise e + try: + group = grp.getgrgid(stat_f.st_gid).gr_name + except Exception as e: + bb.error("Content of /etc/group in sysroot:\n{}".format( + open(d.getVar("RECIPE_SYSROOT") +"/etc/group").read())) + raise e + return "%attr({:o},{},{}) ".format(mode, owner, group) + + def escape_chars(p): + return p.replace("%", "%%").replace("\\", "\\\\").replace('"', '\\"') + + path = rootpath.replace(walkpath, "") + if path.endswith("DEBIAN") or path.endswith("CONTROL"): + continue + + # Treat all symlinks to directories as normal files. + # os.walk() lists them as directories. + def move_to_files(dir): + if os.path.islink(os.path.join(rootpath, dir)): + files.append(dir) + return True + else: + return False + dirs[:] = [dir for dir in dirs if not move_to_files(dir)] + + for dir in dirs: + if dir == "CONTROL" or dir == "DEBIAN": + continue + p = path + '/' + dir + # All packages own the directories their files are in... + target.append(get_attr(dir) + '%dir "' + escape_chars(p) + '"') + + for file in files: + if file == "CONTROL" or file == "DEBIAN": + continue + attr = get_attr(file) + p = path + '/' + file + if conffiles.count(p): + target.append(attr + '%config "' + escape_chars(p) + '"') + else: + target.append(attr + '"' + escape_chars(p) + '"') + + # Prevent the prerm/postrm scripts from being run during an upgrade + def wrap_uninstall(scriptvar): + scr = scriptvar.strip() + if scr.startswith("#!"): + pos = scr.find("\n") + 1 + else: + pos = 0 + scr = scr[:pos] + 'if [ "$1" = "0" ] ; then\n' + scr[pos:] + '\nfi' + return scr + + def get_perfile(varname, pkg, d): + deps = [] + dependsflist_key = 'FILE' + varname + 'FLIST' + ":" + pkg + dependsflist = (d.getVar(dependsflist_key) or "") + for dfile in dependsflist.split(): + key = "FILE" + varname + ":" + dfile + ":" + pkg + depends = d.getVar(key) + if depends: + deps.append(depends) + return " ".join(deps) + + def append_description(spec_preamble, text): + """ + Add the description to the spec file. + """ + import textwrap + dedent_text = textwrap.dedent(text).strip() + # Bitbake saves "\n" as "\\n" + if '\\n' in dedent_text: + for t in dedent_text.split('\\n'): + spec_preamble.append(t.strip()) + else: + spec_preamble.append('%s' % textwrap.fill(dedent_text, width=75)) + + packages = d.getVar('PACKAGES') + if not packages or packages == '': + bb.debug(1, "No packages; nothing to do") + return + + pkgdest = d.getVar('PKGDEST') + if not pkgdest: + bb.fatal("No PKGDEST") + + outspecfile = d.getVar('OUTSPECFILE') + if not outspecfile: + bb.fatal("No OUTSPECFILE") + + # Construct the SPEC file... + srcname = d.getVar('PN') + localdata = bb.data.createCopy(d) + localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + srcname) + srcsummary = (localdata.getVar('SUMMARY') or localdata.getVar('DESCRIPTION') or ".") + srcversion = localdata.getVar('PKGV').replace('-', '+') + srcrelease = localdata.getVar('PKGR') + srcepoch = (localdata.getVar('PKGE') or "") + srclicense = localdata.getVar('LICENSE') + srcsection = localdata.getVar('SECTION') + srcmaintainer = localdata.getVar('MAINTAINER') + srchomepage = localdata.getVar('HOMEPAGE') + srcdescription = localdata.getVar('DESCRIPTION') or "." + srccustomtagschunk = oe.packagedata.get_package_additional_metadata("rpm", localdata) + + srcdepends = d.getVar('DEPENDS') + srcrdepends = "" + srcrrecommends = "" + srcrsuggests = "" + srcrprovides = "" + srcrreplaces = "" + srcrconflicts = "" + srcrobsoletes = "" + + srcrpreinst = [] + srcrpostinst = [] + srcrprerm = [] + srcrpostrm = [] + + spec_preamble_top = [] + spec_preamble_bottom = [] + + spec_scriptlets_top = [] + spec_scriptlets_bottom = [] + + spec_files_top = [] + spec_files_bottom = [] + + perfiledeps = (d.getVar("MERGEPERFILEDEPS") or "0") == "0" + extra_pkgdata = (d.getVar("RPM_EXTRA_PKGDATA") or "0") == "1" + + for pkg in packages.split(): + localdata = bb.data.createCopy(d) + + root = "%s/%s" % (pkgdest, pkg) + + localdata.setVar('ROOT', '') + localdata.setVar('ROOT_%s' % pkg, root) + pkgname = localdata.getVar('PKG:%s' % pkg) + if not pkgname: + pkgname = pkg + localdata.setVar('PKG', pkgname) + + localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + pkg) + + conffiles = oe.package.get_conffiles(pkg, d) + + splitname = pkgname + + splitsummary = (localdata.getVar('SUMMARY') or localdata.getVar('DESCRIPTION') or ".") + splitversion = (localdata.getVar('PKGV') or "").replace('-', '+') + splitrelease = (localdata.getVar('PKGR') or "") + splitepoch = (localdata.getVar('PKGE') or "") + splitlicense = (localdata.getVar('LICENSE') or "") + splitsection = (localdata.getVar('SECTION') or "") + splitdescription = (localdata.getVar('DESCRIPTION') or ".") + splitcustomtagschunk = oe.packagedata.get_package_additional_metadata("rpm", localdata) + + translate_vers('RDEPENDS', localdata) + translate_vers('RRECOMMENDS', localdata) + translate_vers('RSUGGESTS', localdata) + translate_vers('RPROVIDES', localdata) + translate_vers('RREPLACES', localdata) + translate_vers('RCONFLICTS', localdata) + + # Map the dependencies into their final form + oe.packagedata.mapping_rename_hook(localdata) + + splitrdepends = localdata.getVar('RDEPENDS') or "" + splitrrecommends = localdata.getVar('RRECOMMENDS') or "" + splitrsuggests = localdata.getVar('RSUGGESTS') or "" + splitrprovides = localdata.getVar('RPROVIDES') or "" + splitrreplaces = localdata.getVar('RREPLACES') or "" + splitrconflicts = localdata.getVar('RCONFLICTS') or "" + splitrobsoletes = "" + + splitrpreinst = localdata.getVar('pkg_preinst') + splitrpostinst = localdata.getVar('pkg_postinst') + splitrprerm = localdata.getVar('pkg_prerm') + splitrpostrm = localdata.getVar('pkg_postrm') + + + if not perfiledeps: + # Add in summary of per file dependencies + splitrdepends = splitrdepends + " " + get_perfile('RDEPENDS', pkg, d) + splitrprovides = splitrprovides + " " + get_perfile('RPROVIDES', pkg, d) + + splitrdepends = filter_nativesdk_deps(srcname, splitrdepends) + + # Gather special src/first package data + if srcname == splitname: + archiving = d.getVarFlag('ARCHIVER_MODE', 'srpm') == '1' and \ + bb.data.inherits_class('archiver', d) + if archiving and srclicense != splitlicense: + bb.warn("The SRPM produced may not have the correct overall source license in the License tag. This is due to the LICENSE for the primary package and SRPM conflicting.") + + srclicense = splitlicense + srcrdepends = splitrdepends + srcrrecommends = splitrrecommends + srcrsuggests = splitrsuggests + srcrprovides = splitrprovides + srcrreplaces = splitrreplaces + srcrconflicts = splitrconflicts + + srcrpreinst = splitrpreinst + srcrpostinst = splitrpostinst + srcrprerm = splitrprerm + srcrpostrm = splitrpostrm + + file_list = [] + walk_files(root, file_list, conffiles) + if not file_list and localdata.getVar('ALLOW_EMPTY', False) != "1": + bb.note("Not creating empty RPM package for %s" % splitname) + else: + spec_files_top.append('%files') + if extra_pkgdata: + package_rpm_extra_pkgdata(splitname, spec_files_top, localdata) + spec_files_top.append('%defattr(-,-,-,-)') + if file_list: + bb.note("Creating RPM package for %s" % splitname) + spec_files_top.extend(file_list) + else: + bb.note("Creating empty RPM package for %s" % splitname) + spec_files_top.append('') + continue + + # Process subpackage data + spec_preamble_bottom.append('%%package -n %s' % splitname) + spec_preamble_bottom.append('Summary: %s' % splitsummary) + if srcversion != splitversion: + spec_preamble_bottom.append('Version: %s' % splitversion) + if srcrelease != splitrelease: + spec_preamble_bottom.append('Release: %s' % splitrelease) + if srcepoch != splitepoch: + spec_preamble_bottom.append('Epoch: %s' % splitepoch) + spec_preamble_bottom.append('License: %s' % splitlicense) + spec_preamble_bottom.append('Group: %s' % splitsection) + + if srccustomtagschunk != splitcustomtagschunk: + spec_preamble_bottom.append(splitcustomtagschunk) + + # Replaces == Obsoletes && Provides + robsoletes = bb.utils.explode_dep_versions2(splitrobsoletes) + rprovides = bb.utils.explode_dep_versions2(splitrprovides) + rreplaces = bb.utils.explode_dep_versions2(splitrreplaces) + for dep in rreplaces: + if dep not in robsoletes: + robsoletes[dep] = rreplaces[dep] + if dep not in rprovides: + rprovides[dep] = rreplaces[dep] + splitrobsoletes = bb.utils.join_deps(robsoletes, commasep=False) + splitrprovides = bb.utils.join_deps(rprovides, commasep=False) + + print_deps(splitrdepends, "Requires", spec_preamble_bottom, d) + if splitrpreinst: + print_deps(splitrdepends, "Requires(pre)", spec_preamble_bottom, d) + if splitrpostinst: + print_deps(splitrdepends, "Requires(post)", spec_preamble_bottom, d) + if splitrprerm: + print_deps(splitrdepends, "Requires(preun)", spec_preamble_bottom, d) + if splitrpostrm: + print_deps(splitrdepends, "Requires(postun)", spec_preamble_bottom, d) + + print_deps(splitrrecommends, "Recommends", spec_preamble_bottom, d) + print_deps(splitrsuggests, "Suggests", spec_preamble_bottom, d) + print_deps(splitrprovides, "Provides", spec_preamble_bottom, d) + print_deps(splitrobsoletes, "Obsoletes", spec_preamble_bottom, d) + print_deps(splitrconflicts, "Conflicts", spec_preamble_bottom, d) + + spec_preamble_bottom.append('') + + spec_preamble_bottom.append('%%description -n %s' % splitname) + append_description(spec_preamble_bottom, splitdescription) + + spec_preamble_bottom.append('') + + # Now process scriptlets + if splitrpreinst: + spec_scriptlets_bottom.append('%%pre -n %s' % splitname) + spec_scriptlets_bottom.append('# %s - preinst' % splitname) + spec_scriptlets_bottom.append(splitrpreinst) + spec_scriptlets_bottom.append('') + if splitrpostinst: + spec_scriptlets_bottom.append('%%post -n %s' % splitname) + spec_scriptlets_bottom.append('# %s - postinst' % splitname) + spec_scriptlets_bottom.append(splitrpostinst) + spec_scriptlets_bottom.append('') + if splitrprerm: + spec_scriptlets_bottom.append('%%preun -n %s' % splitname) + spec_scriptlets_bottom.append('# %s - prerm' % splitname) + scriptvar = wrap_uninstall(splitrprerm) + spec_scriptlets_bottom.append(scriptvar) + spec_scriptlets_bottom.append('') + if splitrpostrm: + spec_scriptlets_bottom.append('%%postun -n %s' % splitname) + spec_scriptlets_bottom.append('# %s - postrm' % splitname) + scriptvar = wrap_uninstall(splitrpostrm) + spec_scriptlets_bottom.append(scriptvar) + spec_scriptlets_bottom.append('') + + # Now process files + file_list = [] + walk_files(root, file_list, conffiles) + if not file_list and localdata.getVar('ALLOW_EMPTY', False) != "1": + bb.note("Not creating empty RPM package for %s" % splitname) + else: + spec_files_bottom.append('%%files -n %s' % splitname) + if extra_pkgdata: + package_rpm_extra_pkgdata(splitname, spec_files_bottom, localdata) + spec_files_bottom.append('%defattr(-,-,-,-)') + if file_list: + bb.note("Creating RPM package for %s" % splitname) + spec_files_bottom.extend(file_list) + else: + bb.note("Creating empty RPM package for %s" % splitname) + spec_files_bottom.append('') + + del localdata + + add_prep(d, spec_files_bottom) + spec_preamble_top.append('Summary: %s' % srcsummary) + spec_preamble_top.append('Name: %s' % srcname) + spec_preamble_top.append('Version: %s' % srcversion) + spec_preamble_top.append('Release: %s' % srcrelease) + if srcepoch and srcepoch.strip() != "": + spec_preamble_top.append('Epoch: %s' % srcepoch) + spec_preamble_top.append('License: %s' % srclicense) + spec_preamble_top.append('Group: %s' % srcsection) + spec_preamble_top.append('Packager: %s' % srcmaintainer) + if srchomepage: + spec_preamble_top.append('URL: %s' % srchomepage) + if srccustomtagschunk: + spec_preamble_top.append(srccustomtagschunk) + tail_source(d) + + # Replaces == Obsoletes && Provides + robsoletes = bb.utils.explode_dep_versions2(srcrobsoletes) + rprovides = bb.utils.explode_dep_versions2(srcrprovides) + rreplaces = bb.utils.explode_dep_versions2(srcrreplaces) + for dep in rreplaces: + if dep not in robsoletes: + robsoletes[dep] = rreplaces[dep] + if dep not in rprovides: + rprovides[dep] = rreplaces[dep] + srcrobsoletes = bb.utils.join_deps(robsoletes, commasep=False) + srcrprovides = bb.utils.join_deps(rprovides, commasep=False) + + print_deps(srcdepends, "BuildRequires", spec_preamble_top, d) + print_deps(srcrdepends, "Requires", spec_preamble_top, d) + if srcrpreinst: + print_deps(srcrdepends, "Requires(pre)", spec_preamble_top, d) + if srcrpostinst: + print_deps(srcrdepends, "Requires(post)", spec_preamble_top, d) + if srcrprerm: + print_deps(srcrdepends, "Requires(preun)", spec_preamble_top, d) + if srcrpostrm: + print_deps(srcrdepends, "Requires(postun)", spec_preamble_top, d) + + print_deps(srcrrecommends, "Recommends", spec_preamble_top, d) + print_deps(srcrsuggests, "Suggests", spec_preamble_top, d) + print_deps(srcrprovides, "Provides", spec_preamble_top, d) + print_deps(srcrobsoletes, "Obsoletes", spec_preamble_top, d) + print_deps(srcrconflicts, "Conflicts", spec_preamble_top, d) + + spec_preamble_top.append('') + + spec_preamble_top.append('%description') + append_description(spec_preamble_top, srcdescription) + + spec_preamble_top.append('') + + if srcrpreinst: + spec_scriptlets_top.append('%pre') + spec_scriptlets_top.append('# %s - preinst' % srcname) + spec_scriptlets_top.append(srcrpreinst) + spec_scriptlets_top.append('') + if srcrpostinst: + spec_scriptlets_top.append('%post') + spec_scriptlets_top.append('# %s - postinst' % srcname) + spec_scriptlets_top.append(srcrpostinst) + spec_scriptlets_top.append('') + if srcrprerm: + spec_scriptlets_top.append('%preun') + spec_scriptlets_top.append('# %s - prerm' % srcname) + scriptvar = wrap_uninstall(srcrprerm) + spec_scriptlets_top.append(scriptvar) + spec_scriptlets_top.append('') + if srcrpostrm: + spec_scriptlets_top.append('%postun') + spec_scriptlets_top.append('# %s - postrm' % srcname) + scriptvar = wrap_uninstall(srcrpostrm) + spec_scriptlets_top.append(scriptvar) + spec_scriptlets_top.append('') + + # Write the SPEC file + specfile = open(outspecfile, 'w') + + # RPMSPEC_PREAMBLE is a way to add arbitrary text to the top + # of the generated spec file + external_preamble = d.getVar("RPMSPEC_PREAMBLE") + if external_preamble: + specfile.write(external_preamble + "\n") + + for line in spec_preamble_top: + specfile.write(line + "\n") + + for line in spec_preamble_bottom: + specfile.write(line + "\n") + + for line in spec_scriptlets_top: + specfile.write(line + "\n") + + for line in spec_scriptlets_bottom: + specfile.write(line + "\n") + + for line in spec_files_top: + specfile.write(line + "\n") + + for line in spec_files_bottom: + specfile.write(line + "\n") + + specfile.close() +} +# Otherwise allarch packages may change depending on override configuration +write_specfile[vardepsexclude] = "OVERRIDES" + +# Have to list any variables referenced as X_<pkg> that aren't in pkgdata here +RPMEXTRAVARS = "PACKAGE_ADD_METADATA_RPM" +write_specfile[vardeps] += "${@gen_packagevar(d, 'RPMEXTRAVARS')}" + +python do_package_rpm () { + workdir = d.getVar('WORKDIR') + tmpdir = d.getVar('TMPDIR') + pkgd = d.getVar('PKGD') + if not workdir or not pkgd or not tmpdir: + bb.error("Variables incorrectly set, unable to package") + return + + packages = d.getVar('PACKAGES') + if not packages or packages == '': + bb.debug(1, "No packages; nothing to do") + return + + # Construct the spec file... + # If the spec file already exist, and has not been stored into + # pseudo's files.db, it maybe cause rpmbuild src.rpm fail, + # so remove it before doing rpmbuild src.rpm. + srcname = d.getVar('PN') + outspecfile = workdir + "/" + srcname + ".spec" + if os.path.isfile(outspecfile): + os.remove(outspecfile) + d.setVar('OUTSPECFILE', outspecfile) + bb.build.exec_func('write_specfile', d) + + perfiledeps = (d.getVar("MERGEPERFILEDEPS") or "0") == "0" + if perfiledeps: + outdepends, outprovides = write_rpm_perfiledata(srcname, d) + + # Setup the rpmbuild arguments... + rpmbuild = d.getVar('RPMBUILD') + rpmbuild_compmode = d.getVar('RPMBUILD_COMPMODE') + rpmbuild_extra_params = d.getVar('RPMBUILD_EXTRA_PARAMS') or "" + + # Too many places in dnf stack assume that arch-independent packages are "noarch". + # Let's not fight against this. + package_arch = (d.getVar('PACKAGE_ARCH') or "").replace("-", "_") + if package_arch == "all": + package_arch = "noarch" + + d.setVar('PACKAGE_ARCH_EXTEND', package_arch) + pkgwritedir = d.expand('${PKGWRITEDIRRPM}/${PACKAGE_ARCH_EXTEND}') + d.setVar('RPM_PKGWRITEDIR', pkgwritedir) + bb.debug(1, 'PKGWRITEDIR: %s' % d.getVar('RPM_PKGWRITEDIR')) + pkgarch = d.expand('${PACKAGE_ARCH_EXTEND}${HOST_VENDOR}-linux') + bb.utils.mkdirhier(pkgwritedir) + os.chmod(pkgwritedir, 0o755) + + cmd = rpmbuild + cmd = cmd + " --noclean --nodeps --short-circuit --target " + pkgarch + " --buildroot " + pkgd + cmd = cmd + " --define '_topdir " + workdir + "' --define '_rpmdir " + pkgwritedir + "'" + cmd = cmd + " --define '_builddir " + d.getVar('B') + "'" + cmd = cmd + " --define '_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm'" + cmd = cmd + " --define '_use_internal_dependency_generator 0'" + cmd = cmd + " --define '_binaries_in_noarch_packages_terminate_build 0'" + cmd = cmd + " --define '_build_id_links none'" + cmd = cmd + " --define '_source_payload %s'" % rpmbuild_compmode + cmd = cmd + " --define '_binary_payload %s'" % rpmbuild_compmode + cmd = cmd + " --define 'clamp_mtime_to_source_date_epoch 1'" + cmd = cmd + " --define 'use_source_date_epoch_as_buildtime 1'" + cmd = cmd + " --define '_buildhost reproducible'" + cmd = cmd + " --define '__font_provides %{nil}'" + if perfiledeps: + cmd = cmd + " --define '__find_requires " + outdepends + "'" + cmd = cmd + " --define '__find_provides " + outprovides + "'" + else: + cmd = cmd + " --define '__find_requires %{nil}'" + cmd = cmd + " --define '__find_provides %{nil}'" + cmd = cmd + " --define '_unpackaged_files_terminate_build 0'" + cmd = cmd + " --define 'debug_package %{nil}'" + cmd = cmd + " --define '_tmppath " + workdir + "'" + cmd = cmd + " --define '_use_weak_usergroup_deps 1'" + cmd = cmd + " --define '_passwd_path " + "/completely/bogus/path" + "'" + cmd = cmd + " --define '_group_path " + "/completely/bogus/path" + "'" + cmd = cmd + rpmbuild_extra_params + if d.getVarFlag('ARCHIVER_MODE', 'srpm') == '1' and bb.data.inherits_class('archiver', d): + cmd = cmd + " --define '_sourcedir " + d.getVar('ARCHIVER_OUTDIR') + "'" + cmdsrpm = cmd + " --define '_srcrpmdir " + d.getVar('ARCHIVER_RPMOUTDIR') + "'" + cmdsrpm = cmdsrpm + " -bs " + outspecfile + # Build the .src.rpm + d.setVar('SBUILDSPEC', cmdsrpm + "\n") + d.setVarFlag('SBUILDSPEC', 'func', '1') + bb.build.exec_func('SBUILDSPEC', d) + cmd = cmd + " -bb " + outspecfile + + # rpm 4 creates various empty directories in _topdir, let's clean them up + cleanupcmd = "rm -rf %s/BUILDROOT %s/SOURCES %s/SPECS %s/SRPMS" % (workdir, workdir, workdir, workdir) + + # Build the rpm package! + d.setVar('BUILDSPEC', cmd + "\n" + cleanupcmd + "\n") + d.setVarFlag('BUILDSPEC', 'func', '1') + bb.build.exec_func('BUILDSPEC', d) + + if d.getVar('RPM_SIGN_PACKAGES') == '1': + bb.build.exec_func("sign_rpm", d) +} + +python () { + if d.getVar('PACKAGES') != '': + deps = ' rpm-native:do_populate_sysroot virtual/fakeroot-native:do_populate_sysroot' + d.appendVarFlag('do_package_write_rpm', 'depends', deps) + d.setVarFlag('do_package_write_rpm', 'fakeroot', '1') + + # Needed to ensure PKG_xxx renaming of dependency packages works + d.setVarFlag('do_package_write_rpm', 'deptask', "do_packagedata") + d.setVarFlag('do_package_write_rpm', 'rdeptask', "do_packagedata") +} + +SSTATETASKS += "do_package_write_rpm" +do_package_write_rpm[sstate-inputdirs] = "${PKGWRITEDIRRPM}" +do_package_write_rpm[sstate-outputdirs] = "${DEPLOY_DIR_RPM}" +# Take a shared lock, we can write multiple packages at the same time... +# but we need to stop the rootfs/solver from running while we do... +do_package_write_rpm[sstate-lockfile-shared] += "${DEPLOY_DIR_RPM}/rpm.lock" + +python do_package_write_rpm_setscene () { + sstate_setscene(d) +} +addtask do_package_write_rpm_setscene + +python do_package_write_rpm () { + bb.build.exec_func("read_subpackage_metadata", d) + bb.build.exec_func("do_package_rpm", d) +} + +do_package_write_rpm[dirs] = "${PKGWRITEDIRRPM}" +do_package_write_rpm[cleandirs] = "${PKGWRITEDIRRPM}" +do_package_write_rpm[depends] += "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'), 'do_populate_sysroot')}" +addtask package_write_rpm after do_packagedata do_package do_deploy_source_date_epoch before do_build +do_build[rdeptask] += "do_package_write_rpm" + +PACKAGEINDEXDEPS += "rpm-native:do_populate_sysroot" +PACKAGEINDEXDEPS += "createrepo-c-native:do_populate_sysroot" diff --git a/meta/classes-global/packagedata.bbclass b/meta/classes-global/packagedata.bbclass new file mode 100644 index 0000000000..9f72c01d77 --- /dev/null +++ b/meta/classes-global/packagedata.bbclass @@ -0,0 +1,40 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +python read_subpackage_metadata () { + import oe.packagedata + + vars = { + "PN" : d.getVar('PN'), + "PE" : d.getVar('PE'), + "PV" : d.getVar('PV'), + "PR" : d.getVar('PR'), + } + + data = oe.packagedata.read_pkgdata(vars["PN"], d) + + for key in data.keys(): + d.setVar(key, data[key]) + + for pkg in d.getVar('PACKAGES').split(): + sdata = oe.packagedata.read_subpkgdata(pkg, d) + for key in sdata.keys(): + if key in vars: + if sdata[key] != vars[key]: + if key == "PN": + bb.fatal("Recipe %s is trying to create package %s which was already written by recipe %s. This will cause corruption, please resolve this and only provide the package from one recipe or the other or only build one of the recipes." % (vars[key], pkg, sdata[key])) + bb.fatal("Recipe %s is trying to change %s from '%s' to '%s'. This will cause do_package_write_* failures since the incorrect data will be used and they will be unable to find the right workdir." % (vars["PN"], key, vars[key], sdata[key])) + continue + # + # If we set unsuffixed variables here there is a chance they could clobber override versions + # of that variable, e.g. DESCRIPTION could clobber DESCRIPTION:<pkgname> + # We therefore don't clobber for the unsuffixed variable versions + # + if key.endswith(":" + pkg): + d.setVar(key, sdata[key]) + else: + d.setVar(key, sdata[key], parsing=True) +} diff --git a/meta/classes-global/patch.bbclass b/meta/classes-global/patch.bbclass new file mode 100644 index 0000000000..e5786b1c9a --- /dev/null +++ b/meta/classes-global/patch.bbclass @@ -0,0 +1,169 @@ +# Copyright (C) 2006 OpenedHand LTD +# +# SPDX-License-Identifier: MIT + +# Point to an empty file so any user's custom settings don't break things +QUILTRCFILE ?= "${STAGING_ETCDIR_NATIVE}/quiltrc" + +PATCHDEPENDENCY = "${PATCHTOOL}-native:do_populate_sysroot" + +# There is a bug in patch 2.7.3 and earlier where index lines +# in patches can change file modes when they shouldn't: +# http://git.savannah.gnu.org/cgit/patch.git/patch/?id=82b800c9552a088a241457948219d25ce0a407a4 +# This leaks into debug sources in particular. Add the dependency +# to target recipes to avoid this problem until we can rely on 2.7.4 or later. +PATCHDEPENDENCY:append:class-target = " patch-replacement-native:do_populate_sysroot" + +PATCH_GIT_USER_NAME ?= "OpenEmbedded" +PATCH_GIT_USER_EMAIL ?= "oe.patch@oe" + +inherit terminal + +python () { + if d.getVar('PATCHTOOL') == 'git' and d.getVar('PATCH_COMMIT_FUNCTIONS') == '1': + extratasks = bb.build.tasksbetween('do_unpack', 'do_patch', d) + try: + extratasks.remove('do_unpack') + except ValueError: + # For some recipes do_unpack doesn't exist, ignore it + pass + + d.appendVarFlag('do_patch', 'prefuncs', ' patch_task_patch_prefunc') + for task in extratasks: + d.appendVarFlag(task, 'postfuncs', ' patch_task_postfunc') +} + +python patch_task_patch_prefunc() { + # Prefunc for do_patch + srcsubdir = d.getVar('S') + + workdir = os.path.abspath(d.getVar('WORKDIR')) + testsrcdir = os.path.abspath(srcsubdir) + if (testsrcdir + os.sep).startswith(workdir + os.sep): + # Double-check that either workdir or S or some directory in-between is a git repository + found = False + while testsrcdir != workdir: + if os.path.exists(os.path.join(testsrcdir, '.git')): + found = True + break + if testsrcdir == workdir: + break + testsrcdir = os.path.dirname(testsrcdir) + if not found: + bb.fatal('PATCHTOOL = "git" set for source tree that is not a git repository. Refusing to continue as that may result in commits being made in your metadata repository.') + + patchdir = os.path.join(srcsubdir, 'patches') + if os.path.exists(patchdir): + if os.listdir(patchdir): + d.setVar('PATCH_HAS_PATCHES_DIR', '1') + else: + os.rmdir(patchdir) +} + +python patch_task_postfunc() { + # Prefunc for task functions between do_unpack and do_patch + import oe.patch + import shutil + func = d.getVar('BB_RUNTASK') + srcsubdir = d.getVar('S') + + if os.path.exists(srcsubdir): + if func == 'do_patch': + haspatches = (d.getVar('PATCH_HAS_PATCHES_DIR') == '1') + patchdir = os.path.join(srcsubdir, 'patches') + if os.path.exists(patchdir): + shutil.rmtree(patchdir) + if haspatches: + stdout, _ = bb.process.run('git status --porcelain patches', cwd=srcsubdir) + if stdout: + bb.process.run('git checkout patches', cwd=srcsubdir) + stdout, _ = bb.process.run('git status --porcelain .', cwd=srcsubdir) + if stdout: + oe.patch.GitApplyTree.commitIgnored("Add changes from %s" % func, dir=srcsubdir, files=['.'], d=d) +} + +def src_patches(d, all=False, expand=True): + import oe.patch + return oe.patch.src_patches(d, all, expand) + +def should_apply(parm, d): + """Determine if we should apply the given patch""" + import oe.patch + return oe.patch.should_apply(parm, d) + +should_apply[vardepsexclude] = "DATE SRCDATE" + +python patch_do_patch() { + import oe.patch + + patchsetmap = { + "patch": oe.patch.PatchTree, + "quilt": oe.patch.QuiltTree, + "git": oe.patch.GitApplyTree, + } + + cls = patchsetmap[d.getVar('PATCHTOOL') or 'quilt'] + + resolvermap = { + "noop": oe.patch.NOOPResolver, + "user": oe.patch.UserResolver, + } + + rcls = resolvermap[d.getVar('PATCHRESOLVE') or 'user'] + + classes = {} + + s = d.getVar('S') + + os.putenv('PATH', d.getVar('PATH')) + + # We must use one TMPDIR per process so that the "patch" processes + # don't generate the same temp file name. + + import tempfile + process_tmpdir = tempfile.mkdtemp() + os.environ['TMPDIR'] = process_tmpdir + + for patch in src_patches(d): + _, _, local, _, _, parm = bb.fetch.decodeurl(patch) + + if "patchdir" in parm: + patchdir = parm["patchdir"] + if not os.path.isabs(patchdir): + patchdir = os.path.join(s, patchdir) + if not os.path.isdir(patchdir): + bb.fatal("Target directory '%s' not found, patchdir '%s' is incorrect in patch file '%s'" % + (patchdir, parm["patchdir"], parm['patchname'])) + else: + patchdir = s + + if not patchdir in classes: + patchset = cls(patchdir, d) + resolver = rcls(patchset, oe_terminal) + classes[patchdir] = (patchset, resolver) + patchset.Clean() + else: + patchset, resolver = classes[patchdir] + + bb.note("Applying patch '%s' (%s)" % (parm['patchname'], oe.path.format_display(local, d))) + try: + patchset.Import({"file":local, "strippath": parm['striplevel']}, True) + except Exception as exc: + bb.utils.remove(process_tmpdir, True) + bb.fatal("Importing patch '%s' with striplevel '%s'\n%s" % (parm['patchname'], parm['striplevel'], repr(exc).replace("\\n", "\n"))) + try: + resolver.Resolve() + except bb.BBHandledException as e: + bb.utils.remove(process_tmpdir, True) + bb.fatal("Applying patch '%s' on target directory '%s'\n%s" % (parm['patchname'], patchdir, repr(e).replace("\\n", "\n"))) + + bb.utils.remove(process_tmpdir, True) + del os.environ['TMPDIR'] +} +patch_do_patch[vardepsexclude] = "PATCHRESOLVE" + +addtask patch after do_unpack +do_patch[dirs] = "${WORKDIR}" +do_patch[depends] = "${PATCHDEPENDENCY}" + +EXPORT_FUNCTIONS do_patch diff --git a/meta/classes-global/sanity.bbclass b/meta/classes-global/sanity.bbclass new file mode 100644 index 0000000000..180c6b77d8 --- /dev/null +++ b/meta/classes-global/sanity.bbclass @@ -0,0 +1,1033 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +# +# Sanity check the users setup for common misconfigurations +# + +SANITY_REQUIRED_UTILITIES ?= "patch diffstat git bzip2 tar \ + gzip gawk chrpath wget cpio perl file which" + +def bblayers_conf_file(d): + return os.path.join(d.getVar('TOPDIR'), 'conf/bblayers.conf') + +def sanity_conf_read(fn): + with open(fn, 'r') as f: + lines = f.readlines() + return lines + +def sanity_conf_find_line(pattern, lines): + import re + return next(((index, line) + for index, line in enumerate(lines) + if re.search(pattern, line)), (None, None)) + +def sanity_conf_update(fn, lines, version_var_name, new_version): + index, line = sanity_conf_find_line(r"^%s" % version_var_name, lines) + lines[index] = '%s = "%d"\n' % (version_var_name, new_version) + with open(fn, "w") as f: + f.write(''.join(lines)) + +# Functions added to this variable MUST throw a NotImplementedError exception unless +# they successfully changed the config version in the config file. Exceptions +# are used since exec_func doesn't handle return values. +BBLAYERS_CONF_UPDATE_FUNCS += " \ + conf/bblayers.conf:LCONF_VERSION:LAYER_CONF_VERSION:oecore_update_bblayers \ + conf/local.conf:CONF_VERSION:LOCALCONF_VERSION:oecore_update_localconf \ + conf/site.conf:SCONF_VERSION:SITE_CONF_VERSION:oecore_update_siteconf \ +" + +SANITY_DIFF_TOOL ?= "diff -u" + +SANITY_LOCALCONF_SAMPLE ?= "${COREBASE}/meta*/conf/templates/default/local.conf.sample" +python oecore_update_localconf() { + # Check we are using a valid local.conf + current_conf = d.getVar('CONF_VERSION') + conf_version = d.getVar('LOCALCONF_VERSION') + + failmsg = """Your version of local.conf was generated from an older/newer version of +local.conf.sample and there have been updates made to this file. Please compare the two +files and merge any changes before continuing. + +Matching the version numbers will remove this message. + +\"${SANITY_DIFF_TOOL} conf/local.conf ${SANITY_LOCALCONF_SAMPLE}\" + +is a good way to visualise the changes.""" + failmsg = d.expand(failmsg) + + raise NotImplementedError(failmsg) +} + +SANITY_SITECONF_SAMPLE ?= "${COREBASE}/meta*/conf/templates/default/site.conf.sample" +python oecore_update_siteconf() { + # If we have a site.conf, check it's valid + current_sconf = d.getVar('SCONF_VERSION') + sconf_version = d.getVar('SITE_CONF_VERSION') + + failmsg = """Your version of site.conf was generated from an older version of +site.conf.sample and there have been updates made to this file. Please compare the two +files and merge any changes before continuing. + +Matching the version numbers will remove this message. + +\"${SANITY_DIFF_TOOL} conf/site.conf ${SANITY_SITECONF_SAMPLE}\" + +is a good way to visualise the changes.""" + failmsg = d.expand(failmsg) + + raise NotImplementedError(failmsg) +} + +SANITY_BBLAYERCONF_SAMPLE ?= "${COREBASE}/meta*/conf/templates/default/bblayers.conf.sample" +python oecore_update_bblayers() { + # bblayers.conf is out of date, so see if we can resolve that + + current_lconf = int(d.getVar('LCONF_VERSION')) + lconf_version = int(d.getVar('LAYER_CONF_VERSION')) + + failmsg = """Your version of bblayers.conf has the wrong LCONF_VERSION (has ${LCONF_VERSION}, expecting ${LAYER_CONF_VERSION}). +Please compare your file against bblayers.conf.sample and merge any changes before continuing. +"${SANITY_DIFF_TOOL} conf/bblayers.conf ${SANITY_BBLAYERCONF_SAMPLE}" + +is a good way to visualise the changes.""" + failmsg = d.expand(failmsg) + + if not current_lconf: + raise NotImplementedError(failmsg) + + lines = [] + + if current_lconf < 4: + raise NotImplementedError(failmsg) + + bblayers_fn = bblayers_conf_file(d) + lines = sanity_conf_read(bblayers_fn) + + if current_lconf == 4 and lconf_version > 4: + topdir_var = '$' + '{TOPDIR}' + index, bbpath_line = sanity_conf_find_line('BBPATH', lines) + if bbpath_line: + start = bbpath_line.find('"') + if start != -1 and (len(bbpath_line) != (start + 1)): + if bbpath_line[start + 1] == '"': + lines[index] = (bbpath_line[:start + 1] + + topdir_var + bbpath_line[start + 1:]) + else: + if not topdir_var in bbpath_line: + lines[index] = (bbpath_line[:start + 1] + + topdir_var + ':' + bbpath_line[start + 1:]) + else: + raise NotImplementedError(failmsg) + else: + index, bbfiles_line = sanity_conf_find_line('BBFILES', lines) + if bbfiles_line: + lines.insert(index, 'BBPATH = "' + topdir_var + '"\n') + else: + raise NotImplementedError(failmsg) + + current_lconf += 1 + sanity_conf_update(bblayers_fn, lines, 'LCONF_VERSION', current_lconf) + bb.note("Your conf/bblayers.conf has been automatically updated.") + return + + elif current_lconf == 5 and lconf_version > 5: + # Null update, to avoid issues with people switching between poky and other distros + current_lconf = 6 + sanity_conf_update(bblayers_fn, lines, 'LCONF_VERSION', current_lconf) + bb.note("Your conf/bblayers.conf has been automatically updated.") + return + + status.addresult() + + elif current_lconf == 6 and lconf_version > 6: + # Handle rename of meta-yocto -> meta-poky + # This marks the start of separate version numbers but code is needed in OE-Core + # for the migration, one last time. + layers = d.getVar('BBLAYERS').split() + layers = [ os.path.basename(path) for path in layers ] + if 'meta-yocto' in layers: + found = False + while True: + index, meta_yocto_line = sanity_conf_find_line(r'.*meta-yocto[\'"\s\n]', lines) + if meta_yocto_line: + lines[index] = meta_yocto_line.replace('meta-yocto', 'meta-poky') + found = True + else: + break + if not found: + raise NotImplementedError(failmsg) + index, meta_yocto_line = sanity_conf_find_line('LCONF_VERSION.*\n', lines) + if meta_yocto_line: + lines[index] = 'POKY_BBLAYERS_CONF_VERSION = "1"\n' + else: + raise NotImplementedError(failmsg) + with open(bblayers_fn, "w") as f: + f.write(''.join(lines)) + bb.note("Your conf/bblayers.conf has been automatically updated.") + return + current_lconf += 1 + sanity_conf_update(bblayers_fn, lines, 'LCONF_VERSION', current_lconf) + bb.note("Your conf/bblayers.conf has been automatically updated.") + return + + raise NotImplementedError(failmsg) +} + +def raise_sanity_error(msg, d, network_error=False): + if d.getVar("SANITY_USE_EVENTS") == "1": + try: + bb.event.fire(bb.event.SanityCheckFailed(msg, network_error), d) + except TypeError: + bb.event.fire(bb.event.SanityCheckFailed(msg), d) + return + + bb.fatal(""" OE-core's config sanity checker detected a potential misconfiguration. + Either fix the cause of this error or at your own risk disable the checker (see sanity.conf). + Following is the list of potential problems / advisories: + + %s""" % msg) + +# Check a single tune for validity. +def check_toolchain_tune(data, tune, multilib): + tune_errors = [] + if not tune: + return "No tuning found for %s multilib." % multilib + localdata = bb.data.createCopy(data) + if multilib != "default": + # Apply the overrides so we can look at the details. + overrides = localdata.getVar("OVERRIDES", False) + ":virtclass-multilib-" + multilib + localdata.setVar("OVERRIDES", overrides) + bb.debug(2, "Sanity-checking tuning '%s' (%s) features:" % (tune, multilib)) + features = (localdata.getVar("TUNE_FEATURES:tune-%s" % tune) or "").split() + if not features: + return "Tuning '%s' has no defined features, and cannot be used." % tune + valid_tunes = localdata.getVarFlags('TUNEVALID') or {} + conflicts = localdata.getVarFlags('TUNECONFLICTS') or {} + # [doc] is the documentation for the variable, not a real feature + if 'doc' in valid_tunes: + del valid_tunes['doc'] + if 'doc' in conflicts: + del conflicts['doc'] + for feature in features: + if feature in conflicts: + for conflict in conflicts[feature].split(): + if conflict in features: + tune_errors.append("Feature '%s' conflicts with '%s'." % + (feature, conflict)) + if feature in valid_tunes: + bb.debug(2, " %s: %s" % (feature, valid_tunes[feature])) + else: + tune_errors.append("Feature '%s' is not defined." % feature) + if tune_errors: + return "Tuning '%s' has the following errors:\n" % tune + '\n'.join(tune_errors) + +def check_toolchain(data): + tune_error_set = [] + deftune = data.getVar("DEFAULTTUNE") + tune_errors = check_toolchain_tune(data, deftune, 'default') + if tune_errors: + tune_error_set.append(tune_errors) + + multilibs = (data.getVar("MULTILIB_VARIANTS") or "").split() + global_multilibs = (data.getVar("MULTILIB_GLOBAL_VARIANTS") or "").split() + + if multilibs: + seen_libs = [] + seen_tunes = [] + for lib in multilibs: + if lib in seen_libs: + tune_error_set.append("The multilib '%s' appears more than once." % lib) + else: + seen_libs.append(lib) + if not lib in global_multilibs: + tune_error_set.append("Multilib %s is not present in MULTILIB_GLOBAL_VARIANTS" % lib) + tune = data.getVar("DEFAULTTUNE:virtclass-multilib-%s" % lib) + if tune in seen_tunes: + tune_error_set.append("The tuning '%s' appears in more than one multilib." % tune) + else: + seen_libs.append(tune) + if tune == deftune: + tune_error_set.append("Multilib '%s' (%s) is also the default tuning." % (lib, deftune)) + else: + tune_errors = check_toolchain_tune(data, tune, lib) + if tune_errors: + tune_error_set.append(tune_errors) + if tune_error_set: + return "Toolchain tunings invalid:\n" + '\n'.join(tune_error_set) + "\n" + + return "" + +def check_conf_exists(fn, data): + bbpath = [] + fn = data.expand(fn) + vbbpath = data.getVar("BBPATH", False) + if vbbpath: + bbpath += vbbpath.split(":") + for p in bbpath: + currname = os.path.join(data.expand(p), fn) + if os.access(currname, os.R_OK): + return True + return False + +def check_create_long_filename(filepath, pathname): + import string, random + testfile = os.path.join(filepath, ''.join(random.choice(string.ascii_letters) for x in range(200))) + try: + if not os.path.exists(filepath): + bb.utils.mkdirhier(filepath) + f = open(testfile, "w") + f.close() + os.remove(testfile) + except IOError as e: + import errno + err, strerror = e.args + if err == errno.ENAMETOOLONG: + return "Failed to create a file with a long name in %s. Please use a filesystem that does not unreasonably limit filename length.\n" % pathname + else: + return "Failed to create a file in %s: %s.\n" % (pathname, strerror) + except OSError as e: + errno, strerror = e.args + return "Failed to create %s directory in which to run long name sanity check: %s.\n" % (pathname, strerror) + return "" + +def check_path_length(filepath, pathname, limit): + if len(filepath) > limit: + return "The length of %s is longer than %s, this would cause unexpected errors, please use a shorter path.\n" % (pathname, limit) + return "" + +def get_filesystem_id(path): + import subprocess + try: + return subprocess.check_output(["stat", "-f", "-c", "%t", path]).decode('utf-8').strip() + except subprocess.CalledProcessError: + bb.warn("Can't get filesystem id of: %s" % path) + return None + +# Check that the path isn't located on nfs. +def check_not_nfs(path, name): + # The nfs' filesystem id is 6969 + if get_filesystem_id(path) == "6969": + return "The %s: %s can't be located on nfs.\n" % (name, path) + return "" + +# Check that the path is on a case-sensitive file system +def check_case_sensitive(path, name): + import tempfile + with tempfile.NamedTemporaryFile(prefix='TmP', dir=path) as tmp_file: + if os.path.exists(tmp_file.name.lower()): + return "The %s (%s) can't be on a case-insensitive file system.\n" % (name, path) + return "" + +# Check that path isn't a broken symlink +def check_symlink(lnk, data): + if os.path.islink(lnk) and not os.path.exists(lnk): + raise_sanity_error("%s is a broken symlink." % lnk, data) + +def check_connectivity(d): + # URI's to check can be set in the CONNECTIVITY_CHECK_URIS variable + # using the same syntax as for SRC_URI. If the variable is not set + # the check is skipped + test_uris = (d.getVar('CONNECTIVITY_CHECK_URIS') or "").split() + retval = "" + + bbn = d.getVar('BB_NO_NETWORK') + if bbn not in (None, '0', '1'): + return 'BB_NO_NETWORK should be "0" or "1", but it is "%s"' % bbn + + # Only check connectivity if network enabled and the + # CONNECTIVITY_CHECK_URIS are set + network_enabled = not (bbn == '1') + check_enabled = len(test_uris) + if check_enabled and network_enabled: + # Take a copy of the data store and unset MIRRORS and PREMIRRORS + data = bb.data.createCopy(d) + data.delVar('PREMIRRORS') + data.delVar('MIRRORS') + try: + fetcher = bb.fetch2.Fetch(test_uris, data) + fetcher.checkstatus() + except Exception as err: + # Allow the message to be configured so that users can be + # pointed to a support mechanism. + msg = data.getVar('CONNECTIVITY_CHECK_MSG') or "" + if len(msg) == 0: + msg = "%s.\n" % err + msg += " Please ensure your host's network is configured correctly.\n" + msg += " Please ensure CONNECTIVITY_CHECK_URIS is correct and specified URIs are available.\n" + msg += " If your ISP or network is blocking the above URL,\n" + msg += " try with another domain name, for example by setting:\n" + msg += " CONNECTIVITY_CHECK_URIS = \"https://www.example.com/\"" + msg += " You could also set BB_NO_NETWORK = \"1\" to disable network\n" + msg += " access if all required sources are on local disk.\n" + retval = msg + + return retval + +def check_supported_distro(sanity_data): + from fnmatch import fnmatch + + tested_distros = sanity_data.getVar('SANITY_TESTED_DISTROS') + if not tested_distros: + return + + try: + distro = oe.lsb.distro_identifier() + except Exception: + distro = None + + if not distro: + bb.warn('Host distribution could not be determined; you may possibly experience unexpected failures. It is recommended that you use a tested distribution.') + + for supported in [x.strip() for x in tested_distros.split('\\n')]: + if fnmatch(distro, supported): + return + + bb.warn('Host distribution "%s" has not been validated with this version of the build system; you may possibly experience unexpected failures. It is recommended that you use a tested distribution.' % distro) + +# Checks we should only make if MACHINE is set correctly +def check_sanity_validmachine(sanity_data): + messages = "" + + # Check TUNE_ARCH is set + if sanity_data.getVar('TUNE_ARCH') == 'INVALID': + messages = messages + 'TUNE_ARCH is unset. Please ensure your MACHINE configuration includes a valid tune configuration file which will set this correctly.\n' + + # Check TARGET_OS is set + if sanity_data.getVar('TARGET_OS') == 'INVALID': + messages = messages + 'Please set TARGET_OS directly, or choose a MACHINE or DISTRO that does so.\n' + + # Check that we don't have duplicate entries in PACKAGE_ARCHS & that TUNE_PKGARCH is in PACKAGE_ARCHS + pkgarchs = sanity_data.getVar('PACKAGE_ARCHS') + tunepkg = sanity_data.getVar('TUNE_PKGARCH') + defaulttune = sanity_data.getVar('DEFAULTTUNE') + tunefound = False + seen = {} + dups = [] + + for pa in pkgarchs.split(): + if seen.get(pa, 0) == 1: + dups.append(pa) + else: + seen[pa] = 1 + if pa == tunepkg: + tunefound = True + + if len(dups): + messages = messages + "Error, the PACKAGE_ARCHS variable contains duplicates. The following archs are listed more than once: %s" % " ".join(dups) + + if tunefound == False: + messages = messages + "Error, the PACKAGE_ARCHS variable (%s) for DEFAULTTUNE (%s) does not contain TUNE_PKGARCH (%s)." % (pkgarchs, defaulttune, tunepkg) + + return messages + +# Patch before 2.7 can't handle all the features in git-style diffs. Some +# patches may incorrectly apply, and others won't apply at all. +def check_patch_version(sanity_data): + import re, subprocess + + try: + result = subprocess.check_output(["patch", "--version"], stderr=subprocess.STDOUT).decode('utf-8') + version = re.search(r"[0-9.]+", result.splitlines()[0]).group() + if bb.utils.vercmp_string_op(version, "2.7", "<"): + return "Your version of patch is older than 2.7 and has bugs which will break builds. Please install a newer version of patch.\n" + else: + return None + except subprocess.CalledProcessError as e: + return "Unable to execute patch --version, exit code %d:\n%s\n" % (e.returncode, e.output) + +# Glibc needs make 4.0 or later, we may as well match at this point +def check_make_version(sanity_data): + import subprocess + + try: + result = subprocess.check_output(['make', '--version'], stderr=subprocess.STDOUT).decode('utf-8') + except subprocess.CalledProcessError as e: + return "Unable to execute make --version, exit code %d\n%s\n" % (e.returncode, e.output) + version = result.split()[2] + if bb.utils.vercmp_string_op(version, "4.0", "<"): + return "Please install a make version of 4.0 or later.\n" + + if bb.utils.vercmp_string_op(version, "4.2.1", "=="): + distro = oe.lsb.distro_identifier() + if "ubuntu" in distro or "debian" in distro or "linuxmint" in distro: + return None + return "make version 4.2.1 is known to have issues on Centos/OpenSUSE and other non-Ubuntu systems. Please use a buildtools-make-tarball or a newer version of make.\n" + return None + + +# Check if we're running on WSL (Windows Subsystem for Linux). +# WSLv1 is known not to work but WSLv2 should work properly as +# long as the VHDX file is optimized often, let the user know +# upfront. +# More information on installing WSLv2 at: +# https://docs.microsoft.com/en-us/windows/wsl/wsl2-install +def check_wsl(d): + with open("/proc/version", "r") as f: + verdata = f.readlines() + for l in verdata: + if "Microsoft" in l: + return "OpenEmbedded doesn't work under WSLv1, please upgrade to WSLv2 if you want to run builds on Windows" + elif "microsoft" in l: + bb.warn("You are running bitbake under WSLv2, this works properly but you should optimize your VHDX file eventually to avoid running out of storage space") + return None + +# Require at least gcc version 8.0 +# +# This can be fixed on CentOS-7 with devtoolset-6+ +# https://www.softwarecollections.org/en/scls/rhscl/devtoolset-6/ +# +# A less invasive fix is with scripts/install-buildtools (or with user +# built buildtools-extended-tarball) +# +def check_gcc_version(sanity_data): + import subprocess + + build_cc, version = oe.utils.get_host_compiler_version(sanity_data) + if build_cc.strip() == "gcc": + if bb.utils.vercmp_string_op(version, "8.0", "<"): + return "Your version of gcc is older than 8.0 and will break builds. Please install a newer version of gcc (you could use the project's buildtools-extended-tarball or use scripts/install-buildtools).\n" + return None + +# Tar version 1.24 and onwards handle overwriting symlinks correctly +# but earlier versions do not; this needs to work properly for sstate +# Version 1.28 is needed so opkg-build works correctly when reproducible builds are enabled +def check_tar_version(sanity_data): + import subprocess + try: + result = subprocess.check_output(["tar", "--version"], stderr=subprocess.STDOUT).decode('utf-8') + except subprocess.CalledProcessError as e: + return "Unable to execute tar --version, exit code %d\n%s\n" % (e.returncode, e.output) + version = result.split()[3] + if bb.utils.vercmp_string_op(version, "1.28", "<"): + return "Your version of tar is older than 1.28 and does not have the support needed to enable reproducible builds. Please install a newer version of tar (you could use the project's buildtools-tarball from our last release or use scripts/install-buildtools).\n" + + try: + result = subprocess.check_output(["tar", "--help"], stderr=subprocess.STDOUT).decode('utf-8') + if "--xattrs" not in result: + return "Your tar doesn't support --xattrs, please use GNU tar.\n" + except subprocess.CalledProcessError as e: + return "Unable to execute tar --help, exit code %d\n%s\n" % (e.returncode, e.output) + + return None + +# We use git parameters and functionality only found in 1.7.8 or later +# The kernel tools assume git >= 1.8.3.1 (verified needed > 1.7.9.5) see #6162 +# The git fetcher also had workarounds for git < 1.7.9.2 which we've dropped +def check_git_version(sanity_data): + import subprocess + try: + result = subprocess.check_output(["git", "--version"], stderr=subprocess.DEVNULL).decode('utf-8') + except subprocess.CalledProcessError as e: + return "Unable to execute git --version, exit code %d\n%s\n" % (e.returncode, e.output) + version = result.split()[2] + if bb.utils.vercmp_string_op(version, "1.8.3.1", "<"): + return "Your version of git is older than 1.8.3.1 and has bugs which will break builds. Please install a newer version of git.\n" + return None + +# Check the required perl modules which may not be installed by default +def check_perl_modules(sanity_data): + import subprocess + ret = "" + modules = ( "Text::ParseWords", "Thread::Queue", "Data::Dumper", "File::Compare", "File::Copy", "open ':std'", "FindBin" ) + errresult = '' + for m in modules: + try: + subprocess.check_output(["perl", "-e", "use %s" % m]) + except subprocess.CalledProcessError as e: + errresult += bytes.decode(e.output) + ret += "%s " % m + if ret: + return "Required perl module(s) not found: %s\n\n%s\n" % (ret, errresult) + return None + +def sanity_check_conffiles(d): + funcs = d.getVar('BBLAYERS_CONF_UPDATE_FUNCS').split() + for func in funcs: + conffile, current_version, required_version, func = func.split(":") + if check_conf_exists(conffile, d) and d.getVar(current_version) is not None and \ + d.getVar(current_version) != d.getVar(required_version): + try: + bb.build.exec_func(func, d) + except NotImplementedError as e: + bb.fatal(str(e)) + d.setVar("BB_INVALIDCONF", True) + +def drop_v14_cross_builds(d): + import glob + indexes = glob.glob(d.expand("${SSTATE_MANIFESTS}/index-${BUILD_ARCH}_*")) + for i in indexes: + with open(i, "r") as f: + lines = f.readlines() + for l in reversed(lines): + try: + (stamp, manifest, workdir) = l.split() + except ValueError: + bb.fatal("Invalid line '%s' in sstate manifest '%s'" % (l, i)) + for m in glob.glob(manifest + ".*"): + if m.endswith(".postrm"): + continue + sstate_clean_manifest(m, d) + bb.utils.remove(stamp + "*") + bb.utils.remove(workdir, recurse = True) + +def sanity_handle_abichanges(status, d): + # + # Check the 'ABI' of TMPDIR + # + import subprocess + + current_abi = d.getVar('OELAYOUT_ABI') + abifile = d.getVar('SANITY_ABIFILE') + if os.path.exists(abifile): + with open(abifile, "r") as f: + abi = f.read().strip() + if not abi.isdigit(): + with open(abifile, "w") as f: + f.write(current_abi) + elif int(abi) <= 11 and current_abi == "12": + status.addresult("The layout of TMPDIR changed for Recipe Specific Sysroots.\nConversion doesn't make sense and this change will rebuild everything so please delete TMPDIR (%s).\n" % d.getVar("TMPDIR")) + elif int(abi) <= 13 and current_abi == "14": + status.addresult("TMPDIR changed to include path filtering from the pseudo database.\nIt is recommended to use a clean TMPDIR with the new pseudo path filtering so TMPDIR (%s) would need to be removed to continue.\n" % d.getVar("TMPDIR")) + elif int(abi) == 14 and current_abi == "15": + drop_v14_cross_builds(d) + with open(abifile, "w") as f: + f.write(current_abi) + elif (abi != current_abi): + # Code to convert from one ABI to another could go here if possible. + status.addresult("Error, TMPDIR has changed its layout version number (%s to %s) and you need to either rebuild, revert or adjust it at your own risk.\n" % (abi, current_abi)) + else: + with open(abifile, "w") as f: + f.write(current_abi) + +def check_sanity_sstate_dir_change(sstate_dir, data): + # Sanity checks to be done when the value of SSTATE_DIR changes + + # Check that SSTATE_DIR isn't on a filesystem with limited filename length (eg. eCryptFS) + testmsg = "" + if sstate_dir != "": + testmsg = check_create_long_filename(sstate_dir, "SSTATE_DIR") + # If we don't have permissions to SSTATE_DIR, suggest the user set it as an SSTATE_MIRRORS + try: + err = testmsg.split(': ')[1].strip() + if err == "Permission denied.": + testmsg = testmsg + "You could try using %s in SSTATE_MIRRORS rather than as an SSTATE_CACHE.\n" % (sstate_dir) + except IndexError: + pass + return testmsg + +def check_sanity_version_change(status, d): + # Sanity checks to be done when SANITY_VERSION or NATIVELSBSTRING changes + # In other words, these tests run once in a given build directory and then + # never again until the sanity version or host distribution id/version changes. + + # Check the python install is complete. Examples that are often removed in + # minimal installations: glib-2.0-natives requires xml.parsers.expat + try: + import xml.parsers.expat + except ImportError as e: + status.addresult('Your Python 3 is not a full install. Please install the module %s (see the Getting Started guide for further information).\n' % e.name) + + status.addresult(check_gcc_version(d)) + status.addresult(check_make_version(d)) + status.addresult(check_patch_version(d)) + status.addresult(check_tar_version(d)) + status.addresult(check_git_version(d)) + status.addresult(check_perl_modules(d)) + status.addresult(check_wsl(d)) + + missing = "" + + if not check_app_exists("${MAKE}", d): + missing = missing + "GNU make," + + if not check_app_exists('${BUILD_CC}', d): + missing = missing + "C Compiler (%s)," % d.getVar("BUILD_CC") + + if not check_app_exists('${BUILD_CXX}', d): + missing = missing + "C++ Compiler (%s)," % d.getVar("BUILD_CXX") + + required_utilities = d.getVar('SANITY_REQUIRED_UTILITIES') + + for util in required_utilities.split(): + if not check_app_exists(util, d): + missing = missing + "%s," % util + + if missing: + missing = missing.rstrip(',') + status.addresult("Please install the following missing utilities: %s\n" % missing) + + assume_provided = d.getVar('ASSUME_PROVIDED').split() + # Check user doesn't have ASSUME_PROVIDED = instead of += in local.conf + if "diffstat-native" not in assume_provided: + status.addresult('Please use ASSUME_PROVIDED +=, not ASSUME_PROVIDED = in your local.conf\n') + + # Check that TMPDIR isn't on a filesystem with limited filename length (eg. eCryptFS) + import stat + tmpdir = d.getVar('TMPDIR') + status.addresult(check_create_long_filename(tmpdir, "TMPDIR")) + tmpdirmode = os.stat(tmpdir).st_mode + if (tmpdirmode & stat.S_ISGID): + status.addresult("TMPDIR is setgid, please don't build in a setgid directory") + if (tmpdirmode & stat.S_ISUID): + status.addresult("TMPDIR is setuid, please don't build in a setuid directory") + + # Check that a user isn't building in a path in PSEUDO_IGNORE_PATHS + pseudoignorepaths = d.getVar('PSEUDO_IGNORE_PATHS', expand=True).split(",") + workdir = d.getVar('WORKDIR', expand=True) + for i in pseudoignorepaths: + if i and workdir.startswith(i): + status.addresult("You are building in a path included in PSEUDO_IGNORE_PATHS " + str(i) + " please locate the build outside this path.\n") + + # Check if PSEUDO_IGNORE_PATHS and paths under pseudo control overlap + pseudoignorepaths = d.getVar('PSEUDO_IGNORE_PATHS', expand=True).split(",") + pseudo_control_dir = "${D},${PKGD},${PKGDEST},${IMAGEROOTFS},${SDK_OUTPUT}" + pseudocontroldir = d.expand(pseudo_control_dir).split(",") + for i in pseudoignorepaths: + for j in pseudocontroldir: + if i and j: + if j.startswith(i): + status.addresult("A path included in PSEUDO_IGNORE_PATHS " + str(i) + " and the path " + str(j) + " overlap and this will break pseudo permission and ownership tracking. Please set the path " + str(j) + " to a different directory which does not overlap with pseudo controlled directories. \n") + + # Some third-party software apparently relies on chmod etc. being suid root (!!) + import stat + suid_check_bins = "chown chmod mknod".split() + for bin_cmd in suid_check_bins: + bin_path = bb.utils.which(os.environ["PATH"], bin_cmd) + if bin_path: + bin_stat = os.stat(bin_path) + if bin_stat.st_uid == 0 and bin_stat.st_mode & stat.S_ISUID: + status.addresult('%s has the setuid bit set. This interferes with pseudo and may cause other issues that break the build process.\n' % bin_path) + + # Check that we can fetch from various network transports + netcheck = check_connectivity(d) + status.addresult(netcheck) + if netcheck: + status.network_error = True + + nolibs = d.getVar('NO32LIBS') + if not nolibs: + lib32path = '/lib' + if os.path.exists('/lib64') and ( os.path.islink('/lib64') or os.path.islink('/lib') ): + lib32path = '/lib32' + + if os.path.exists('%s/libc.so.6' % lib32path) and not os.path.exists('/usr/include/gnu/stubs-32.h'): + status.addresult("You have a 32-bit libc, but no 32-bit headers. You must install the 32-bit libc headers.\n") + + bbpaths = d.getVar('BBPATH').split(":") + if ("." in bbpaths or "./" in bbpaths or "" in bbpaths): + status.addresult("BBPATH references the current directory, either through " \ + "an empty entry, a './' or a '.'.\n\t This is unsafe and means your "\ + "layer configuration is adding empty elements to BBPATH.\n\t "\ + "Please check your layer.conf files and other BBPATH " \ + "settings to remove the current working directory " \ + "references.\n" \ + "Parsed BBPATH is" + str(bbpaths)); + + oes_bb_conf = d.getVar( 'OES_BITBAKE_CONF') + if not oes_bb_conf: + status.addresult('You are not using the OpenEmbedded version of conf/bitbake.conf. This means your environment is misconfigured, in particular check BBPATH.\n') + + # The length of TMPDIR can't be longer than 410 + status.addresult(check_path_length(tmpdir, "TMPDIR", 410)) + + # Check that TMPDIR isn't located on nfs + status.addresult(check_not_nfs(tmpdir, "TMPDIR")) + + # Check for case-insensitive file systems (such as Linux in Docker on + # macOS with default HFS+ file system) + status.addresult(check_case_sensitive(tmpdir, "TMPDIR")) + +def sanity_check_locale(d): + """ + Currently bitbake switches locale to en_US.UTF-8 so check that this locale actually exists. + """ + import locale + try: + locale.setlocale(locale.LC_ALL, "en_US.UTF-8") + except locale.Error: + raise_sanity_error("Your system needs to support the en_US.UTF-8 locale.", d) + +def check_sanity_everybuild(status, d): + import os, stat + # Sanity tests which test the users environment so need to run at each build (or are so cheap + # it makes sense to always run them. + + if 0 == os.getuid(): + raise_sanity_error("Do not use Bitbake as root.", d) + + # Check the Python version, we now have a minimum of Python 3.8 + import sys + if sys.hexversion < 0x030800F0: + status.addresult('The system requires at least Python 3.8 to run. Please update your Python interpreter.\n') + + # Check the bitbake version meets minimum requirements + minversion = d.getVar('BB_MIN_VERSION') + if bb.utils.vercmp_string_op(bb.__version__, minversion, "<"): + status.addresult('Bitbake version %s is required and version %s was found\n' % (minversion, bb.__version__)) + + sanity_check_locale(d) + + paths = d.getVar('PATH').split(":") + if "." in paths or "./" in paths or "" in paths: + status.addresult("PATH contains '.', './' or '' (empty element), which will break the build, please remove this.\nParsed PATH is " + str(paths) + "\n") + + #Check if bitbake is present in PATH environment variable + bb_check = bb.utils.which(d.getVar('PATH'), 'bitbake') + if not bb_check: + bb.warn("bitbake binary is not found in PATH, did you source the script?") + + # Check whether 'inherit' directive is found (used for a class to inherit) + # in conf file it's supposed to be uppercase INHERIT + inherit = d.getVar('inherit') + if inherit: + status.addresult("Please don't use inherit directive in your local.conf. The directive is supposed to be used in classes and recipes only to inherit of bbclasses. Here INHERIT should be used.\n") + + # Check that the DISTRO is valid, if set + # need to take into account DISTRO renaming DISTRO + distro = d.getVar('DISTRO') + if distro and distro != "nodistro": + if not ( check_conf_exists("conf/distro/${DISTRO}.conf", d) or check_conf_exists("conf/distro/include/${DISTRO}.inc", d) ): + status.addresult("DISTRO '%s' not found. Please set a valid DISTRO in your local.conf\n" % d.getVar("DISTRO")) + + # Check that these variables don't use tilde-expansion as we don't do that + for v in ("TMPDIR", "DL_DIR", "SSTATE_DIR"): + if d.getVar(v).startswith("~"): + status.addresult("%s uses ~ but Bitbake will not expand this, use an absolute path or variables." % v) + + # Check that DL_DIR is set, exists and is writable. In theory, we should never even hit the check if DL_DIR isn't + # set, since so much relies on it being set. + dldir = d.getVar('DL_DIR') + if not dldir: + status.addresult("DL_DIR is not set. Your environment is misconfigured, check that DL_DIR is set, and if the directory exists, that it is writable. \n") + if os.path.exists(dldir) and not os.access(dldir, os.W_OK): + status.addresult("DL_DIR: %s exists but you do not appear to have write access to it. \n" % dldir) + check_symlink(dldir, d) + + # Check that the MACHINE is valid, if it is set + machinevalid = True + if d.getVar('MACHINE'): + if not check_conf_exists("conf/machine/${MACHINE}.conf", d): + status.addresult('MACHINE=%s is invalid. Please set a valid MACHINE in your local.conf, environment or other configuration file.\n' % (d.getVar('MACHINE'))) + machinevalid = False + else: + status.addresult(check_sanity_validmachine(d)) + else: + status.addresult('Please set a MACHINE in your local.conf or environment\n') + machinevalid = False + if machinevalid: + status.addresult(check_toolchain(d)) + + # Check that the SDKMACHINE is valid, if it is set + if d.getVar('SDKMACHINE'): + if not check_conf_exists("conf/machine-sdk/${SDKMACHINE}.conf", d): + status.addresult('Specified SDKMACHINE value is not valid\n') + elif d.getVar('SDK_ARCH', False) == "${BUILD_ARCH}": + status.addresult('SDKMACHINE is set, but SDK_ARCH has not been changed as a result - SDKMACHINE may have been set too late (e.g. in the distro configuration)\n') + + # If SDK_VENDOR looks like "-my-sdk" then the triples are badly formed so fail early + sdkvendor = d.getVar("SDK_VENDOR") + if not (sdkvendor.startswith("-") and sdkvendor.count("-") == 1): + status.addresult("SDK_VENDOR should be of the form '-foosdk' with a single dash; found '%s'\n" % sdkvendor) + + check_supported_distro(d) + + omask = os.umask(0o022) + if omask & 0o755: + status.addresult("Please use a umask which allows a+rx and u+rwx\n") + os.umask(omask) + + # Ensure /tmp is NOT mounted with noexec + if os.statvfs("/tmp").f_flag & os.ST_NOEXEC: + raise_sanity_error("/tmp shouldn't be mounted with noexec.", d) + + if d.getVar('TARGET_ARCH') == "arm": + # This path is no longer user-readable in modern (very recent) Linux + try: + if os.path.exists("/proc/sys/vm/mmap_min_addr"): + f = open("/proc/sys/vm/mmap_min_addr", "r") + try: + if (int(f.read().strip()) > 65536): + status.addresult("/proc/sys/vm/mmap_min_addr is not <= 65536. This will cause problems with qemu so please fix the value (as root).\n\nTo fix this in later reboots, set vm.mmap_min_addr = 65536 in /etc/sysctl.conf.\n") + finally: + f.close() + except: + pass + + for checkdir in ['COREBASE', 'TMPDIR']: + val = d.getVar(checkdir) + if val.find('..') != -1: + status.addresult("Error, you have '..' in your %s directory path. Please ensure the variable contains an absolute path as this can break some recipe builds in obtuse ways." % checkdir) + if val.find('+') != -1: + status.addresult("Error, you have an invalid character (+) in your %s directory path. Please move the installation to a directory which doesn't include any + characters." % checkdir) + if val.find('@') != -1: + status.addresult("Error, you have an invalid character (@) in your %s directory path. Please move the installation to a directory which doesn't include any @ characters." % checkdir) + if val.find(' ') != -1: + status.addresult("Error, you have a space in your %s directory path. Please move the installation to a directory which doesn't include a space since autotools doesn't support this." % checkdir) + if val.find('%') != -1: + status.addresult("Error, you have an invalid character (%) in your %s directory path which causes problems with python string formatting. Please move the installation to a directory which doesn't include any % characters." % checkdir) + + # Check the format of MIRRORS, PREMIRRORS and SSTATE_MIRRORS + import re + mirror_vars = ['MIRRORS', 'PREMIRRORS', 'SSTATE_MIRRORS'] + protocols = ['http', 'ftp', 'file', 'https', \ + 'git', 'gitsm', 'hg', 'osc', 'p4', 'svn', \ + 'bzr', 'cvs', 'npm', 'sftp', 'ssh', 's3', \ + 'az', 'ftps', 'crate', 'gs'] + for mirror_var in mirror_vars: + mirrors = (d.getVar(mirror_var) or '').replace('\\n', ' ').split() + + # Split into pairs + if len(mirrors) % 2 != 0: + bb.warn('Invalid mirror variable value for %s: %s, should contain paired members.' % (mirror_var, str(mirrors))) + continue + mirrors = list(zip(*[iter(mirrors)]*2)) + + for mirror_entry in mirrors: + pattern, mirror = mirror_entry + + decoded = bb.fetch2.decodeurl(pattern) + try: + pattern_scheme = re.compile(decoded[0]) + except re.error as exc: + bb.warn('Invalid scheme regex (%s) in %s; %s' % (pattern, mirror_var, mirror_entry)) + continue + + if not any(pattern_scheme.match(protocol) for protocol in protocols): + bb.warn('Invalid protocol (%s) in %s: %s' % (decoded[0], mirror_var, mirror_entry)) + continue + + if not any(mirror.startswith(protocol + '://') for protocol in protocols): + bb.warn('Invalid protocol in %s: %s' % (mirror_var, mirror_entry)) + continue + + if mirror.startswith('file://'): + import urllib + check_symlink(urllib.parse.urlparse(mirror).path, d) + # SSTATE_MIRROR ends with a /PATH string + if mirror.endswith('/PATH'): + # remove /PATH$ from SSTATE_MIRROR to get a working + # base directory path + mirror_base = urllib.parse.urlparse(mirror[:-1*len('/PATH')]).path + check_symlink(mirror_base, d) + + # Check sstate mirrors aren't being used with a local hash server and no remote + hashserv = d.getVar("BB_HASHSERVE") + if d.getVar("SSTATE_MIRRORS") and hashserv and hashserv.startswith("unix://") and not d.getVar("BB_HASHSERVE_UPSTREAM"): + bb.warn("You are using a local hash equivalence server but have configured an sstate mirror. This will likely mean no sstate will match from the mirror. You may wish to disable the hash equivalence use (BB_HASHSERVE), or use a hash equivalence server alongside the sstate mirror.") + + # Check that TMPDIR hasn't changed location since the last time we were run + tmpdir = d.getVar('TMPDIR') + checkfile = os.path.join(tmpdir, "saved_tmpdir") + if os.path.exists(checkfile): + with open(checkfile, "r") as f: + saved_tmpdir = f.read().strip() + if (saved_tmpdir != tmpdir): + status.addresult("Error, TMPDIR has changed location. You need to either move it back to %s or delete it and rebuild\n" % saved_tmpdir) + else: + bb.utils.mkdirhier(tmpdir) + # Remove setuid, setgid and sticky bits from TMPDIR + try: + os.chmod(tmpdir, os.stat(tmpdir).st_mode & ~ stat.S_ISUID) + os.chmod(tmpdir, os.stat(tmpdir).st_mode & ~ stat.S_ISGID) + os.chmod(tmpdir, os.stat(tmpdir).st_mode & ~ stat.S_ISVTX) + except OSError as exc: + bb.warn("Unable to chmod TMPDIR: %s" % exc) + with open(checkfile, "w") as f: + f.write(tmpdir) + + # If /bin/sh is a symlink, check that it points to dash or bash + if os.path.islink('/bin/sh'): + real_sh = os.path.realpath('/bin/sh') + # Due to update-alternatives, the shell name may take various + # forms, such as /bin/dash, bin/bash, /bin/bash.bash ... + if '/dash' not in real_sh and '/bash' not in real_sh: + status.addresult("Error, /bin/sh links to %s, must be dash or bash\n" % real_sh) + +def check_sanity(sanity_data): + class SanityStatus(object): + def __init__(self): + self.messages = "" + self.network_error = False + + def addresult(self, message): + if message: + self.messages = self.messages + message + + status = SanityStatus() + + tmpdir = sanity_data.getVar('TMPDIR') + sstate_dir = sanity_data.getVar('SSTATE_DIR') + + check_symlink(sstate_dir, sanity_data) + + # Check saved sanity info + last_sanity_version = 0 + last_tmpdir = "" + last_sstate_dir = "" + last_nativelsbstr = "" + sanityverfile = sanity_data.expand("${TOPDIR}/cache/sanity_info") + if os.path.exists(sanityverfile): + with open(sanityverfile, 'r') as f: + for line in f: + if line.startswith('SANITY_VERSION'): + last_sanity_version = int(line.split()[1]) + if line.startswith('TMPDIR'): + last_tmpdir = line.split()[1] + if line.startswith('SSTATE_DIR'): + last_sstate_dir = line.split()[1] + if line.startswith('NATIVELSBSTRING'): + last_nativelsbstr = line.split()[1] + + check_sanity_everybuild(status, sanity_data) + + sanity_version = int(sanity_data.getVar('SANITY_VERSION') or 1) + network_error = False + # NATIVELSBSTRING var may have been overridden with "universal", so + # get actual host distribution id and version + nativelsbstr = lsb_distro_identifier(sanity_data) + if last_sanity_version < sanity_version or last_nativelsbstr != nativelsbstr: + check_sanity_version_change(status, sanity_data) + status.addresult(check_sanity_sstate_dir_change(sstate_dir, sanity_data)) + else: + if last_sstate_dir != sstate_dir: + status.addresult(check_sanity_sstate_dir_change(sstate_dir, sanity_data)) + + if os.path.exists(os.path.dirname(sanityverfile)) and not status.messages: + with open(sanityverfile, 'w') as f: + f.write("SANITY_VERSION %s\n" % sanity_version) + f.write("TMPDIR %s\n" % tmpdir) + f.write("SSTATE_DIR %s\n" % sstate_dir) + f.write("NATIVELSBSTRING %s\n" % nativelsbstr) + + sanity_handle_abichanges(status, sanity_data) + + if status.messages != "": + raise_sanity_error(sanity_data.expand(status.messages), sanity_data, status.network_error) + +addhandler config_reparse_eventhandler +config_reparse_eventhandler[eventmask] = "bb.event.ConfigParsed" +python config_reparse_eventhandler() { + sanity_check_conffiles(e.data) +} + +addhandler check_sanity_eventhandler +check_sanity_eventhandler[eventmask] = "bb.event.SanityCheck bb.event.NetworkTest" +python check_sanity_eventhandler() { + if bb.event.getName(e) == "SanityCheck": + sanity_data = bb.data.createCopy(e.data) + check_sanity(sanity_data) + if e.generateevents: + sanity_data.setVar("SANITY_USE_EVENTS", "1") + bb.event.fire(bb.event.SanityCheckPassed(), e.data) + elif bb.event.getName(e) == "NetworkTest": + sanity_data = bb.data.createCopy(e.data) + if e.generateevents: + sanity_data.setVar("SANITY_USE_EVENTS", "1") + bb.event.fire(bb.event.NetworkTestFailed() if check_connectivity(sanity_data) else bb.event.NetworkTestPassed(), e.data) + + return +} diff --git a/meta/classes-global/sstate.bbclass b/meta/classes-global/sstate.bbclass new file mode 100644 index 0000000000..04539bbb99 --- /dev/null +++ b/meta/classes-global/sstate.bbclass @@ -0,0 +1,1390 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +SSTATE_VERSION = "12" + +SSTATE_ZSTD_CLEVEL ??= "8" + +SSTATE_MANIFESTS ?= "${TMPDIR}/sstate-control" +SSTATE_MANFILEPREFIX = "${SSTATE_MANIFESTS}/manifest-${SSTATE_MANMACH}-${PN}" + +def generate_sstatefn(spec, hash, taskname, siginfo, d): + if taskname is None: + return "" + extension = ".tar.zst" + # 8 chars reserved for siginfo + limit = 254 - 8 + if siginfo: + limit = 254 + extension = ".tar.zst.siginfo" + if not hash: + hash = "INVALID" + fn = spec + hash + "_" + taskname + extension + # If the filename is too long, attempt to reduce it + if len(fn) > limit: + components = spec.split(":") + # Fields 0,5,6 are mandatory, 1 is most useful, 2,3,4 are just for information + # 7 is for the separators + avail = (limit - len(hash + "_" + taskname + extension) - len(components[0]) - len(components[1]) - len(components[5]) - len(components[6]) - 7) // 3 + components[2] = components[2][:avail] + components[3] = components[3][:avail] + components[4] = components[4][:avail] + spec = ":".join(components) + fn = spec + hash + "_" + taskname + extension + if len(fn) > limit: + bb.fatal("Unable to reduce sstate name to less than 255 chararacters") + return hash[:2] + "/" + hash[2:4] + "/" + fn + +SSTATE_PKGARCH = "${PACKAGE_ARCH}" +SSTATE_PKGSPEC = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:" +SSTATE_SWSPEC = "sstate:${PN}::${PV}:${PR}::${SSTATE_VERSION}:" +SSTATE_PKGNAME = "${SSTATE_EXTRAPATH}${@generate_sstatefn(d.getVar('SSTATE_PKGSPEC'), d.getVar('BB_UNIHASH'), d.getVar('SSTATE_CURRTASK'), False, d)}" +SSTATE_PKG = "${SSTATE_DIR}/${SSTATE_PKGNAME}" +SSTATE_EXTRAPATH = "" +SSTATE_EXTRAPATHWILDCARD = "" +SSTATE_PATHSPEC = "${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/*/${SSTATE_PKGSPEC}*_${SSTATE_PATH_CURRTASK}.tar.zst*" + +# explicitly make PV to depend on evaluated value of PV variable +PV[vardepvalue] = "${PV}" + +# We don't want the sstate to depend on things like the distro string +# of the system, we let the sstate paths take care of this. +SSTATE_EXTRAPATH[vardepvalue] = "" +SSTATE_EXTRAPATHWILDCARD[vardepvalue] = "" + +# Avoid docbook/sgml catalog warnings for now +SSTATE_ALLOW_OVERLAP_FILES += "${STAGING_ETCDIR_NATIVE}/sgml ${STAGING_DATADIR_NATIVE}/sgml" +# sdk-provides-dummy-nativesdk and nativesdk-buildtools-perl-dummy overlap for different SDKMACHINE +SSTATE_ALLOW_OVERLAP_FILES += "${DEPLOY_DIR_RPM}/sdk_provides_dummy_nativesdk/ ${DEPLOY_DIR_IPK}/sdk-provides-dummy-nativesdk/" +SSTATE_ALLOW_OVERLAP_FILES += "${DEPLOY_DIR_RPM}/buildtools_dummy_nativesdk/ ${DEPLOY_DIR_IPK}/buildtools-dummy-nativesdk/" +# target-sdk-provides-dummy overlaps that allarch is disabled when multilib is used +SSTATE_ALLOW_OVERLAP_FILES += "${COMPONENTS_DIR}/sdk-provides-dummy-target/ ${DEPLOY_DIR_RPM}/sdk_provides_dummy_target/ ${DEPLOY_DIR_IPK}/sdk-provides-dummy-target/" +# Archive the sources for many architectures in one deploy folder +SSTATE_ALLOW_OVERLAP_FILES += "${DEPLOY_DIR_SRC}" +# ovmf/grub-efi/systemd-boot/intel-microcode multilib recipes can generate identical overlapping files +SSTATE_ALLOW_OVERLAP_FILES += "${DEPLOY_DIR_IMAGE}/ovmf" +SSTATE_ALLOW_OVERLAP_FILES += "${DEPLOY_DIR_IMAGE}/grub-efi" +SSTATE_ALLOW_OVERLAP_FILES += "${DEPLOY_DIR_IMAGE}/systemd-boot" +SSTATE_ALLOW_OVERLAP_FILES += "${DEPLOY_DIR_IMAGE}/microcode" + +SSTATE_SCAN_FILES ?= "*.la *-config *_config postinst-*" +SSTATE_SCAN_CMD ??= 'find ${SSTATE_BUILDDIR} \( -name "${@"\" -o -name \"".join(d.getVar("SSTATE_SCAN_FILES").split())}" \) -type f' +SSTATE_SCAN_CMD_NATIVE ??= 'grep -Irl -e ${RECIPE_SYSROOT} -e ${RECIPE_SYSROOT_NATIVE} -e ${HOSTTOOLS_DIR} ${SSTATE_BUILDDIR}' +SSTATE_HASHEQUIV_FILEMAP ?= " \ + populate_sysroot:*/postinst-useradd-*:${TMPDIR} \ + populate_sysroot:*/postinst-useradd-*:${COREBASE} \ + populate_sysroot:*/postinst-useradd-*:regex-\s(PATH|PSEUDO_IGNORE_PATHS|HOME|LOGNAME|OMP_NUM_THREADS|USER)=.*\s \ + populate_sysroot:*/crossscripts/*:${TMPDIR} \ + populate_sysroot:*/crossscripts/*:${COREBASE} \ + " + +BB_HASHFILENAME = "False ${SSTATE_PKGSPEC} ${SSTATE_SWSPEC}" + +SSTATE_ARCHS_TUNEPKG ??= "${TUNE_PKGARCH}" +SSTATE_ARCHS = " \ + ${BUILD_ARCH} \ + ${BUILD_ARCH}_${ORIGNATIVELSBSTRING} \ + ${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS} \ + ${SDK_ARCH}_${SDK_OS} \ + ${SDK_ARCH}_${SDK_ARCH}-${SDKPKGSUFFIX} \ + allarch \ + ${SSTATE_ARCHS_TUNEPKG} \ + ${PACKAGE_EXTRA_ARCHS} \ + ${MACHINE_ARCH}" +SSTATE_ARCHS[vardepsexclude] = "ORIGNATIVELSBSTRING" + +SSTATE_MANMACH ?= "${SSTATE_PKGARCH}" + +SSTATECREATEFUNCS += "sstate_hardcode_path" +SSTATECREATEFUNCS[vardeps] = "SSTATE_SCAN_FILES" +SSTATEPOSTCREATEFUNCS = "" +SSTATEPREINSTFUNCS = "" +SSTATEPOSTUNPACKFUNCS = "sstate_hardcode_path_unpack" +SSTATEPOSTINSTFUNCS = "" +EXTRA_STAGING_FIXMES ?= "HOSTTOOLS_DIR" + +# Check whether sstate exists for tasks that support sstate and are in the +# locked signatures file. +SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK ?= 'error' + +# Check whether the task's computed hash matches the task's hash in the +# locked signatures file. +SIGGEN_LOCKEDSIGS_TASKSIG_CHECK ?= "error" + +# The GnuPG key ID and passphrase to use to sign sstate archives (or unset to +# not sign) +SSTATE_SIG_KEY ?= "" +SSTATE_SIG_PASSPHRASE ?= "" +# Whether to verify the GnUPG signatures when extracting sstate archives +SSTATE_VERIFY_SIG ?= "0" +# List of signatures to consider valid. +SSTATE_VALID_SIGS ??= "" +SSTATE_VALID_SIGS[vardepvalue] = "" + +SSTATE_HASHEQUIV_METHOD ?= "oe.sstatesig.OEOuthashBasic" +SSTATE_HASHEQUIV_METHOD[doc] = "The fully-qualified function used to calculate \ + the output hash for a task, which in turn is used to determine equivalency. \ + " + +SSTATE_HASHEQUIV_REPORT_TASKDATA ?= "0" +SSTATE_HASHEQUIV_REPORT_TASKDATA[doc] = "Report additional useful data to the \ + hash equivalency server, such as PN, PV, taskname, etc. This information \ + is very useful for developers looking at task data, but may leak sensitive \ + data if the equivalence server is public. \ + " + +python () { + if bb.data.inherits_class('native', d): + d.setVar('SSTATE_PKGARCH', d.getVar('BUILD_ARCH', False)) + elif bb.data.inherits_class('crosssdk', d): + d.setVar('SSTATE_PKGARCH', d.expand("${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS}")) + elif bb.data.inherits_class('cross', d): + d.setVar('SSTATE_PKGARCH', d.expand("${BUILD_ARCH}")) + elif bb.data.inherits_class('nativesdk', d): + d.setVar('SSTATE_PKGARCH', d.expand("${SDK_ARCH}_${SDK_OS}")) + elif bb.data.inherits_class('cross-canadian', d): + d.setVar('SSTATE_PKGARCH', d.expand("${SDK_ARCH}_${PACKAGE_ARCH}")) + elif bb.data.inherits_class('allarch', d) and d.getVar("PACKAGE_ARCH") == "all": + d.setVar('SSTATE_PKGARCH', "allarch") + else: + d.setVar('SSTATE_MANMACH', d.expand("${PACKAGE_ARCH}")) + + if bb.data.inherits_class('native', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross', d): + d.setVar('SSTATE_EXTRAPATH', "${NATIVELSBSTRING}/") + d.setVar('BB_HASHFILENAME', "True ${SSTATE_PKGSPEC} ${SSTATE_SWSPEC}") + d.setVar('SSTATE_EXTRAPATHWILDCARD', "${NATIVELSBSTRING}/") + + unique_tasks = sorted(set((d.getVar('SSTATETASKS') or "").split())) + d.setVar('SSTATETASKS', " ".join(unique_tasks)) + for task in unique_tasks: + d.prependVarFlag(task, 'prefuncs', "sstate_task_prefunc ") + d.appendVarFlag(task, 'postfuncs', " sstate_task_postfunc") + d.setVarFlag(task, 'network', '1') + d.setVarFlag(task + "_setscene", 'network', '1') +} + +def sstate_init(task, d): + ss = {} + ss['task'] = task + ss['dirs'] = [] + ss['plaindirs'] = [] + ss['lockfiles'] = [] + ss['lockfiles-shared'] = [] + return ss + +def sstate_state_fromvars(d, task = None): + if task is None: + task = d.getVar('BB_CURRENTTASK') + if not task: + bb.fatal("sstate code running without task context?!") + task = task.replace("_setscene", "") + + if task.startswith("do_"): + task = task[3:] + inputs = (d.getVarFlag("do_" + task, 'sstate-inputdirs') or "").split() + outputs = (d.getVarFlag("do_" + task, 'sstate-outputdirs') or "").split() + plaindirs = (d.getVarFlag("do_" + task, 'sstate-plaindirs') or "").split() + lockfiles = (d.getVarFlag("do_" + task, 'sstate-lockfile') or "").split() + lockfilesshared = (d.getVarFlag("do_" + task, 'sstate-lockfile-shared') or "").split() + interceptfuncs = (d.getVarFlag("do_" + task, 'sstate-interceptfuncs') or "").split() + fixmedir = d.getVarFlag("do_" + task, 'sstate-fixmedir') or "" + if not task or len(inputs) != len(outputs): + bb.fatal("sstate variables not setup correctly?!") + + if task == "populate_lic": + d.setVar("SSTATE_PKGSPEC", "${SSTATE_SWSPEC}") + d.setVar("SSTATE_EXTRAPATH", "") + d.setVar('SSTATE_EXTRAPATHWILDCARD', "") + + ss = sstate_init(task, d) + for i in range(len(inputs)): + sstate_add(ss, inputs[i], outputs[i], d) + ss['lockfiles'] = lockfiles + ss['lockfiles-shared'] = lockfilesshared + ss['plaindirs'] = plaindirs + ss['interceptfuncs'] = interceptfuncs + ss['fixmedir'] = fixmedir + return ss + +def sstate_add(ss, source, dest, d): + if not source.endswith("/"): + source = source + "/" + if not dest.endswith("/"): + dest = dest + "/" + source = os.path.normpath(source) + dest = os.path.normpath(dest) + srcbase = os.path.basename(source) + ss['dirs'].append([srcbase, source, dest]) + return ss + +def sstate_install(ss, d): + import oe.path + import oe.sstatesig + import subprocess + + sharedfiles = [] + shareddirs = [] + bb.utils.mkdirhier(d.expand("${SSTATE_MANIFESTS}")) + + sstateinst = d.expand("${WORKDIR}/sstate-install-%s/" % ss['task']) + + manifest, d2 = oe.sstatesig.sstate_get_manifest_filename(ss['task'], d) + + if os.access(manifest, os.R_OK): + bb.fatal("Package already staged (%s)?!" % manifest) + + d.setVar("SSTATE_INST_POSTRM", manifest + ".postrm") + + locks = [] + for lock in ss['lockfiles-shared']: + locks.append(bb.utils.lockfile(lock, True)) + for lock in ss['lockfiles']: + locks.append(bb.utils.lockfile(lock)) + + for state in ss['dirs']: + bb.debug(2, "Staging files from %s to %s" % (state[1], state[2])) + for walkroot, dirs, files in os.walk(state[1]): + for file in files: + srcpath = os.path.join(walkroot, file) + dstpath = srcpath.replace(state[1], state[2]) + #bb.debug(2, "Staging %s to %s" % (srcpath, dstpath)) + sharedfiles.append(dstpath) + for dir in dirs: + srcdir = os.path.join(walkroot, dir) + dstdir = srcdir.replace(state[1], state[2]) + #bb.debug(2, "Staging %s to %s" % (srcdir, dstdir)) + if os.path.islink(srcdir): + sharedfiles.append(dstdir) + continue + if not dstdir.endswith("/"): + dstdir = dstdir + "/" + shareddirs.append(dstdir) + + # Check the file list for conflicts against files which already exist + overlap_allowed = (d.getVar("SSTATE_ALLOW_OVERLAP_FILES") or "").split() + match = [] + for f in sharedfiles: + if os.path.exists(f): + f = os.path.normpath(f) + realmatch = True + for w in overlap_allowed: + w = os.path.normpath(w) + if f.startswith(w): + realmatch = False + break + if realmatch: + match.append(f) + sstate_search_cmd = "grep -rlF '%s' %s --exclude=index-* | sed -e 's:^.*/::'" % (f, d.expand("${SSTATE_MANIFESTS}")) + search_output = subprocess.Popen(sstate_search_cmd, shell=True, stdout=subprocess.PIPE).communicate()[0] + if search_output: + match.append(" (matched in %s)" % search_output.decode('utf-8').rstrip()) + else: + match.append(" (not matched to any task)") + if match: + bb.fatal("Recipe %s is trying to install files into a shared " \ + "area when those files already exist. The files and the manifests listing " \ + "them are:\n %s\n" + "Please adjust the recipes so only one recipe provides a given file. " % \ + (d.getVar('PN'), "\n ".join(match))) + + if ss['fixmedir'] and os.path.exists(ss['fixmedir'] + "/fixmepath.cmd"): + sharedfiles.append(ss['fixmedir'] + "/fixmepath.cmd") + sharedfiles.append(ss['fixmedir'] + "/fixmepath") + + # Write out the manifest + f = open(manifest, "w") + for file in sharedfiles: + f.write(file + "\n") + + # We want to ensure that directories appear at the end of the manifest + # so that when we test to see if they should be deleted any contents + # added by the task will have been removed first. + dirs = sorted(shareddirs, key=len) + # Must remove children first, which will have a longer path than the parent + for di in reversed(dirs): + f.write(di + "\n") + f.close() + + # Append to the list of manifests for this PACKAGE_ARCH + + i = d2.expand("${SSTATE_MANIFESTS}/index-${SSTATE_MANMACH}") + l = bb.utils.lockfile(i + ".lock") + filedata = d.getVar("STAMP") + " " + d2.getVar("SSTATE_MANFILEPREFIX") + " " + d.getVar("WORKDIR") + "\n" + manifests = [] + if os.path.exists(i): + with open(i, "r") as f: + manifests = f.readlines() + # We append new entries, we don't remove older entries which may have the same + # manifest name but different versions from stamp/workdir. See below. + if filedata not in manifests: + with open(i, "a+") as f: + f.write(filedata) + bb.utils.unlockfile(l) + + # Run the actual file install + for state in ss['dirs']: + if os.path.exists(state[1]): + oe.path.copyhardlinktree(state[1], state[2]) + + for postinst in (d.getVar('SSTATEPOSTINSTFUNCS') or '').split(): + # All hooks should run in the SSTATE_INSTDIR + bb.build.exec_func(postinst, d, (sstateinst,)) + + for lock in locks: + bb.utils.unlockfile(lock) + +sstate_install[vardepsexclude] += "SSTATE_ALLOW_OVERLAP_FILES SSTATE_MANMACH SSTATE_MANFILEPREFIX" +sstate_install[vardeps] += "${SSTATEPOSTINSTFUNCS}" + +def sstate_installpkg(ss, d): + from oe.gpg_sign import get_signer + + sstateinst = d.expand("${WORKDIR}/sstate-install-%s/" % ss['task']) + d.setVar("SSTATE_CURRTASK", ss['task']) + sstatefetch = d.getVar('SSTATE_PKGNAME') + sstatepkg = d.getVar('SSTATE_PKG') + verify_sig = bb.utils.to_boolean(d.getVar("SSTATE_VERIFY_SIG"), False) + + if not os.path.exists(sstatepkg) or (verify_sig and not os.path.exists(sstatepkg + '.sig')): + pstaging_fetch(sstatefetch, d) + + if not os.path.isfile(sstatepkg): + bb.note("Sstate package %s does not exist" % sstatepkg) + return False + + sstate_clean(ss, d) + + d.setVar('SSTATE_INSTDIR', sstateinst) + + if verify_sig: + if not os.path.isfile(sstatepkg + '.sig'): + bb.warn("No signature file for sstate package %s, skipping acceleration..." % sstatepkg) + return False + signer = get_signer(d, 'local') + if not signer.verify(sstatepkg + '.sig', d.getVar("SSTATE_VALID_SIGS")): + bb.warn("Cannot verify signature on sstate package %s, skipping acceleration..." % sstatepkg) + return False + + # Empty sstateinst directory, ensure its clean + if os.path.exists(sstateinst): + oe.path.remove(sstateinst) + bb.utils.mkdirhier(sstateinst) + + sstateinst = d.getVar("SSTATE_INSTDIR") + d.setVar('SSTATE_FIXMEDIR', ss['fixmedir']) + + for f in (d.getVar('SSTATEPREINSTFUNCS') or '').split() + ['sstate_unpack_package']: + # All hooks should run in the SSTATE_INSTDIR + bb.build.exec_func(f, d, (sstateinst,)) + + return sstate_installpkgdir(ss, d) + +def sstate_installpkgdir(ss, d): + import oe.path + import subprocess + + sstateinst = d.getVar("SSTATE_INSTDIR") + d.setVar('SSTATE_FIXMEDIR', ss['fixmedir']) + + for f in (d.getVar('SSTATEPOSTUNPACKFUNCS') or '').split(): + # All hooks should run in the SSTATE_INSTDIR + bb.build.exec_func(f, d, (sstateinst,)) + + def prepdir(dir): + # remove dir if it exists, ensure any parent directories do exist + if os.path.exists(dir): + oe.path.remove(dir) + bb.utils.mkdirhier(dir) + oe.path.remove(dir) + + for state in ss['dirs']: + prepdir(state[1]) + bb.utils.rename(sstateinst + state[0], state[1]) + sstate_install(ss, d) + + for plain in ss['plaindirs']: + workdir = d.getVar('WORKDIR') + sharedworkdir = os.path.join(d.getVar('TMPDIR'), "work-shared") + src = sstateinst + "/" + plain.replace(workdir, '') + if sharedworkdir in plain: + src = sstateinst + "/" + plain.replace(sharedworkdir, '') + dest = plain + bb.utils.mkdirhier(src) + prepdir(dest) + bb.utils.rename(src, dest) + + return True + +python sstate_hardcode_path_unpack () { + # Fixup hardcoded paths + # + # Note: The logic below must match the reverse logic in + # sstate_hardcode_path(d) + import subprocess + + sstateinst = d.getVar('SSTATE_INSTDIR') + sstatefixmedir = d.getVar('SSTATE_FIXMEDIR') + fixmefn = sstateinst + "fixmepath" + if os.path.isfile(fixmefn): + staging_target = d.getVar('RECIPE_SYSROOT') + staging_host = d.getVar('RECIPE_SYSROOT_NATIVE') + + if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross-canadian', d): + sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRHOST:%s:g'" % (staging_host) + elif bb.data.inherits_class('cross', d) or bb.data.inherits_class('crosssdk', d): + sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRTARGET:%s:g; s:FIXMESTAGINGDIRHOST:%s:g'" % (staging_target, staging_host) + else: + sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRTARGET:%s:g'" % (staging_target) + + extra_staging_fixmes = d.getVar('EXTRA_STAGING_FIXMES') or '' + for fixmevar in extra_staging_fixmes.split(): + fixme_path = d.getVar(fixmevar) + sstate_sed_cmd += " -e 's:FIXME_%s:%s:g'" % (fixmevar, fixme_path) + + # Add sstateinst to each filename in fixmepath, use xargs to efficiently call sed + sstate_hardcode_cmd = "sed -e 's:^:%s:g' %s | xargs %s" % (sstateinst, fixmefn, sstate_sed_cmd) + + # Defer do_populate_sysroot relocation command + if sstatefixmedir: + bb.utils.mkdirhier(sstatefixmedir) + with open(sstatefixmedir + "/fixmepath.cmd", "w") as f: + sstate_hardcode_cmd = sstate_hardcode_cmd.replace(fixmefn, sstatefixmedir + "/fixmepath") + sstate_hardcode_cmd = sstate_hardcode_cmd.replace(sstateinst, "FIXMEFINALSSTATEINST") + sstate_hardcode_cmd = sstate_hardcode_cmd.replace(staging_host, "FIXMEFINALSSTATEHOST") + sstate_hardcode_cmd = sstate_hardcode_cmd.replace(staging_target, "FIXMEFINALSSTATETARGET") + f.write(sstate_hardcode_cmd) + bb.utils.copyfile(fixmefn, sstatefixmedir + "/fixmepath") + return + + bb.note("Replacing fixme paths in sstate package: %s" % (sstate_hardcode_cmd)) + subprocess.check_call(sstate_hardcode_cmd, shell=True) + + # Need to remove this or we'd copy it into the target directory and may + # conflict with another writer + os.remove(fixmefn) +} + +def sstate_clean_cachefile(ss, d): + import oe.path + + if d.getVarFlag('do_%s' % ss['task'], 'task'): + d.setVar("SSTATE_PATH_CURRTASK", ss['task']) + sstatepkgfile = d.getVar('SSTATE_PATHSPEC') + bb.note("Removing %s" % sstatepkgfile) + oe.path.remove(sstatepkgfile) + +def sstate_clean_cachefiles(d): + for task in (d.getVar('SSTATETASKS') or "").split(): + ld = d.createCopy() + ss = sstate_state_fromvars(ld, task) + sstate_clean_cachefile(ss, ld) + +def sstate_clean_manifest(manifest, d, canrace=False, prefix=None): + import oe.path + + mfile = open(manifest) + entries = mfile.readlines() + mfile.close() + + for entry in entries: + entry = entry.strip() + if prefix and not entry.startswith("/"): + entry = prefix + "/" + entry + bb.debug(2, "Removing manifest: %s" % entry) + # We can race against another package populating directories as we're removing them + # so we ignore errors here. + try: + if entry.endswith("/"): + if os.path.islink(entry[:-1]): + os.remove(entry[:-1]) + elif os.path.exists(entry) and len(os.listdir(entry)) == 0 and not canrace: + # Removing directories whilst builds are in progress exposes a race. Only + # do it in contexts where it is safe to do so. + os.rmdir(entry[:-1]) + else: + os.remove(entry) + except OSError: + pass + + postrm = manifest + ".postrm" + if os.path.exists(manifest + ".postrm"): + import subprocess + os.chmod(postrm, 0o755) + subprocess.check_call(postrm, shell=True) + oe.path.remove(postrm) + + oe.path.remove(manifest) + +def sstate_clean(ss, d): + import oe.path + import glob + + d2 = d.createCopy() + stamp_clean = d.getVar("STAMPCLEAN") + extrainf = d.getVarFlag("do_" + ss['task'], 'stamp-extra-info') + if extrainf: + d2.setVar("SSTATE_MANMACH", extrainf) + wildcard_stfile = "%s.do_%s*.%s" % (stamp_clean, ss['task'], extrainf) + else: + wildcard_stfile = "%s.do_%s*" % (stamp_clean, ss['task']) + + manifest = d2.expand("${SSTATE_MANFILEPREFIX}.%s" % ss['task']) + + if os.path.exists(manifest): + locks = [] + for lock in ss['lockfiles-shared']: + locks.append(bb.utils.lockfile(lock)) + for lock in ss['lockfiles']: + locks.append(bb.utils.lockfile(lock)) + + sstate_clean_manifest(manifest, d, canrace=True) + + for lock in locks: + bb.utils.unlockfile(lock) + + # Remove the current and previous stamps, but keep the sigdata. + # + # The glob() matches do_task* which may match multiple tasks, for + # example: do_package and do_package_write_ipk, so we need to + # exactly match *.do_task.* and *.do_task_setscene.* + rm_stamp = '.do_%s.' % ss['task'] + rm_setscene = '.do_%s_setscene.' % ss['task'] + # For BB_SIGNATURE_HANDLER = "noop" + rm_nohash = ".do_%s" % ss['task'] + for stfile in glob.glob(wildcard_stfile): + # Keep the sigdata + if ".sigdata." in stfile or ".sigbasedata." in stfile: + continue + # Preserve taint files in the stamps directory + if stfile.endswith('.taint'): + continue + if rm_stamp in stfile or rm_setscene in stfile or \ + stfile.endswith(rm_nohash): + oe.path.remove(stfile) + +sstate_clean[vardepsexclude] = "SSTATE_MANFILEPREFIX" + +CLEANFUNCS += "sstate_cleanall" + +python sstate_cleanall() { + bb.note("Removing shared state for package %s" % d.getVar('PN')) + + manifest_dir = d.getVar('SSTATE_MANIFESTS') + if not os.path.exists(manifest_dir): + return + + tasks = d.getVar('SSTATETASKS').split() + for name in tasks: + ld = d.createCopy() + shared_state = sstate_state_fromvars(ld, name) + sstate_clean(shared_state, ld) +} + +python sstate_hardcode_path () { + import subprocess, platform + + # Need to remove hardcoded paths and fix these when we install the + # staging packages. + # + # Note: the logic in this function needs to match the reverse logic + # in sstate_installpkg(ss, d) + + staging_target = d.getVar('RECIPE_SYSROOT') + staging_host = d.getVar('RECIPE_SYSROOT_NATIVE') + sstate_builddir = d.getVar('SSTATE_BUILDDIR') + + sstate_sed_cmd = "sed -i -e 's:%s:FIXMESTAGINGDIRHOST:g'" % staging_host + if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross-canadian', d): + sstate_grep_cmd = "grep -l -e '%s'" % (staging_host) + elif bb.data.inherits_class('cross', d) or bb.data.inherits_class('crosssdk', d): + sstate_grep_cmd = "grep -l -e '%s' -e '%s'" % (staging_target, staging_host) + sstate_sed_cmd += " -e 's:%s:FIXMESTAGINGDIRTARGET:g'" % staging_target + else: + sstate_grep_cmd = "grep -l -e '%s' -e '%s'" % (staging_target, staging_host) + sstate_sed_cmd += " -e 's:%s:FIXMESTAGINGDIRTARGET:g'" % staging_target + + extra_staging_fixmes = d.getVar('EXTRA_STAGING_FIXMES') or '' + for fixmevar in extra_staging_fixmes.split(): + fixme_path = d.getVar(fixmevar) + sstate_sed_cmd += " -e 's:%s:FIXME_%s:g'" % (fixme_path, fixmevar) + sstate_grep_cmd += " -e '%s'" % (fixme_path) + + fixmefn = sstate_builddir + "fixmepath" + + sstate_scan_cmd = d.getVar('SSTATE_SCAN_CMD') + sstate_filelist_cmd = "tee %s" % (fixmefn) + + # fixmepath file needs relative paths, drop sstate_builddir prefix + sstate_filelist_relative_cmd = "sed -i -e 's:^%s::g' %s" % (sstate_builddir, fixmefn) + + xargs_no_empty_run_cmd = '--no-run-if-empty' + if platform.system() == 'Darwin': + xargs_no_empty_run_cmd = '' + + # Limit the fixpaths and sed operations based on the initial grep search + # This has the side effect of making sure the vfs cache is hot + sstate_hardcode_cmd = "%s | xargs %s | %s | xargs %s %s" % (sstate_scan_cmd, sstate_grep_cmd, sstate_filelist_cmd, xargs_no_empty_run_cmd, sstate_sed_cmd) + + bb.note("Removing hardcoded paths from sstate package: '%s'" % (sstate_hardcode_cmd)) + subprocess.check_output(sstate_hardcode_cmd, shell=True, cwd=sstate_builddir) + + # If the fixmefn is empty, remove it.. + if os.stat(fixmefn).st_size == 0: + os.remove(fixmefn) + else: + bb.note("Replacing absolute paths in fixmepath file: '%s'" % (sstate_filelist_relative_cmd)) + subprocess.check_output(sstate_filelist_relative_cmd, shell=True) +} + +def sstate_package(ss, d): + import oe.path + import time + + tmpdir = d.getVar('TMPDIR') + + fixtime = False + if ss['task'] == "package": + fixtime = True + + def fixtimestamp(root, path): + f = os.path.join(root, path) + if os.lstat(f).st_mtime > sde: + os.utime(f, (sde, sde), follow_symlinks=False) + + sstatebuild = d.expand("${WORKDIR}/sstate-build-%s/" % ss['task']) + sde = int(d.getVar("SOURCE_DATE_EPOCH") or time.time()) + d.setVar("SSTATE_CURRTASK", ss['task']) + bb.utils.remove(sstatebuild, recurse=True) + bb.utils.mkdirhier(sstatebuild) + for state in ss['dirs']: + if not os.path.exists(state[1]): + continue + srcbase = state[0].rstrip("/").rsplit('/', 1)[0] + # Find and error for absolute symlinks. We could attempt to relocate but its not + # clear where the symlink is relative to in this context. We could add that markup + # to sstate tasks but there aren't many of these so better just avoid them entirely. + for walkroot, dirs, files in os.walk(state[1]): + for file in files + dirs: + if fixtime: + fixtimestamp(walkroot, file) + srcpath = os.path.join(walkroot, file) + if not os.path.islink(srcpath): + continue + link = os.readlink(srcpath) + if not os.path.isabs(link): + continue + if not link.startswith(tmpdir): + continue + bb.error("sstate found an absolute path symlink %s pointing at %s. Please replace this with a relative link." % (srcpath, link)) + bb.debug(2, "Preparing tree %s for packaging at %s" % (state[1], sstatebuild + state[0])) + bb.utils.rename(state[1], sstatebuild + state[0]) + + workdir = d.getVar('WORKDIR') + sharedworkdir = os.path.join(d.getVar('TMPDIR'), "work-shared") + for plain in ss['plaindirs']: + pdir = plain.replace(workdir, sstatebuild) + if sharedworkdir in plain: + pdir = plain.replace(sharedworkdir, sstatebuild) + bb.utils.mkdirhier(plain) + bb.utils.mkdirhier(pdir) + bb.utils.rename(plain, pdir) + if fixtime: + fixtimestamp(pdir, "") + for walkroot, dirs, files in os.walk(pdir): + for file in files + dirs: + fixtimestamp(walkroot, file) + + d.setVar('SSTATE_BUILDDIR', sstatebuild) + d.setVar('SSTATE_INSTDIR', sstatebuild) + + if d.getVar('SSTATE_SKIP_CREATION') == '1': + return + + sstate_create_package = ['sstate_report_unihash', 'sstate_create_and_sign_package'] + + for f in (d.getVar('SSTATECREATEFUNCS') or '').split() + \ + sstate_create_package + \ + (d.getVar('SSTATEPOSTCREATEFUNCS') or '').split(): + # All hooks should run in SSTATE_BUILDDIR. + bb.build.exec_func(f, d, (sstatebuild,)) + + # SSTATE_PKG may have been changed by sstate_report_unihash + siginfo = d.getVar('SSTATE_PKG') + ".siginfo" + if not os.path.exists(siginfo): + bb.siggen.dump_this_task(siginfo, d) + else: + try: + os.utime(siginfo, None) + except PermissionError: + pass + except OSError as e: + # Handle read-only file systems gracefully + import errno + if e.errno != errno.EROFS: + raise e + + return + +sstate_package[vardepsexclude] += "SSTATE_SIG_KEY" + +def pstaging_fetch(sstatefetch, d): + import bb.fetch2 + + # Only try and fetch if the user has configured a mirror + mirrors = d.getVar('SSTATE_MIRRORS') + if not mirrors: + return + + # Copy the data object and override DL_DIR and SRC_URI + localdata = bb.data.createCopy(d) + + dldir = localdata.expand("${SSTATE_DIR}") + bb.utils.mkdirhier(dldir) + + localdata.delVar('MIRRORS') + localdata.setVar('FILESPATH', dldir) + localdata.setVar('DL_DIR', dldir) + localdata.setVar('PREMIRRORS', mirrors) + + # if BB_NO_NETWORK is set but we also have SSTATE_MIRROR_ALLOW_NETWORK, + # we'll want to allow network access for the current set of fetches. + if bb.utils.to_boolean(localdata.getVar('BB_NO_NETWORK')) and \ + bb.utils.to_boolean(localdata.getVar('SSTATE_MIRROR_ALLOW_NETWORK')): + localdata.delVar('BB_NO_NETWORK') + + # Try a fetch from the sstate mirror, if it fails just return and + # we will build the package + uris = ['file://{0};downloadfilename={0}'.format(sstatefetch), + 'file://{0}.siginfo;downloadfilename={0}.siginfo'.format(sstatefetch)] + if bb.utils.to_boolean(d.getVar("SSTATE_VERIFY_SIG"), False): + uris += ['file://{0}.sig;downloadfilename={0}.sig'.format(sstatefetch)] + + for srcuri in uris: + localdata.delVar('SRC_URI') + localdata.setVar('SRC_URI', srcuri) + try: + fetcher = bb.fetch2.Fetch([srcuri], localdata, cache=False) + fetcher.checkstatus() + fetcher.download() + + except bb.fetch2.BBFetchException: + pass + +def sstate_setscene(d): + shared_state = sstate_state_fromvars(d) + accelerate = sstate_installpkg(shared_state, d) + if not accelerate: + msg = "No sstate archive obtainable, will run full task instead." + bb.warn(msg) + raise bb.BBHandledException(msg) + +python sstate_task_prefunc () { + shared_state = sstate_state_fromvars(d) + sstate_clean(shared_state, d) +} +sstate_task_prefunc[dirs] = "${WORKDIR}" + +python sstate_task_postfunc () { + shared_state = sstate_state_fromvars(d) + + for intercept in shared_state['interceptfuncs']: + bb.build.exec_func(intercept, d, (d.getVar("WORKDIR"),)) + + omask = os.umask(0o002) + if omask != 0o002: + bb.note("Using umask 0o002 (not %0o) for sstate packaging" % omask) + sstate_package(shared_state, d) + os.umask(omask) + + sstateinst = d.getVar("SSTATE_INSTDIR") + d.setVar('SSTATE_FIXMEDIR', shared_state['fixmedir']) + + sstate_installpkgdir(shared_state, d) + + bb.utils.remove(d.getVar("SSTATE_BUILDDIR"), recurse=True) +} +sstate_task_postfunc[dirs] = "${WORKDIR}" + +# Create a sstate package +# If enabled, sign the package. +# Package and signature are created in a sub-directory +# and renamed in place once created. +python sstate_create_and_sign_package () { + from pathlib import Path + + # Best effort touch + def touch(file): + try: + file.touch() + except: + pass + + def update_file(src, dst, force=False): + if dst.is_symlink() and not dst.exists(): + force=True + try: + # This relies on that src is a temporary file that can be renamed + # or left as is. + if force: + src.rename(dst) + else: + os.link(src, dst) + return True + except: + pass + + if dst.exists(): + touch(dst) + + return False + + sign_pkg = ( + bb.utils.to_boolean(d.getVar("SSTATE_VERIFY_SIG")) and + bool(d.getVar("SSTATE_SIG_KEY")) + ) + + sstate_pkg = Path(d.getVar("SSTATE_PKG")) + sstate_pkg_sig = Path(str(sstate_pkg) + ".sig") + if sign_pkg: + if sstate_pkg.exists() and sstate_pkg_sig.exists(): + touch(sstate_pkg) + touch(sstate_pkg_sig) + return + else: + if sstate_pkg.exists(): + touch(sstate_pkg) + return + + # Create the required sstate directory if it is not present. + if not sstate_pkg.parent.is_dir(): + with bb.utils.umask(0o002): + bb.utils.mkdirhier(str(sstate_pkg.parent)) + + if sign_pkg: + from tempfile import TemporaryDirectory + with TemporaryDirectory(dir=sstate_pkg.parent) as tmp_dir: + tmp_pkg = Path(tmp_dir) / sstate_pkg.name + d.setVar("TMP_SSTATE_PKG", str(tmp_pkg)) + bb.build.exec_func('sstate_archive_package', d) + + from oe.gpg_sign import get_signer + signer = get_signer(d, 'local') + signer.detach_sign(str(tmp_pkg), d.getVar('SSTATE_SIG_KEY'), None, + d.getVar('SSTATE_SIG_PASSPHRASE'), armor=False) + + tmp_pkg_sig = Path(tmp_dir) / sstate_pkg_sig.name + if not update_file(tmp_pkg_sig, sstate_pkg_sig): + # If the created signature file could not be copied into place, + # then we should not use the sstate package either. + return + + # If the .sig file was updated, then the sstate package must also + # be updated. + update_file(tmp_pkg, sstate_pkg, force=True) + else: + from tempfile import NamedTemporaryFile + with NamedTemporaryFile(prefix=sstate_pkg.name, dir=sstate_pkg.parent) as tmp_pkg_fd: + tmp_pkg = tmp_pkg_fd.name + d.setVar("TMP_SSTATE_PKG", str(tmp_pkg)) + bb.build.exec_func('sstate_archive_package',d) + update_file(tmp_pkg, sstate_pkg) + # update_file() may have renamed tmp_pkg, which must exist when the + # NamedTemporaryFile() context handler ends. + touch(Path(tmp_pkg)) + +} + +# Shell function to generate a sstate package from a directory +# set as SSTATE_BUILDDIR. Will be run from within SSTATE_BUILDDIR. +# The calling function handles moving the sstate package into the final +# destination. +sstate_archive_package () { + OPT="-cS" + ZSTD="zstd -${SSTATE_ZSTD_CLEVEL} -T${ZSTD_THREADS}" + # Use pzstd if available + if [ -x "$(command -v pzstd)" ]; then + ZSTD="pzstd -${SSTATE_ZSTD_CLEVEL} -p ${ZSTD_THREADS}" + fi + + # Need to handle empty directories + if [ "$(ls -A)" ]; then + set +e + tar -I "$ZSTD" $OPT -f ${TMP_SSTATE_PKG} * + ret=$? + if [ $ret -ne 0 ] && [ $ret -ne 1 ]; then + exit 1 + fi + set -e + else + tar -I "$ZSTD" $OPT --file=${TMP_SSTATE_PKG} --files-from=/dev/null + fi + chmod 0664 ${TMP_SSTATE_PKG} +} + + +python sstate_report_unihash() { + report_unihash = getattr(bb.parse.siggen, 'report_unihash', None) + + if report_unihash: + ss = sstate_state_fromvars(d) + report_unihash(os.getcwd(), ss['task'], d) +} + +# +# Shell function to decompress and prepare a package for installation +# Will be run from within SSTATE_INSTDIR. +# +sstate_unpack_package () { + ZSTD="zstd -T${ZSTD_THREADS}" + # Use pzstd if available + if [ -x "$(command -v pzstd)" ]; then + ZSTD="pzstd -p ${ZSTD_THREADS}" + fi + + tar -I "$ZSTD" -xvpf ${SSTATE_PKG} + # update .siginfo atime on local/NFS mirror if it is a symbolic link + [ ! -h ${SSTATE_PKG}.siginfo ] || [ ! -e ${SSTATE_PKG}.siginfo ] || touch -a ${SSTATE_PKG}.siginfo 2>/dev/null || true + # update each symbolic link instead of any referenced file + touch --no-dereference ${SSTATE_PKG} 2>/dev/null || true + [ ! -e ${SSTATE_PKG}.sig ] || touch --no-dereference ${SSTATE_PKG}.sig 2>/dev/null || true + [ ! -e ${SSTATE_PKG}.siginfo ] || touch --no-dereference ${SSTATE_PKG}.siginfo 2>/dev/null || true +} + +BB_HASHCHECK_FUNCTION = "sstate_checkhashes" + +def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, **kwargs): + import itertools + + found = set() + missed = set() + + def gethash(task): + return sq_data['unihash'][task] + + def getpathcomponents(task, d): + # Magic data from BB_HASHFILENAME + splithashfn = sq_data['hashfn'][task].split(" ") + spec = splithashfn[1] + if splithashfn[0] == "True": + extrapath = d.getVar("NATIVELSBSTRING") + "/" + else: + extrapath = "" + + tname = bb.runqueue.taskname_from_tid(task)[3:] + + if tname in ["fetch", "unpack", "patch", "populate_lic", "preconfigure"] and splithashfn[2]: + spec = splithashfn[2] + extrapath = "" + + return spec, extrapath, tname + + def getsstatefile(tid, siginfo, d): + spec, extrapath, tname = getpathcomponents(tid, d) + return extrapath + generate_sstatefn(spec, gethash(tid), tname, siginfo, d) + + for tid in sq_data['hash']: + + sstatefile = d.expand("${SSTATE_DIR}/" + getsstatefile(tid, siginfo, d)) + + if os.path.exists(sstatefile): + oe.utils.touch(sstatefile) + found.add(tid) + bb.debug(2, "SState: Found valid sstate file %s" % sstatefile) + else: + missed.add(tid) + bb.debug(2, "SState: Looked for but didn't find file %s" % sstatefile) + + foundLocal = len(found) + mirrors = d.getVar("SSTATE_MIRRORS") + if mirrors: + # Copy the data object and override DL_DIR and SRC_URI + localdata = bb.data.createCopy(d) + + dldir = localdata.expand("${SSTATE_DIR}") + localdata.delVar('MIRRORS') + localdata.setVar('FILESPATH', dldir) + localdata.setVar('DL_DIR', dldir) + localdata.setVar('PREMIRRORS', mirrors) + + bb.debug(2, "SState using premirror of: %s" % mirrors) + + # if BB_NO_NETWORK is set but we also have SSTATE_MIRROR_ALLOW_NETWORK, + # we'll want to allow network access for the current set of fetches. + if bb.utils.to_boolean(localdata.getVar('BB_NO_NETWORK')) and \ + bb.utils.to_boolean(localdata.getVar('SSTATE_MIRROR_ALLOW_NETWORK')): + localdata.delVar('BB_NO_NETWORK') + + from bb.fetch2 import FetchConnectionCache + def checkstatus_init(): + while not connection_cache_pool.full(): + connection_cache_pool.put(FetchConnectionCache()) + + def checkstatus_end(): + while not connection_cache_pool.empty(): + connection_cache = connection_cache_pool.get() + connection_cache.close_connections() + + def checkstatus(arg): + (tid, sstatefile) = arg + + connection_cache = connection_cache_pool.get() + localdata2 = bb.data.createCopy(localdata) + srcuri = "file://" + sstatefile + localdata2.setVar('SRC_URI', srcuri) + bb.debug(2, "SState: Attempting to fetch %s" % srcuri) + + import traceback + + try: + fetcher = bb.fetch2.Fetch(srcuri.split(), localdata2, + connection_cache=connection_cache) + fetcher.checkstatus() + bb.debug(2, "SState: Successful fetch test for %s" % srcuri) + found.add(tid) + missed.remove(tid) + except bb.fetch2.FetchError as e: + bb.debug(2, "SState: Unsuccessful fetch test for %s (%s)\n%s" % (srcuri, repr(e), traceback.format_exc())) + except Exception as e: + bb.error("SState: cannot test %s: %s\n%s" % (srcuri, repr(e), traceback.format_exc())) + + connection_cache_pool.put(connection_cache) + + if progress: + bb.event.fire(bb.event.ProcessProgress(msg, next(cnt_tasks_done)), d) + bb.event.check_for_interrupts(d) + + tasklist = [] + for tid in missed: + sstatefile = d.expand(getsstatefile(tid, siginfo, d)) + tasklist.append((tid, sstatefile)) + + if tasklist: + nproc = min(int(d.getVar("BB_NUMBER_THREADS")), len(tasklist)) + + ## thread-safe counter + cnt_tasks_done = itertools.count(start = 1) + progress = len(tasklist) >= 100 + if progress: + msg = "Checking sstate mirror object availability" + bb.event.fire(bb.event.ProcessStarted(msg, len(tasklist)), d) + + # Have to setup the fetcher environment here rather than in each thread as it would race + fetcherenv = bb.fetch2.get_fetcher_environment(d) + with bb.utils.environment(**fetcherenv): + bb.event.enable_threadlock() + import concurrent.futures + from queue import Queue + connection_cache_pool = Queue(nproc) + checkstatus_init() + with concurrent.futures.ThreadPoolExecutor(max_workers=nproc) as executor: + executor.map(checkstatus, tasklist.copy()) + checkstatus_end() + bb.event.disable_threadlock() + + if progress: + bb.event.fire(bb.event.ProcessFinished(msg), d) + + inheritlist = d.getVar("INHERIT") + if "toaster" in inheritlist: + evdata = {'missed': [], 'found': []}; + for tid in missed: + sstatefile = d.expand(getsstatefile(tid, False, d)) + evdata['missed'].append((bb.runqueue.fn_from_tid(tid), bb.runqueue.taskname_from_tid(tid), gethash(tid), sstatefile ) ) + for tid in found: + sstatefile = d.expand(getsstatefile(tid, False, d)) + evdata['found'].append((bb.runqueue.fn_from_tid(tid), bb.runqueue.taskname_from_tid(tid), gethash(tid), sstatefile ) ) + bb.event.fire(bb.event.MetadataEvent("MissedSstate", evdata), d) + + if summary: + # Print some summary statistics about the current task completion and how much sstate + # reuse there was. Avoid divide by zero errors. + total = len(sq_data['hash']) + complete = 0 + if currentcount: + complete = (len(found) + currentcount) / (total + currentcount) * 100 + match = 0 + if total: + match = len(found) / total * 100 + bb.plain("Sstate summary: Wanted %d Local %d Mirrors %d Missed %d Current %d (%d%% match, %d%% complete)" % + (total, foundLocal, len(found)-foundLocal, len(missed), currentcount, match, complete)) + + if hasattr(bb.parse.siggen, "checkhashes"): + bb.parse.siggen.checkhashes(sq_data, missed, found, d) + + return found +setscene_depvalid[vardepsexclude] = "SSTATE_EXCLUDEDEPS_SYSROOT" + +BB_SETSCENE_DEPVALID = "setscene_depvalid" + +def setscene_depvalid(task, taskdependees, notneeded, d, log=None): + # taskdependees is a dict of tasks which depend on task, each being a 3 item list of [PN, TASKNAME, FILENAME] + # task is included in taskdependees too + # Return - False - We need this dependency + # - True - We can skip this dependency + import re + + def logit(msg, log): + if log is not None: + log.append(msg) + else: + bb.debug(2, msg) + + logit("Considering setscene task: %s" % (str(taskdependees[task])), log) + + directtasks = ["do_populate_lic", "do_deploy_source_date_epoch", "do_shared_workdir", "do_stash_locale", "do_gcc_stash_builddir", "do_create_spdx", "do_deploy_archives"] + + def isNativeCross(x): + return x.endswith("-native") or "-cross-" in x or "-crosssdk" in x or x.endswith("-cross") + + # We only need to trigger deploy_source_date_epoch through direct dependencies + if taskdependees[task][1] in directtasks: + return True + + # We only need to trigger packagedata through direct dependencies + # but need to preserve packagedata on packagedata links + if taskdependees[task][1] == "do_packagedata": + for dep in taskdependees: + if taskdependees[dep][1] == "do_packagedata": + return False + return True + + for dep in taskdependees: + logit(" considering dependency: %s" % (str(taskdependees[dep])), log) + if task == dep: + continue + if dep in notneeded: + continue + # do_package_write_* and do_package doesn't need do_package + if taskdependees[task][1] == "do_package" and taskdependees[dep][1] in ['do_package', 'do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package_qa']: + continue + # do_package_write_* need do_populate_sysroot as they're mainly postinstall dependencies + if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm']: + return False + # do_package/packagedata/package_qa/deploy don't need do_populate_sysroot + if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package', 'do_packagedata', 'do_package_qa', 'do_deploy']: + continue + # Native/Cross packages don't exist and are noexec anyway + if isNativeCross(taskdependees[dep][0]) and taskdependees[dep][1] in ['do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package', 'do_package_qa']: + continue + + # Consider sysroot depending on sysroot tasks + if taskdependees[task][1] == 'do_populate_sysroot' and taskdependees[dep][1] == 'do_populate_sysroot': + # Allow excluding certain recursive dependencies. If a recipe needs it should add a + # specific dependency itself, rather than relying on one of its dependees to pull + # them in. + # See also http://lists.openembedded.org/pipermail/openembedded-core/2018-January/146324.html + not_needed = False + excludedeps = d.getVar('_SSTATE_EXCLUDEDEPS_SYSROOT') + if excludedeps is None: + # Cache the regular expressions for speed + excludedeps = [] + for excl in (d.getVar('SSTATE_EXCLUDEDEPS_SYSROOT') or "").split(): + excludedeps.append((re.compile(excl.split('->', 1)[0]), re.compile(excl.split('->', 1)[1]))) + d.setVar('_SSTATE_EXCLUDEDEPS_SYSROOT', excludedeps) + for excl in excludedeps: + if excl[0].match(taskdependees[dep][0]): + if excl[1].match(taskdependees[task][0]): + not_needed = True + break + if not_needed: + continue + # For meta-extsdk-toolchain we want all sysroot dependencies + if taskdependees[dep][0] == 'meta-extsdk-toolchain': + return False + # Native/Cross populate_sysroot need their dependencies + if isNativeCross(taskdependees[task][0]) and isNativeCross(taskdependees[dep][0]): + return False + # Target populate_sysroot depended on by cross tools need to be installed + if isNativeCross(taskdependees[dep][0]): + return False + # Native/cross tools depended upon by target sysroot are not needed + # Add an exception for shadow-native as required by useradd.bbclass + if isNativeCross(taskdependees[task][0]) and taskdependees[task][0] != 'shadow-native': + continue + # Target populate_sysroot need their dependencies + return False + + if taskdependees[dep][1] in directtasks: + continue + + # Safe fallthrough default + logit(" Default setscene dependency fall through due to dependency: %s" % (str(taskdependees[dep])), log) + return False + return True + +addhandler sstate_eventhandler +sstate_eventhandler[eventmask] = "bb.build.TaskSucceeded" +python sstate_eventhandler() { + d = e.data + writtensstate = d.getVar('SSTATE_CURRTASK') + if not writtensstate: + taskname = d.getVar("BB_RUNTASK")[3:] + spec = d.getVar('SSTATE_PKGSPEC') + swspec = d.getVar('SSTATE_SWSPEC') + if taskname in ["fetch", "unpack", "patch", "populate_lic", "preconfigure"] and swspec: + d.setVar("SSTATE_PKGSPEC", "${SSTATE_SWSPEC}") + d.setVar("SSTATE_EXTRAPATH", "") + d.setVar("SSTATE_CURRTASK", taskname) + siginfo = d.getVar('SSTATE_PKG') + ".siginfo" + if not os.path.exists(siginfo): + bb.siggen.dump_this_task(siginfo, d) + else: + oe.utils.touch(siginfo) +} + +SSTATE_PRUNE_OBSOLETEWORKDIR ?= "1" + +# +# Event handler which removes manifests and stamps file for recipes which are no +# longer 'reachable' in a build where they once were. 'Reachable' refers to +# whether a recipe is parsed so recipes in a layer which was removed would no +# longer be reachable. Switching between systemd and sysvinit where recipes +# became skipped would be another example. +# +# Also optionally removes the workdir of those tasks/recipes +# +addhandler sstate_eventhandler_reachablestamps +sstate_eventhandler_reachablestamps[eventmask] = "bb.event.ReachableStamps" +python sstate_eventhandler_reachablestamps() { + import glob + d = e.data + stamps = e.stamps.values() + removeworkdir = (d.getVar("SSTATE_PRUNE_OBSOLETEWORKDIR", False) == "1") + preservestampfile = d.expand('${SSTATE_MANIFESTS}/preserve-stamps') + preservestamps = [] + if os.path.exists(preservestampfile): + with open(preservestampfile, 'r') as f: + preservestamps = f.readlines() + seen = [] + + # The machine index contains all the stamps this machine has ever seen in this build directory. + # We should only remove things which this machine once accessed but no longer does. + machineindex = set() + bb.utils.mkdirhier(d.expand("${SSTATE_MANIFESTS}")) + mi = d.expand("${SSTATE_MANIFESTS}/index-machine-${MACHINE}") + if os.path.exists(mi): + with open(mi, "r") as f: + machineindex = set(line.strip() for line in f.readlines()) + + for a in sorted(list(set(d.getVar("SSTATE_ARCHS").split()))): + toremove = [] + i = d.expand("${SSTATE_MANIFESTS}/index-" + a) + if not os.path.exists(i): + continue + manseen = set() + ignore = [] + with open(i, "r") as f: + lines = f.readlines() + for l in reversed(lines): + try: + (stamp, manifest, workdir) = l.split() + # The index may have multiple entries for the same manifest as the code above only appends + # new entries and there may be an entry with matching manifest but differing version in stamp/workdir. + # The last entry in the list is the valid one, any earlier entries with matching manifests + # should be ignored. + if manifest in manseen: + ignore.append(l) + continue + manseen.add(manifest) + if stamp not in stamps and stamp not in preservestamps and stamp in machineindex: + toremove.append(l) + if stamp not in seen: + bb.debug(2, "Stamp %s is not reachable, removing related manifests" % stamp) + seen.append(stamp) + except ValueError: + bb.fatal("Invalid line '%s' in sstate manifest '%s'" % (l, i)) + + if toremove: + msg = "Removing %d recipes from the %s sysroot" % (len(toremove), a) + bb.event.fire(bb.event.ProcessStarted(msg, len(toremove)), d) + + removed = 0 + for r in toremove: + (stamp, manifest, workdir) = r.split() + for m in glob.glob(manifest + ".*"): + if m.endswith(".postrm"): + continue + sstate_clean_manifest(m, d) + bb.utils.remove(stamp + "*") + if removeworkdir: + bb.utils.remove(workdir, recurse = True) + lines.remove(r) + removed = removed + 1 + bb.event.fire(bb.event.ProcessProgress(msg, removed), d) + bb.event.check_for_interrupts(d) + + bb.event.fire(bb.event.ProcessFinished(msg), d) + + with open(i, "w") as f: + for l in lines: + if l in ignore: + continue + f.write(l) + machineindex |= set(stamps) + with open(mi, "w") as f: + for l in machineindex: + f.write(l + "\n") + + if preservestamps: + os.remove(preservestampfile) +} + + +# +# Bitbake can generate an event showing which setscene tasks are 'stale', +# i.e. which ones will be rerun. These are ones where a stamp file is present but +# it is stable (e.g. taskhash doesn't match). With that list we can go through +# the manifests for matching tasks and "uninstall" those manifests now. We do +# this now rather than mid build since the distribution of files between sstate +# objects may have changed, new tasks may run first and if those new tasks overlap +# with the stale tasks, we'd see overlapping files messages and failures. Thankfully +# removing these files is fast. +# +addhandler sstate_eventhandler_stalesstate +sstate_eventhandler_stalesstate[eventmask] = "bb.event.StaleSetSceneTasks" +python sstate_eventhandler_stalesstate() { + d = e.data + tasks = e.tasks + + bb.utils.mkdirhier(d.expand("${SSTATE_MANIFESTS}")) + + for a in list(set(d.getVar("SSTATE_ARCHS").split())): + toremove = [] + i = d.expand("${SSTATE_MANIFESTS}/index-" + a) + if not os.path.exists(i): + continue + with open(i, "r") as f: + lines = f.readlines() + for l in lines: + try: + (stamp, manifest, workdir) = l.split() + for tid in tasks: + for s in tasks[tid]: + if s.startswith(stamp): + taskname = bb.runqueue.taskname_from_tid(tid)[3:] + manname = manifest + "." + taskname + if os.path.exists(manname): + bb.debug(2, "Sstate for %s is stale, removing related manifest %s" % (tid, manname)) + toremove.append((manname, tid, tasks[tid])) + break + except ValueError: + bb.fatal("Invalid line '%s' in sstate manifest '%s'" % (l, i)) + + if toremove: + msg = "Removing %d stale sstate objects for arch %s" % (len(toremove), a) + bb.event.fire(bb.event.ProcessStarted(msg, len(toremove)), d) + + removed = 0 + for (manname, tid, stamps) in toremove: + sstate_clean_manifest(manname, d) + for stamp in stamps: + bb.utils.remove(stamp) + removed = removed + 1 + bb.event.fire(bb.event.ProcessProgress(msg, removed), d) + bb.event.check_for_interrupts(d) + + bb.event.fire(bb.event.ProcessFinished(msg), d) +} diff --git a/meta/classes-global/staging.bbclass b/meta/classes-global/staging.bbclass new file mode 100644 index 0000000000..3678a1b441 --- /dev/null +++ b/meta/classes-global/staging.bbclass @@ -0,0 +1,695 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +# These directories will be staged in the sysroot +SYSROOT_DIRS = " \ + ${includedir} \ + ${libdir} \ + ${base_libdir} \ + ${nonarch_base_libdir} \ + ${datadir} \ + /sysroot-only \ +" + +# These directories are also staged in the sysroot when they contain files that +# are usable on the build system +SYSROOT_DIRS_NATIVE = " \ + ${bindir} \ + ${sbindir} \ + ${base_bindir} \ + ${base_sbindir} \ + ${libexecdir} \ + ${sysconfdir} \ + ${localstatedir} \ +" +SYSROOT_DIRS:append:class-native = " ${SYSROOT_DIRS_NATIVE}" +SYSROOT_DIRS:append:class-cross = " ${SYSROOT_DIRS_NATIVE}" +SYSROOT_DIRS:append:class-crosssdk = " ${SYSROOT_DIRS_NATIVE}" + +# These directories will not be staged in the sysroot +SYSROOT_DIRS_IGNORE = " \ + ${mandir} \ + ${docdir} \ + ${infodir} \ + ${datadir}/X11/locale \ + ${datadir}/applications \ + ${datadir}/bash-completion \ + ${datadir}/fonts \ + ${datadir}/gtk-doc/html \ + ${datadir}/installed-tests \ + ${datadir}/locale \ + ${datadir}/pixmaps \ + ${datadir}/terminfo \ + ${libdir}/${BPN}/ptest \ +" + +sysroot_stage_dir() { + src="$1" + dest="$2" + # if the src doesn't exist don't do anything + if [ ! -d "$src" ]; then + return + fi + + mkdir -p "$dest" + rdest=$(realpath --relative-to="$src" "$dest") + ( + cd $src + find . -print0 | cpio --null -pdlu $rdest + ) +} + +sysroot_stage_dirs() { + from="$1" + to="$2" + + for dir in ${SYSROOT_DIRS}; do + sysroot_stage_dir "$from$dir" "$to$dir" + done + + # Remove directories we do not care about + for dir in ${SYSROOT_DIRS_IGNORE}; do + rm -rf "$to$dir" + done +} + +sysroot_stage_all() { + sysroot_stage_dirs ${D} ${SYSROOT_DESTDIR} +} + +python sysroot_strip () { + inhibit_sysroot = d.getVar('INHIBIT_SYSROOT_STRIP') + if inhibit_sysroot and oe.types.boolean(inhibit_sysroot): + return + + dstdir = d.getVar('SYSROOT_DESTDIR') + pn = d.getVar('PN') + libdir = d.getVar("libdir") + base_libdir = d.getVar("base_libdir") + qa_already_stripped = 'already-stripped' in (d.getVar('INSANE_SKIP:' + pn) or "").split() + strip_cmd = d.getVar("STRIP") + + max_process = oe.utils.get_bb_number_threads(d) + oe.package.strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, max_process, + qa_already_stripped=qa_already_stripped) +} + +do_populate_sysroot[dirs] = "${SYSROOT_DESTDIR}" + +addtask populate_sysroot after do_install + +SYSROOT_PREPROCESS_FUNCS ?= "" +SYSROOT_DESTDIR = "${WORKDIR}/sysroot-destdir" + +python do_populate_sysroot () { + # SYSROOT 'version' 2 + bb.build.exec_func("sysroot_stage_all", d) + bb.build.exec_func("sysroot_strip", d) + for f in (d.getVar('SYSROOT_PREPROCESS_FUNCS') or '').split(): + bb.build.exec_func(f, d) + pn = d.getVar("PN") + multiprov = d.getVar("BB_MULTI_PROVIDER_ALLOWED").split() + provdir = d.expand("${SYSROOT_DESTDIR}${base_prefix}/sysroot-providers/") + bb.utils.mkdirhier(provdir) + for p in d.getVar("PROVIDES").split(): + if p in multiprov: + continue + p = p.replace("/", "_") + with open(provdir + p, "w") as f: + f.write(pn) +} + +do_populate_sysroot[vardeps] += "${SYSROOT_PREPROCESS_FUNCS}" +do_populate_sysroot[vardepsexclude] += "BB_MULTI_PROVIDER_ALLOWED" + +POPULATESYSROOTDEPS = "" +POPULATESYSROOTDEPS:class-target = "virtual/${HOST_PREFIX}binutils:do_populate_sysroot" +POPULATESYSROOTDEPS:class-nativesdk = "virtual/${HOST_PREFIX}binutils:do_populate_sysroot" +do_populate_sysroot[depends] += "${POPULATESYSROOTDEPS}" + +SSTATETASKS += "do_populate_sysroot" +do_populate_sysroot[cleandirs] = "${SYSROOT_DESTDIR}" +do_populate_sysroot[sstate-inputdirs] = "${SYSROOT_DESTDIR}" +do_populate_sysroot[sstate-outputdirs] = "${COMPONENTS_DIR}/${PACKAGE_ARCH}/${PN}" +do_populate_sysroot[sstate-fixmedir] = "${COMPONENTS_DIR}/${PACKAGE_ARCH}/${PN}" + +python do_populate_sysroot_setscene () { + sstate_setscene(d) +} +addtask do_populate_sysroot_setscene + +def staging_copyfile(c, target, dest, postinsts, seendirs): + import errno + + destdir = os.path.dirname(dest) + if destdir not in seendirs: + bb.utils.mkdirhier(destdir) + seendirs.add(destdir) + if "/usr/bin/postinst-" in c: + postinsts.append(dest) + if os.path.islink(c): + linkto = os.readlink(c) + if os.path.lexists(dest): + if not os.path.islink(dest): + raise OSError(errno.EEXIST, "Link %s already exists as a file" % dest, dest) + if os.readlink(dest) == linkto: + return dest + raise OSError(errno.EEXIST, "Link %s already exists to a different location? (%s vs %s)" % (dest, os.readlink(dest), linkto), dest) + os.symlink(linkto, dest) + #bb.warn(c) + else: + try: + os.link(c, dest) + except OSError as err: + if err.errno == errno.EXDEV: + bb.utils.copyfile(c, dest) + else: + raise + return dest + +def staging_copydir(c, target, dest, seendirs): + if dest not in seendirs: + bb.utils.mkdirhier(dest) + seendirs.add(dest) + +def staging_processfixme(fixme, target, recipesysroot, recipesysrootnative, d): + import subprocess + + if not fixme: + return + cmd = "sed -e 's:^[^/]*/:%s/:g' %s | xargs sed -i -e 's:FIXMESTAGINGDIRTARGET:%s:g; s:FIXMESTAGINGDIRHOST:%s:g'" % (target, " ".join(fixme), recipesysroot, recipesysrootnative) + for fixmevar in ['PSEUDO_SYSROOT', 'HOSTTOOLS_DIR', 'PKGDATA_DIR', 'PSEUDO_LOCALSTATEDIR', 'LOGFIFO']: + fixme_path = d.getVar(fixmevar) + cmd += " -e 's:FIXME_%s:%s:g'" % (fixmevar, fixme_path) + bb.debug(2, cmd) + subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) + + +def staging_populate_sysroot_dir(targetsysroot, nativesysroot, native, d): + import glob + import subprocess + import errno + + fixme = [] + postinsts = [] + seendirs = set() + stagingdir = d.getVar("STAGING_DIR") + if native: + pkgarchs = ['${BUILD_ARCH}', '${BUILD_ARCH}_*'] + targetdir = nativesysroot + else: + pkgarchs = ['${MACHINE_ARCH}'] + pkgarchs = pkgarchs + list(reversed(d.getVar("PACKAGE_EXTRA_ARCHS").split())) + pkgarchs.append('allarch') + targetdir = targetsysroot + + bb.utils.mkdirhier(targetdir) + for pkgarch in pkgarchs: + for manifest in glob.glob(d.expand("${SSTATE_MANIFESTS}/manifest-%s-*.populate_sysroot" % pkgarch)): + if manifest.endswith("-initial.populate_sysroot"): + # skip libgcc-initial due to file overlap + continue + if not native and (manifest.endswith("-native.populate_sysroot") or "nativesdk-" in manifest): + continue + if native and not (manifest.endswith("-native.populate_sysroot") or manifest.endswith("-cross.populate_sysroot") or "-cross-" in manifest): + continue + tmanifest = targetdir + "/" + os.path.basename(manifest) + if os.path.exists(tmanifest): + continue + try: + os.link(manifest, tmanifest) + except OSError as err: + if err.errno == errno.EXDEV: + bb.utils.copyfile(manifest, tmanifest) + else: + raise + with open(manifest, "r") as f: + for l in f: + l = l.strip() + if l.endswith("/fixmepath"): + fixme.append(l) + continue + if l.endswith("/fixmepath.cmd"): + continue + dest = l.replace(stagingdir, "") + dest = targetdir + "/" + "/".join(dest.split("/")[3:]) + if l.endswith("/"): + staging_copydir(l, targetdir, dest, seendirs) + continue + try: + staging_copyfile(l, targetdir, dest, postinsts, seendirs) + except FileExistsError: + continue + + staging_processfixme(fixme, targetdir, targetsysroot, nativesysroot, d) + for p in sorted(postinsts): + bb.note("Running postinst {}, output:\n{}".format(p, subprocess.check_output(p, shell=True, stderr=subprocess.STDOUT))) + +# +# Manifests here are complicated. The main sysroot area has the unpacked sstate +# which us unrelocated and tracked by the main sstate manifests. Each recipe +# specific sysroot has manifests for each dependency that is installed there. +# The task hash is used to tell whether the data needs to be reinstalled. We +# use a symlink to point to the currently installed hash. There is also a +# "complete" stamp file which is used to mark if installation completed. If +# something fails (e.g. a postinst), this won't get written and we would +# remove and reinstall the dependency. This also means partially installed +# dependencies should get cleaned up correctly. +# + +python extend_recipe_sysroot() { + import copy + import subprocess + import errno + import collections + import glob + + taskdepdata = d.getVar("BB_TASKDEPDATA", False) + mytaskname = d.getVar("BB_RUNTASK") + if mytaskname.endswith("_setscene"): + mytaskname = mytaskname.replace("_setscene", "") + workdir = d.getVar("WORKDIR") + #bb.warn(str(taskdepdata)) + pn = d.getVar("PN") + stagingdir = d.getVar("STAGING_DIR") + sharedmanifests = d.getVar("COMPONENTS_DIR") + "/manifests" + # only needed by multilib cross-canadian since it redefines RECIPE_SYSROOT + manifestprefix = d.getVar("RECIPE_SYSROOT_MANIFEST_SUBDIR") + if manifestprefix: + sharedmanifests = sharedmanifests + "/" + manifestprefix + recipesysroot = d.getVar("RECIPE_SYSROOT") + recipesysrootnative = d.getVar("RECIPE_SYSROOT_NATIVE") + + # Detect bitbake -b usage + nodeps = d.getVar("BB_LIMITEDDEPS") or False + if nodeps: + lock = bb.utils.lockfile(recipesysroot + "/sysroot.lock") + staging_populate_sysroot_dir(recipesysroot, recipesysrootnative, True, d) + staging_populate_sysroot_dir(recipesysroot, recipesysrootnative, False, d) + bb.utils.unlockfile(lock) + return + + start = None + configuredeps = [] + owntaskdeps = [] + for dep in taskdepdata: + data = taskdepdata[dep] + if data[1] == mytaskname and data[0] == pn: + start = dep + elif data[0] == pn: + owntaskdeps.append(data[1]) + if start is None: + bb.fatal("Couldn't find ourself in BB_TASKDEPDATA?") + + # We need to figure out which sysroot files we need to expose to this task. + # This needs to match what would get restored from sstate, which is controlled + # ultimately by calls from bitbake to setscene_depvalid(). + # That function expects a setscene dependency tree. We build a dependency tree + # condensed to inter-sstate task dependencies, similar to that used by setscene + # tasks. We can then call into setscene_depvalid() and decide + # which dependencies we can "see" and should expose in the recipe specific sysroot. + setscenedeps = copy.deepcopy(taskdepdata) + + start = set([start]) + + sstatetasks = d.getVar("SSTATETASKS").split() + # Add recipe specific tasks referenced by setscene_depvalid() + sstatetasks.append("do_stash_locale") + sstatetasks.append("do_deploy") + + def print_dep_tree(deptree): + data = "" + for dep in deptree: + deps = " " + "\n ".join(deptree[dep][3]) + "\n" + data = data + "%s:\n %s\n %s\n%s %s\n %s\n" % (deptree[dep][0], deptree[dep][1], deptree[dep][2], deps, deptree[dep][4], deptree[dep][5]) + return data + + #bb.note("Full dep tree is:\n%s" % print_dep_tree(taskdepdata)) + + #bb.note(" start2 is %s" % str(start)) + + # If start is an sstate task (like do_package) we need to add in its direct dependencies + # else the code below won't recurse into them. + for dep in set(start): + for dep2 in setscenedeps[dep][3]: + start.add(dep2) + start.remove(dep) + + #bb.note(" start3 is %s" % str(start)) + + # Create collapsed do_populate_sysroot -> do_populate_sysroot tree + for dep in taskdepdata: + data = setscenedeps[dep] + if data[1] not in sstatetasks: + for dep2 in setscenedeps: + data2 = setscenedeps[dep2] + if dep in data2[3]: + data2[3].update(setscenedeps[dep][3]) + data2[3].remove(dep) + if dep in start: + start.update(setscenedeps[dep][3]) + start.remove(dep) + del setscenedeps[dep] + + # Remove circular references + for dep in setscenedeps: + if dep in setscenedeps[dep][3]: + setscenedeps[dep][3].remove(dep) + + #bb.note("Computed dep tree is:\n%s" % print_dep_tree(setscenedeps)) + #bb.note(" start is %s" % str(start)) + + # Direct dependencies should be present and can be depended upon + for dep in sorted(set(start)): + if setscenedeps[dep][1] == "do_populate_sysroot": + if dep not in configuredeps: + configuredeps.append(dep) + bb.note("Direct dependencies are %s" % str(configuredeps)) + #bb.note(" or %s" % str(start)) + + msgbuf = [] + # Call into setscene_depvalid for each sub-dependency and only copy sysroot files + # for ones that would be restored from sstate. + done = list(start) + next = list(start) + while next: + new = [] + for dep in next: + data = setscenedeps[dep] + for datadep in data[3]: + if datadep in done: + continue + taskdeps = {} + taskdeps[dep] = setscenedeps[dep][:2] + taskdeps[datadep] = setscenedeps[datadep][:2] + retval = setscene_depvalid(datadep, taskdeps, [], d, msgbuf) + if retval: + msgbuf.append("Skipping setscene dependency %s for installation into the sysroot" % datadep) + continue + done.append(datadep) + new.append(datadep) + if datadep not in configuredeps and setscenedeps[datadep][1] == "do_populate_sysroot": + configuredeps.append(datadep) + msgbuf.append("Adding dependency on %s" % setscenedeps[datadep][0]) + else: + msgbuf.append("Following dependency on %s" % setscenedeps[datadep][0]) + next = new + + # This logging is too verbose for day to day use sadly + #bb.debug(2, "\n".join(msgbuf)) + + depdir = recipesysrootnative + "/installeddeps" + bb.utils.mkdirhier(depdir) + bb.utils.mkdirhier(sharedmanifests) + + lock = bb.utils.lockfile(recipesysroot + "/sysroot.lock") + + fixme = {} + seendirs = set() + postinsts = [] + multilibs = {} + manifests = {} + # All files that we're going to be installing, to find conflicts. + fileset = {} + + invalidate_tasks = set() + for f in os.listdir(depdir): + removed = [] + if not f.endswith(".complete"): + continue + f = depdir + "/" + f + if os.path.islink(f) and not os.path.exists(f): + bb.note("%s no longer exists, removing from sysroot" % f) + lnk = os.readlink(f.replace(".complete", "")) + sstate_clean_manifest(depdir + "/" + lnk, d, canrace=True, prefix=workdir) + os.unlink(f) + os.unlink(f.replace(".complete", "")) + removed.append(os.path.basename(f.replace(".complete", ""))) + + # If we've removed files from the sysroot above, the task that installed them may still + # have a stamp file present for the task. This is probably invalid right now but may become + # valid again if the user were to change configuration back for example. Since we've removed + # the files a task might need, remove the stamp file too to force it to rerun. + # YOCTO #14790 + if removed: + for i in glob.glob(depdir + "/index.*"): + if i.endswith("." + mytaskname): + continue + with open(i, "r") as f: + for l in f: + if l.startswith("TaskDeps:"): + continue + l = l.strip() + if l in removed: + invalidate_tasks.add(i.rsplit(".", 1)[1]) + break + for t in invalidate_tasks: + bb.note("Invalidating stamps for task %s" % t) + bb.build.clean_stamp(t, d) + + installed = [] + for dep in configuredeps: + c = setscenedeps[dep][0] + if mytaskname in ["do_sdk_depends", "do_populate_sdk_ext"] and c.endswith("-initial"): + bb.note("Skipping initial setscene dependency %s for installation into the sysroot" % c) + continue + installed.append(c) + + # We want to remove anything which this task previously installed but is no longer a dependency + taskindex = depdir + "/" + "index." + mytaskname + if os.path.exists(taskindex): + potential = [] + with open(taskindex, "r") as f: + for l in f: + l = l.strip() + if l not in installed: + fl = depdir + "/" + l + if not os.path.exists(fl): + # Was likely already uninstalled + continue + potential.append(l) + # We need to ensure no other task needs this dependency. We hold the sysroot + # lock so we ca search the indexes to check + if potential: + for i in glob.glob(depdir + "/index.*"): + if i.endswith("." + mytaskname): + continue + with open(i, "r") as f: + for l in f: + if l.startswith("TaskDeps:"): + prevtasks = l.split()[1:] + if mytaskname in prevtasks: + # We're a dependency of this task so we can clear items out the sysroot + break + l = l.strip() + if l in potential: + potential.remove(l) + for l in potential: + fl = depdir + "/" + l + bb.note("Task %s no longer depends on %s, removing from sysroot" % (mytaskname, l)) + lnk = os.readlink(fl) + sstate_clean_manifest(depdir + "/" + lnk, d, canrace=True, prefix=workdir) + os.unlink(fl) + os.unlink(fl + ".complete") + + msg_exists = [] + msg_adding = [] + + # Handle all removals first since files may move between recipes + for dep in configuredeps: + c = setscenedeps[dep][0] + if c not in installed: + continue + taskhash = setscenedeps[dep][5] + taskmanifest = depdir + "/" + c + "." + taskhash + + if os.path.exists(depdir + "/" + c): + lnk = os.readlink(depdir + "/" + c) + if lnk == c + "." + taskhash and os.path.exists(depdir + "/" + c + ".complete"): + continue + else: + bb.note("%s exists in sysroot, but is stale (%s vs. %s), removing." % (c, lnk, c + "." + taskhash)) + sstate_clean_manifest(depdir + "/" + lnk, d, canrace=True, prefix=workdir) + os.unlink(depdir + "/" + c) + if os.path.lexists(depdir + "/" + c + ".complete"): + os.unlink(depdir + "/" + c + ".complete") + elif os.path.lexists(depdir + "/" + c): + os.unlink(depdir + "/" + c) + + binfiles = {} + # Now handle installs + for dep in sorted(configuredeps): + c = setscenedeps[dep][0] + if c not in installed: + continue + taskhash = setscenedeps[dep][5] + taskmanifest = depdir + "/" + c + "." + taskhash + + if os.path.exists(depdir + "/" + c): + lnk = os.readlink(depdir + "/" + c) + if lnk == c + "." + taskhash and os.path.exists(depdir + "/" + c + ".complete"): + msg_exists.append(c) + continue + + msg_adding.append(c) + + os.symlink(c + "." + taskhash, depdir + "/" + c) + + manifest, d2 = oe.sstatesig.find_sstate_manifest(c, setscenedeps[dep][2], "populate_sysroot", d, multilibs) + if d2 is not d: + # If we don't do this, the recipe sysroot will be placed in the wrong WORKDIR for multilibs + # We need a consistent WORKDIR for the image + d2.setVar("WORKDIR", d.getVar("WORKDIR")) + destsysroot = d2.getVar("RECIPE_SYSROOT") + # We put allarch recipes into the default sysroot + if manifest and "allarch" in manifest: + destsysroot = d.getVar("RECIPE_SYSROOT") + + native = False + if c.endswith("-native") or "-cross-" in c or "-crosssdk" in c: + native = True + + if manifest: + newmanifest = collections.OrderedDict() + targetdir = destsysroot + if native: + targetdir = recipesysrootnative + if targetdir not in fixme: + fixme[targetdir] = [] + fm = fixme[targetdir] + + with open(manifest, "r") as f: + manifests[dep] = manifest + for l in f: + l = l.strip() + if l.endswith("/fixmepath"): + fm.append(l) + continue + if l.endswith("/fixmepath.cmd"): + continue + dest = l.replace(stagingdir, "") + dest = "/" + "/".join(dest.split("/")[3:]) + newmanifest[l] = targetdir + dest + + # Check if files have already been installed by another + # recipe and abort if they have, explaining what recipes are + # conflicting. + hashname = targetdir + dest + if not hashname.endswith("/"): + if hashname in fileset: + bb.fatal("The file %s is installed by both %s and %s, aborting" % (dest, c, fileset[hashname])) + else: + fileset[hashname] = c + + # Having multiple identical manifests in each sysroot eats diskspace so + # create a shared pool of them and hardlink if we can. + # We create the manifest in advance so that if something fails during installation, + # or the build is interrupted, subsequent exeuction can cleanup. + sharedm = sharedmanifests + "/" + os.path.basename(taskmanifest) + if not os.path.exists(sharedm): + smlock = bb.utils.lockfile(sharedm + ".lock") + # Can race here. You'd think it just means we may not end up with all copies hardlinked to each other + # but python can lose file handles so we need to do this under a lock. + if not os.path.exists(sharedm): + with open(sharedm, 'w') as m: + for l in newmanifest: + dest = newmanifest[l] + m.write(dest.replace(workdir + "/", "") + "\n") + bb.utils.unlockfile(smlock) + try: + os.link(sharedm, taskmanifest) + except OSError as err: + if err.errno == errno.EXDEV: + bb.utils.copyfile(sharedm, taskmanifest) + else: + raise + # Finally actually install the files + for l in newmanifest: + dest = newmanifest[l] + if l.endswith("/"): + staging_copydir(l, targetdir, dest, seendirs) + continue + if "/bin/" in l or "/sbin/" in l: + # defer /*bin/* files until last in case they need libs + binfiles[l] = (targetdir, dest) + else: + staging_copyfile(l, targetdir, dest, postinsts, seendirs) + + # Handle deferred binfiles + for l in binfiles: + (targetdir, dest) = binfiles[l] + staging_copyfile(l, targetdir, dest, postinsts, seendirs) + + bb.note("Installed into sysroot: %s" % str(msg_adding)) + bb.note("Skipping as already exists in sysroot: %s" % str(msg_exists)) + + for f in fixme: + staging_processfixme(fixme[f], f, recipesysroot, recipesysrootnative, d) + + for p in sorted(postinsts): + bb.note("Running postinst {}, output:\n{}".format(p, subprocess.check_output(p, shell=True, stderr=subprocess.STDOUT))) + + for dep in manifests: + c = setscenedeps[dep][0] + os.symlink(manifests[dep], depdir + "/" + c + ".complete") + + with open(taskindex, "w") as f: + f.write("TaskDeps: " + " ".join(owntaskdeps) + "\n") + for l in sorted(installed): + f.write(l + "\n") + + bb.utils.unlockfile(lock) +} +extend_recipe_sysroot[vardepsexclude] += "MACHINE_ARCH PACKAGE_EXTRA_ARCHS SDK_ARCH BUILD_ARCH SDK_OS BB_TASKDEPDATA" + +do_prepare_recipe_sysroot[deptask] = "do_populate_sysroot" +python do_prepare_recipe_sysroot () { + bb.build.exec_func("extend_recipe_sysroot", d) +} +addtask do_prepare_recipe_sysroot before do_configure after do_fetch + +python staging_taskhandler() { + bbtasks = e.tasklist + for task in bbtasks: + deps = d.getVarFlag(task, "depends") + if task != 'do_prepare_recipe_sysroot' and (task == "do_configure" or (deps and "populate_sysroot" in deps)): + d.prependVarFlag(task, "prefuncs", "extend_recipe_sysroot ") +} +staging_taskhandler[eventmask] = "bb.event.RecipeTaskPreProcess" +addhandler staging_taskhandler + + +# +# Target build output, stored in do_populate_sysroot or do_package can depend +# not only upon direct dependencies but also indirect ones. A good example is +# linux-libc-headers. The toolchain depends on this but most target recipes do +# not. There are some headers which are not used by the toolchain build and do +# not change the toolchain task output, hence the task hashes can change without +# changing the sysroot output of that recipe yet they can influence others. +# +# A specific example is rtc.h which can change rtcwake.c in util-linux but is not +# used in the glibc or gcc build. To account for this, we need to account for the +# populate_sysroot hashes in the task output hashes. +# +python target_add_sysroot_deps () { + current_task = "do_" + d.getVar("BB_CURRENTTASK") + if current_task not in ["do_populate_sysroot", "do_package"]: + return + + pn = d.getVar("PN") + if pn.endswith("-native"): + return + + taskdepdata = d.getVar("BB_TASKDEPDATA", False) + deps = {} + for dep in taskdepdata.values(): + if dep[1] == "do_populate_sysroot" and not dep[0].endswith(("-native", "-initial")) and "-cross-" not in dep[0] and dep[0] != pn: + deps[dep[0]] = dep[6] + + d.setVar("HASHEQUIV_EXTRA_SIGDATA", "\n".join("%s: %s" % (k, deps[k]) for k in sorted(deps.keys()))) +} +SSTATECREATEFUNCS += "target_add_sysroot_deps" + diff --git a/meta/classes-global/uninative.bbclass b/meta/classes-global/uninative.bbclass new file mode 100644 index 0000000000..d2297b53f5 --- /dev/null +++ b/meta/classes-global/uninative.bbclass @@ -0,0 +1,181 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +UNINATIVE_LOADER ?= "${UNINATIVE_STAGING_DIR}-uninative/${BUILD_ARCH}-linux/lib/${@bb.utils.contains('BUILD_ARCH', 'x86_64', 'ld-linux-x86-64.so.2', '', d)}${@bb.utils.contains('BUILD_ARCH', 'i686', 'ld-linux.so.2', '', d)}${@bb.utils.contains('BUILD_ARCH', 'aarch64', 'ld-linux-aarch64.so.1', '', d)}${@bb.utils.contains('BUILD_ARCH', 'ppc64le', 'ld64.so.2', '', d)}${@bb.utils.contains('BUILD_ARCH', 'riscv64', 'ld-linux-riscv64-lp64d.so.1', '', d)}" +UNINATIVE_STAGING_DIR ?= "${STAGING_DIR}" + +UNINATIVE_URL ?= "unset" +UNINATIVE_TARBALL ?= "${BUILD_ARCH}-nativesdk-libc-${UNINATIVE_VERSION}.tar.xz" +# Example checksums +#UNINATIVE_CHECKSUM[aarch64] = "dead" +#UNINATIVE_CHECKSUM[i686] = "dead" +#UNINATIVE_CHECKSUM[x86_64] = "dead" +UNINATIVE_DLDIR ?= "${DL_DIR}/uninative/" + +# Enabling uninative will change the following variables so they need to go the parsing ignored variables list to prevent multiple recipe parsing +BB_HASHCONFIG_IGNORE_VARS += "NATIVELSBSTRING SSTATEPOSTUNPACKFUNCS BUILD_LDFLAGS" + +addhandler uninative_event_fetchloader +uninative_event_fetchloader[eventmask] = "bb.event.BuildStarted" + +addhandler uninative_event_enable +uninative_event_enable[eventmask] = "bb.event.ConfigParsed" + +python uninative_event_fetchloader() { + """ + This event fires on the parent and will try to fetch the tarball if the + loader isn't already present. + """ + + chksum = d.getVarFlag("UNINATIVE_CHECKSUM", d.getVar("BUILD_ARCH")) + if not chksum: + bb.fatal("Uninative selected but not configured correctly, please set UNINATIVE_CHECKSUM[%s]" % d.getVar("BUILD_ARCH")) + + loader = d.getVar("UNINATIVE_LOADER") + loaderchksum = loader + ".chksum" + if os.path.exists(loader) and os.path.exists(loaderchksum): + with open(loaderchksum, "r") as f: + readchksum = f.read().strip() + if readchksum == chksum: + if "uninative" not in d.getVar("SSTATEPOSTUNPACKFUNCS"): + enable_uninative(d) + return + + import subprocess + try: + # Save and restore cwd as Fetch.download() does a chdir() + olddir = os.getcwd() + + tarball = d.getVar("UNINATIVE_TARBALL") + tarballdir = os.path.join(d.getVar("UNINATIVE_DLDIR"), chksum) + tarballpath = os.path.join(tarballdir, tarball) + + if not os.path.exists(tarballpath + ".done"): + bb.utils.mkdirhier(tarballdir) + if d.getVar("UNINATIVE_URL") == "unset": + bb.fatal("Uninative selected but not configured, please set UNINATIVE_URL") + + localdata = bb.data.createCopy(d) + localdata.setVar('FILESPATH', "") + localdata.setVar('DL_DIR', tarballdir) + # Our games with path manipulation of DL_DIR mean standard PREMIRRORS don't work + # and we can't easily put 'chksum' into the url path from a url parameter with + # the current fetcher url handling + premirrors = bb.fetch2.mirror_from_string(localdata.getVar("PREMIRRORS")) + for line in premirrors: + try: + (find, replace) = line + except ValueError: + continue + if find.startswith("http"): + localdata.appendVar("PREMIRRORS", " ${UNINATIVE_URL}${UNINATIVE_TARBALL} %s/uninative/%s/${UNINATIVE_TARBALL}" % (replace, chksum)) + + srcuri = d.expand("${UNINATIVE_URL}${UNINATIVE_TARBALL};sha256sum=%s" % chksum) + bb.note("Fetching uninative binary shim %s (will check PREMIRRORS first)" % srcuri) + + fetcher = bb.fetch2.Fetch([srcuri], localdata, cache=False) + fetcher.download() + localpath = fetcher.localpath(srcuri) + if localpath != tarballpath and os.path.exists(localpath) and not os.path.exists(tarballpath): + # Follow the symlink behavior from the bitbake fetch2. + # This will cover the case where an existing symlink is broken + # as well as if there are two processes trying to create it + # at the same time. + if os.path.islink(tarballpath): + # Broken symbolic link + os.unlink(tarballpath) + + # Deal with two processes trying to make symlink at once + try: + os.symlink(localpath, tarballpath) + except FileExistsError: + pass + + # ldd output is "ldd (Ubuntu GLIBC 2.23-0ubuntu10) 2.23", extract last option from first line + glibcver = subprocess.check_output(["ldd", "--version"]).decode('utf-8').split('\n')[0].split()[-1] + if bb.utils.vercmp_string(d.getVar("UNINATIVE_MAXGLIBCVERSION"), glibcver) < 0: + raise RuntimeError("Your host glibc version (%s) is newer than that in uninative (%s). Disabling uninative so that sstate is not corrupted." % (glibcver, d.getVar("UNINATIVE_MAXGLIBCVERSION"))) + + cmd = d.expand("\ +mkdir -p ${UNINATIVE_STAGING_DIR}-uninative; \ +cd ${UNINATIVE_STAGING_DIR}-uninative; \ +tar -xJf ${UNINATIVE_DLDIR}/%s/${UNINATIVE_TARBALL}; \ +${UNINATIVE_STAGING_DIR}-uninative/relocate_sdk.py \ + ${UNINATIVE_STAGING_DIR}-uninative/${BUILD_ARCH}-linux \ + ${UNINATIVE_LOADER} \ + ${UNINATIVE_LOADER} \ + ${UNINATIVE_STAGING_DIR}-uninative/${BUILD_ARCH}-linux/${bindir_native}/patchelf-uninative \ + ${UNINATIVE_STAGING_DIR}-uninative/${BUILD_ARCH}-linux${base_libdir_native}/libc*.so*" % chksum) + subprocess.check_output(cmd, shell=True) + + with open(loaderchksum, "w") as f: + f.write(chksum) + + enable_uninative(d) + + except RuntimeError as e: + bb.warn(str(e)) + except bb.fetch2.BBFetchException as exc: + bb.warn("Disabling uninative as unable to fetch uninative tarball: %s" % str(exc)) + bb.warn("To build your own uninative loader, please bitbake uninative-tarball and set UNINATIVE_TARBALL appropriately.") + except subprocess.CalledProcessError as exc: + bb.warn("Disabling uninative as unable to install uninative tarball: %s" % str(exc)) + bb.warn("To build your own uninative loader, please bitbake uninative-tarball and set UNINATIVE_TARBALL appropriately.") + finally: + os.chdir(olddir) +} + +python uninative_event_enable() { + """ + This event handler is called in the workers and is responsible for setting + up uninative if a loader is found. + """ + enable_uninative(d) +} + +def enable_uninative(d): + loader = d.getVar("UNINATIVE_LOADER") + if os.path.exists(loader): + bb.debug(2, "Enabling uninative") + d.setVar("NATIVELSBSTRING", "universal%s" % oe.utils.host_gcc_version(d)) + d.appendVar("SSTATEPOSTUNPACKFUNCS", " uninative_changeinterp") + d.appendVarFlag("SSTATEPOSTUNPACKFUNCS", "vardepvalueexclude", "| uninative_changeinterp") + d.appendVar("BUILD_LDFLAGS", " -Wl,--allow-shlib-undefined -Wl,--dynamic-linker=${UNINATIVE_LOADER} -pthread") + d.appendVarFlag("BUILD_LDFLAGS", "vardepvalueexclude", "| -Wl,--allow-shlib-undefined -Wl,--dynamic-linker=${UNINATIVE_LOADER} -pthread") + d.appendVarFlag("BUILD_LDFLAGS", "vardepsexclude", "UNINATIVE_LOADER") + d.prependVar("PATH", "${UNINATIVE_STAGING_DIR}-uninative/${BUILD_ARCH}-linux${bindir_native}:") + +python uninative_changeinterp () { + import subprocess + import stat + import oe.qa + + if not (bb.data.inherits_class('native', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross', d)): + return + + sstateinst = d.getVar('SSTATE_INSTDIR') + for walkroot, dirs, files in os.walk(sstateinst): + for file in files: + if file.endswith(".so") or ".so." in file: + continue + f = os.path.join(walkroot, file) + if os.path.islink(f): + continue + s = os.stat(f) + if not ((s[stat.ST_MODE] & stat.S_IXUSR) or (s[stat.ST_MODE] & stat.S_IXGRP) or (s[stat.ST_MODE] & stat.S_IXOTH)): + continue + elf = oe.qa.ELFFile(f) + try: + elf.open() + except oe.qa.NotELFFileError: + continue + if not elf.isDynamic(): + continue + + os.chmod(f, s[stat.ST_MODE] | stat.S_IWUSR) + subprocess.check_output(("patchelf-uninative", "--set-interpreter", d.getVar("UNINATIVE_LOADER"), f), stderr=subprocess.STDOUT) + os.chmod(f, s[stat.ST_MODE]) +} diff --git a/meta/classes-global/utility-tasks.bbclass b/meta/classes-global/utility-tasks.bbclass new file mode 100644 index 0000000000..ae2da330b8 --- /dev/null +++ b/meta/classes-global/utility-tasks.bbclass @@ -0,0 +1,60 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +addtask listtasks +do_listtasks[nostamp] = "1" +python do_listtasks() { + taskdescs = {} + maxlen = 0 + for e in d.keys(): + if d.getVarFlag(e, 'task'): + maxlen = max(maxlen, len(e)) + if e.endswith('_setscene'): + desc = "%s (setscene version)" % (d.getVarFlag(e[:-9], 'doc') or '') + else: + desc = d.getVarFlag(e, 'doc') or '' + taskdescs[e] = desc + + tasks = sorted(taskdescs.keys()) + for taskname in tasks: + bb.plain("%s %s" % (taskname.ljust(maxlen), taskdescs[taskname])) +} + +CLEANFUNCS ?= "" + +T:task-clean = "${LOG_DIR}/cleanlogs/${PN}" +addtask clean +do_clean[nostamp] = "1" +python do_clean() { + """clear the build and temp directories""" + dir = d.expand("${WORKDIR}") + bb.note("Removing " + dir) + oe.path.remove(dir) + + dir = "%s.*" % d.getVar('STAMP') + bb.note("Removing " + dir) + oe.path.remove(dir) + + for f in (d.getVar('CLEANFUNCS') or '').split(): + bb.build.exec_func(f, d) +} + +addtask checkuri +do_checkuri[nostamp] = "1" +do_checkuri[network] = "1" +python do_checkuri() { + src_uri = (d.getVar('SRC_URI') or "").split() + if len(src_uri) == 0: + return + + try: + fetcher = bb.fetch2.Fetch(src_uri, d) + fetcher.checkstatus() + except bb.fetch2.BBFetchException as e: + bb.fatal(str(e)) +} + + diff --git a/meta/classes-global/utils.bbclass b/meta/classes-global/utils.bbclass new file mode 100644 index 0000000000..957389928f --- /dev/null +++ b/meta/classes-global/utils.bbclass @@ -0,0 +1,369 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +oe_soinstall() { + # Purpose: Install shared library file and + # create the necessary links + # Example: oe_soinstall libfoo.so.1.2.3 ${D}${libdir} + libname=`basename $1` + case "$libname" in + *.so) + bbfatal "oe_soinstall: Shared library must haved versioned filename (e.g. libfoo.so.1.2.3)" + ;; + esac + install -m 755 $1 $2/$libname + sonamelink=`${READELF} -d $1 |grep 'Library soname:' |sed -e 's/.*\[\(.*\)\].*/\1/'` + if [ -z $sonamelink ]; then + bbfatal "oe_soinstall: $libname is missing ELF tag 'SONAME'." + fi + solink=`echo $libname | sed -e 's/\.so\..*/.so/'` + ln -sf $libname $2/$sonamelink + ln -sf $libname $2/$solink +} + +oe_libinstall() { + # Purpose: Install a library, in all its forms + # Example + # + # oe_libinstall libltdl ${STAGING_LIBDIR}/ + # oe_libinstall -C src/libblah libblah ${D}/${libdir}/ + dir="" + libtool="" + silent="" + require_static="" + require_shared="" + while [ "$#" -gt 0 ]; do + case "$1" in + -C) + shift + dir="$1" + ;; + -s) + silent=1 + ;; + -a) + require_static=1 + ;; + -so) + require_shared=1 + ;; + -*) + bbfatal "oe_libinstall: unknown option: $1" + ;; + *) + break; + ;; + esac + shift + done + + libname="$1" + shift + destpath="$1" + if [ -z "$destpath" ]; then + bbfatal "oe_libinstall: no destination path specified" + fi + + __runcmd () { + if [ -z "$silent" ]; then + echo >&2 "oe_libinstall: $*" + fi + $* + } + + if [ -z "$dir" ]; then + dir=`pwd` + fi + + dotlai=$libname.lai + + # Sanity check that the libname.lai is unique + number_of_files=`(cd $dir; find . -name "$dotlai") | wc -l` + if [ $number_of_files -gt 1 ]; then + bbfatal "oe_libinstall: $dotlai is not unique in $dir" + fi + + + dir=$dir`(cd $dir;find . -name "$dotlai") | sed "s/^\.//;s/\/$dotlai\$//;q"` + olddir=`pwd` + __runcmd cd $dir + + lafile=$libname.la + + # If such file doesn't exist, try to cut version suffix + if [ ! -f "$lafile" ]; then + libname1=`echo "$libname" | sed 's/-[0-9.]*$//'` + lafile1=$libname.la + if [ -f "$lafile1" ]; then + libname=$libname1 + lafile=$lafile1 + fi + fi + + if [ -f "$lafile" ]; then + # libtool archive + eval `cat $lafile|grep "^library_names="` + libtool=1 + else + library_names="$libname.so* $libname.dll.a $libname.*.dylib" + fi + + __runcmd install -d $destpath/ + dota=$libname.a + if [ -f "$dota" -o -n "$require_static" ]; then + rm -f $destpath/$dota + __runcmd install -m 0644 $dota $destpath/ + fi + if [ -f "$dotlai" -a -n "$libtool" ]; then + rm -f $destpath/$libname.la + __runcmd install -m 0644 $dotlai $destpath/$libname.la + fi + + for name in $library_names; do + files=`eval echo $name` + for f in $files; do + if [ ! -e "$f" ]; then + if [ -n "$libtool" ]; then + bbfatal "oe_libinstall: $dir/$f not found." + fi + elif [ -L "$f" ]; then + __runcmd cp -P "$f" $destpath/ + elif [ ! -L "$f" ]; then + libfile="$f" + rm -f $destpath/$libfile + __runcmd install -m 0755 $libfile $destpath/ + fi + done + done + + if [ -z "$libfile" ]; then + if [ -n "$require_shared" ]; then + bbfatal "oe_libinstall: unable to locate shared library" + fi + elif [ -z "$libtool" ]; then + # special case hack for non-libtool .so.#.#.# links + baselibfile=`basename "$libfile"` + if (echo $baselibfile | grep -qE '^lib.*\.so\.[0-9.]*$'); then + sonamelink=`${READELF} -d $libfile |grep 'Library soname:' |sed -e 's/.*\[\(.*\)\].*/\1/'` + solink=`echo $baselibfile | sed -e 's/\.so\..*/.so/'` + if [ -n "$sonamelink" -a x"$baselibfile" != x"$sonamelink" ]; then + __runcmd ln -sf $baselibfile $destpath/$sonamelink + fi + __runcmd ln -sf $baselibfile $destpath/$solink + fi + fi + + __runcmd cd "$olddir" +} + +create_cmdline_wrapper () { + # Create a wrapper script where commandline options are needed + # + # These are useful to work around relocation issues, by passing extra options + # to a program + # + # Usage: create_cmdline_wrapper FILENAME <extra-options> + + cmd=$1 + shift + + echo "Generating wrapper script for $cmd" + + mv $cmd $cmd.real + cmdname=`basename $cmd` + dirname=`dirname $cmd` + cmdoptions=$@ + if [ "${base_prefix}" != "" ]; then + relpath=`python3 -c "import os; print(os.path.relpath('${D}${base_prefix}', '$dirname'))"` + cmdoptions=`echo $@ | sed -e "s:${base_prefix}:\\$realdir/$relpath:g"` + fi + cat <<END >$cmd +#!/bin/bash +realpath=\`readlink -fn \$0\` +realdir=\`dirname \$realpath\` +exec -a \$realdir/$cmdname \$realdir/$cmdname.real $cmdoptions "\$@" +END + chmod +x $cmd +} + +create_cmdline_shebang_wrapper () { + # Create a wrapper script where commandline options are needed + # + # These are useful to work around shebang relocation issues, where shebangs are too + # long or have arguments in them, thus preventing them from using the /usr/bin/env + # shebang + # + # Usage: create_cmdline_wrapper FILENAME <extra-options> + + cmd=$1 + shift + + echo "Generating wrapper script for $cmd" + + # Strip #! and get remaining interpreter + arg + argument="$(sed -ne 's/^#! *//p;q' $cmd)" + # strip the shebang from the real script as we do not want it to be usable anyway + tail -n +2 $cmd > $cmd.real + chown --reference=$cmd $cmd.real + chmod --reference=$cmd $cmd.real + rm -f $cmd + cmdname=$(basename $cmd) + dirname=$(dirname $cmd) + cmdoptions=$@ + if [ "${base_prefix}" != "" ]; then + relpath=`python3 -c "import os; print(os.path.relpath('${D}${base_prefix}', '$dirname'))"` + cmdoptions=`echo $@ | sed -e "s:${base_prefix}:\\$realdir/$relpath:g"` + fi + cat <<END >$cmd +#!/usr/bin/env bash +realpath=\`readlink -fn \$0\` +realdir=\`dirname \$realpath\` +exec -a \$realdir/$cmdname $argument \$realdir/$cmdname.real $cmdoptions "\$@" +END + chmod +x $cmd +} + +create_wrapper () { + # Create a wrapper script where extra environment variables are needed + # + # These are useful to work around relocation issues, by setting environment + # variables which point to paths in the filesystem. + # + # Usage: create_wrapper FILENAME [[VAR=VALUE]..] + + cmd=$1 + shift + + echo "Generating wrapper script for $cmd" + + mv $cmd $cmd.real + cmdname=`basename $cmd` + dirname=`dirname $cmd` + exportstring=$@ + if [ "${base_prefix}" != "" ]; then + relpath=`python3 -c "import os; print(os.path.relpath('${D}${base_prefix}', '$dirname'))"` + exportstring=`echo $@ | sed -e "s:${base_prefix}:\\$realdir/$relpath:g"` + fi + cat <<END >$cmd +#!/bin/bash +realpath=\`readlink -fn \$0\` +realdir=\`dirname \$realpath\` +export $exportstring +exec -a "\$0" \$realdir/$cmdname.real "\$@" +END + chmod +x $cmd +} + +# Copy files/directories from $1 to $2 but using hardlinks +# (preserve symlinks) +hardlinkdir () { + from=$1 + to=$2 + (cd $from; find . -print0 | cpio --null -pdlu $to) +} + + +def check_app_exists(app, d): + app = d.expand(app).split()[0].strip() + path = d.getVar('PATH') + return bool(bb.utils.which(path, app)) + +def explode_deps(s): + return bb.utils.explode_deps(s) + +def base_set_filespath(path, d): + filespath = [] + extrapaths = (d.getVar("FILESEXTRAPATHS") or "") + # Remove default flag which was used for checking + extrapaths = extrapaths.replace("__default:", "") + # Don't prepend empty strings to the path list + if extrapaths != "": + path = extrapaths.split(":") + path + # The ":" ensures we have an 'empty' override + overrides = (":" + (d.getVar("FILESOVERRIDES") or "")).split(":") + overrides.reverse() + for o in overrides: + for p in path: + if p != "": + filespath.append(os.path.join(p, o)) + return ":".join(filespath) + +def extend_variants(d, var, extend, delim=':'): + """Return a string of all bb class extend variants for the given extend""" + variants = [] + whole = d.getVar(var) or "" + for ext in whole.split(): + eext = ext.split(delim) + if len(eext) > 1 and eext[0] == extend: + variants.append(eext[1]) + return " ".join(variants) + +def multilib_pkg_extend(d, pkg): + variants = (d.getVar("MULTILIB_VARIANTS") or "").split() + if not variants: + return pkg + pkgs = pkg + for v in variants: + pkgs = pkgs + " " + v + "-" + pkg + return pkgs + +def get_multilib_datastore(variant, d): + return oe.utils.get_multilib_datastore(variant, d) + +def all_multilib_tune_values(d, var, unique = True, need_split = True, delim = ' '): + """Return a string of all ${var} in all multilib tune configuration""" + values = [] + variants = (d.getVar("MULTILIB_VARIANTS") or "").split() + [''] + for item in variants: + localdata = get_multilib_datastore(item, d) + # We need WORKDIR to be consistent with the original datastore + localdata.setVar("WORKDIR", d.getVar("WORKDIR")) + value = localdata.getVar(var) or "" + if value != "": + if need_split: + for item in value.split(delim): + values.append(item) + else: + values.append(value) + if unique: + #we do this to keep order as much as possible + ret = [] + for value in values: + if not value in ret: + ret.append(value) + else: + ret = values + return " ".join(ret) + +def all_multilib_tune_list(vars, d): + """ + Return a list of ${VAR} for each variable VAR in vars from each + multilib tune configuration. + Is safe to be called from a multilib recipe/context as it can + figure out the original tune and remove the multilib overrides. + """ + values = {} + for v in vars: + values[v] = [] + values['ml'] = [''] + + variants = (d.getVar("MULTILIB_VARIANTS") or "").split() + [''] + for item in variants: + localdata = get_multilib_datastore(item, d) + values[v].append(localdata.getVar(v)) + values['ml'].append(item) + return values +all_multilib_tune_list[vardepsexclude] = "OVERRIDES" + +# If the user hasn't set up their name/email, set some defaults +check_git_config() { + if ! git config user.email > /dev/null ; then + git config --local user.email "${PATCH_GIT_USER_EMAIL}" + fi + if ! git config user.name > /dev/null ; then + git config --local user.name "${PATCH_GIT_USER_NAME}" + fi +} |