diff options
Diffstat (limited to 'meta/classes')
33 files changed, 1393 insertions, 124 deletions
diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass index 19604a4646..3cae577a0e 100644 --- a/meta/classes/base.bbclass +++ b/meta/classes/base.bbclass @@ -139,7 +139,7 @@ def setup_hosttools_dir(dest, toolsvar, d, fatal=True): # /usr/local/bin/ccache/gcc -> /usr/bin/ccache, then which(gcc) # would return /usr/local/bin/ccache/gcc, but what we need is # /usr/bin/gcc, this code can check and fix that. - if "ccache" in srctool: + if os.path.islink(srctool) and os.path.basename(os.readlink(srctool)) == 'ccache': srctool = bb.utils.which(path, tool, executable=True, direction=1) if srctool: os.symlink(srctool, desttool) diff --git a/meta/classes/bin_package.bbclass b/meta/classes/bin_package.bbclass index cbc9b1fa13..c1954243ee 100644 --- a/meta/classes/bin_package.bbclass +++ b/meta/classes/bin_package.bbclass @@ -30,8 +30,9 @@ bin_package_do_install () { bbfatal bin_package has nothing to install. Be sure the SRC_URI unpacks into S. fi cd ${S} + install -d ${D}${base_prefix} tar --no-same-owner --exclude='./patches' --exclude='./.pc' -cpf - . \ - | tar --no-same-owner -xpf - -C ${D} + | tar --no-same-owner -xpf - -C ${D}${base_prefix} } FILES_${PN} = "/" diff --git a/meta/classes/create-spdx-2.2.bbclass b/meta/classes/create-spdx-2.2.bbclass new file mode 100644 index 0000000000..42b693d586 --- /dev/null +++ b/meta/classes/create-spdx-2.2.bbclass @@ -0,0 +1,1067 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: GPL-2.0-only +# + +DEPLOY_DIR_SPDX ??= "${DEPLOY_DIR}/spdx/${MACHINE}" + +# The product name that the CVE database uses. Defaults to BPN, but may need to +# be overriden per recipe (for example tiff.bb sets CVE_PRODUCT=libtiff). +CVE_PRODUCT ??= "${BPN}" +CVE_VERSION ??= "${PV}" + +SPDXDIR ??= "${WORKDIR}/spdx" +SPDXDEPLOY = "${SPDXDIR}/deploy" +SPDXWORK = "${SPDXDIR}/work" +SPDXIMAGEWORK = "${SPDXDIR}/image-work" +SPDXSDKWORK = "${SPDXDIR}/sdk-work" + +SPDX_TOOL_NAME ??= "oe-spdx-creator" +SPDX_TOOL_VERSION ??= "1.0" + +SPDXRUNTIMEDEPLOY = "${SPDXDIR}/runtime-deploy" + +SPDX_INCLUDE_SOURCES ??= "0" +SPDX_ARCHIVE_SOURCES ??= "0" +SPDX_ARCHIVE_PACKAGED ??= "0" + +SPDX_UUID_NAMESPACE ??= "sbom.openembedded.org" +SPDX_NAMESPACE_PREFIX ??= "http://spdx.org/spdxdoc" +SPDX_PRETTY ??= "0" + +SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json" + +SPDX_CUSTOM_ANNOTATION_VARS ??= "" + +SPDX_ORG ??= "OpenEmbedded ()" +SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}" +SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created from \ + this recipe. For SPDX documents create using this class during the build, this \ + is the contact information for the person or organization who is doing the \ + build." + +def extract_licenses(filename): + import re + + lic_regex = re.compile(rb'^\W*SPDX-License-Identifier:\s*([ \w\d.()+-]+?)(?:\s+\W*)?$', re.MULTILINE) + + try: + with open(filename, 'rb') as f: + size = min(15000, os.stat(filename).st_size) + txt = f.read(size) + licenses = re.findall(lic_regex, txt) + if licenses: + ascii_licenses = [lic.decode('ascii') for lic in licenses] + return ascii_licenses + except Exception as e: + bb.warn(f"Exception reading {filename}: {e}") + return None + +def get_doc_namespace(d, doc): + import uuid + namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE")) + return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), doc.name, str(uuid.uuid5(namespace_uuid, doc.name))) + +def create_annotation(d, comment): + from datetime import datetime, timezone + + creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + annotation = oe.spdx.SPDXAnnotation() + annotation.annotationDate = creation_time + annotation.annotationType = "OTHER" + annotation.annotator = "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) + annotation.comment = comment + return annotation + +def recipe_spdx_is_native(d, recipe): + return any(a.annotationType == "OTHER" and + a.annotator == "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) and + a.comment == "isNative" for a in recipe.annotations) + +def is_work_shared_spdx(d): + return bb.data.inherits_class('kernel', d) or ('work-shared' in d.getVar('WORKDIR')) + +def get_json_indent(d): + if d.getVar("SPDX_PRETTY") == "1": + return 2 + return None + +python() { + import json + if d.getVar("SPDX_LICENSE_DATA"): + return + + with open(d.getVar("SPDX_LICENSES"), "r") as f: + data = json.load(f) + # Transform the license array to a dictionary + data["licenses"] = {l["licenseId"]: l for l in data["licenses"]} + d.setVar("SPDX_LICENSE_DATA", data) +} + +def convert_license_to_spdx(lic, document, d, existing={}): + from pathlib import Path + import oe.spdx + + license_data = d.getVar("SPDX_LICENSE_DATA") + extracted = {} + + def add_extracted_license(ident, name): + nonlocal document + + if name in extracted: + return + + extracted_info = oe.spdx.SPDXExtractedLicensingInfo() + extracted_info.name = name + extracted_info.licenseId = ident + extracted_info.extractedText = None + + if name == "PD": + # Special-case this. + extracted_info.extractedText = "Software released to the public domain" + else: + # Seach for the license in COMMON_LICENSE_DIR and LICENSE_PATH + for directory in [d.getVar('COMMON_LICENSE_DIR')] + (d.getVar('LICENSE_PATH') or '').split(): + try: + with (Path(directory) / name).open(errors="replace") as f: + extracted_info.extractedText = f.read() + break + except FileNotFoundError: + pass + if extracted_info.extractedText is None: + # If it's not SPDX or PD, then NO_GENERIC_LICENSE must be set + filename = d.getVarFlag('NO_GENERIC_LICENSE', name) + if filename: + filename = d.expand("${S}/" + filename) + with open(filename, errors="replace") as f: + extracted_info.extractedText = f.read() + else: + bb.error("Cannot find any text for license %s" % name) + + extracted[name] = extracted_info + document.hasExtractedLicensingInfos.append(extracted_info) + + def convert(l): + if l == "(" or l == ")": + return l + + if l == "&": + return "AND" + + if l == "|": + return "OR" + + if l == "CLOSED": + return "NONE" + + spdx_license = d.getVarFlag("SPDXLICENSEMAP", l) or l + if spdx_license in license_data["licenses"]: + return spdx_license + + try: + spdx_license = existing[l] + except KeyError: + spdx_license = "LicenseRef-" + l + add_extracted_license(spdx_license, l) + + return spdx_license + + lic_split = lic.replace("(", " ( ").replace(")", " ) ").split() + + return ' '.join(convert(l) for l in lic_split) + +def process_sources(d): + pn = d.getVar('PN') + assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split() + if pn in assume_provided: + for p in d.getVar("PROVIDES").split(): + if p != pn: + pn = p + break + + # glibc-locale: do_fetch, do_unpack and do_patch tasks have been deleted, + # so avoid archiving source here. + if pn.startswith('glibc-locale'): + return False + if d.getVar('PN') == "libtool-cross": + return False + if d.getVar('PN') == "libgcc-initial": + return False + if d.getVar('PN') == "shadow-sysroot": + return False + + # We just archive gcc-source for all the gcc related recipes + if d.getVar('BPN') in ['gcc', 'libgcc']: + bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn) + return False + + return True + + +def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archive=None, ignore_dirs=[], ignore_top_level_dirs=[]): + from pathlib import Path + import oe.spdx + import hashlib + + source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") + if source_date_epoch: + source_date_epoch = int(source_date_epoch) + + sha1s = [] + spdx_files = [] + + file_counter = 1 + for subdir, dirs, files in os.walk(topdir): + dirs[:] = [d for d in dirs if d not in ignore_dirs] + if subdir == str(topdir): + dirs[:] = [d for d in dirs if d not in ignore_top_level_dirs] + + for file in files: + filepath = Path(subdir) / file + filename = str(filepath.relative_to(topdir)) + + if not filepath.is_symlink() and filepath.is_file(): + spdx_file = oe.spdx.SPDXFile() + spdx_file.SPDXID = get_spdxid(file_counter) + for t in get_types(filepath): + spdx_file.fileTypes.append(t) + spdx_file.fileName = filename + + if archive is not None: + with filepath.open("rb") as f: + info = archive.gettarinfo(fileobj=f) + info.name = filename + info.uid = 0 + info.gid = 0 + info.uname = "root" + info.gname = "root" + + if source_date_epoch is not None and info.mtime > source_date_epoch: + info.mtime = source_date_epoch + + archive.addfile(info, f) + + sha1 = bb.utils.sha1_file(filepath) + sha1s.append(sha1) + spdx_file.checksums.append(oe.spdx.SPDXChecksum( + algorithm="SHA1", + checksumValue=sha1, + )) + spdx_file.checksums.append(oe.spdx.SPDXChecksum( + algorithm="SHA256", + checksumValue=bb.utils.sha256_file(filepath), + )) + + if "SOURCE" in spdx_file.fileTypes: + extracted_lics = extract_licenses(filepath) + if extracted_lics: + spdx_file.licenseInfoInFiles = extracted_lics + + doc.files.append(spdx_file) + doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file) + spdx_pkg.hasFiles.append(spdx_file.SPDXID) + + spdx_files.append(spdx_file) + + file_counter += 1 + + sha1s.sort() + verifier = hashlib.sha1() + for v in sha1s: + verifier.update(v.encode("utf-8")) + spdx_pkg.packageVerificationCode.packageVerificationCodeValue = verifier.hexdigest() + + return spdx_files + + +def add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources): + from pathlib import Path + import hashlib + import oe.packagedata + import oe.spdx + + debug_search_paths = [ + Path(d.getVar('PKGD')), + Path(d.getVar('STAGING_DIR_TARGET')), + Path(d.getVar('STAGING_DIR_NATIVE')), + Path(d.getVar('STAGING_KERNEL_DIR')), + ] + + pkg_data = oe.packagedata.read_subpkgdata_extended(package, d) + + if pkg_data is None: + return + + for file_path, file_data in pkg_data["files_info"].items(): + if not "debugsrc" in file_data: + continue + + for pkg_file in package_files: + if file_path.lstrip("/") == pkg_file.fileName.lstrip("/"): + break + else: + bb.fatal("No package file found for %s" % str(file_path)) + continue + + for debugsrc in file_data["debugsrc"]: + ref_id = "NOASSERTION" + for search in debug_search_paths: + if debugsrc.startswith("/usr/src/kernel"): + debugsrc_path = search / debugsrc.replace('/usr/src/kernel/', '') + else: + debugsrc_path = search / debugsrc.lstrip("/") + if not debugsrc_path.exists(): + continue + + file_sha256 = bb.utils.sha256_file(debugsrc_path) + + if file_sha256 in sources: + source_file = sources[file_sha256] + + doc_ref = package_doc.find_external_document_ref(source_file.doc.documentNamespace) + if doc_ref is None: + doc_ref = oe.spdx.SPDXExternalDocumentRef() + doc_ref.externalDocumentId = "DocumentRef-dependency-" + source_file.doc.name + doc_ref.spdxDocument = source_file.doc.documentNamespace + doc_ref.checksum.algorithm = "SHA1" + doc_ref.checksum.checksumValue = source_file.doc_sha1 + package_doc.externalDocumentRefs.append(doc_ref) + + ref_id = "%s:%s" % (doc_ref.externalDocumentId, source_file.file.SPDXID) + else: + bb.debug(1, "Debug source %s with SHA256 %s not found in any dependency" % (str(debugsrc_path), file_sha256)) + break + else: + bb.debug(1, "Debug source %s not found" % debugsrc) + + package_doc.add_relationship(pkg_file, "GENERATED_FROM", ref_id, comment=debugsrc) + +def collect_dep_recipes(d, doc, spdx_recipe): + from pathlib import Path + import oe.sbom + import oe.spdx + + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + + dep_recipes = [] + taskdepdata = d.getVar("BB_TASKDEPDATA", False) + deps = sorted(set( + dep[0] for dep in taskdepdata.values() if + dep[1] == "do_create_spdx" and dep[0] != d.getVar("PN") + )) + for dep_pn in deps: + dep_recipe_path = deploy_dir_spdx / "recipes" / ("recipe-%s.spdx.json" % dep_pn) + + spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_recipe_path) + + for pkg in spdx_dep_doc.packages: + if pkg.name == dep_pn: + spdx_dep_recipe = pkg + break + else: + continue + + dep_recipes.append(oe.sbom.DepRecipe(spdx_dep_doc, spdx_dep_sha1, spdx_dep_recipe)) + + dep_recipe_ref = oe.spdx.SPDXExternalDocumentRef() + dep_recipe_ref.externalDocumentId = "DocumentRef-dependency-" + spdx_dep_doc.name + dep_recipe_ref.spdxDocument = spdx_dep_doc.documentNamespace + dep_recipe_ref.checksum.algorithm = "SHA1" + dep_recipe_ref.checksum.checksumValue = spdx_dep_sha1 + + doc.externalDocumentRefs.append(dep_recipe_ref) + + doc.add_relationship( + "%s:%s" % (dep_recipe_ref.externalDocumentId, spdx_dep_recipe.SPDXID), + "BUILD_DEPENDENCY_OF", + spdx_recipe + ) + + return dep_recipes + +collect_dep_recipes[vardepsexclude] += "BB_TASKDEPDATA" +collect_dep_recipes[vardeps] += "DEPENDS" + +def collect_dep_sources(d, dep_recipes): + import oe.sbom + + sources = {} + for dep in dep_recipes: + # Don't collect sources from native recipes as they + # match non-native sources also. + if recipe_spdx_is_native(d, dep.recipe): + continue + recipe_files = set(dep.recipe.hasFiles) + + for spdx_file in dep.doc.files: + if spdx_file.SPDXID not in recipe_files: + continue + + if "SOURCE" in spdx_file.fileTypes: + for checksum in spdx_file.checksums: + if checksum.algorithm == "SHA256": + sources[checksum.checksumValue] = oe.sbom.DepSource(dep.doc, dep.doc_sha1, dep.recipe, spdx_file) + break + + return sources + +def add_download_packages(d, doc, recipe): + import os.path + from bb.fetch2 import decodeurl, CHECKSUM_LIST + import bb.process + import oe.spdx + import oe.sbom + + for download_idx, src_uri in enumerate(d.getVar('SRC_URI').split()): + f = bb.fetch2.FetchData(src_uri, d) + + for name in f.names: + package = oe.spdx.SPDXPackage() + package.name = "%s-source-%d" % (d.getVar("PN"), download_idx + 1) + package.SPDXID = oe.sbom.get_download_spdxid(d, download_idx + 1) + + if f.type == "file": + continue + + uri = f.type + proto = getattr(f, "proto", None) + if proto is not None: + uri = uri + "+" + proto + uri = uri + "://" + f.host + f.path + + if f.method.supports_srcrev(): + uri = uri + "@" + f.revisions[name] + + if f.method.supports_checksum(f): + for checksum_id in CHECKSUM_LIST: + if checksum_id.upper() not in oe.spdx.SPDXPackage.ALLOWED_CHECKSUMS: + continue + + expected_checksum = getattr(f, "%s_expected" % checksum_id) + if expected_checksum is None: + continue + + c = oe.spdx.SPDXChecksum() + c.algorithm = checksum_id.upper() + c.checksumValue = expected_checksum + package.checksums.append(c) + + package.downloadLocation = uri + doc.packages.append(package) + doc.add_relationship(doc, "DESCRIBES", package) + # In the future, we might be able to do more fancy dependencies, + # but this should be sufficient for now + doc.add_relationship(package, "BUILD_DEPENDENCY_OF", recipe) + +python do_create_spdx() { + from datetime import datetime, timezone + import oe.sbom + import oe.spdx + import uuid + from pathlib import Path + from contextlib import contextmanager + import oe.cve_check + + @contextmanager + def optional_tarfile(name, guard, mode="w"): + import tarfile + import gzip + + if guard: + name.parent.mkdir(parents=True, exist_ok=True) + with gzip.open(name, mode=mode + "b") as f: + with tarfile.open(fileobj=f, mode=mode + "|") as tf: + yield tf + else: + yield None + + + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + spdx_workdir = Path(d.getVar("SPDXWORK")) + include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1" + archive_sources = d.getVar("SPDX_ARCHIVE_SOURCES") == "1" + archive_packaged = d.getVar("SPDX_ARCHIVE_PACKAGED") == "1" + + creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + doc = oe.spdx.SPDXDocument() + + doc.name = "recipe-" + d.getVar("PN") + doc.documentNamespace = get_doc_namespace(d, doc) + doc.creationInfo.created = creation_time + doc.creationInfo.comment = "This document was created by analyzing recipe files during the build." + doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] + doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") + doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) + doc.creationInfo.creators.append("Person: N/A ()") + + recipe = oe.spdx.SPDXPackage() + recipe.name = d.getVar("PN") + recipe.versionInfo = d.getVar("PV") + recipe.SPDXID = oe.sbom.get_recipe_spdxid(d) + recipe.supplier = d.getVar("SPDX_SUPPLIER") + if bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d): + recipe.annotations.append(create_annotation(d, "isNative")) + + homepage = d.getVar("HOMEPAGE") + if homepage: + recipe.homepage = homepage + + license = d.getVar("LICENSE") + if license: + recipe.licenseDeclared = convert_license_to_spdx(license, doc, d) + + summary = d.getVar("SUMMARY") + if summary: + recipe.summary = summary + + description = d.getVar("DESCRIPTION") + if description: + recipe.description = description + + if d.getVar("SPDX_CUSTOM_ANNOTATION_VARS"): + for var in d.getVar('SPDX_CUSTOM_ANNOTATION_VARS').split(): + recipe.annotations.append(create_annotation(d, var + "=" + d.getVar(var))) + + # Some CVEs may be patched during the build process without incrementing the version number, + # so querying for CVEs based on the CPE id can lead to false positives. To account for this, + # save the CVEs fixed by patches to source information field in the SPDX. + patched_cves = oe.cve_check.get_patched_cves(d) + patched_cves = list(patched_cves) + patched_cves = ' '.join(patched_cves) + if patched_cves: + recipe.sourceInfo = "CVEs fixed: " + patched_cves + + cpe_ids = oe.cve_check.get_cpe_ids(d.getVar("CVE_PRODUCT"), d.getVar("CVE_VERSION")) + if cpe_ids: + for cpe_id in cpe_ids: + cpe = oe.spdx.SPDXExternalReference() + cpe.referenceCategory = "SECURITY" + cpe.referenceType = "http://spdx.org/rdf/references/cpe23Type" + cpe.referenceLocator = cpe_id + recipe.externalRefs.append(cpe) + + doc.packages.append(recipe) + doc.add_relationship(doc, "DESCRIBES", recipe) + + add_download_packages(d, doc, recipe) + + if process_sources(d) and include_sources: + recipe_archive = deploy_dir_spdx / "recipes" / (doc.name + ".tar.gz") + with optional_tarfile(recipe_archive, archive_sources) as archive: + spdx_get_src(d) + + add_package_files( + d, + doc, + recipe, + spdx_workdir, + lambda file_counter: "SPDXRef-SourceFile-%s-%d" % (d.getVar("PN"), file_counter), + lambda filepath: ["SOURCE"], + ignore_dirs=[".git"], + ignore_top_level_dirs=["temp"], + archive=archive, + ) + + if archive is not None: + recipe.packageFileName = str(recipe_archive.name) + + dep_recipes = collect_dep_recipes(d, doc, recipe) + + doc_sha1 = oe.sbom.write_doc(d, doc, "recipes", indent=get_json_indent(d)) + dep_recipes.append(oe.sbom.DepRecipe(doc, doc_sha1, recipe)) + + recipe_ref = oe.spdx.SPDXExternalDocumentRef() + recipe_ref.externalDocumentId = "DocumentRef-recipe-" + recipe.name + recipe_ref.spdxDocument = doc.documentNamespace + recipe_ref.checksum.algorithm = "SHA1" + recipe_ref.checksum.checksumValue = doc_sha1 + + sources = collect_dep_sources(d, dep_recipes) + found_licenses = {license.name:recipe_ref.externalDocumentId + ":" + license.licenseId for license in doc.hasExtractedLicensingInfos} + + if not recipe_spdx_is_native(d, recipe): + bb.build.exec_func("read_subpackage_metadata", d) + + pkgdest = Path(d.getVar("PKGDEST")) + for package in d.getVar("PACKAGES").split(): + if not oe.packagedata.packaged(package, d): + continue + + package_doc = oe.spdx.SPDXDocument() + pkg_name = d.getVar("PKG:%s" % package) or package + package_doc.name = pkg_name + package_doc.documentNamespace = get_doc_namespace(d, package_doc) + package_doc.creationInfo.created = creation_time + package_doc.creationInfo.comment = "This document was created by analyzing packages created during the build." + package_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] + package_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") + package_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) + package_doc.creationInfo.creators.append("Person: N/A ()") + package_doc.externalDocumentRefs.append(recipe_ref) + + package_license = d.getVar("LICENSE:%s" % package) or d.getVar("LICENSE") + + spdx_package = oe.spdx.SPDXPackage() + + spdx_package.SPDXID = oe.sbom.get_package_spdxid(pkg_name) + spdx_package.name = pkg_name + spdx_package.versionInfo = d.getVar("PV") + spdx_package.licenseDeclared = convert_license_to_spdx(package_license, package_doc, d, found_licenses) + spdx_package.supplier = d.getVar("SPDX_SUPPLIER") + + package_doc.packages.append(spdx_package) + + package_doc.add_relationship(spdx_package, "GENERATED_FROM", "%s:%s" % (recipe_ref.externalDocumentId, recipe.SPDXID)) + package_doc.add_relationship(package_doc, "DESCRIBES", spdx_package) + + package_archive = deploy_dir_spdx / "packages" / (package_doc.name + ".tar.gz") + with optional_tarfile(package_archive, archive_packaged) as archive: + package_files = add_package_files( + d, + package_doc, + spdx_package, + pkgdest / package, + lambda file_counter: oe.sbom.get_packaged_file_spdxid(pkg_name, file_counter), + lambda filepath: ["BINARY"], + ignore_top_level_dirs=['CONTROL', 'DEBIAN'], + archive=archive, + ) + + if archive is not None: + spdx_package.packageFileName = str(package_archive.name) + + add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources) + + oe.sbom.write_doc(d, package_doc, "packages", indent=get_json_indent(d)) +} +# NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source +addtask do_create_spdx after do_package do_packagedata do_unpack before do_populate_sdk do_build do_rm_work + +SSTATETASKS += "do_create_spdx" +do_create_spdx[sstate-inputdirs] = "${SPDXDEPLOY}" +do_create_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" + +python do_create_spdx_setscene () { + sstate_setscene(d) +} +addtask do_create_spdx_setscene + +do_create_spdx[dirs] = "${SPDXWORK}" +do_create_spdx[cleandirs] = "${SPDXDEPLOY} ${SPDXWORK}" +do_create_spdx[depends] += "${PATCHDEPENDENCY}" +do_create_spdx[deptask] = "do_create_spdx" + +def collect_package_providers(d): + from pathlib import Path + import oe.sbom + import oe.spdx + import json + + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + + providers = {} + + taskdepdata = d.getVar("BB_TASKDEPDATA", False) + deps = sorted(set( + dep[0] for dep in taskdepdata.values() if dep[0] != d.getVar("PN") + )) + deps.append(d.getVar("PN")) + + for dep_pn in deps: + recipe_data = oe.packagedata.read_pkgdata(dep_pn, d) + + for pkg in recipe_data.get("PACKAGES", "").split(): + + pkg_data = oe.packagedata.read_subpkgdata_dict(pkg, d) + rprovides = set(n for n, _ in bb.utils.explode_dep_versions2(pkg_data.get("RPROVIDES", "")).items()) + rprovides.add(pkg) + + for r in rprovides: + providers[r] = pkg + + return providers + +collect_package_providers[vardepsexclude] += "BB_TASKDEPDATA" + +python do_create_runtime_spdx() { + from datetime import datetime, timezone + import oe.sbom + import oe.spdx + import oe.packagedata + from pathlib import Path + + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + spdx_deploy = Path(d.getVar("SPDXRUNTIMEDEPLOY")) + is_native = bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d) + + creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + providers = collect_package_providers(d) + + if not is_native: + bb.build.exec_func("read_subpackage_metadata", d) + + dep_package_cache = {} + + pkgdest = Path(d.getVar("PKGDEST")) + for package in d.getVar("PACKAGES").split(): + localdata = bb.data.createCopy(d) + pkg_name = d.getVar("PKG:%s" % package) or package + localdata.setVar("PKG", pkg_name) + localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + package) + + if not oe.packagedata.packaged(package, localdata): + continue + + pkg_spdx_path = deploy_dir_spdx / "packages" / (pkg_name + ".spdx.json") + + package_doc, package_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path) + + for p in package_doc.packages: + if p.name == pkg_name: + spdx_package = p + break + else: + bb.fatal("Package '%s' not found in %s" % (pkg_name, pkg_spdx_path)) + + runtime_doc = oe.spdx.SPDXDocument() + runtime_doc.name = "runtime-" + pkg_name + runtime_doc.documentNamespace = get_doc_namespace(localdata, runtime_doc) + runtime_doc.creationInfo.created = creation_time + runtime_doc.creationInfo.comment = "This document was created by analyzing package runtime dependencies." + runtime_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] + runtime_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") + runtime_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) + runtime_doc.creationInfo.creators.append("Person: N/A ()") + + package_ref = oe.spdx.SPDXExternalDocumentRef() + package_ref.externalDocumentId = "DocumentRef-package-" + package + package_ref.spdxDocument = package_doc.documentNamespace + package_ref.checksum.algorithm = "SHA1" + package_ref.checksum.checksumValue = package_doc_sha1 + + runtime_doc.externalDocumentRefs.append(package_ref) + + runtime_doc.add_relationship( + runtime_doc.SPDXID, + "AMENDS", + "%s:%s" % (package_ref.externalDocumentId, package_doc.SPDXID) + ) + + deps = bb.utils.explode_dep_versions2(localdata.getVar("RDEPENDS") or "") + seen_deps = set() + for dep, _ in deps.items(): + if dep in seen_deps: + continue + + if dep not in providers: + continue + + dep = providers[dep] + + if not oe.packagedata.packaged(dep, localdata): + continue + + dep_pkg_data = oe.packagedata.read_subpkgdata_dict(dep, d) + dep_pkg = dep_pkg_data["PKG"] + + if dep in dep_package_cache: + (dep_spdx_package, dep_package_ref) = dep_package_cache[dep] + else: + dep_path = deploy_dir_spdx / "packages" / ("%s.spdx.json" % dep_pkg) + + spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_path) + + for pkg in spdx_dep_doc.packages: + if pkg.name == dep_pkg: + dep_spdx_package = pkg + break + else: + bb.fatal("Package '%s' not found in %s" % (dep_pkg, dep_path)) + + dep_package_ref = oe.spdx.SPDXExternalDocumentRef() + dep_package_ref.externalDocumentId = "DocumentRef-runtime-dependency-" + spdx_dep_doc.name + dep_package_ref.spdxDocument = spdx_dep_doc.documentNamespace + dep_package_ref.checksum.algorithm = "SHA1" + dep_package_ref.checksum.checksumValue = spdx_dep_sha1 + + dep_package_cache[dep] = (dep_spdx_package, dep_package_ref) + + runtime_doc.externalDocumentRefs.append(dep_package_ref) + + runtime_doc.add_relationship( + "%s:%s" % (dep_package_ref.externalDocumentId, dep_spdx_package.SPDXID), + "RUNTIME_DEPENDENCY_OF", + "%s:%s" % (package_ref.externalDocumentId, spdx_package.SPDXID) + ) + seen_deps.add(dep) + + oe.sbom.write_doc(d, runtime_doc, "runtime", spdx_deploy, indent=get_json_indent(d)) +} + +addtask do_create_runtime_spdx after do_create_spdx before do_build do_rm_work +SSTATETASKS += "do_create_runtime_spdx" +do_create_runtime_spdx[sstate-inputdirs] = "${SPDXRUNTIMEDEPLOY}" +do_create_runtime_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" + +python do_create_runtime_spdx_setscene () { + sstate_setscene(d) +} +addtask do_create_runtime_spdx_setscene + +do_create_runtime_spdx[dirs] = "${SPDXRUNTIMEDEPLOY}" +do_create_runtime_spdx[cleandirs] = "${SPDXRUNTIMEDEPLOY}" +do_create_runtime_spdx[rdeptask] = "do_create_spdx" + +def spdx_get_src(d): + """ + save patched source of the recipe in SPDX_WORKDIR. + """ + import shutil + spdx_workdir = d.getVar('SPDXWORK') + spdx_sysroot_native = d.getVar('STAGING_DIR_NATIVE') + pn = d.getVar('PN') + + workdir = d.getVar("WORKDIR") + + try: + # The kernel class functions require it to be on work-shared, so we dont change WORKDIR + if not is_work_shared_spdx(d): + # Change the WORKDIR to make do_unpack do_patch run in another dir. + d.setVar('WORKDIR', spdx_workdir) + # Restore the original path to recipe's native sysroot (it's relative to WORKDIR). + d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native) + + # The changed 'WORKDIR' also caused 'B' changed, create dir 'B' for the + # possibly requiring of the following tasks (such as some recipes's + # do_patch required 'B' existed). + bb.utils.mkdirhier(d.getVar('B')) + + bb.build.exec_func('do_unpack', d) + # Copy source of kernel to spdx_workdir + if is_work_shared_spdx(d): + share_src = d.getVar('WORKDIR') + d.setVar('WORKDIR', spdx_workdir) + d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native) + src_dir = spdx_workdir + "/" + d.getVar('PN')+ "-" + d.getVar('PV') + "-" + d.getVar('PR') + bb.utils.mkdirhier(src_dir) + if bb.data.inherits_class('kernel',d): + share_src = d.getVar('STAGING_KERNEL_DIR') + cmd_copy_share = "cp -rf " + share_src + "/* " + src_dir + "/" + cmd_copy_shared_res = os.popen(cmd_copy_share).read() + bb.note("cmd_copy_shared_result = " + cmd_copy_shared_res) + + git_path = src_dir + "/.git" + if os.path.exists(git_path): + shutils.rmtree(git_path) + + # Make sure gcc and kernel sources are patched only once + if not (d.getVar('SRC_URI') == "" or is_work_shared_spdx(d)): + bb.build.exec_func('do_patch', d) + + # Some userland has no source. + if not os.path.exists( spdx_workdir ): + bb.utils.mkdirhier(spdx_workdir) + finally: + d.setVar("WORKDIR", workdir) + +do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx" +do_rootfs[cleandirs] += "${SPDXIMAGEWORK}" + +ROOTFS_POSTUNINSTALL_COMMAND =+ "image_combine_spdx ; " + +do_populate_sdk[recrdeptask] += "do_create_spdx do_create_runtime_spdx" +do_populate_sdk[cleandirs] += "${SPDXSDKWORK}" +POPULATE_SDK_POST_HOST_COMMAND:append:task-populate-sdk = " sdk_host_combine_spdx; " +POPULATE_SDK_POST_TARGET_COMMAND:append:task-populate-sdk = " sdk_target_combine_spdx; " + +python image_combine_spdx() { + import os + import oe.sbom + from pathlib import Path + from oe.rootfs import image_list_installed_packages + + image_name = d.getVar("IMAGE_NAME") + image_link_name = d.getVar("IMAGE_LINK_NAME") + imgdeploydir = Path(d.getVar("IMGDEPLOYDIR")) + img_spdxid = oe.sbom.get_image_spdxid(image_name) + packages = image_list_installed_packages(d) + + combine_spdx(d, image_name, imgdeploydir, img_spdxid, packages, Path(d.getVar("SPDXIMAGEWORK"))) + + def make_image_link(target_path, suffix): + if image_link_name: + link = imgdeploydir / (image_link_name + suffix) + if link != target_path: + link.symlink_to(os.path.relpath(target_path, link.parent)) + + spdx_tar_path = imgdeploydir / (image_name + ".spdx.tar.gz") + make_image_link(spdx_tar_path, ".spdx.tar.gz") +} + +python sdk_host_combine_spdx() { + sdk_combine_spdx(d, "host") +} + +python sdk_target_combine_spdx() { + sdk_combine_spdx(d, "target") +} + +def sdk_combine_spdx(d, sdk_type): + import oe.sbom + from pathlib import Path + from oe.sdk import sdk_list_installed_packages + + sdk_name = d.getVar("SDK_NAME") + "-" + sdk_type + sdk_deploydir = Path(d.getVar("SDKDEPLOYDIR")) + sdk_spdxid = oe.sbom.get_sdk_spdxid(sdk_name) + sdk_packages = sdk_list_installed_packages(d, sdk_type == "target") + combine_spdx(d, sdk_name, sdk_deploydir, sdk_spdxid, sdk_packages, Path(d.getVar('SPDXSDKWORK'))) + +def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx_workdir): + import os + import oe.spdx + import oe.sbom + import io + import json + from datetime import timezone, datetime + from pathlib import Path + import tarfile + import gzip + + creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") + + doc = oe.spdx.SPDXDocument() + doc.name = rootfs_name + doc.documentNamespace = get_doc_namespace(d, doc) + doc.creationInfo.created = creation_time + doc.creationInfo.comment = "This document was created by analyzing the source of the Yocto recipe during the build." + doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] + doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") + doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) + doc.creationInfo.creators.append("Person: N/A ()") + + image = oe.spdx.SPDXPackage() + image.name = d.getVar("PN") + image.versionInfo = d.getVar("PV") + image.SPDXID = rootfs_spdxid + image.supplier = d.getVar("SPDX_SUPPLIER") + + doc.packages.append(image) + + for name in sorted(packages.keys()): + pkg_spdx_path = deploy_dir_spdx / "packages" / (name + ".spdx.json") + pkg_doc, pkg_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path) + + for p in pkg_doc.packages: + if p.name == name: + pkg_ref = oe.spdx.SPDXExternalDocumentRef() + pkg_ref.externalDocumentId = "DocumentRef-%s" % pkg_doc.name + pkg_ref.spdxDocument = pkg_doc.documentNamespace + pkg_ref.checksum.algorithm = "SHA1" + pkg_ref.checksum.checksumValue = pkg_doc_sha1 + + doc.externalDocumentRefs.append(pkg_ref) + doc.add_relationship(image, "CONTAINS", "%s:%s" % (pkg_ref.externalDocumentId, p.SPDXID)) + break + else: + bb.fatal("Unable to find package with name '%s' in SPDX file %s" % (name, pkg_spdx_path)) + + runtime_spdx_path = deploy_dir_spdx / "runtime" / ("runtime-" + name + ".spdx.json") + runtime_doc, runtime_doc_sha1 = oe.sbom.read_doc(runtime_spdx_path) + + runtime_ref = oe.spdx.SPDXExternalDocumentRef() + runtime_ref.externalDocumentId = "DocumentRef-%s" % runtime_doc.name + runtime_ref.spdxDocument = runtime_doc.documentNamespace + runtime_ref.checksum.algorithm = "SHA1" + runtime_ref.checksum.checksumValue = runtime_doc_sha1 + + # "OTHER" isn't ideal here, but I can't find a relationship that makes sense + doc.externalDocumentRefs.append(runtime_ref) + doc.add_relationship( + image, + "OTHER", + "%s:%s" % (runtime_ref.externalDocumentId, runtime_doc.SPDXID), + comment="Runtime dependencies for %s" % name + ) + + image_spdx_path = spdx_workdir / (rootfs_name + ".spdx.json") + + with image_spdx_path.open("wb") as f: + doc.to_json(f, sort_keys=True, indent=get_json_indent(d)) + + num_threads = int(d.getVar("BB_NUMBER_THREADS")) + + visited_docs = set() + + index = {"documents": []} + + spdx_tar_path = rootfs_deploydir / (rootfs_name + ".spdx.tar.gz") + with gzip.open(spdx_tar_path, "w") as f: + with tarfile.open(fileobj=f, mode="w|") as tar: + def collect_spdx_document(path): + nonlocal tar + nonlocal deploy_dir_spdx + nonlocal source_date_epoch + nonlocal index + + if path in visited_docs: + return + + visited_docs.add(path) + + with path.open("rb") as f: + doc, sha1 = oe.sbom.read_doc(f) + f.seek(0) + + if doc.documentNamespace in visited_docs: + return + + bb.note("Adding SPDX document %s" % path) + visited_docs.add(doc.documentNamespace) + info = tar.gettarinfo(fileobj=f) + + info.name = doc.name + ".spdx.json" + info.uid = 0 + info.gid = 0 + info.uname = "root" + info.gname = "root" + + if source_date_epoch is not None and info.mtime > int(source_date_epoch): + info.mtime = int(source_date_epoch) + + tar.addfile(info, f) + + index["documents"].append({ + "filename": info.name, + "documentNamespace": doc.documentNamespace, + "sha1": sha1, + }) + + for ref in doc.externalDocumentRefs: + ref_path = deploy_dir_spdx / "by-namespace" / ref.spdxDocument.replace("/", "_") + collect_spdx_document(ref_path) + + collect_spdx_document(image_spdx_path) + + index["documents"].sort(key=lambda x: x["filename"]) + + index_str = io.BytesIO(json.dumps( + index, + sort_keys=True, + indent=get_json_indent(d), + ).encode("utf-8")) + + info = tarfile.TarInfo() + info.name = "index.json" + info.size = len(index_str.getvalue()) + info.uid = 0 + info.gid = 0 + info.uname = "root" + info.gname = "root" + + tar.addfile(info, fileobj=index_str) diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass new file mode 100644 index 0000000000..19c6c0ff0b --- /dev/null +++ b/meta/classes/create-spdx.bbclass @@ -0,0 +1,8 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: GPL-2.0-only +# +# Include this class when you don't care what version of SPDX you get; it will +# be updated to the latest stable version that is supported +inherit create-spdx-2.2 diff --git a/meta/classes/cve-check.bbclass b/meta/classes/cve-check.bbclass index 9eb9a95574..5e6bae1757 100644 --- a/meta/classes/cve-check.bbclass +++ b/meta/classes/cve-check.bbclass @@ -26,7 +26,7 @@ CVE_PRODUCT ??= "${BPN}" CVE_VERSION ??= "${PV}" CVE_CHECK_DB_DIR ?= "${DL_DIR}/CVE_CHECK" -CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/nvdcve_1.1.db" +CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/nvdcve_2.db" CVE_CHECK_DB_FILE_LOCK ?= "${CVE_CHECK_DB_FILE}.lock" CVE_CHECK_LOG ?= "${T}/cve.log" @@ -42,8 +42,8 @@ CVE_CHECK_LOG_JSON ?= "${T}/cve.json" CVE_CHECK_DIR ??= "${DEPLOY_DIR}/cve" CVE_CHECK_RECIPE_FILE ?= "${CVE_CHECK_DIR}/${PN}" CVE_CHECK_RECIPE_FILE_JSON ?= "${CVE_CHECK_DIR}/${PN}_cve.json" -CVE_CHECK_MANIFEST ?= "${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.cve" -CVE_CHECK_MANIFEST_JSON ?= "${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.json" +CVE_CHECK_MANIFEST ?= "${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.cve" +CVE_CHECK_MANIFEST_JSON ?= "${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.json" CVE_CHECK_COPY_FILES ??= "1" CVE_CHECK_CREATE_MANIFEST ??= "1" @@ -97,6 +97,8 @@ def generate_json_report(d, out_path, link_path): cve_check_merge_jsons(summary, data) filename = f.readline() + summary["package"].sort(key=lambda d: d['name']) + with open(out_path, "w") as f: json.dump(summary, f, indent=2) @@ -138,22 +140,23 @@ python do_cve_check () { """ from oe.cve_check import get_patched_cves - if os.path.exists(d.getVar("CVE_CHECK_DB_FILE")): - try: - patched_cves = get_patched_cves(d) - except FileNotFoundError: - bb.fatal("Failure in searching patches") - whitelisted, patched, unpatched, status = check_cves(d, patched_cves) - if patched or unpatched or (d.getVar("CVE_CHECK_COVERAGE") == "1" and status): - cve_data = get_cve_info(d, patched + unpatched + whitelisted) - cve_write_data(d, patched, unpatched, whitelisted, cve_data, status) - else: - bb.note("No CVE database found, skipping CVE check") + with bb.utils.fileslocked([d.getVar("CVE_CHECK_DB_FILE_LOCK")], shared=True): + if os.path.exists(d.getVar("CVE_CHECK_DB_FILE")): + try: + patched_cves = get_patched_cves(d) + except FileNotFoundError: + bb.fatal("Failure in searching patches") + ignored, patched, unpatched, status = check_cves(d, patched_cves) + if patched or unpatched or (d.getVar("CVE_CHECK_COVERAGE") == "1" and status): + cve_data = get_cve_info(d, patched + unpatched + ignored) + cve_write_data(d, patched, unpatched, ignored, cve_data, status) + else: + bb.note("No CVE database found, skipping CVE check") } addtask cve_check before do_build -do_cve_check[depends] = "cve-update-db-native:do_fetch" +do_cve_check[depends] = "cve-update-nvd2-native:do_fetch" do_cve_check[nostamp] = "1" python cve_check_cleanup () { @@ -194,7 +197,7 @@ python cve_check_write_rootfs_manifest () { recipies.add(pkg_data["PN"]) bb.note("Writing rootfs CVE manifest") - deploy_dir = d.getVar("DEPLOY_DIR_IMAGE") + deploy_dir = d.getVar("IMGDEPLOYDIR") link_name = d.getVar("IMAGE_LINK_NAME") json_data = {"version":"1", "package": []} @@ -252,7 +255,7 @@ def check_cves(d, patched_cves): """ Connect to the NVD database and find unpatched cves. """ - from oe.cve_check import Version + from oe.cve_check import Version, convert_cve_version pn = d.getVar("PN") real_pv = d.getVar("PV") @@ -289,7 +292,8 @@ def check_cves(d, patched_cves): vendor = "%" # Find all relevant CVE IDs. - for cverow in conn.execute("SELECT DISTINCT ID FROM PRODUCTS WHERE PRODUCT IS ? AND VENDOR LIKE ?", (product, vendor)): + cve_cursor = conn.execute("SELECT DISTINCT ID FROM PRODUCTS WHERE PRODUCT IS ? AND VENDOR LIKE ?", (product, vendor)) + for cverow in cve_cursor: cve = cverow[0] if cve in cve_whitelist: @@ -308,12 +312,16 @@ def check_cves(d, patched_cves): vulnerable = False ignored = False - for row in conn.execute("SELECT * FROM PRODUCTS WHERE ID IS ? AND PRODUCT IS ? AND VENDOR LIKE ?", (cve, product, vendor)): + product_cursor = conn.execute("SELECT * FROM PRODUCTS WHERE ID IS ? AND PRODUCT IS ? AND VENDOR LIKE ?", (cve, product, vendor)) + for row in product_cursor: (_, _, _, version_start, operator_start, version_end, operator_end) = row #bb.debug(2, "Evaluating row " + str(row)) if cve in cve_whitelist: ignored = True + version_start = convert_cve_version(version_start) + version_end = convert_cve_version(version_end) + if (operator_start == '=' and pv == version_start) or version_start == '-': vulnerable = True else: @@ -352,10 +360,12 @@ def check_cves(d, patched_cves): bb.note("%s-%s is vulnerable to %s" % (pn, real_pv, cve)) cves_unpatched.append(cve) break + product_cursor.close() if not vulnerable: bb.note("%s-%s is not vulnerable to %s" % (pn, real_pv, cve)) patched_cves.add(cve) + cve_cursor.close() if not cves_in_product: bb.note("No CVE records found for product %s, pn %s" % (product, pn)) @@ -377,14 +387,15 @@ def get_cve_info(d, cves): conn = sqlite3.connect(db_file, uri=True) for cve in cves: - for row in conn.execute("SELECT * FROM NVD WHERE ID IS ?", (cve,)): + cursor = conn.execute("SELECT * FROM NVD WHERE ID IS ?", (cve,)) + for row in cursor: cve_data[row[0]] = {} cve_data[row[0]]["summary"] = row[1] cve_data[row[0]]["scorev2"] = row[2] cve_data[row[0]]["scorev3"] = row[3] cve_data[row[0]]["modified"] = row[4] cve_data[row[0]]["vector"] = row[5] - + cursor.close() conn.close() return cve_data diff --git a/meta/classes/devshell.bbclass b/meta/classes/devshell.bbclass index b6212ebd89..76dd0b42ee 100644 --- a/meta/classes/devshell.bbclass +++ b/meta/classes/devshell.bbclass @@ -2,8 +2,6 @@ inherit terminal DEVSHELL = "${SHELL}" -PATH:prepend:task-devshell = "${COREBASE}/scripts/git-intercept:" - python do_devshell () { if d.getVarFlag("do_devshell", "manualfakeroot"): d.prependVar("DEVSHELL", "pseudo ") diff --git a/meta/classes/externalsrc.bbclass b/meta/classes/externalsrc.bbclass index 0e0a3ae89c..9c9451e528 100644 --- a/meta/classes/externalsrc.bbclass +++ b/meta/classes/externalsrc.bbclass @@ -60,7 +60,7 @@ python () { if externalsrcbuild: d.setVar('B', externalsrcbuild) else: - d.setVar('B', '${WORKDIR}/${BPN}-${PV}/') + d.setVar('B', '${WORKDIR}/${BPN}-${PV}') local_srcuri = [] fetch = bb.fetch2.Fetch((d.getVar('SRC_URI') or '').split(), d) @@ -207,8 +207,8 @@ def srctree_hash_files(d, srcdir=None): try: git_dir = os.path.join(s_dir, subprocess.check_output(['git', '-C', s_dir, 'rev-parse', '--git-dir'], stderr=subprocess.DEVNULL).decode("utf-8").rstrip()) - top_git_dir = os.path.join(s_dir, subprocess.check_output(['git', '-C', d.getVar("TOPDIR"), 'rev-parse', '--git-dir'], - stderr=subprocess.DEVNULL).decode("utf-8").rstrip()) + top_git_dir = os.path.join(d.getVar("TOPDIR"), + subprocess.check_output(['git', '-C', d.getVar("TOPDIR"), 'rev-parse', '--git-dir'], stderr=subprocess.DEVNULL).decode("utf-8").rstrip()) if git_dir == top_git_dir: git_dir = None except subprocess.CalledProcessError: @@ -225,15 +225,16 @@ def srctree_hash_files(d, srcdir=None): env['GIT_INDEX_FILE'] = tmp_index.name subprocess.check_output(['git', 'add', '-A', '.'], cwd=s_dir, env=env) git_sha1 = subprocess.check_output(['git', 'write-tree'], cwd=s_dir, env=env).decode("utf-8") - submodule_helper = subprocess.check_output(['git', 'submodule--helper', 'list'], cwd=s_dir, env=env).decode("utf-8") - for line in submodule_helper.splitlines(): - module_dir = os.path.join(s_dir, line.rsplit(maxsplit=1)[1]) - if os.path.isdir(module_dir): - proc = subprocess.Popen(['git', 'add', '-A', '.'], cwd=module_dir, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - proc.communicate() - proc = subprocess.Popen(['git', 'write-tree'], cwd=module_dir, env=env, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) - stdout, _ = proc.communicate() - git_sha1 += stdout.decode("utf-8") + if os.path.exists(os.path.join(s_dir, ".gitmodules")) and os.path.getsize(os.path.join(s_dir, ".gitmodules")) > 0: + submodule_helper = subprocess.check_output(["git", "config", "--file", ".gitmodules", "--get-regexp", "path"], cwd=s_dir, env=env).decode("utf-8") + for line in submodule_helper.splitlines(): + module_dir = os.path.join(s_dir, line.rsplit(maxsplit=1)[1]) + if os.path.isdir(module_dir): + proc = subprocess.Popen(['git', 'add', '-A', '.'], cwd=module_dir, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + proc.communicate() + proc = subprocess.Popen(['git', 'write-tree'], cwd=module_dir, env=env, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + stdout, _ = proc.communicate() + git_sha1 += stdout.decode("utf-8") sha1 = hashlib.sha1(git_sha1.encode("utf-8")).hexdigest() with open(oe_hash_file, 'w') as fobj: fobj.write(sha1) diff --git a/meta/classes/fs-uuid.bbclass b/meta/classes/fs-uuid.bbclass index 9b53dfba7a..731ea575bd 100644 --- a/meta/classes/fs-uuid.bbclass +++ b/meta/classes/fs-uuid.bbclass @@ -4,7 +4,7 @@ def get_rootfs_uuid(d): import subprocess rootfs = d.getVar('ROOTFS') - output = subprocess.check_output(['tune2fs', '-l', rootfs]) + output = subprocess.check_output(['tune2fs', '-l', rootfs], text=True) for line in output.split('\n'): if line.startswith('Filesystem UUID:'): uuid = line.split()[-1] diff --git a/meta/classes/go.bbclass b/meta/classes/go.bbclass index 16e46398b1..21b1a0271e 100644 --- a/meta/classes/go.bbclass +++ b/meta/classes/go.bbclass @@ -118,7 +118,7 @@ go_do_install() { tar -C ${B} -cf - --exclude-vcs --exclude '*.test' --exclude 'testdata' pkg | \ tar -C ${D}${libdir}/go --no-same-owner -xf - - if [ -n "`ls ${B}/${GO_BUILD_BINDIR}/`" ]; then + if ls ${B}/${GO_BUILD_BINDIR}/* >/dev/null 2>/dev/null ; then install -d ${D}${bindir} install -m 0755 ${B}/${GO_BUILD_BINDIR}/* ${D}${bindir}/ fi diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass index 0d77d2f676..fbf7206d04 100644 --- a/meta/classes/image.bbclass +++ b/meta/classes/image.bbclass @@ -311,7 +311,7 @@ fakeroot python do_image_qa () { except oe.utils.ImageQAFailed as e: qamsg = qamsg + '\tImage QA function %s failed: %s\n' % (e.name, e.description) except Exception as e: - qamsg = qamsg + '\tImage QA function %s failed\n' % cmd + qamsg = qamsg + '\tImage QA function %s failed: %s\n' % (cmd, e) if qamsg: imgname = d.getVar('IMAGE_NAME') @@ -437,7 +437,7 @@ python () { localdata.delVar('DATETIME') localdata.delVar('DATE') localdata.delVar('TMPDIR') - vardepsexclude = (d.getVarFlag('IMAGE_CMD_' + realt, 'vardepsexclude', True) or '').split() + vardepsexclude = (d.getVarFlag('IMAGE_CMD_' + realt, 'vardepsexclude') or '').split() for dep in vardepsexclude: localdata.delVar(dep) diff --git a/meta/classes/kernel-arch.bbclass b/meta/classes/kernel-arch.bbclass index 07ec242e63..4cd08b96fb 100644 --- a/meta/classes/kernel-arch.bbclass +++ b/meta/classes/kernel-arch.bbclass @@ -61,8 +61,8 @@ HOST_LD_KERNEL_ARCH ?= "${TARGET_LD_KERNEL_ARCH}" TARGET_AR_KERNEL_ARCH ?= "" HOST_AR_KERNEL_ARCH ?= "${TARGET_AR_KERNEL_ARCH}" -KERNEL_CC = "${CCACHE}${HOST_PREFIX}gcc ${HOST_CC_KERNEL_ARCH} -fuse-ld=bfd ${DEBUG_PREFIX_MAP} -fdebug-prefix-map=${STAGING_KERNEL_DIR}=${KERNEL_SRC_PATH}" +KERNEL_CC = "${CCACHE}${HOST_PREFIX}gcc ${HOST_CC_KERNEL_ARCH} -fuse-ld=bfd ${DEBUG_PREFIX_MAP} -fdebug-prefix-map=${STAGING_KERNEL_DIR}=${KERNEL_SRC_PATH} -fdebug-prefix-map=${STAGING_KERNEL_BUILDDIR}=${KERNEL_SRC_PATH}" KERNEL_LD = "${CCACHE}${HOST_PREFIX}ld.bfd ${HOST_LD_KERNEL_ARCH}" KERNEL_AR = "${CCACHE}${HOST_PREFIX}ar ${HOST_AR_KERNEL_ARCH}" -TOOLCHAIN = "gcc" +TOOLCHAIN ?= "gcc" diff --git a/meta/classes/kernel-fitimage.bbclass b/meta/classes/kernel-fitimage.bbclass index 7c0d93625b..7c7bcd3fc0 100644 --- a/meta/classes/kernel-fitimage.bbclass +++ b/meta/classes/kernel-fitimage.bbclass @@ -1,5 +1,7 @@ inherit kernel-uboot kernel-artifact-names uboot-sign +KERNEL_IMAGETYPE_REPLACEMENT = "" + python __anonymous () { kerneltypes = d.getVar('KERNEL_IMAGETYPES') or "" if 'fitImage' in kerneltypes.split(): @@ -21,6 +23,8 @@ python __anonymous () { else: replacementtype = "zImage" + d.setVar("KERNEL_IMAGETYPE_REPLACEMENT", replacementtype) + # Override KERNEL_IMAGETYPE_FOR_MAKE variable, which is internal # to kernel.bbclass . We have to override it, since we pack zImage # (at least for now) into the fitImage . @@ -45,6 +49,8 @@ python __anonymous () { if d.getVar('UBOOT_SIGN_ENABLE') == "1" and d.getVar('UBOOT_DTB_BINARY'): uboot_pn = d.getVar('PREFERRED_PROVIDER_u-boot') or 'u-boot' d.appendVarFlag('do_assemble_fitimage', 'depends', ' %s:do_populate_sysroot' % uboot_pn) + if d.getVar('INITRAMFS_IMAGE_BUNDLE') == "1": + d.appendVarFlag('do_assemble_fitimage_initramfs', 'depends', ' %s:do_populate_sysroot' % uboot_pn) } # Options for the device tree compiler passed to mkimage '-D' feature: @@ -59,6 +65,9 @@ FIT_SIGN_ALG ?= "rsa2048" # fitImage Padding Algo FIT_PAD_ALG ?= "pkcs-1.5" +# Arguments passed to mkimage for signing +UBOOT_MKIMAGE_SIGN_ARGS ?= "" + # # Emit the fitImage ITS header # @@ -178,6 +187,43 @@ EOF } # +# Emit the fitImage ITS u-boot script section +# +# $1 ... .its filename +# $2 ... Image counter +# $3 ... Path to boot script image +fitimage_emit_section_boot_script() { + + bootscr_csum="${FIT_HASH_ALG}" + bootscr_sign_algo="${FIT_SIGN_ALG}" + bootscr_sign_keyname="${UBOOT_SIGN_IMG_KEYNAME}" + + cat << EOF >> $1 + bootscr-$2 { + description = "U-boot script"; + data = /incbin/("$3"); + type = "script"; + arch = "${UBOOT_ARCH}"; + compression = "none"; + hash-1 { + algo = "$bootscr_csum"; + }; + }; +EOF + + if [ "${UBOOT_SIGN_ENABLE}" = "1" -a "${FIT_SIGN_INDIVIDUAL}" = "1" -a -n "$bootscr_sign_keyname" ] ; then + sed -i '$ d' $1 + cat << EOF >> $1 + signature-1 { + algo = "$bootscr_csum,$bootscr_sign_algo"; + key-name-hint = "$bootscr_sign_keyname"; + }; + }; +EOF + fi +} + +# # Emit the fitImage ITS setup section # # $1 ... .its filename @@ -247,8 +293,9 @@ EOF # $2 ... Linux kernel ID # $3 ... DTB image name # $4 ... ramdisk ID -# $5 ... config ID -# $6 ... default flag +# $5 ... u-boot script ID +# $6 ... config ID +# $7 ... default flag fitimage_emit_section_config() { conf_csum="${FIT_HASH_ALG}" @@ -264,6 +311,7 @@ fitimage_emit_section_config() { kernel_line="" fdt_line="" ramdisk_line="" + bootscr_line="" setup_line="" default_line="" @@ -286,21 +334,28 @@ fitimage_emit_section_config() { fi if [ -n "${5}" ]; then + conf_desc="${conf_desc}${sep}u-boot script" + sep=", " + bootscr_line="bootscr = \"bootscr-${5}\";" + fi + + if [ -n "${6}" ]; then conf_desc="${conf_desc}${sep}setup" - setup_line="setup = \"setup-${5}\";" + setup_line="setup = \"setup-${6}\";" fi - if [ "${6}" = "1" ]; then + if [ "${7}" = "1" ]; then default_line="default = \"conf-${3}\";" fi cat << EOF >> ${1} ${default_line} conf-${3} { - description = "${6} ${conf_desc}"; + description = "${7} ${conf_desc}"; ${kernel_line} ${fdt_line} ${ramdisk_line} + ${bootscr_line} ${setup_line} hash-1 { algo = "${conf_csum}"; @@ -328,6 +383,11 @@ EOF fi if [ -n "${5}" ]; then + sign_line="${sign_line}${sep}\"bootscr\"" + sep=", " + fi + + if [ -n "${6}" ]; then sign_line="${sign_line}${sep}\"setup\"" fi @@ -360,6 +420,7 @@ fitimage_assemble() { DTBS="" ramdiskcount=${3} setupcount="" + bootscr_id="" rm -f ${1} arch/${ARCH}/boot/${2} fitimage_emit_fit_header ${1} @@ -370,7 +431,7 @@ fitimage_assemble() { fitimage_emit_section_maint ${1} imagestart uboot_prep_kimage - fitimage_emit_section_kernel ${1} "${kernelcount}" linux.bin "${linux_comp}" + fitimage_emit_section_kernel $1 $kernelcount linux.bin "$linux_comp" # # Step 2: Prepare a DTB image section @@ -404,7 +465,21 @@ fitimage_assemble() { fi # - # Step 3: Prepare a setup section. (For x86) + # Step 3: Prepare a u-boot script section + # + + if [ -n "${UBOOT_ENV}" ] && [ -d "${STAGING_DIR_HOST}/boot" ]; then + if [ -e "${STAGING_DIR_HOST}/boot/${UBOOT_ENV_BINARY}" ]; then + cp ${STAGING_DIR_HOST}/boot/${UBOOT_ENV_BINARY} ${B} + bootscr_id="${UBOOT_ENV_BINARY}" + fitimage_emit_section_boot_script ${1} "${bootscr_id}" ${UBOOT_ENV_BINARY} + else + bbwarn "${STAGING_DIR_HOST}/boot/${UBOOT_ENV_BINARY} not found." + fi + fi + + # + # Step 4: Prepare a setup section. (For x86) # if [ -e arch/${ARCH}/boot/setup.bin ]; then setupcount=1 @@ -412,9 +487,9 @@ fitimage_assemble() { fi # - # Step 4: Prepare a ramdisk section. + # Step 5: Prepare a ramdisk section. # - if [ "x${ramdiskcount}" = "x1" ] ; then + if [ "x${ramdiskcount}" = "x1" ] && [ "${INITRAMFS_IMAGE_BUNDLE}" != "1" ]; then # Find and use the first initramfs image archive type we find for img in cpio.lz4 cpio.lzo cpio.lzma cpio.xz cpio.gz ext2.gz cpio; do initramfs_path="${DEPLOY_DIR_IMAGE}/${INITRAMFS_IMAGE_NAME}.${img}" @@ -435,7 +510,7 @@ fitimage_assemble() { fi # - # Step 5: Prepare a configurations section + # Step 6: Prepare a configurations section # fitimage_emit_section_maint ${1} confstart @@ -444,9 +519,9 @@ fitimage_assemble() { for DTB in ${DTBS}; do dtb_ext=${DTB##*.} if [ "${dtb_ext}" = "dtbo" ]; then - fitimage_emit_section_config ${1} "" "${DTB}" "" "" "`expr ${i} = ${dtbcount}`" + fitimage_emit_section_config ${1} "" "${DTB}" "" "${bootscr_id}" "" "`expr ${i} = ${dtbcount}`" else - fitimage_emit_section_config ${1} "${kernelcount}" "${DTB}" "${ramdiskcount}" "${setupcount}" "`expr ${i} = ${dtbcount}`" + fitimage_emit_section_config ${1} "${kernelcount}" "${DTB}" "${ramdiskcount}" "${bootscr_id}" "${setupcount}" "`expr ${i} = ${dtbcount}`" fi i=`expr ${i} + 1` done @@ -457,7 +532,7 @@ fitimage_assemble() { fitimage_emit_section_maint ${1} fitend # - # Step 6: Assemble the image + # Step 7: Assemble the image # uboot-mkimage \ ${@'-D "${UBOOT_MKIMAGE_DTCOPTS}"' if len('${UBOOT_MKIMAGE_DTCOPTS}') else ''} \ @@ -465,7 +540,7 @@ fitimage_assemble() { arch/${ARCH}/boot/${2} # - # Step 7: Sign the image and add public key to U-Boot dtb + # Step 8: Sign the image and add public key to U-Boot dtb # if [ "x${UBOOT_SIGN_ENABLE}" = "x1" ] ; then add_key_to_u_boot="" @@ -479,7 +554,8 @@ fitimage_assemble() { ${@'-D "${UBOOT_MKIMAGE_DTCOPTS}"' if len('${UBOOT_MKIMAGE_DTCOPTS}') else ''} \ -F -k "${UBOOT_SIGN_KEYDIR}" \ $add_key_to_u_boot \ - -r arch/${ARCH}/boot/${2} + -r arch/${ARCH}/boot/${2} \ + ${UBOOT_MKIMAGE_SIGN_ARGS} fi } @@ -496,7 +572,11 @@ do_assemble_fitimage_initramfs() { if echo ${KERNEL_IMAGETYPES} | grep -wq "fitImage" && \ test -n "${INITRAMFS_IMAGE}" ; then cd ${B} - fitimage_assemble fit-image-${INITRAMFS_IMAGE}.its fitImage-${INITRAMFS_IMAGE} 1 + if [ "${INITRAMFS_IMAGE_BUNDLE}" = "1" ]; then + fitimage_assemble fit-image-${INITRAMFS_IMAGE}.its fitImage "" + else + fitimage_assemble fit-image-${INITRAMFS_IMAGE}.its fitImage-${INITRAMFS_IMAGE} 1 + fi fi } @@ -507,22 +587,32 @@ kernel_do_deploy[vardepsexclude] = "DATETIME" kernel_do_deploy_append() { # Update deploy directory if echo ${KERNEL_IMAGETYPES} | grep -wq "fitImage"; then - echo "Copying fit-image.its source file..." - install -m 0644 ${B}/fit-image.its "$deployDir/fitImage-its-${KERNEL_FIT_NAME}.its" - ln -snf fitImage-its-${KERNEL_FIT_NAME}.its "$deployDir/fitImage-its-${KERNEL_FIT_LINK_NAME}" + if [ "${INITRAMFS_IMAGE_BUNDLE}" != "1" ]; then + echo "Copying fit-image.its source file..." + install -m 0644 ${B}/fit-image.its "$deployDir/fitImage-its-${KERNEL_FIT_NAME}.its" + if [ -n "${KERNEL_FIT_LINK_NAME}" ] ; then + ln -snf fitImage-its-${KERNEL_FIT_NAME}.its "$deployDir/fitImage-its-${KERNEL_FIT_LINK_NAME}" + fi - echo "Copying linux.bin file..." - install -m 0644 ${B}/linux.bin $deployDir/fitImage-linux.bin-${KERNEL_FIT_NAME}.bin - ln -snf fitImage-linux.bin-${KERNEL_FIT_NAME}.bin "$deployDir/fitImage-linux.bin-${KERNEL_FIT_LINK_NAME}" + echo "Copying linux.bin file..." + install -m 0644 ${B}/linux.bin $deployDir/fitImage-linux.bin-${KERNEL_FIT_NAME}.bin + if [ -n "${KERNEL_FIT_LINK_NAME}" ] ; then + ln -snf fitImage-linux.bin-${KERNEL_FIT_NAME}.bin "$deployDir/fitImage-linux.bin-${KERNEL_FIT_LINK_NAME}" + fi + fi if [ -n "${INITRAMFS_IMAGE}" ]; then echo "Copying fit-image-${INITRAMFS_IMAGE}.its source file..." install -m 0644 ${B}/fit-image-${INITRAMFS_IMAGE}.its "$deployDir/fitImage-its-${INITRAMFS_IMAGE_NAME}-${KERNEL_FIT_NAME}.its" ln -snf fitImage-its-${INITRAMFS_IMAGE_NAME}-${KERNEL_FIT_NAME}.its "$deployDir/fitImage-its-${INITRAMFS_IMAGE_NAME}-${KERNEL_FIT_LINK_NAME}" - echo "Copying fitImage-${INITRAMFS_IMAGE} file..." - install -m 0644 ${B}/arch/${ARCH}/boot/fitImage-${INITRAMFS_IMAGE} "$deployDir/fitImage-${INITRAMFS_IMAGE_NAME}-${KERNEL_FIT_NAME}.bin" - ln -snf fitImage-${INITRAMFS_IMAGE_NAME}-${KERNEL_FIT_NAME}.bin "$deployDir/fitImage-${INITRAMFS_IMAGE_NAME}-${KERNEL_FIT_LINK_NAME}" + if [ "${INITRAMFS_IMAGE_BUNDLE}" != "1" ]; then + echo "Copying fitImage-${INITRAMFS_IMAGE} file..." + install -m 0644 ${B}/arch/${ARCH}/boot/fitImage-${INITRAMFS_IMAGE} "$deployDir/fitImage-${INITRAMFS_IMAGE_NAME}-${KERNEL_FIT_NAME}.bin" + if [ -n "${KERNEL_FIT_LINK_NAME}" ] ; then + ln -snf fitImage-${INITRAMFS_IMAGE_NAME}-${KERNEL_FIT_NAME}.bin "$deployDir/fitImage-${INITRAMFS_IMAGE_NAME}-${KERNEL_FIT_LINK_NAME}" + fi + fi fi if [ "${UBOOT_SIGN_ENABLE}" = "1" -a -n "${UBOOT_DTB_BINARY}" ] ; then # UBOOT_DTB_IMAGE is a realfile, but we can't use @@ -532,3 +622,13 @@ kernel_do_deploy_append() { fi fi } + +# The function below performs the following in case of initramfs bundles: +# - Removes do_assemble_fitimage. FIT generation is done through +# do_assemble_fitimage_initramfs. do_assemble_fitimage is not needed +# and should not be part of the tasks to be executed. +python () { + d.appendVarFlag('do_compile', 'vardeps', ' INITRAMFS_IMAGE_BUNDLE') + if d.getVar('INITRAMFS_IMAGE_BUNDLE') == "1": + bb.build.deltask('do_assemble_fitimage', d) +} diff --git a/meta/classes/kernel-yocto.bbclass b/meta/classes/kernel-yocto.bbclass index 2a6231803b..2abbc2ff66 100644 --- a/meta/classes/kernel-yocto.bbclass +++ b/meta/classes/kernel-yocto.bbclass @@ -194,7 +194,7 @@ do_kernel_metadata() { # SRC_URI. If they were supplied, we convert them into include directives # for the update part of the process for f in ${feat_dirs}; do - if [ -d "${WORKDIR}/$f/meta" ]; then + if [ -d "${WORKDIR}/$f/kernel-meta" ]; then includes="$includes -I${WORKDIR}/$f/kernel-meta" elif [ -d "${WORKDIR}/../oe-local-files/$f" ]; then includes="$includes -I${WORKDIR}/../oe-local-files/$f" diff --git a/meta/classes/kernel.bbclass b/meta/classes/kernel.bbclass index 2a3cb21fc0..ca7530095e 100644 --- a/meta/classes/kernel.bbclass +++ b/meta/classes/kernel.bbclass @@ -75,7 +75,7 @@ python __anonymous () { # KERNEL_IMAGETYPES may contain a mixture of image types supported directly # by the kernel build system and types which are created by post-processing # the output of the kernel build system (e.g. compressing vmlinux -> - # vmlinux.gz in kernel_do_compile()). + # vmlinux.gz in kernel_do_transform_kernel()). # KERNEL_IMAGETYPE_FOR_MAKE should contain only image types supported # directly by the kernel build system. if not d.getVar('KERNEL_IMAGETYPE_FOR_MAKE'): @@ -106,6 +106,8 @@ python __anonymous () { # standalone for use by wic and other tools. if image: d.appendVarFlag('do_bundle_initramfs', 'depends', ' ${INITRAMFS_IMAGE}:do_image_complete') + if image and bb.utils.to_boolean(d.getVar('INITRAMFS_IMAGE_BUNDLE')): + bb.build.addtask('do_transform_bundled_initramfs', 'do_deploy', 'do_bundle_initramfs', d) # NOTE: setting INITRAMFS_TASK is for backward compatibility # The preferred method is to set INITRAMFS_IMAGE, because @@ -141,13 +143,14 @@ do_unpack[cleandirs] += " ${S} ${STAGING_KERNEL_DIR} ${B} ${STAGING_KERNEL_BUILD do_clean[cleandirs] += " ${S} ${STAGING_KERNEL_DIR} ${B} ${STAGING_KERNEL_BUILDDIR}" python do_symlink_kernsrc () { s = d.getVar("S") - if s[-1] == '/': - # drop trailing slash, so that os.symlink(kernsrc, s) doesn't use s as directory name and fail - s=s[:-1] kernsrc = d.getVar("STAGING_KERNEL_DIR") if s != kernsrc: bb.utils.mkdirhier(kernsrc) bb.utils.remove(kernsrc, recurse=True) + if s[-1] == '/': + # drop trailing slash, so that os.symlink(kernsrc, s) doesn't use s as + # directory name and fail + s = s[:-1] if d.getVar("EXTERNALSRC"): # With EXTERNALSRC S will not be wiped so we can symlink to it os.symlink(s, kernsrc) @@ -280,6 +283,14 @@ do_bundle_initramfs () { } do_bundle_initramfs[dirs] = "${B}" +kernel_do_transform_bundled_initramfs() { + # vmlinux.gz is not built by kernel + if (echo "${KERNEL_IMAGETYPES}" | grep -wq "vmlinux\.gz"); then + gzip -9cn < ${KERNEL_OUTPUT_DIR}/vmlinux.initramfs > ${KERNEL_OUTPUT_DIR}/vmlinux.gz.initramfs + fi +} +do_transform_bundled_initramfs[dirs] = "${B}" + python do_devshell_prepend () { os.environ["LDFLAGS"] = '' } @@ -311,6 +322,10 @@ kernel_do_compile() { export KBUILD_BUILD_TIMESTAMP="$ts" export KCONFIG_NOTIMESTAMP=1 bbnote "KBUILD_BUILD_TIMESTAMP: $ts" + else + ts=`LC_ALL=C date` + export KBUILD_BUILD_TIMESTAMP="$ts" + bbnote "KBUILD_BUILD_TIMESTAMP: $ts" fi # The $use_alternate_initrd is only set from # do_bundle_initramfs() This variable is specifically for the @@ -329,12 +344,17 @@ kernel_do_compile() { for typeformake in ${KERNEL_IMAGETYPE_FOR_MAKE} ; do oe_runmake ${typeformake} CC="${KERNEL_CC} $cc_extra " LD="${KERNEL_LD}" ${KERNEL_EXTRA_ARGS} $use_alternate_initrd done +} + +kernel_do_transform_kernel() { # vmlinux.gz is not built by kernel if (echo "${KERNEL_IMAGETYPES}" | grep -wq "vmlinux\.gz"); then mkdir -p "${KERNEL_OUTPUT_DIR}" gzip -9cn < ${B}/vmlinux > "${KERNEL_OUTPUT_DIR}/vmlinux.gz" fi } +do_transform_kernel[dirs] = "${B}" +addtask transform_kernel after do_compile before do_install do_compile_kernelmodules() { unset CFLAGS CPPFLAGS CXXFLAGS LDFLAGS MACHINE @@ -352,6 +372,10 @@ do_compile_kernelmodules() { export KBUILD_BUILD_TIMESTAMP="$ts" export KCONFIG_NOTIMESTAMP=1 bbnote "KBUILD_BUILD_TIMESTAMP: $ts" + else + ts=`LC_ALL=C date` + export KBUILD_BUILD_TIMESTAMP="$ts" + bbnote "KBUILD_BUILD_TIMESTAMP: $ts" fi if (grep -q -i -e '^CONFIG_MODULES=y$' ${B}/.config); then cc_extra=$(get_cc_option) @@ -381,8 +405,8 @@ kernel_do_install() { unset CFLAGS CPPFLAGS CXXFLAGS LDFLAGS MACHINE if (grep -q -i -e '^CONFIG_MODULES=y$' .config); then oe_runmake DEPMOD=echo MODLIB=${D}${nonarch_base_libdir}/modules/${KERNEL_VERSION} INSTALL_FW_PATH=${D}${nonarch_base_libdir}/firmware modules_install - rm "${D}${nonarch_base_libdir}/modules/${KERNEL_VERSION}/build" - rm "${D}${nonarch_base_libdir}/modules/${KERNEL_VERSION}/source" + rm -f "${D}${nonarch_base_libdir}/modules/${KERNEL_VERSION}/build" + rm -f "${D}${nonarch_base_libdir}/modules/${KERNEL_VERSION}/source" # If the kernel/ directory is empty remove it to prevent QA issues rmdir --ignore-fail-on-non-empty "${D}${nonarch_base_libdir}/modules/${KERNEL_VERSION}/kernel" else @@ -394,12 +418,26 @@ kernel_do_install() { # install -d ${D}/${KERNEL_IMAGEDEST} install -d ${D}/boot + + # + # When including an initramfs bundle inside a FIT image, the fitImage is created after the install task + # by do_assemble_fitimage_initramfs. + # This happens after the generation of the initramfs bundle (done by do_bundle_initramfs). + # So, at the level of the install task we should not try to install the fitImage. fitImage is still not + # generated yet. + # After the generation of the fitImage, the deploy task copies the fitImage from the build directory to + # the deploy folder. + # + for imageType in ${KERNEL_IMAGETYPES} ; do - install -m 0644 ${KERNEL_OUTPUT_DIR}/${imageType} ${D}/${KERNEL_IMAGEDEST}/${imageType}-${KERNEL_VERSION} - if [ "${KERNEL_PACKAGE_NAME}" = "kernel" ]; then - ln -sf ${imageType}-${KERNEL_VERSION} ${D}/${KERNEL_IMAGEDEST}/${imageType} + if [ $imageType != "fitImage" ] || [ "${INITRAMFS_IMAGE_BUNDLE}" != "1" ] ; then + install -m 0644 ${KERNEL_OUTPUT_DIR}/${imageType} ${D}/${KERNEL_IMAGEDEST}/${imageType}-${KERNEL_VERSION} + if [ "${KERNEL_PACKAGE_NAME}" = "kernel" ]; then + ln -sf ${imageType}-${KERNEL_VERSION} ${D}/${KERNEL_IMAGEDEST}/${imageType} + fi fi done + install -m 0644 System.map ${D}/boot/System.map-${KERNEL_VERSION} install -m 0644 .config ${D}/boot/config-${KERNEL_VERSION} install -m 0644 vmlinux ${D}/boot/vmlinux-${KERNEL_VERSION} @@ -572,11 +610,11 @@ do_savedefconfig() { do_savedefconfig[nostamp] = "1" addtask savedefconfig after do_configure -inherit cml1 +inherit cml1 pkgconfig KCONFIG_CONFIG_COMMAND_append = " LD='${KERNEL_LD}' HOSTLDFLAGS='${BUILD_LDFLAGS}'" -EXPORT_FUNCTIONS do_compile do_install do_configure +EXPORT_FUNCTIONS do_compile do_transform_kernel do_transform_bundled_initramfs do_install do_configure # kernel-base becomes kernel-${KERNEL_VERSION} # kernel-image becomes kernel-image-${KERNEL_VERSION} @@ -721,7 +759,7 @@ kernel_do_deploy() { fi if [ ! -z "${INITRAMFS_IMAGE}" -a x"${INITRAMFS_IMAGE_BUNDLE}" = x1 ]; then - for imageType in ${KERNEL_IMAGETYPE_FOR_MAKE} ; do + for imageType in ${KERNEL_IMAGETYPES} ; do if [ "$imageType" = "fitImage" ] ; then continue fi diff --git a/meta/classes/libc-package.bbclass b/meta/classes/libc-package.bbclass index 1143f538d6..72f489d673 100644 --- a/meta/classes/libc-package.bbclass +++ b/meta/classes/libc-package.bbclass @@ -45,6 +45,7 @@ PACKAGE_NO_GCONV ?= "0" OVERRIDES_append = ":${TARGET_ARCH}-${TARGET_OS}" locale_base_postinst_ontarget() { +mkdir ${libdir}/locale localedef --inputfile=${datadir}/i18n/locales/%s --charmap=%s %s } diff --git a/meta/classes/license_image.bbclass b/meta/classes/license_image.bbclass index 9f3a0c3727..325b3cbba7 100644 --- a/meta/classes/license_image.bbclass +++ b/meta/classes/license_image.bbclass @@ -211,7 +211,7 @@ def get_deployed_dependencies(d): deploy = {} # Get all the dependencies for the current task (rootfs). taskdata = d.getVar("BB_TASKDEPDATA", False) - pn = d.getVar("PN", True) + pn = d.getVar("PN") depends = list(set([dep[0] for dep in list(taskdata.values()) if not dep[0].endswith("-native") and not dep[0] == pn])) diff --git a/meta/classes/multilib.bbclass b/meta/classes/multilib.bbclass index 9a8b02d4f6..b5c59ac593 100644 --- a/meta/classes/multilib.bbclass +++ b/meta/classes/multilib.bbclass @@ -45,6 +45,7 @@ python multilib_virtclass_handler () { e.data.setVar("RECIPE_SYSROOT", "${WORKDIR}/recipe-sysroot") e.data.setVar("STAGING_DIR_TARGET", "${WORKDIR}/recipe-sysroot") e.data.setVar("STAGING_DIR_HOST", "${WORKDIR}/recipe-sysroot") + e.data.setVar("RECIPE_SYSROOT_MANIFEST_SUBDIR", "nativesdk-" + variant) e.data.setVar("MLPREFIX", variant + "-") override = ":virtclass-multilib-" + variant e.data.setVar("OVERRIDES", e.data.getVar("OVERRIDES", False) + override) diff --git a/meta/classes/nativesdk.bbclass b/meta/classes/nativesdk.bbclass index 7f2692c51a..dc5a9756b6 100644 --- a/meta/classes/nativesdk.bbclass +++ b/meta/classes/nativesdk.bbclass @@ -113,3 +113,5 @@ do_packagedata[stamp-extra-info] = "" USE_NLS = "${SDKUSE_NLS}" OLDEST_KERNEL = "${SDK_OLDEST_KERNEL}" + +PATH_prepend = "${COREBASE}/scripts/nativesdk-intercept:" diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass index 702427fecc..49d30caef7 100644 --- a/meta/classes/package.bbclass +++ b/meta/classes/package.bbclass @@ -1140,6 +1140,14 @@ python split_and_strip_files () { # Modified the file so clear the cache cpath.updatecache(file) + def strip_pkgd_prefix(f): + nonlocal dvar + + if f.startswith(dvar): + return f[len(dvar):] + + return f + # # First lets process debug splitting # @@ -1153,6 +1161,8 @@ python split_and_strip_files () { for file in staticlibs: results.append( (file,source_info(file, d)) ) + d.setVar("PKGDEBUGSOURCES", {strip_pkgd_prefix(f): sorted(s) for f, s in results}) + sources = set() for r in results: sources.update(r[1]) @@ -1460,6 +1470,7 @@ PKGDATA_VARS = "PN PE PV PR PKGE PKGV PKGR LICENSE DESCRIPTION SUMMARY RDEPENDS python emit_pkgdata() { from glob import glob import json + import gzip def process_postinst_on_target(pkg, mlprefix): pkgval = d.getVar('PKG_%s' % pkg) @@ -1532,6 +1543,8 @@ fi with open(data_file, 'w') as fd: fd.write("PACKAGES: %s\n" % packages) + pkgdebugsource = d.getVar("PKGDEBUGSOURCES") or [] + pn = d.getVar('PN') global_variants = (d.getVar('MULTILIB_GLOBAL_VARIANTS') or "").split() variants = (d.getVar('MULTILIB_VARIANTS') or "").split() @@ -1551,17 +1564,32 @@ fi pkgval = pkg d.setVar('PKG_%s' % pkg, pkg) + extended_data = { + "files_info": {} + } + pkgdestpkg = os.path.join(pkgdest, pkg) files = {} + files_extra = {} total_size = 0 seen = set() for f in pkgfiles[pkg]: - relpth = os.path.relpath(f, pkgdestpkg) + fpath = os.sep + os.path.relpath(f, pkgdestpkg) + fstat = os.lstat(f) - files[os.sep + relpth] = fstat.st_size + files[fpath] = fstat.st_size + + extended_data["files_info"].setdefault(fpath, {}) + extended_data["files_info"][fpath]['size'] = fstat.st_size + if fstat.st_ino not in seen: seen.add(fstat.st_ino) total_size += fstat.st_size + + if fpath in pkgdebugsource: + extended_data["files_info"][fpath]['debugsrc'] = pkgdebugsource[fpath] + del pkgdebugsource[fpath] + d.setVar('FILES_INFO', json.dumps(files, sort_keys=True)) process_postinst_on_target(pkg, d.getVar("MLPREFIX")) @@ -1582,6 +1610,10 @@ fi sf.write('%s_%s: %d\n' % ('PKGSIZE', pkg, total_size)) + subdata_extended_file = pkgdatadir + "/extended/%s.json.gz" % pkg + with gzip.open(subdata_extended_file, "wt", encoding="utf-8") as f: + json.dump(extended_data, f, sort_keys=True, separators=(",", ":")) + # Symlinks needed for rprovides lookup rprov = d.getVar('RPROVIDES_%s' % pkg) or d.getVar('RPROVIDES') if rprov: @@ -1612,7 +1644,8 @@ fi write_extra_runtime_pkgs(global_variants, packages, pkgdatadir) } -emit_pkgdata[dirs] = "${PKGDESTWORK}/runtime ${PKGDESTWORK}/runtime-reverse ${PKGDESTWORK}/runtime-rprovides" +emit_pkgdata[dirs] = "${PKGDESTWORK}/runtime ${PKGDESTWORK}/runtime-reverse ${PKGDESTWORK}/runtime-rprovides ${PKGDESTWORK}/extended" +emit_pkgdata[vardepsexclude] = "BB_NUMBER_THREADS" ldconfig_postinst_fragment() { if [ x"$D" = "x" ]; then diff --git a/meta/classes/populate_sdk_base.bbclass b/meta/classes/populate_sdk_base.bbclass index 396792f0f7..49fdfaa93d 100644 --- a/meta/classes/populate_sdk_base.bbclass +++ b/meta/classes/populate_sdk_base.bbclass @@ -51,6 +51,8 @@ TOOLCHAIN_OUTPUTNAME ?= "${SDK_NAME}-toolchain-${SDK_VERSION}" SDK_ARCHIVE_TYPE ?= "tar.xz" SDK_XZ_COMPRESSION_LEVEL ?= "-9" SDK_XZ_OPTIONS ?= "${XZ_DEFAULTS} ${SDK_XZ_COMPRESSION_LEVEL}" +SDK_ZIP_OPTIONS ?= "-y" + # To support different sdk type according to SDK_ARCHIVE_TYPE, now support zip and tar.xz python () { @@ -58,7 +60,7 @@ python () { d.setVar('SDK_ARCHIVE_DEPENDS', 'zip-native') # SDK_ARCHIVE_CMD used to generate archived sdk ${TOOLCHAIN_OUTPUTNAME}.${SDK_ARCHIVE_TYPE} from input dir ${SDK_OUTPUT}/${SDKPATH} to output dir ${SDKDEPLOYDIR} # recommand to cd into input dir first to avoid archive with buildpath - d.setVar('SDK_ARCHIVE_CMD', 'cd ${SDK_OUTPUT}/${SDKPATH}; zip -r -y ${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.${SDK_ARCHIVE_TYPE} .') + d.setVar('SDK_ARCHIVE_CMD', 'cd ${SDK_OUTPUT}/${SDKPATH}; zip -r ${SDK_ZIP_OPTIONS} ${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.${SDK_ARCHIVE_TYPE} .') else: d.setVar('SDK_ARCHIVE_DEPENDS', 'xz-native') d.setVar('SDK_ARCHIVE_CMD', 'cd ${SDK_OUTPUT}/${SDKPATH}; tar ${SDKTAROPTS} -cf - . | xz ${SDK_XZ_OPTIONS} > ${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.${SDK_ARCHIVE_TYPE}') diff --git a/meta/classes/populate_sdk_ext.bbclass b/meta/classes/populate_sdk_ext.bbclass index aa00d5397c..1bdfd92847 100644 --- a/meta/classes/populate_sdk_ext.bbclass +++ b/meta/classes/populate_sdk_ext.bbclass @@ -117,7 +117,7 @@ python write_host_sdk_ext_manifest () { f.write("%s %s %s\n" % (info[1], info[2], info[3])) } -SDK_POSTPROCESS_COMMAND_append_task-populate-sdk-ext = "write_target_sdk_ext_manifest; write_host_sdk_ext_manifest; " +SDK_POSTPROCESS_COMMAND_append_task-populate-sdk-ext = " write_target_sdk_ext_manifest; write_host_sdk_ext_manifest; " SDK_TITLE_task-populate-sdk-ext = "${@d.getVar('DISTRO_NAME') or d.getVar('DISTRO')} Extensible SDK" @@ -363,7 +363,8 @@ python copy_buildsystem () { f.write('BUILDCFG_HEADER = ""\n\n') # Write METADATA_REVISION - f.write('METADATA_REVISION = "%s"\n\n' % d.getVar('METADATA_REVISION')) + # Needs distro override so it can override the value set in the bbclass code (later than local.conf) + f.write('METADATA_REVISION:%s = "%s"\n\n' % (d.getVar('DISTRO'), d.getVar('METADATA_REVISION'))) f.write('# Provide a flag to indicate we are in the EXT_SDK Context\n') f.write('WITHIN_EXT_SDK = "1"\n\n') @@ -669,7 +670,7 @@ sdk_ext_postinst() { # A bit of another hack, but we need this in the path only for devtool # so put it at the end of $PATH. - echo "export PATH=$target_sdk_dir/sysroots/${SDK_SYS}${bindir_nativesdk}:\$PATH" >> $env_setup_script + echo "export PATH=\"$target_sdk_dir/sysroots/${SDK_SYS}${bindir_nativesdk}:\$PATH\"" >> $env_setup_script echo "printf 'SDK environment now set up; additionally you may now run devtool to perform development tasks.\nRun devtool --help for further details.\n'" >> $env_setup_script diff --git a/meta/classes/pypi.bbclass b/meta/classes/pypi.bbclass index 87b4c85fc0..c68367449a 100644 --- a/meta/classes/pypi.bbclass +++ b/meta/classes/pypi.bbclass @@ -24,3 +24,5 @@ S = "${WORKDIR}/${PYPI_PACKAGE}-${PV}" UPSTREAM_CHECK_URI ?= "https://pypi.org/project/${PYPI_PACKAGE}/" UPSTREAM_CHECK_REGEX ?= "/${PYPI_PACKAGE}/(?P<pver>(\d+[\.\-_]*)+)/" + +CVE_PRODUCT ?= "python:${PYPI_PACKAGE}" diff --git a/meta/classes/qemuboot.bbclass b/meta/classes/qemuboot.bbclass index 648af09b6e..92ae69d9f2 100644 --- a/meta/classes/qemuboot.bbclass +++ b/meta/classes/qemuboot.bbclass @@ -7,6 +7,7 @@ # QB_OPT_APPEND: options to append to qemu, e.g., "-show-cursor" # # QB_DEFAULT_KERNEL: default kernel to boot, e.g., "bzImage" +# e.g., "bzImage-initramfs-qemux86-64.bin" if INITRAMFS_IMAGE_BUNDLE is set to 1. # # QB_DEFAULT_FSTYPE: default FSTYPE to boot, e.g., "ext4" # @@ -75,7 +76,7 @@ QB_MEM ?= "-m 256" QB_SERIAL_OPT ?= "-serial mon:stdio -serial null" -QB_DEFAULT_KERNEL ?= "${KERNEL_IMAGETYPE}" +QB_DEFAULT_KERNEL ?= "${@bb.utils.contains("INITRAMFS_IMAGE_BUNDLE", "1", "${KERNEL_IMAGETYPE}-${INITRAMFS_LINK_NAME}.bin", "${KERNEL_IMAGETYPE}", d)}" QB_DEFAULT_FSTYPE ?= "ext4" QB_OPT_APPEND ?= "-show-cursor" QB_NETWORK_DEVICE ?= "-device virtio-net-pci,netdev=net0,mac=@MAC@" diff --git a/meta/classes/rm_work.bbclass b/meta/classes/rm_work.bbclass index 2d5a56c238..24051aa378 100644 --- a/meta/classes/rm_work.bbclass +++ b/meta/classes/rm_work.bbclass @@ -27,6 +27,13 @@ BB_SCHEDULER ?= "completion" BB_TASK_IONICE_LEVEL_task-rm_work = "3.0" do_rm_work () { + # Force using the HOSTTOOLS 'rm' - otherwise the SYSROOT_NATIVE 'rm' can be selected depending on PATH + # Avoids race-condition accessing 'rm' when deleting WORKDIR folders at the end of this function + RM_BIN="$(PATH=${HOSTTOOLS_DIR} command -v rm)" + if [ -z "${RM_BIN}" ]; then + bbfatal "Binary 'rm' not found in HOSTTOOLS_DIR, cannot remove WORKDIR data." + fi + # If the recipe name is in the RM_WORK_EXCLUDE, skip the recipe. for p in ${RM_WORK_EXCLUDE}; do if [ "$p" = "${PN}" ]; then @@ -73,7 +80,7 @@ do_rm_work () { # sstate version since otherwise we'd need to leave 'plaindirs' around # such as 'packages' and 'packages-split' and these can be large. No end # of chain tasks depend directly on do_package anymore. - rm -f -- $i; + "${RM_BIN}" -f -- $i; ;; *_setscene*) # Skip stamps which are already setscene versions @@ -90,7 +97,7 @@ do_rm_work () { ;; esac done - rm -f -- $i + "${RM_BIN}" -f -- $i esac done @@ -100,9 +107,9 @@ do_rm_work () { # Retain only logs and other files in temp, safely ignore # failures of removing pseudo folers on NFS2/3 server. if [ $dir = 'pseudo' ]; then - rm -rf -- $dir 2> /dev/null || true + "${RM_BIN}" -rf -- $dir 2> /dev/null || true elif ! echo "$excludes" | grep -q -w "$dir"; then - rm -rf -- $dir + "${RM_BIN}" -rf -- $dir fi done } diff --git a/meta/classes/rootfs-postcommands.bbclass b/meta/classes/rootfs-postcommands.bbclass index 0fef52af40..943534c57a 100644 --- a/meta/classes/rootfs-postcommands.bbclass +++ b/meta/classes/rootfs-postcommands.bbclass @@ -1,6 +1,6 @@ # Zap the root password if debug-tweaks feature is not enabled -ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains_any("IMAGE_FEATURES", [ 'debug-tweaks', 'empty-root-password' ], "", "zap_empty_root_password ; ",d)}' +ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains_any("IMAGE_FEATURES", [ 'debug-tweaks', 'empty-root-password' ], "", "zap_empty_root_password; ",d)}' # Allow dropbear/openssh to accept logins from accounts with an empty password string if debug-tweaks or allow-empty-password is enabled ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains_any("IMAGE_FEATURES", [ 'debug-tweaks', 'allow-empty-password' ], "ssh_allow_empty_password; ", "",d)}' @@ -12,7 +12,7 @@ ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains_any("IMAGE_FEATURES", [ 'deb ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains_any("IMAGE_FEATURES", [ 'debug-tweaks', 'post-install-logging' ], "postinst_enable_logging; ", "",d)}' # Create /etc/timestamp during image construction to give a reasonably sane default time setting -ROOTFS_POSTPROCESS_COMMAND += "rootfs_update_timestamp ; " +ROOTFS_POSTPROCESS_COMMAND += "rootfs_update_timestamp; " # Tweak the mount options for rootfs in /etc/fstab if read-only-rootfs is enabled ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", "read_only_rootfs_hook; ", "",d)}' @@ -26,7 +26,7 @@ ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains("IMAGE_FEATURES", "read-only APPEND_append = '${@bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", " ro", "", d)}' # Generates test data file with data store variables expanded in json format -ROOTFS_POSTPROCESS_COMMAND += "write_image_test_data ; " +ROOTFS_POSTPROCESS_COMMAND += "write_image_test_data; " # Write manifest IMAGE_MANIFEST = "${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.manifest" @@ -305,7 +305,7 @@ rootfs_trim_schemas () { } rootfs_check_host_user_contaminated () { - contaminated="${WORKDIR}/host-user-contaminated.txt" + contaminated="${S}/host-user-contaminated.txt" HOST_USER_UID="$(PSEUDO_UNLOAD=1 id -u)" HOST_USER_GID="$(PSEUDO_UNLOAD=1 id -g)" diff --git a/meta/classes/rootfsdebugfiles.bbclass b/meta/classes/rootfsdebugfiles.bbclass index e2ba4e3647..85c7ec7434 100644 --- a/meta/classes/rootfsdebugfiles.bbclass +++ b/meta/classes/rootfsdebugfiles.bbclass @@ -28,7 +28,7 @@ ROOTFS_DEBUG_FILES ?= "" ROOTFS_DEBUG_FILES[doc] = "Lists additional files or directories to be installed with 'cp -a' in the format 'source1 target1;source2 target2;...'" -ROOTFS_POSTPROCESS_COMMAND += "rootfs_debug_files ;" +ROOTFS_POSTPROCESS_COMMAND += "rootfs_debug_files;" rootfs_debug_files () { #!/bin/sh -e echo "${ROOTFS_DEBUG_FILES}" | sed -e 's/;/\n/g' | while read source target mode; do diff --git a/meta/classes/sanity.bbclass b/meta/classes/sanity.bbclass index 37354af9d5..33e5e5952f 100644 --- a/meta/classes/sanity.bbclass +++ b/meta/classes/sanity.bbclass @@ -561,6 +561,14 @@ def check_tar_version(sanity_data): version = result.split()[3] if LooseVersion(version) < LooseVersion("1.28"): return "Your version of tar is older than 1.28 and does not have the support needed to enable reproducible builds. Please install a newer version of tar (you could use the project's buildtools-tarball from our last release or use scripts/install-buildtools).\n" + + try: + result = subprocess.check_output(["tar", "--help"], stderr=subprocess.STDOUT).decode('utf-8') + if "--xattrs" not in result: + return "Your tar doesn't support --xattrs, please use GNU tar.\n" + except subprocess.CalledProcessError as e: + return "Unable to execute tar --help, exit code %d\n%s\n" % (e.returncode, e.output) + return None # We use git parameters and functionality only found in 1.7.8 or later diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass index 3d6fb84d63..1058778980 100644 --- a/meta/classes/sstate.bbclass +++ b/meta/classes/sstate.bbclass @@ -20,7 +20,7 @@ def generate_sstatefn(spec, hash, taskname, siginfo, d): components = spec.split(":") # Fields 0,5,6 are mandatory, 1 is most useful, 2,3,4 are just for information # 7 is for the separators - avail = (254 - len(hash + "_" + taskname + extension) - len(components[0]) - len(components[1]) - len(components[5]) - len(components[6]) - 7) // 3 + avail = (limit - len(hash + "_" + taskname + extension) - len(components[0]) - len(components[1]) - len(components[5]) - len(components[6]) - 7) // 3 components[2] = components[2][:avail] components[3] = components[3][:avail] components[4] = components[4][:avail] diff --git a/meta/classes/staging.bbclass b/meta/classes/staging.bbclass index 78eb914921..21523c8f75 100644 --- a/meta/classes/staging.bbclass +++ b/meta/classes/staging.bbclass @@ -267,6 +267,10 @@ python extend_recipe_sysroot() { pn = d.getVar("PN") stagingdir = d.getVar("STAGING_DIR") sharedmanifests = d.getVar("COMPONENTS_DIR") + "/manifests" + # only needed by multilib cross-canadian since it redefines RECIPE_SYSROOT + manifestprefix = d.getVar("RECIPE_SYSROOT_MANIFEST_SUBDIR") + if manifestprefix: + sharedmanifests = sharedmanifests + "/" + manifestprefix recipesysroot = d.getVar("RECIPE_SYSROOT") recipesysrootnative = d.getVar("RECIPE_SYSROOT_NATIVE") diff --git a/meta/classes/testimage.bbclass b/meta/classes/testimage.bbclass index b1aef626f7..7c8b2b30a1 100644 --- a/meta/classes/testimage.bbclass +++ b/meta/classes/testimage.bbclass @@ -99,30 +99,9 @@ TESTIMAGE_DUMP_DIR ?= "${LOG_DIR}/runtime-hostdump/" TESTIMAGE_UPDATE_VARS ?= "DL_DIR WORKDIR DEPLOY_DIR" testimage_dump_target () { - top -bn1 - ps - free - df - # The next command will export the default gateway IP - export DEFAULT_GATEWAY=$(ip route | awk '/default/ { print $3}') - ping -c3 $DEFAULT_GATEWAY - dmesg - netstat -an - ip address - # Next command will dump logs from /var/log/ - find /var/log/ -type f 2>/dev/null -exec echo "====================" \; -exec echo {} \; -exec echo "====================" \; -exec cat {} \; -exec echo "" \; } testimage_dump_host () { - top -bn1 - iostat -x -z -N -d -p ALL 20 2 - ps -ef - free - df - memstat - dmesg - ip -s link - netstat -an } python do_testimage() { diff --git a/meta/classes/toolchain-scripts.bbclass b/meta/classes/toolchain-scripts.bbclass index db1d3215ef..21762b803b 100644 --- a/meta/classes/toolchain-scripts.bbclass +++ b/meta/classes/toolchain-scripts.bbclass @@ -29,7 +29,7 @@ toolchain_create_sdk_env_script () { echo '# http://tldp.org/HOWTO/Program-Library-HOWTO/shared-libraries.html#AEN80' >> $script echo '# http://xahlee.info/UnixResource_dir/_/ldpath.html' >> $script echo '# Only disable this check if you are absolutely know what you are doing!' >> $script - echo 'if [ ! -z "$LD_LIBRARY_PATH" ]; then' >> $script + echo 'if [ ! -z "${LD_LIBRARY_PATH:-}" ]; then' >> $script echo " echo \"Your environment is misconfigured, you probably need to 'unset LD_LIBRARY_PATH'\"" >> $script echo " echo \"but please check why this was set in the first place and that it's safe to unset.\"" >> $script echo ' echo "The SDK will not operate correctly in most cases when LD_LIBRARY_PATH is set."' >> $script @@ -44,7 +44,7 @@ toolchain_create_sdk_env_script () { for i in ${CANADIANEXTRAOS}; do EXTRAPATH="$EXTRAPATH:$sdkpathnative$bindir/${TARGET_ARCH}${TARGET_VENDOR}-$i" done - echo "export PATH=$sdkpathnative$bindir:$sdkpathnative$sbindir:$sdkpathnative$base_bindir:$sdkpathnative$base_sbindir:$sdkpathnative$bindir/../${HOST_SYS}/bin:$sdkpathnative$bindir/${TARGET_SYS}"$EXTRAPATH':$PATH' >> $script + echo "export PATH=$sdkpathnative$bindir:$sdkpathnative$sbindir:$sdkpathnative$base_bindir:$sdkpathnative$base_sbindir:$sdkpathnative$bindir/../${HOST_SYS}/bin:$sdkpathnative$bindir/${TARGET_SYS}"$EXTRAPATH':"$PATH"' >> $script echo 'export PKG_CONFIG_SYSROOT_DIR=$SDKTARGETSYSROOT' >> $script echo 'export PKG_CONFIG_PATH=$SDKTARGETSYSROOT'"$libdir"'/pkgconfig:$SDKTARGETSYSROOT'"$prefix"'/share/pkgconfig' >> $script echo 'export CONFIG_SITE=${SDKPATH}/site-config-'"${multimach_target_sys}" >> $script diff --git a/meta/classes/uninative.bbclass b/meta/classes/uninative.bbclass index 4412d7c567..4d4f53ad4d 100644 --- a/meta/classes/uninative.bbclass +++ b/meta/classes/uninative.bbclass @@ -34,6 +34,8 @@ python uninative_event_fetchloader() { with open(loaderchksum, "r") as f: readchksum = f.read().strip() if readchksum == chksum: + if "uninative" not in d.getVar("SSTATEPOSTUNPACKFUNCS"): + enable_uninative(d) return import subprocess @@ -167,5 +169,7 @@ python uninative_changeinterp () { if not elf.isDynamic(): continue + os.chmod(f, s[stat.ST_MODE] | stat.S_IWUSR) subprocess.check_output(("patchelf-uninative", "--set-interpreter", d.getVar("UNINATIVE_LOADER"), f), stderr=subprocess.STDOUT) + os.chmod(f, s[stat.ST_MODE]) } diff --git a/meta/classes/useradd-staticids.bbclass b/meta/classes/useradd-staticids.bbclass index 3a1b5f1320..908b24969f 100644 --- a/meta/classes/useradd-staticids.bbclass +++ b/meta/classes/useradd-staticids.bbclass @@ -41,7 +41,7 @@ def update_useradd_static_config(d): def handle_missing_id(id, type, pkg, files, var, value): # For backwards compatibility we accept "1" in addition to "error" error_dynamic = d.getVar('USERADD_ERROR_DYNAMIC') - msg = "%s - %s: %sname %s does not have a static ID defined." % (d.getVar('PN'), pkg, type, id) + msg = 'Recipe %s, package %s: %sname "%s" does not have a static ID defined.' % (d.getVar('PN'), pkg, type, id) if files: msg += " Add %s to one of these files: %s" % (id, files) else: |