# # Populate builds using prebuilt packages where possible to speed up builds # and allow staging to be reconstructed. # # To use it add that line to conf/local.conf: # # INHERIT += "packaged-staging" # # # bitbake.conf set PSTAGING_ACTIVE = "0", this class sets to "1" if we're active # PSTAGE_PKGVERSION = "${PV}-${PR}" PSTAGE_PKGARCH = "${BUILD_SYS}" PSTAGE_EXTRAPATH ?= "" PSTAGE_PKGPATH = "${DISTRO}${PSTAGE_EXTRAPATH}" PSTAGE_PKGPN = "${@bb.data.expand('staging-${PN}-${MULTIMACH_ARCH}${TARGET_VENDOR}-${TARGET_OS}', d).replace('_', '-')}" PSTAGE_PKGNAME = "${PSTAGE_PKGPN}_${PSTAGE_PKGVERSION}_${PSTAGE_PKGARCH}.ipk" PSTAGE_PKG = "${DEPLOY_DIR_PSTAGE}/${PSTAGE_PKGPATH}/${PSTAGE_PKGNAME}" # multimachine.bbclass will override this but add a default in case we're not using it MULTIMACH_ARCH ?= "${PACKAGE_ARCH}" PSTAGE_NATIVEDEPENDS = "\ pkgconfig-native \ autoconf-native \ automake-native \ curl-native \ zlib-native \ libtool-native \ gnu-config-native \ shasum-native \ libtool-native \ automake-native \ update-alternatives-cworth-native \ ipkg-utils-native \ opkg-native \ m4-native \ quilt-native \ stagemanager-native \ " python () { import bb pstage_allowed = True # These classes encode staging paths into the binary data so can only be # reused if the path doesn't change/ if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross', d) or bb.data.inherits_class('sdk', d): path = bb.data.getVar('PSTAGE_PKGPATH', d, 1) path = path + bb.data.getVar('TMPDIR', d, 1).replace('/', '-') bb.data.setVar('PSTAGE_PKGPATH', path, d) # PSTAGE_NATIVEDEPENDS lists the packages we need before we can use packaged # staging. There will always be some packages we depend on. if bb.data.inherits_class('native', d): pn = bb.data.getVar('PN', d, True) nativedeps = bb.data.getVar('PSTAGE_NATIVEDEPENDS', d, True).split() if pn in nativedeps: pstage_allowed = False # Images aren't of interest to us if bb.data.inherits_class('image', d): pstage_allowed = False # Add task dependencies if we're active, otherwise mark packaged staging # as inactive. if pstage_allowed: deps = bb.data.getVarFlag('do_populate_staging', 'depends', d) or "" deps += " stagemanager-native:do_populate_staging" bb.data.setVarFlag('do_populate_staging', 'depends', deps, d) deps = bb.data.getVarFlag('do_prepackaged_stage', 'depends', d) or "" deps += " opkg-native:do_populate_staging ipkg-utils-native:do_populate_staging" bb.data.setVarFlag('do_prepackaged_stage', 'depends', deps, d) bb.data.setVar("PSTAGING_ACTIVE", "1", d) else: bb.data.setVar("PSTAGING_ACTIVE", "0", d) } DEPLOY_DIR_PSTAGE = "${DEPLOY_DIR}/pstage" PSTAGE_MACHCONFIG = "${DEPLOY_DIR_PSTAGE}/opkg.conf" PSTAGE_BUILD_CMD = "${IPKGBUILDCMD}" PSTAGE_INSTALL_CMD = "opkg-cl install -force-depends -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR}" PSTAGE_UPDATE_CMD = "opkg-cl update -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR}" PSTAGE_REMOVE_CMD = "opkg-cl remove -force-depends -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR}" PSTAGE_LIST_CMD = "opkg-cl list_installed -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR}" PSTAGE_TMPDIR_STAGE = "${WORKDIR}/staging-pkg" do_clean_append() { """ Clear the build and temp directories """ bb.note("Uninstalling package from staging...") path = bb.data.getVar("PATH", d, 1) removecmd = bb.data.getVar("PSTAGE_REMOVE_CMD", d, 1) removepkg = bb.data.expand("${PSTAGE_PKGPN}", d) ret = os.system("PATH=\"%s\" %s %s" % (path, removecmd, removepkg)) if ret != 0: bb.note("Failure removing staging package") stagepkg = bb.data.expand("${PSTAGE_PKG}", d) bb.note("Removing staging package %s" % stagepkg) os.system('rm -rf ' + stagepkg) } staging_helper () { # Assemble appropriate opkg.conf conffile=${PSTAGE_MACHCONFIG} mkdir -p ${DEPLOY_DIR_PSTAGE}/pstaging_lists if [ ! -e $conffile ]; then ipkgarchs="${BUILD_SYS}" priority=1 for arch in $ipkgarchs; do echo "arch $arch $priority" >> $conffile priority=$(expr $priority + 5) done fi } PSTAGE_TASKS_COVERED = "fetch unpack munge patch configure qa_configure rig_locales compile sizecheck install deploy package populate_staging package_write_deb package_write_ipk package_write package_stage qa_staging" python do_prepackaged_stage () { import os if bb.data.getVar("PSTAGING_ACTIVE", d, 1) == "0": bb.build.make_stamp("do_prepackaged_stage", d) return bb.note("Uninstalling any existing package from staging...") path = bb.data.getVar("PATH", d, 1) removecmd = bb.data.getVar("PSTAGE_REMOVE_CMD", d, 1) removepkg = bb.data.expand("${PSTAGE_PKGPN}", d) lf = bb.utils.lockfile(bb.data.expand("${STAGING_DIR}/staging.lock", d)) ret = os.system("PATH=\"%s\" %s %s" % (path, removecmd, removepkg)) bb.utils.unlockfile(lf) if ret != 0: bb.note("Failure attempting to remove staging package") stagepkg = bb.data.expand("${PSTAGE_PKG}", d) if os.path.exists(stagepkg): bb.note("Following speedup\n") path = bb.data.getVar("PATH", d, 1) installcmd = bb.data.getVar("PSTAGE_INSTALL_CMD", d, 1) bb.build.exec_func("staging_helper", d) bb.debug(1, "Staging stuff already packaged, using that instead") lf = bb.utils.lockfile(bb.data.expand("${STAGING_DIR}/staging.lock", d)) ret = os.system("PATH=\"%s\" %s %s" % (path, installcmd, stagepkg)) bb.utils.unlockfile(lf) if ret != 0: bb.note("Failure installing prestage package") #bb.build.make_stamp("do_prepackaged_stage", d) #for task in bb.data.getVar("PSTAGE_TASKS_COVERED", d, 1).split(): # bb.build.make_stamp("do_" + task, d) bb.build.make_stamp("do_stage_package_populated", d) else: bb.build.make_stamp("do_prepackaged_stage", d) } do_prepackaged_stage[cleandirs] = "${PSTAGE_TMPDIR_STAGE}" do_prepackaged_stage[selfstamp] = "1" addtask prepackaged_stage before do_fetch addhandler packagedstage_stampfixing_eventhandler python packagedstage_stampfixing_eventhandler() { from bb.event import getName import os if getName(e) == "StampUpdate": taskscovered = bb.data.getVar("PSTAGE_TASKS_COVERED", e.data, 1).split() for (fn, task) in e.targets: # strip off 'do_' task = task[3:] if task in taskscovered: stamp = "%s.do_stage_package_populated" % e.stampPrefix[fn] if os.path.exists(stamp): # We're targetting a task which was skipped with packaged staging # so we need to remove the autogenerated stamps. for task in taskscovered: dir = "%s.do_%s" % (e.stampPrefix[fn], task) os.system('rm -f ' + dir) os.system('rm -f ' + stamp) return NotHandled } populate_staging_preamble () { if [ "$PSTAGING_ACTIVE" = "1" ]; then stage-manager -p ${STAGING_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-staging -u || true stage-manager -p ${CROSS_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-cross -u || true fi } populate_staging_postamble () { if [ "$PSTAGING_ACTIVE" = "1" ]; then # list the packages currently installed in staging ${PSTAGE_LIST_CMD} | awk '{print $1}' > ${DEPLOY_DIR_PSTAGE}/installed-list set +e stage-manager -p ${STAGING_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-staging -u -d ${PSTAGE_TMPDIR_STAGE}/staging stage-manager -p ${CROSS_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-cross -u -d ${PSTAGE_TMPDIR_STAGE}/cross set -e fi } do_populate_staging[lockfiles] = "${STAGING_DIR}/staging.lock" do_populate_staging[dirs] =+ "${DEPLOY_DIR_PSTAGE}" python do_populate_staging_prepend() { bb.build.exec_func("populate_staging_preamble", d) } python do_populate_staging_append() { bb.build.exec_func("populate_staging_postamble", d) } staging_packager () { mkdir -p ${PSTAGE_TMPDIR_STAGE}/CONTROL mkdir -p ${DEPLOY_DIR_PSTAGE}/${PSTAGE_PKGPATH} echo "Package: ${PSTAGE_PKGPN}" > ${PSTAGE_TMPDIR_STAGE}/CONTROL/control echo "Version: ${PSTAGE_PKGVERSION}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control echo "Description: ${DESCRIPTION}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control echo "Section: ${SECTION}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control echo "Priority: Optional" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control echo "Maintainer: ${MAINTAINER}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control echo "Architecture: ${PSTAGE_PKGARCH}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control echo "Source: ${SRC_URI}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control ${PSTAGE_BUILD_CMD} ${PSTAGE_TMPDIR_STAGE} ${DEPLOY_DIR_PSTAGE}/${PSTAGE_PKGPATH} ${PSTAGE_INSTALL_CMD} ${PSTAGE_PKG} } staging_package_installer () { ${PSTAGE_INSTALL_CMD} ${PSTAGE_PKG} } python do_package_stage () { if bb.data.getVar("PSTAGING_ACTIVE", d, 1) != "1": return # # Handle deploy/ packages # bb.build.exec_func("read_subpackage_metadata", d) stagepath = bb.data.getVar("PSTAGE_TMPDIR_STAGE", d, 1) tmpdir = bb.data.getVar("TMPDIR", d, True) packages = (bb.data.getVar('PACKAGES', d, 1) or "").split() if len(packages) > 0: if bb.data.inherits_class('package_ipk', d): ipkpath = bb.data.getVar('DEPLOY_DIR_IPK', d, True).replace(tmpdir, stagepath) if bb.data.inherits_class('package_deb', d): debpath = bb.data.getVar('DEPLOY_DIR_DEB', d, True).replace(tmpdir, stagepath) for pkg in packages: pkgname = bb.data.getVar('PKG_%s' % pkg, d, 1) if not pkgname: pkgname = pkg arch = bb.data.getVar('PACKAGE_ARCH_%s' % pkg, d, 1) if not arch: arch = bb.data.getVar('PACKAGE_ARCH', d, 1) pr = bb.data.getVar('PR_%s' % pkg, d, 1) if not pr: pr = bb.data.getVar('PR', d, 1) if not packaged(pkg, d): continue if bb.data.inherits_class('package_ipk', d): srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "_" + arch + ".ipk", d) srcfile = bb.data.expand("${DEPLOY_DIR_IPK}/" + arch + "/" + srcname, d) if not os.path.exists(srcfile): bb.fatal("Package %s does not exist yet it should" % srcfile) destpath = ipkpath + "/" + arch + "/" bb.mkdirhier(destpath) bb.copyfile(srcfile, destpath + srcname) if bb.data.inherits_class('package_deb', d): if arch == 'all': srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "_all.deb", d) else: srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "_${DPKG_ARCH}.deb", d) srcfile = bb.data.expand("${DEPLOY_DIR_DEB}/" + arch + "/" + srcname, d) if not os.path.exists(srcfile): bb.fatal("Package %s does not exist yet it should" % srcfile) destpath = debpath + "/" + arch + "/" bb.mkdirhier(destpath) bb.copyfile(srcfile, destpath + srcname) # # Handle stamps/ files # stampfn = bb.data.getVar("STAMP", d, True) destdir = os.path.dirname(stampfn.replace(tmpdir, stagepath)) bb.mkdirhier(destdir) # We need to include the package_stage stamp in the staging package so create one bb.build.make_stamp("do_package_stage", d) os.system("cp %s.do_* %s/" % (stampfn, destdir)) bb.build.exec_func("staging_helper", d) bb.build.exec_func("staging_packager", d) lf = bb.utils.lockfile(bb.data.expand("${STAGING_DIR}/staging.lock", d)) bb.build.exec_func("staging_package_installer", d) bb.utils.unlockfile(lf) } # # Note an assumption here is that do_deploy runs before do_package_write/do_populate_staging # addtask package_stage after do_package_write do_populate_staging before do_build