summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorStephano Cetola <stephano.cetola@linux.intel.com>2016-08-10 13:03:16 -0700
committerRichard Purdie <richard.purdie@linuxfoundation.org>2016-08-11 13:36:06 +0100
commita92c196449c516fe51786d429078bbb1213bb029 (patch)
treeed7a7309c9bf4c675c7e42cafcc68eff618c0f70
parentc47288615c05aeec46ced39b2853cdd436703dec (diff)
downloadopenembedded-core-a92c196449c516fe51786d429078bbb1213bb029.tar.gz
Allow for simultaneous do_rootfs tasks with rpmuninative-1.3
Give each rootfs its own RPM channel to use. This puts the RPM metadata in a private subdirectory of $WORKDIR, rather than living in DEPLOY_DIR where other tasks may race with it. This allows us to reduce the time that the rpm.lock is held to only the time needed to hardlink the RPMs, allowing the majority of the rootfs operation to run in parallel. Also, this fixes the smart tests by generating an index for all packages at the time of the test, rather than using the one provided by the rootfs process. Original credit for the enhancement should go to Steven Walter stevenrwalter@gmail.com. Signed-off-by: Stephano Cetola <stephano.cetola@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta/classes/rootfs_rpm.bbclass5
-rw-r--r--meta/lib/oe/package_manager.py17
-rw-r--r--meta/lib/oeqa/runtime/smart.py41
3 files changed, 54 insertions, 9 deletions
diff --git a/meta/classes/rootfs_rpm.bbclass b/meta/classes/rootfs_rpm.bbclass
index 38b3c99495..37730a7104 100644
--- a/meta/classes/rootfs_rpm.bbclass
+++ b/meta/classes/rootfs_rpm.bbclass
@@ -24,11 +24,6 @@ do_populate_sdk[depends] += "${RPMROOTFSDEPENDS}"
do_rootfs[recrdeptask] += "do_package_write_rpm"
do_rootfs[vardeps] += "PACKAGE_FEED_URIS"
-# RPM doesn't work with multiple rootfs generation at once due to collisions in the use of files
-# in ${DEPLOY_DIR_RPM}. This can be removed if package_update_index_rpm can be called concurrently
-do_rootfs[lockfiles] += "${DEPLOY_DIR_RPM}/rpm.lock"
-do_populate_sdk[lockfiles] += "${DEPLOY_DIR_RPM}/rpm.lock"
-
python () {
if d.getVar('BUILD_IMAGES_FROM_FEEDS', True):
flags = d.getVarFlag('do_rootfs', 'recrdeptask', True)
diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
index 47f6831f98..2802254f1f 100644
--- a/meta/lib/oe/package_manager.py
+++ b/meta/lib/oe/package_manager.py
@@ -9,6 +9,7 @@ import collections
import bb
import tempfile
import oe.utils
+import oe.path
import string
from oe.gpg_sign import get_signer
@@ -175,7 +176,7 @@ class RpmIndexer(Indexer):
dbpath = os.path.join(self.d.getVar('WORKDIR', True), 'rpmdb', arch)
if os.path.exists(dbpath):
bb.utils.remove(dbpath, True)
- arch_dir = os.path.join(self.deploy_dir, arch)
+ arch_dir = os.path.join(self.d.getVar('WORKDIR', True), 'rpms', arch)
if not os.path.isdir(arch_dir):
continue
@@ -1010,8 +1011,18 @@ class RpmPM(PackageManager):
ch_already_added = []
for canonical_arch in platform_extra:
arch = canonical_arch.split('-')[0]
- arch_channel = os.path.join(self.deploy_dir, arch)
- if os.path.exists(arch_channel) and not arch in ch_already_added:
+ arch_channel = os.path.join(self.d.getVar('WORKDIR', True), 'rpms', arch)
+ oe.path.remove(arch_channel)
+ deploy_arch_dir = os.path.join(self.deploy_dir, arch)
+ if not os.path.exists(deploy_arch_dir):
+ continue
+
+ lockfilename = self.d.getVar('DEPLOY_DIR_RPM', True) + "/rpm.lock"
+ lf = bb.utils.lockfile(lockfilename, False)
+ oe.path.copyhardlinktree(deploy_arch_dir, arch_channel)
+ bb.utils.unlockfile(lf)
+
+ if not arch in ch_already_added:
bb.note('Adding Smart channel %s (%s)' %
(arch, channel_priority))
self._invoke_smart('channel --add %s type=rpm-md baseurl=%s -y'
diff --git a/meta/lib/oeqa/runtime/smart.py b/meta/lib/oeqa/runtime/smart.py
index c7a5753991..c8ba433cdd 100644
--- a/meta/lib/oeqa/runtime/smart.py
+++ b/meta/lib/oeqa/runtime/smart.py
@@ -1,5 +1,7 @@
import unittest
import re
+import oe
+import subprocess
from oeqa.oetest import oeRuntimeTest, skipModule
from oeqa.utils.decorators import *
from oeqa.utils.httpserver import HTTPService
@@ -53,9 +55,46 @@ class SmartBasicTest(SmartTest):
class SmartRepoTest(SmartTest):
@classmethod
+ def create_index(self, arg):
+ index_cmd = arg
+ try:
+ bb.note("Executing '%s' ..." % index_cmd)
+ result = subprocess.check_output(index_cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8")
+ except subprocess.CalledProcessError as e:
+ return("Index creation command '%s' failed with return code %d:\n%s" %
+ (e.cmd, e.returncode, e.output.decode("utf-8")))
+ if result:
+ bb.note(result)
+ return None
+
+ @classmethod
def setUpClass(self):
self.repolist = []
- self.repo_server = HTTPService(oeRuntimeTest.tc.d.getVar('DEPLOY_DIR', True), oeRuntimeTest.tc.target.server_ip)
+
+ # Index RPMs
+ rpm_createrepo = bb.utils.which(os.getenv('PATH'), "createrepo")
+ index_cmds = []
+ rpm_dirs_found = False
+ archs = (oeRuntimeTest.tc.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS', True) or "").replace('-', '_').split()
+ for arch in archs:
+ rpm_dir = os.path.join(oeRuntimeTest.tc.d.getVar('DEPLOY_DIR_RPM', True), arch)
+ idx_path = os.path.join(oeRuntimeTest.tc.d.getVar('WORKDIR', True), 'rpm', arch)
+ db_path = os.path.join(oeRuntimeTest.tc.d.getVar('WORKDIR', True), 'rpmdb', arch)
+ if not os.path.isdir(rpm_dir):
+ continue
+ if os.path.exists(db_path):
+ bb.utils.remove(dbpath, True)
+ lockfilename = oeRuntimeTest.tc.d.getVar('DEPLOY_DIR_RPM', True) + "/rpm.lock"
+ lf = bb.utils.lockfile(lockfilename, False)
+ oe.path.copyhardlinktree(rpm_dir, idx_path)
+ bb.utils.unlockfile(lf)
+ index_cmds.append("%s --dbpath %s --update -q %s" % (rpm_createrepo, db_path, idx_path))
+ rpm_dirs_found = True
+ # Create repodata¬
+ result = oe.utils.multiprocess_exec(index_cmds, self.create_index)
+ if result:
+ bb.fatal('%s' % ('\n'.join(result)))
+ self.repo_server = HTTPService(oeRuntimeTest.tc.d.getVar('WORKDIR', True), oeRuntimeTest.tc.target.server_ip)
self.repo_server.start()
@classmethod