From 3d35631121f0e030bc8151f5c23d84008d06f44b Mon Sep 17 00:00:00 2001 From: Paul Eggleton Date: Sat, 23 Jan 2016 00:59:51 +1300 Subject: devtool: sdk-update: improve SDK update process robustness Make the following improvements to the SDK update process: * Use a manifest file with sha256sums to track files other than sstate and metadata that we need to update - e.g. conf files. This allows us to handle where files such as auto.conf may or may not be present, as well as the configuration changing without affecting task signatures - we still want the config files copied in that case rather than it saying nothing needs to be done. * Write the SSTATE_MIRRORS_append to site.conf rather than local.conf so that local.conf remains static (since we don't want to trigger an update every time). Also, If there is an SSTATE_MIRRORS value already set in the configuration we can skip this and assume it contains the needed packages. * Allow the update process to be run in any directory, don't assume we're already at the base of the SDK * Where practical, fetch remote files into a temporary location and then move them to the desired location at the end, to avoid a failed update leaving the SDK in a broken state. * Update all installed do_populate_sysroot / do_packagedata tasks instead of using the SDK targets. This ensures any item installed through dependencies after installation (e.g. when running "devtool build") won't go stale. Signed-off-by: Paul Eggleton Signed-off-by: Richard Purdie --- scripts/lib/devtool/sdk.py | 118 ++++++++++++++++++++++++++++++++------------- 1 file changed, 84 insertions(+), 34 deletions(-) (limited to 'scripts/lib/devtool') diff --git a/scripts/lib/devtool/sdk.py b/scripts/lib/devtool/sdk.py index 4fcd36a0df..80ea8711cc 100644 --- a/scripts/lib/devtool/sdk.py +++ b/scripts/lib/devtool/sdk.py @@ -75,6 +75,21 @@ def install_sstate_objects(sstate_objects, src_sdk, dest_sdk): logger.debug("Copying %s to %s" % (sb, dst)) shutil.copy(sb, dst) +def check_manifest(fn, basepath): + import bb.utils + changedfiles = [] + with open(fn, 'r') as f: + for line in f: + splitline = line.split() + if len(splitline) > 1: + chksum = splitline[0] + fpath = splitline[1] + curr_chksum = bb.utils.sha256_file(os.path.join(basepath, fpath)) + if chksum != curr_chksum: + logger.debug('File %s changed: old csum = %s, new = %s' % (os.path.join(basepath, fpath), curr_chksum, chksum)) + changedfiles.append(fpath) + return changedfiles + def sdk_update(args, config, basepath, workspace): # Fetch locked-sigs.inc file from remote/local destination updateserver = args.updateserver @@ -98,6 +113,18 @@ def sdk_update(args, config, basepath, workspace): else: is_remote = False + layers_dir = os.path.join(basepath, 'layers') + conf_dir = os.path.join(basepath, 'conf') + + # Grab variable values + tinfoil = setup_tinfoil(config_only=True, basepath=basepath) + try: + stamps_dir = tinfoil.config_data.getVar('STAMPS_DIR', True) + sstate_mirrors = tinfoil.config_data.getVar('SSTATE_MIRRORS', True) + site_conf_version = tinfoil.config_data.getVar('SITE_CONF_VERSION', True) + finally: + tinfoil.shutdown() + if not is_remote: # devtool sdk-update /local/path/to/latest/sdk new_locked_sig_file_path = os.path.join(updateserver, 'conf/locked-sigs.inc') @@ -121,16 +148,14 @@ def sdk_update(args, config, basepath, workspace): install_sstate_objects(sstate_objects, updateserver.rstrip('/'), basepath) logger.info("Updating configuration files") new_conf_dir = os.path.join(updateserver, 'conf') - old_conf_dir = os.path.join(basepath, 'conf') - shutil.rmtree(old_conf_dir) - shutil.copytree(new_conf_dir, old_conf_dir) + shutil.rmtree(conf_dir) + shutil.copytree(new_conf_dir, conf_dir) logger.info("Updating layers") new_layers_dir = os.path.join(updateserver, 'layers') - old_layers_dir = os.path.join(basepath, 'layers') - shutil.rmtree(old_layers_dir) - ret = subprocess.call("cp -a %s %s" % (new_layers_dir, old_layers_dir), shell=True) + shutil.rmtree(layers_dir) + ret = subprocess.call("cp -a %s %s" % (new_layers_dir, layers_dir), shell=True) if ret != 0: - logger.error("Copying %s to %s failed" % (new_layers_dir, old_layers_dir)) + logger.error("Copying %s to %s failed" % (new_layers_dir, layers_dir)) return ret else: # devtool sdk-update http://myhost/sdk @@ -138,50 +163,75 @@ def sdk_update(args, config, basepath, workspace): try: os.makedirs(os.path.join(tmpsdk_dir, 'conf')) new_locked_sig_file_path = os.path.join(tmpsdk_dir, 'conf', 'locked-sigs.inc') - # Fetch locked-sigs.inc from update server - ret = subprocess.call("wget -q -O - %s/conf/locked-sigs.inc > %s" % (updateserver, new_locked_sig_file_path), shell=True) - if ret != 0: - logger.error("Fetching conf/locked-sigs.inc from %s to %s failed" % (updateserver, new_locked_sig_file_path)) - return ret - else: - logger.info("Fetching conf/locked-sigs.inc from %s to %s succeeded" % (updateserver, new_locked_sig_file_path)) - update_dict = generate_update_dict(new_locked_sig_file_path, old_locked_sig_file_path) - logger.debug("update_dict = %s" % update_dict) - if len(update_dict) == 0: - logger.info("No need to update.") + # Fetch manifest from server + tmpmanifest = os.path.join(tmpsdk_dir, 'conf', 'sdk-conf-manifest') + ret = subprocess.call("wget -q -O %s %s/conf/sdk-conf-manifest" % (tmpmanifest, updateserver), shell=True) + changedfiles = check_manifest(tmpmanifest, basepath) + if not changedfiles: + logger.info("Already up-to-date") return 0 # Update metadata - logger.debug("Updating meta data via git ...") + logger.debug("Updating metadata via git ...") # Try using 'git pull', if failed, use 'git clone' if os.path.exists(os.path.join(basepath, 'layers/.git')): - ret = subprocess.call("cd layers && git pull %s/layers/.git" % updateserver, shell=True) + ret = subprocess.call("git pull %s/layers/.git" % updateserver, shell=True, cwd=layers_dir) else: ret = -1 if ret != 0: - ret = subprocess.call("rm -rf layers && git clone %s/layers/.git" % updateserver, shell=True) - if ret != 0: - logger.error("Updating meta data via git failed") - return ret + ret = subprocess.call("git clone %s/layers/.git" % updateserver, shell=True, cwd=tmpsdk_dir) + if ret != 0: + logger.error("Updating metadata via git failed") + return ret logger.debug("Updating conf files ...") - conf_files = ['local.conf', 'bblayers.conf', 'devtool.conf', 'locked-sigs.inc'] - for conf in conf_files: - ret = subprocess.call("wget -q -O - %s/conf/%s > conf/%s" % (updateserver, conf, conf), shell=True) + for changedfile in changedfiles: + ret = subprocess.call("wget -q -O %s %s/%s" % (changedfile, updateserver, changedfile), shell=True, cwd=tmpsdk_dir) if ret != 0: - logger.error("Update %s failed" % conf) + logger.error("Updating %s failed" % changedfile) return ret - with open(os.path.join(basepath, 'conf/local.conf'), 'a') as f: - f.write('SSTATE_MIRRORS_append = " file://.* %s/sstate-cache/PATH \\n "\n' % updateserver) + + # Ok, all is well at this point - move everything over + tmplayers_dir = os.path.join(tmpsdk_dir, 'layers') + if os.path.exists(tmplayers_dir): + shutil.rmtree(layers_dir) + shutil.move(tmplayers_dir, layers_dir) + for changedfile in changedfiles: + destfile = os.path.join(basepath, changedfile) + os.remove(destfile) + shutil.move(os.path.join(tmpsdk_dir, changedfile), destfile) + os.remove(os.path.join(conf_dir, 'sdk-conf-manifest')) + shutil.move(tmpmanifest, conf_dir) + + if not sstate_mirrors: + with open(os.path.join(conf_dir, 'site.conf'), 'a') as f: + f.write('SCONF_VERSION = "%s"\n' % site_conf_version) + f.write('SSTATE_MIRRORS_append = " file://.* %s/sstate-cache/PATH \\n "\n' % updateserver) finally: shutil.rmtree(tmpsdk_dir) if not args.skip_prepare: + # Find all potentially updateable tasks + sdk_update_targets = [] + tasks = ['do_populate_sysroot', 'do_packagedata'] + for root, _, files in os.walk(stamps_dir): + for fn in files: + if not '.sigdata.' in fn: + for task in tasks: + if '.%s.' % task in fn or '.%s_setscene.' % task in fn: + sdk_update_targets.append('%s:%s' % (os.path.basename(root), task)) # Run bitbake command for the whole SDK - sdk_update_targets = config.get('SDK', 'sdk_update_targets', config.get('SDK', 'sdk_targets')) logger.info("Preparing build system... (This may take some time.)") try: - exec_build_env_command(config.init_path, basepath, 'bitbake %s --setscene-only' % sdk_update_targets) - except: - logger.error('bitbake %s failed' % sdk_update_targets) + exec_build_env_command(config.init_path, basepath, 'bitbake --setscene-only %s' % ' '.join(sdk_update_targets), stderr=subprocess.STDOUT) + output, _ = exec_build_env_command(config.init_path, basepath, 'bitbake -n %s' % ' '.join(sdk_update_targets), stderr=subprocess.STDOUT) + runlines = [] + for line in output.splitlines(): + if 'Running task ' in line: + runlines.append(line) + if runlines: + logger.error('Unexecuted tasks found in preparation log:\n %s' % '\n '.join(runlines)) + return -1 + except bb.process.ExecutionError as e: + logger.error('Preparation failed:\n%s' % e.stdout) return -1 return 0 -- cgit 1.2.3-korg