From bb4685af1bffe17b3aa92a6d21398f38a44ea874 Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Fri, 20 May 2016 11:17:05 +0100 Subject: classes/lib: Update to use python3 command pipeline decoding In python3, strings are unicode by default. We need to encode/decode from command pipelines and other places where we interface with the real world using the correct locales. This patch updates various call sites to use the correct encoding/decodings. Signed-off-by: Richard Purdie --- meta/lib/oe/gpg_sign.py | 10 ++--- meta/lib/oe/package.py | 4 +- meta/lib/oe/package_manager.py | 88 +++++++++++++++++++++--------------------- meta/lib/oe/packagedata.py | 2 +- 4 files changed, 53 insertions(+), 51 deletions(-) (limited to 'meta/lib/oe') diff --git a/meta/lib/oe/gpg_sign.py b/meta/lib/oe/gpg_sign.py index b83ee86728..a8a478aa95 100644 --- a/meta/lib/oe/gpg_sign.py +++ b/meta/lib/oe/gpg_sign.py @@ -24,7 +24,7 @@ class LocalSigner(object): status, output = oe.utils.getstatusoutput(cmd) if status: raise bb.build.FuncFailed('Failed to export gpg public key (%s): %s' % - (keyid, output)) + (keyid, output.decode("utf-8"))) def sign_rpms(self, files, keyid, passphrase): """Sign RPM files""" @@ -39,7 +39,7 @@ class LocalSigner(object): status, output = oe.utils.getstatusoutput(cmd) if status: - raise bb.build.FuncFailed("Failed to sign RPM packages: %s" % output) + raise bb.build.FuncFailed("Failed to sign RPM packages: %s" % output.decode("utf-8")) def detach_sign(self, input_file, keyid, passphrase_file, passphrase=None, armor=True): """Create a detached signature of a file""" @@ -71,11 +71,11 @@ class LocalSigner(object): passphrase = fobj.readline(); job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE) - (_, stderr) = job.communicate(passphrase) + (_, stderr) = job.communicate(passphrase.encode("utf-8")) if job.returncode: raise bb.build.FuncFailed("GPG exited with code %d: %s" % - (job.returncode, stderr)) + (job.returncode, stderr.decode("utf-8"))) except IOError as e: bb.error("IO error (%s): %s" % (e.errno, e.strerror)) @@ -90,7 +90,7 @@ class LocalSigner(object): """Return the gpg version""" import subprocess try: - return subprocess.check_output((self.gpg_bin, "--version")).split()[2] + return subprocess.check_output((self.gpg_bin, "--version")).split()[2].decode("utf-8") except subprocess.CalledProcessError as e: raise bb.build.FuncFailed("Could not get gpg version: %s" % e) diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py index 252e32d1df..5bb15bbc09 100644 --- a/meta/lib/oe/package.py +++ b/meta/lib/oe/package.py @@ -64,8 +64,8 @@ def filedeprunner(arg): def process_deps(pipe, pkg, pkgdest, provides, requires): for line in pipe: - f = line.split(" ", 1)[0].strip() - line = line.split(" ", 1)[1].strip() + f = line.decode("utf-8").split(" ", 1)[0].strip() + line = line.decode("utf-8").split(" ", 1)[1].strip() if line.startswith("Requires:"): i = requires diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py index 3bc4ebf5b4..abe9f6878b 100644 --- a/meta/lib/oe/package_manager.py +++ b/meta/lib/oe/package_manager.py @@ -17,10 +17,10 @@ def create_index(arg): try: bb.note("Executing '%s' ..." % index_cmd) - result = subprocess.check_output(index_cmd, stderr=subprocess.STDOUT, shell=True) + result = subprocess.check_output(index_cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8") except subprocess.CalledProcessError as e: return("Index creation command '%s' failed with return code %d:\n%s" % - (e.cmd, e.returncode, e.output)) + (e.cmd, e.returncode, e.output.decode("utf-8"))) if result: bb.note(result) @@ -367,10 +367,10 @@ class RpmPkgsList(PkgsList): # Determine rpm version cmd = "%s --version" % self.rpm_cmd try: - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8") except subprocess.CalledProcessError as e: bb.fatal("Getting rpm version failed. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) ''' Translate the RPM/Smart format names to the OE multilib format names @@ -411,10 +411,10 @@ class RpmPkgsList(PkgsList): "-t", self.image_rpmlib] try: - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).strip() + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).strip().decode("utf-8") except subprocess.CalledProcessError as e: bb.fatal("Cannot get the package dependencies. Command '%s' " - "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output)) + "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8"))) return output @@ -425,10 +425,10 @@ class RpmPkgsList(PkgsList): try: # bb.note(cmd) - tmp_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).strip() + tmp_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).strip().decode("utf-8") except subprocess.CalledProcessError as e: bb.fatal("Cannot get the installed packages list. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) output = dict() deps = dict() @@ -485,6 +485,8 @@ class OpkgPkgsList(PkgsList): # output streams separately and check for empty stderr. p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) cmd_output, cmd_stderr = p.communicate() + cmd_output = cmd_output.decode("utf-8") + cmd_stderr = cmd_stderr.decode("utf-8") if p.returncode or cmd_stderr: bb.fatal("Cannot get the installed packages list. Command '%s' " "returned %d and stderr:\n%s" % (cmd, p.returncode, cmd_stderr)) @@ -502,10 +504,10 @@ class DpkgPkgsList(PkgsList): cmd.append("-f=Package: ${Package}\nArchitecture: ${PackageArch}\nVersion: ${Version}\nFile: ${Package}_${Version}_${Architecture}.deb\nDepends: ${Depends}\nRecommends: ${Recommends}\n\n") try: - cmd_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).strip() + cmd_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).strip().decode("utf-8") except subprocess.CalledProcessError as e: bb.fatal("Cannot get the installed packages list. Command '%s' " - "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output)) + "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8"))) return opkg_query(cmd_output) @@ -608,11 +610,11 @@ class PackageManager(object): try: bb.note("Installing complementary packages ...") bb.note('Running %s' % cmd) - complementary_pkgs = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + complementary_pkgs = subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode("utf-8") except subprocess.CalledProcessError as e: bb.fatal("Could not compute complementary packages list. Command " "'%s' returned %d:\n%s" % - (' '.join(cmd), e.returncode, e.output)) + (' '.join(cmd), e.returncode, e.output.decode("utf-8"))) self.install(complementary_pkgs.split(), attempt_only=True) os.remove(installed_pkgs_file) @@ -784,12 +786,12 @@ class RpmPM(PackageManager): try: complementary_pkgs = subprocess.check_output(cmd, stderr=subprocess.STDOUT, - shell=True) + shell=True).decode("utf-8") # bb.note(complementary_pkgs) return complementary_pkgs except subprocess.CalledProcessError as e: bb.fatal("Could not invoke smart. Command " - "'%s' returned %d:\n%s" % (cmd, e.returncode, e.output)) + "'%s' returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) def _search_pkg_name_in_feeds(self, pkg, feed_archs): for arch in feed_archs: @@ -808,7 +810,7 @@ class RpmPM(PackageManager): (self.smart_cmd, self.smart_opt, pkg) cmd += " | sed -ne 's/ *Provides://p'" bb.note('cmd: %s' % cmd) - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8") # Found a provider if output: bb.note('Found providers for %s: %s' % (pkg, output)) @@ -956,7 +958,7 @@ class RpmPM(PackageManager): subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError as e: bb.fatal("Create rpm database failed. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) # Import GPG key to RPM database of the target system if self.d.getVar('RPM_SIGN_PACKAGES', True) == '1': pubkey_path = self.d.getVar('RPM_GPG_PUBKEY', True) @@ -1203,11 +1205,11 @@ class RpmPM(PackageManager): cmd = "%s %s install --attempt -y %s" % \ (self.smart_cmd, self.smart_opt, ' '.join(pkgs)) try: - output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) + output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT).decode("utf-8") bb.note(output) except subprocess.CalledProcessError as e: bb.fatal("Unable to install packages. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) ''' Remove pkgs with smart, the pkg name is smart/rpm format @@ -1233,11 +1235,11 @@ class RpmPM(PackageManager): try: bb.note(cmd) - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8") bb.note(output) except subprocess.CalledProcessError as e: bb.note("Unable to remove packages. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) def upgrade(self): bb.note('smart upgrade') @@ -1310,7 +1312,7 @@ class RpmPM(PackageManager): install_pkgs.append(pkg) except subprocess.CalledProcessError as e: bb.note("Unable to dump install packages. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) # Recovery rpmsys channel self._invoke_smart('channel --enable rpmsys') return install_pkgs @@ -1352,7 +1354,7 @@ class RpmPM(PackageManager): available_pkgs.append(pkg.strip()) except subprocess.CalledProcessError as e: bb.note("Unable to list all available packages. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) self.fullpkglist = available_pkgs @@ -1379,12 +1381,12 @@ class RpmPM(PackageManager): try: bb.note(cmd) - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).strip() + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).strip().decode("utf-8") bb.note(output) - os.chmod(saved_dir, 0755) + os.chmod(saved_dir, 0o755) except subprocess.CalledProcessError as e: bb.fatal("Invoke save_rpmpostinst failed. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) '''Write common configuration for target usage''' def rpm_setup_smart_target_config(self): @@ -1417,7 +1419,7 @@ class RpmPM(PackageManager): output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError as e: bb.fatal("Unable to list available packages. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) # Set default values to avoid UnboundLocalError arch = "" @@ -1482,7 +1484,7 @@ class RpmPM(PackageManager): except subprocess.CalledProcessError as e: bb.utils.remove(tmp_dir, recurse=True) bb.fatal("Unable to extract %s package. Command '%s' " - "returned %d:\n%s" % (pkg_path, cmd, e.returncode, e.output)) + "returned %d:\n%s" % (pkg_path, cmd, e.returncode, e.output.decode("utf-8"))) except OSError as e: bb.utils.remove(tmp_dir, recurse=True) bb.fatal("Unable to extract %s package. Command '%s' " @@ -1512,7 +1514,7 @@ class OpkgDpkgPM(PackageManager): output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError as e: bb.fatal("Unable to list available packages. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) return opkg_query(output) """ @@ -1544,7 +1546,7 @@ class OpkgDpkgPM(PackageManager): except subprocess.CalledProcessError as e: bb.utils.remove(tmp_dir, recurse=True) bb.fatal("Unable to extract %s package. Command '%s' " - "returned %d:\n%s" % (pkg_path, cmd, e.returncode, e.output)) + "returned %d:\n%s" % (pkg_path, cmd, e.returncode, e.output.decode("utf-8"))) except OSError as e: bb.utils.remove(tmp_dir, recurse=True) bb.fatal("Unable to extract %s package. Command '%s' " @@ -1733,7 +1735,7 @@ class OpkgPM(OpkgDpkgPM): except subprocess.CalledProcessError as e: self.deploy_dir_unlock() bb.fatal("Unable to update the package index files. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) self.deploy_dir_unlock() @@ -1754,12 +1756,12 @@ class OpkgPM(OpkgDpkgPM): try: bb.note("Installing the following packages: %s" % ' '.join(pkgs)) bb.note(cmd) - output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) + output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT).decode("utf-8") bb.note(output) except subprocess.CalledProcessError as e: (bb.fatal, bb.note)[attempt_only]("Unable to install packages. " "Command '%s' returned %d:\n%s" % - (cmd, e.returncode, e.output)) + (cmd, e.returncode, e.output.decode("utf-8"))) def remove(self, pkgs, with_dependencies=True): if with_dependencies: @@ -1771,11 +1773,11 @@ class OpkgPM(OpkgDpkgPM): try: bb.note(cmd) - output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) + output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT).decode("utf-8") bb.note(output) except subprocess.CalledProcessError as e: bb.fatal("Unable to remove packages. Command '%s' " - "returned %d:\n%s" % (e.cmd, e.returncode, e.output)) + "returned %d:\n%s" % (e.cmd, e.returncode, e.output.decode("utf-8"))) def write_index(self): self.deploy_dir_lock() @@ -1818,10 +1820,10 @@ class OpkgPM(OpkgDpkgPM): pkg_info = cmd + pkg try: - output = subprocess.check_output(pkg_info.split(), stderr=subprocess.STDOUT).strip() + output = subprocess.check_output(pkg_info.split(), stderr=subprocess.STDOUT).strip().decode("utf-8") except subprocess.CalledProcessError as e: bb.fatal("Cannot get package info. Command '%s' " - "returned %d:\n%s" % (pkg_info, e.returncode, e.output)) + "returned %d:\n%s" % (pkg_info, e.returncode, e.output.decode("utf-8"))) if output == "": bb.note("Ignored bad recommendation: '%s' is " @@ -1858,7 +1860,7 @@ class OpkgPM(OpkgDpkgPM): subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError as e: bb.fatal("Unable to update. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) # Dummy installation cmd = "%s %s --noaction install %s " % (self.opkg_cmd, @@ -1868,7 +1870,7 @@ class OpkgPM(OpkgDpkgPM): output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError as e: bb.fatal("Unable to dummy install packages. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) bb.utils.remove(temp_rootfs, True) @@ -2012,7 +2014,7 @@ class DpkgPM(OpkgDpkgPM): subprocess.check_output(p_full, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: bb.note("%s for package %s failed with %d:\n%s" % - (suffix[1], pkg_name, e.returncode, e.output)) + (suffix[1], pkg_name, e.returncode, e.output.decode("utf-8"))) failed_pkgs.append(pkg_name) break @@ -2030,7 +2032,7 @@ class DpkgPM(OpkgDpkgPM): subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: bb.fatal("Unable to update the package index files. Command '%s' " - "returned %d:\n%s" % (e.cmd, e.returncode, e.output)) + "returned %d:\n%s" % (e.cmd, e.returncode, e.output.decode("utf-8"))) self.deploy_dir_unlock() @@ -2049,7 +2051,7 @@ class DpkgPM(OpkgDpkgPM): except subprocess.CalledProcessError as e: (bb.fatal, bb.note)[attempt_only]("Unable to install packages. " "Command '%s' returned %d:\n%s" % - (cmd, e.returncode, e.output)) + (cmd, e.returncode, e.output.decode("utf-8"))) # rename *.dpkg-new files/dirs for root, dirs, files in os.walk(self.target_rootfs): @@ -2080,7 +2082,7 @@ class DpkgPM(OpkgDpkgPM): subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: bb.fatal("Unable to remove packages. Command '%s' " - "returned %d:\n%s" % (e.cmd, e.returncode, e.output)) + "returned %d:\n%s" % (e.cmd, e.returncode, e.output.decode("utf-8"))) def write_index(self): self.deploy_dir_lock() @@ -2213,7 +2215,7 @@ class DpkgPM(OpkgDpkgPM): subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: bb.fatal("Cannot fix broken dependencies. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) def list_installed(self): return DpkgPkgsList(self.d, self.target_rootfs).list_pkgs() diff --git a/meta/lib/oe/packagedata.py b/meta/lib/oe/packagedata.py index bc0fd06bce..df1b4c52e3 100644 --- a/meta/lib/oe/packagedata.py +++ b/meta/lib/oe/packagedata.py @@ -8,7 +8,7 @@ def read_pkgdatafile(fn): pkgdata = {} def decode(str): - c = codecs.getdecoder("string_escape") + c = codecs.getdecoder("unicode_escape") return c(str)[0] if os.access(fn, os.R_OK): -- cgit 1.2.3-korg