From a7309d5790f5dac46e84d3c14959943eb2496fda Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Fri, 20 May 2016 11:17:05 +0100 Subject: classes/lib: Update to use python3 command pipeline decoding In python3, strings are unicode by default. We need to encode/decode from command pipelines and other places where we interface with the real world using the correct locales. This patch updates various call sites to use the correct encoding/decodings. (From OE-Core rev: bb4685af1bffe17b3aa92a6d21398f38a44ea874) Signed-off-by: Richard Purdie --- meta/classes/buildhistory.bbclass | 12 ++++-------- meta/classes/chrpath.bbclass | 2 ++ meta/classes/externalsrc.bbclass | 2 +- meta/classes/insane.bbclass | 8 ++++---- meta/classes/libc-package.bbclass | 3 +++ meta/classes/package.bbclass | 6 +++--- meta/classes/package_deb.bbclass | 26 +++++++++++--------------- meta/classes/toaster.bbclass | 1 + 8 files changed, 29 insertions(+), 31 deletions(-) (limited to 'meta/classes') diff --git a/meta/classes/buildhistory.bbclass b/meta/classes/buildhistory.bbclass index 581d532693..e3b5c44a09 100644 --- a/meta/classes/buildhistory.bbclass +++ b/meta/classes/buildhistory.bbclass @@ -233,7 +233,7 @@ python buildhistory_emit_pkghistory() { key = item[0] if key.endswith('_' + pkg): key = key[:-len(pkg)-1] - pkgdata[key] = item[1].decode('utf-8').decode('string_escape') + pkgdata[key] = item[1] pkge = pkgdata.get('PKGE', '0') pkgv = pkgdata['PKGV'] @@ -288,14 +288,12 @@ python buildhistory_emit_pkghistory() { def write_recipehistory(rcpinfo, d): - import codecs - bb.debug(2, "Writing recipe history") pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE', True) infofile = os.path.join(pkghistdir, "latest") - with codecs.open(infofile, "w", encoding='utf8') as f: + with open(infofile, "w") as f: if rcpinfo.pe != "0": f.write(u"PE = %s\n" % rcpinfo.pe) f.write(u"PV = %s\n" % rcpinfo.pv) @@ -305,8 +303,6 @@ def write_recipehistory(rcpinfo, d): def write_pkghistory(pkginfo, d): - import codecs - bb.debug(2, "Writing package history for package %s" % pkginfo.name) pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE', True) @@ -316,7 +312,7 @@ def write_pkghistory(pkginfo, d): bb.utils.mkdirhier(pkgpath) infofile = os.path.join(pkgpath, "latest") - with codecs.open(infofile, "w", encoding='utf8') as f: + with open(infofile, "w") as f: if pkginfo.pe != "0": f.write(u"PE = %s\n" % pkginfo.pe) f.write(u"PV = %s\n" % pkginfo.pv) @@ -349,7 +345,7 @@ def write_pkghistory(pkginfo, d): filevarpath = os.path.join(pkgpath, "latest.%s" % filevar) val = pkginfo.filevars[filevar] if val: - with codecs.open(filevarpath, "w", encoding='utf8') as f: + with open(filevarpath, "w") as f: f.write(val) else: if os.path.exists(filevarpath): diff --git a/meta/classes/chrpath.bbclass b/meta/classes/chrpath.bbclass index 9c68855ab2..cdd7f27600 100644 --- a/meta/classes/chrpath.bbclass +++ b/meta/classes/chrpath.bbclass @@ -10,6 +10,8 @@ def process_file_linux(cmd, fpath, rootdir, baseprefix, tmpdir, d): if p.returncode != 0: return + err = err.decode('utf-8') + # Handle RUNPATH as well as RPATH err = err.replace("RUNPATH=","RPATH=") # Throw away everything other than the rpath list diff --git a/meta/classes/externalsrc.bbclass b/meta/classes/externalsrc.bbclass index da7eb4781c..b7140a321d 100644 --- a/meta/classes/externalsrc.bbclass +++ b/meta/classes/externalsrc.bbclass @@ -145,7 +145,7 @@ def srctree_hash_files(d): env = os.environ.copy() env['GIT_INDEX_FILE'] = tmp_index.name subprocess.check_output(['git', 'add', '.'], cwd=s_dir, env=env) - sha1 = subprocess.check_output(['git', 'write-tree'], cwd=s_dir, env=env) + sha1 = subprocess.check_output(['git', 'write-tree'], cwd=s_dir, env=env).decode("utf-8") with open(oe_hash_file, 'w') as fobj: fobj.write(sha1) ret = oe_hash_file + ':True' diff --git a/meta/classes/insane.bbclass b/meta/classes/insane.bbclass index 71999ada34..9b2337cdbf 100644 --- a/meta/classes/insane.bbclass +++ b/meta/classes/insane.bbclass @@ -399,7 +399,7 @@ def package_qa_check_unsafe_references_in_binaries(path, name, d, elf, messages) sysroot_path_usr = sysroot_path + exec_prefix try: - ldd_output = bb.process.Popen(["prelink-rtld", "--root", sysroot_path, path], stdout=sub.PIPE).stdout.read() + ldd_output = bb.process.Popen(["prelink-rtld", "--root", sysroot_path, path], stdout=sub.PIPE).stdout.read().decode("utf-8") except bb.process.CmdError: error_msg = pn + ": prelink-rtld aborted when processing %s" % path package_qa_handle_error("unsafe-references-in-binaries", error_msg, d) @@ -986,12 +986,12 @@ def package_qa_check_expanded_d(path,name,d,elf,messages): return sane def package_qa_check_encoding(keys, encode, d): - def check_encoding(key,enc): + def check_encoding(key, enc): sane = True value = d.getVar(key, True) if value: try: - s = unicode(value, enc) + s = value.encode(enc) except UnicodeDecodeError as e: error_msg = "%s has non %s characters" % (key,enc) sane = False @@ -1217,7 +1217,7 @@ Missing inherit gettext?""" % (gt, config)) try: flag = "WARNING: unrecognized options:" log = os.path.join(d.getVar('B', True), 'config.log') - output = subprocess.check_output(['grep', '-F', flag, log]).replace(', ', ' ') + output = subprocess.check_output(['grep', '-F', flag, log]).decode("utf-8").replace(', ', ' ') options = set() for line in output.splitlines(): options |= set(line.partition(flag)[2].split()) diff --git a/meta/classes/libc-package.bbclass b/meta/classes/libc-package.bbclass index 467d567923..70f479bb41 100644 --- a/meta/classes/libc-package.bbclass +++ b/meta/classes/libc-package.bbclass @@ -150,6 +150,7 @@ python package_do_split_gconvs () { c_re = re.compile('^copy "(.*)"') i_re = re.compile('^include "(\w+)".*') for l in f.readlines(): + l = l.decode("latin-1") m = c_re.match(l) or i_re.match(l) if m: dp = legitimize_package_name('%s%s-gconv-%s' % (mlprefix, bpn, m.group(1))) @@ -171,6 +172,7 @@ python package_do_split_gconvs () { c_re = re.compile('^copy "(.*)"') i_re = re.compile('^include "(\w+)".*') for l in f.readlines(): + l = l.decode("latin-1") m = c_re.match(l) or i_re.match(l) if m: dp = legitimize_package_name('%s%s-charmap-%s' % (mlprefix, bpn, m.group(1))) @@ -191,6 +193,7 @@ python package_do_split_gconvs () { c_re = re.compile('^copy "(.*)"') i_re = re.compile('^include "(\w+)".*') for l in f.readlines(): + l = l.decode("latin-1") m = c_re.match(l) or i_re.match(l) if m: dp = legitimize_package_name(mlprefix+bpn+'-localedata-%s' % m.group(1)) diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass index a4125a0e98..501004ed48 100644 --- a/meta/classes/package.bbclass +++ b/meta/classes/package.bbclass @@ -63,7 +63,7 @@ def legitimize_package_name(s): def fixutf(m): cp = m.group(1) if cp: - return ('\u%s' % cp).decode('unicode_escape').encode('utf-8') + return ('\\u%s' % cp).encode('latin-1').decode('unicode_escape') # Handle unicode codepoints encoded as , as in glibc locale files. s = re.sub('', fixutf, s) @@ -1259,8 +1259,8 @@ python emit_pkgdata() { def write_if_exists(f, pkg, var): def encode(str): import codecs - c = codecs.getencoder("string_escape") - return c(str)[0] + c = codecs.getencoder("unicode_escape") + return c(str)[0].decode("latin1") val = d.getVar('%s_%s' % (var, pkg), True) if val: diff --git a/meta/classes/package_deb.bbclass b/meta/classes/package_deb.bbclass index bb5220ed00..e35f427ea2 100644 --- a/meta/classes/package_deb.bbclass +++ b/meta/classes/package_deb.bbclass @@ -173,7 +173,7 @@ python do_package_deb () { # Special behavior for description... if 'DESCRIPTION' in fs: summary = localdata.getVar('SUMMARY', True) or localdata.getVar('DESCRIPTION', True) or "." - ctrlfile.write('Description: %s\n' % unicode(summary,'utf-8')) + ctrlfile.write('Description: %s\n' % summary) description = localdata.getVar('DESCRIPTION', True) or "." description = textwrap.dedent(description).strip() if '\\n' in description: @@ -182,29 +182,25 @@ python do_package_deb () { # We don't limit the width when manually indent, but we do # need the textwrap.fill() to set the initial_indent and # subsequent_indent, so set a large width - ctrlfile.write('%s\n' % unicode(textwrap.fill(t, width=100000, initial_indent=' ', subsequent_indent=' '),'utf-8')) + ctrlfile.write('%s\n' % textwrap.fill(t, width=100000, initial_indent=' ', subsequent_indent=' ')) else: # Auto indent - ctrlfile.write('%s\n' % unicode(textwrap.fill(description.strip(), width=74, initial_indent=' ', subsequent_indent=' '),'utf-8')) + ctrlfile.write('%s\n' % textwrap.fill(description.strip(), width=74, initial_indent=' ', subsequent_indent=' ')) else: - ctrlfile.write(unicode(c % tuple(pullData(fs, localdata)),'utf-8')) + ctrlfile.write(c % tuple(pullData(fs, localdata))) except KeyError: import sys (type, value, traceback) = sys.exc_info() bb.utils.unlockfile(lf) ctrlfile.close() raise bb.build.FuncFailed("Missing field for deb generation: %s" % value) - except UnicodeDecodeError: - bb.utils.unlockfile(lf) - ctrlfile.close() - raise bb.build.FuncFailed("Non UTF-8 characters found in one of the fields") # more fields custom_fields_chunk = get_package_additional_metadata("deb", localdata) if custom_fields_chunk is not None: - ctrlfile.write(unicode(custom_fields_chunk)) + ctrlfile.write(custom_fields_chunk) ctrlfile.write("\n") mapping_rename_hook(localdata) @@ -255,17 +251,17 @@ python do_package_deb () { rconflicts = bb.utils.explode_dep_versions2(localdata.getVar("RCONFLICTS", True) or "") debian_cmp_remap(rconflicts) if rdepends: - ctrlfile.write("Depends: %s\n" % unicode(bb.utils.join_deps(rdepends))) + ctrlfile.write("Depends: %s\n" % bb.utils.join_deps(rdepends)) if rsuggests: - ctrlfile.write("Suggests: %s\n" % unicode(bb.utils.join_deps(rsuggests))) + ctrlfile.write("Suggests: %s\n" % bb.utils.join_deps(rsuggests)) if rrecommends: - ctrlfile.write("Recommends: %s\n" % unicode(bb.utils.join_deps(rrecommends))) + ctrlfile.write("Recommends: %s\n" % bb.utils.join_deps(rrecommends)) if rprovides: - ctrlfile.write("Provides: %s\n" % unicode(bb.utils.join_deps(rprovides))) + ctrlfile.write("Provides: %s\n" % bb.utils.join_deps(rprovides)) if rreplaces: - ctrlfile.write("Replaces: %s\n" % unicode(bb.utils.join_deps(rreplaces))) + ctrlfile.write("Replaces: %s\n" % bb.utils.join_deps(rreplaces)) if rconflicts: - ctrlfile.write("Conflicts: %s\n" % unicode(bb.utils.join_deps(rconflicts))) + ctrlfile.write("Conflicts: %s\n" % bb.utils.join_deps(rconflicts)) ctrlfile.close() for script in ["preinst", "postinst", "prerm", "postrm"]: diff --git a/meta/classes/toaster.bbclass b/meta/classes/toaster.bbclass index 1a70f14a92..1878fe095d 100644 --- a/meta/classes/toaster.bbclass +++ b/meta/classes/toaster.bbclass @@ -33,6 +33,7 @@ python toaster_layerinfo_dumpdata() { def _get_git_branch(layer_path): branch = subprocess.Popen("git symbolic-ref HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0] + branch = branch.decode('utf-8') branch = branch.replace('refs/heads/', '').rstrip() return branch -- cgit 1.2.3-korg