diff options
Diffstat (limited to 'meta/lib/oe/distro_check.py')
-rw-r--r-- | meta/lib/oe/distro_check.py | 379 |
1 files changed, 154 insertions, 225 deletions
diff --git a/meta/lib/oe/distro_check.py b/meta/lib/oe/distro_check.py index 8ed5b0ec80..88e46c354d 100644 --- a/meta/lib/oe/distro_check.py +++ b/meta/lib/oe/distro_check.py @@ -1,43 +1,32 @@ -def get_links_from_url(url): +# +# SPDX-License-Identifier: GPL-2.0-only +# + +def create_socket(url, d): + import urllib + from bb.utils import export_proxies + + export_proxies(d) + return urllib.request.urlopen(url) + +def get_links_from_url(url, d): "Return all the href links found on the web location" - import urllib, sgmllib - - class LinksParser(sgmllib.SGMLParser): - def parse(self, s): - "Parse the given string 's'." - self.feed(s) - self.close() - - def __init__(self, verbose=0): - "Initialise an object passing 'verbose' to the superclass." - sgmllib.SGMLParser.__init__(self, verbose) - self.hyperlinks = [] - - def start_a(self, attributes): - "Process a hyperlink and its 'attributes'." - for name, value in attributes: - if name == "href": - self.hyperlinks.append(value.strip('/')) - - def get_hyperlinks(self): - "Return the list of hyperlinks." - return self.hyperlinks - - sock = urllib.urlopen(url) - webpage = sock.read() - sock.close() - - linksparser = LinksParser() - linksparser.parse(webpage) - return linksparser.get_hyperlinks() - -def find_latest_numeric_release(url): + from bs4 import BeautifulSoup, SoupStrainer + + soup = BeautifulSoup(create_socket(url,d), "html.parser", parse_only=SoupStrainer("a")) + hyperlinks = [] + for line in soup.find_all('a', href=True): + hyperlinks.append(line['href'].strip('/')) + return hyperlinks + +def find_latest_numeric_release(url, d): "Find the latest listed numeric release on the given url" max=0 maxstr="" - for link in get_links_from_url(url): + for link in get_links_from_url(url, d): try: + # TODO use LooseVersion release = float(link) except: release = 0 @@ -48,194 +37,150 @@ def find_latest_numeric_release(url): def is_src_rpm(name): "Check if the link is pointing to a src.rpm file" - if name[-8:] == ".src.rpm": - return True - else: - return False + return name.endswith(".src.rpm") def package_name_from_srpm(srpm): "Strip out the package name from the src.rpm filename" - strings = srpm.split('-') - package_name = strings[0] - for i in range(1, len (strings) - 1): - str = strings[i] - if not str[0].isdigit(): - package_name += '-' + str - return package_name - -def clean_package_list(package_list): - "Removes multiple entries of packages and sorts the list" - set = {} - map(set.__setitem__, package_list, []) - return set.keys() - -def get_latest_released_meego_source_package_list(): - "Returns list of all the name os packages in the latest meego distro" + # ca-certificates-2016.2.7-1.0.fc24.src.rpm + # ^name ^ver ^release^removed + (name, version, release) = srpm.replace(".src.rpm", "").rsplit("-", 2) + return name - package_names = [] - try: - f = open("/tmp/Meego-1.1", "r") - for line in f: - package_names.append(line[:-1] + ":" + "main") # Also strip the '\n' at the end - except IOError: pass - package_list=clean_package_list(package_names) - return "1.0", package_list - -def get_source_package_list_from_url(url, section): +def get_source_package_list_from_url(url, section, d): "Return a sectioned list of package names from a URL list" bb.note("Reading %s: %s" % (url, section)) - links = get_links_from_url(url) + links = get_links_from_url(url, d) srpms = filter(is_src_rpm, links) names_list = map(package_name_from_srpm, srpms) - new_pkgs = [] + new_pkgs = set() for pkgs in names_list: - new_pkgs.append(pkgs + ":" + section) - + new_pkgs.add(pkgs + ":" + section) return new_pkgs -def get_latest_released_fedora_source_package_list(): - "Returns list of all the name os packages in the latest fedora distro" - latest = find_latest_numeric_release("http://archive.fedoraproject.org/pub/fedora/linux/releases/") - - package_names = get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Fedora/source/SRPMS/" % latest, "main") - -# package_names += get_source_package_list_from_url("http://download.fedora.redhat.com/pub/fedora/linux/releases/%s/Everything/source/SPRMS/" % latest, "everything") - package_names += get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates") +def get_source_package_list_from_url_by_letter(url, section, d): + import string + from urllib.error import HTTPError + packages = set() + for letter in (string.ascii_lowercase + string.digits): + # Not all subfolders may exist, so silently handle 404 + try: + packages |= get_source_package_list_from_url(url + "/" + letter, section, d) + except HTTPError as e: + if e.code != 404: raise + return packages - package_list=clean_package_list(package_names) - - return latest, package_list +def get_latest_released_fedora_source_package_list(d): + "Returns list of all the name os packages in the latest fedora distro" + latest = find_latest_numeric_release("http://archive.fedoraproject.org/pub/fedora/linux/releases/", d) + package_names = get_source_package_list_from_url_by_letter("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Everything/source/tree/Packages/" % latest, "main", d) + package_names |= get_source_package_list_from_url_by_letter("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates", d) + return latest, package_names -def get_latest_released_opensuse_source_package_list(): +def get_latest_released_opensuse_source_package_list(d): "Returns list of all the name os packages in the latest opensuse distro" - latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/") - - package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/%s/repo/oss/suse/src/" % latest, "main") - package_names += get_source_package_list_from_url("http://download.opensuse.org/update/%s/rpm/src/" % latest, "updates") - - package_list=clean_package_list(package_names) - return latest, package_list + latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/leap", d) -def get_latest_released_mandriva_source_package_list(): - "Returns list of all the name os packages in the latest mandriva distro" - latest = find_latest_numeric_release("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/") - package_names = get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/release/" % latest, "main") -# package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/contrib/release/" % latest, "contrib") - package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/updates/" % latest, "updates") + package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/leap/%s/repo/oss/suse/src/" % latest, "main", d) + package_names |= get_source_package_list_from_url("http://download.opensuse.org/update/leap/%s/oss/src/" % latest, "updates", d) + return latest, package_names - package_list=clean_package_list(package_names) - return latest, package_list +def get_latest_released_clear_source_package_list(d): + latest = find_latest_numeric_release("https://download.clearlinux.org/releases/", d) + package_names = get_source_package_list_from_url("https://download.clearlinux.org/releases/%s/clear/source/SRPMS/" % latest, "main", d) + return latest, package_names -def find_latest_debian_release(url): +def find_latest_debian_release(url, d): "Find the latest listed debian release on the given url" - releases = [] - for link in get_links_from_url(url): - if link[:6] == "Debian": - if ';' not in link: - releases.append(link) + releases = [link.replace("Debian", "") + for link in get_links_from_url(url, d) + if link.startswith("Debian")] releases.sort() try: - return releases.pop()[6:] + return releases[-1] except: return "_NotFound_" -def get_debian_style_source_package_list(url, section): +def get_debian_style_source_package_list(url, section, d): "Return the list of package-names stored in the debian style Sources.gz file" - import urllib - sock = urllib.urlopen(url) - import tempfile - tmpfile = tempfile.NamedTemporaryFile(mode='wb', prefix='oecore.', suffix='.tmp', delete=False) - tmpfilename=tmpfile.name - tmpfile.write(sock.read()) - sock.close() - tmpfile.close() import gzip - bb.note("Reading %s: %s" % (url, section)) - - f = gzip.open(tmpfilename) - package_names = [] - for line in f: - if line[:9] == "Package: ": - package_names.append(line[9:-1] + ":" + section) # Also strip the '\n' at the end - os.unlink(tmpfilename) + package_names = set() + for line in gzip.open(create_socket(url, d), mode="rt"): + if line.startswith("Package:"): + pkg = line.split(":", 1)[1].strip() + package_names.add(pkg + ":" + section) return package_names -def get_latest_released_debian_source_package_list(): - "Returns list of all the name os packages in the latest debian distro" - latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/") - url = "http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz" - package_names = get_debian_style_source_package_list(url, "main") -# url = "http://ftp.debian.org/debian/dists/stable/contrib/source/Sources.gz" -# package_names += get_debian_style_source_package_list(url, "contrib") - url = "http://ftp.debian.org/debian/dists/stable-proposed-updates/main/source/Sources.gz" - package_names += get_debian_style_source_package_list(url, "updates") - package_list=clean_package_list(package_names) - return latest, package_list - -def find_latest_ubuntu_release(url): - "Find the latest listed ubuntu release on the given url" +def get_latest_released_debian_source_package_list(d): + "Returns list of all the name of packages in the latest debian distro" + latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/", d) + url = "http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz" + package_names = get_debian_style_source_package_list(url, "main", d) + url = "http://ftp.debian.org/debian/dists/stable-proposed-updates/main/source/Sources.gz" + package_names |= get_debian_style_source_package_list(url, "updates", d) + return latest, package_names + +def find_latest_ubuntu_release(url, d): + """ + Find the latest listed Ubuntu release on the given ubuntu/dists/ URL. + + To avoid matching development releases look for distributions that have + updates, so the resulting distro could be any supported release. + """ url += "?C=M;O=D" # Descending Sort by Last Modified - for link in get_links_from_url(url): - if link[-8:] == "-updates": - return link[:-8] + for link in get_links_from_url(url, d): + if "-updates" in link: + distro = link.replace("-updates", "") + return distro return "_NotFound_" -def get_latest_released_ubuntu_source_package_list(): +def get_latest_released_ubuntu_source_package_list(d): "Returns list of all the name os packages in the latest ubuntu distro" - latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/") + latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/", d) url = "http://archive.ubuntu.com/ubuntu/dists/%s/main/source/Sources.gz" % latest - package_names = get_debian_style_source_package_list(url, "main") -# url = "http://archive.ubuntu.com/ubuntu/dists/%s/multiverse/source/Sources.gz" % latest -# package_names += get_debian_style_source_package_list(url, "multiverse") -# url = "http://archive.ubuntu.com/ubuntu/dists/%s/universe/source/Sources.gz" % latest -# package_names += get_debian_style_source_package_list(url, "universe") + package_names = get_debian_style_source_package_list(url, "main", d) url = "http://archive.ubuntu.com/ubuntu/dists/%s-updates/main/source/Sources.gz" % latest - package_names += get_debian_style_source_package_list(url, "updates") - package_list=clean_package_list(package_names) - return latest, package_list + package_names |= get_debian_style_source_package_list(url, "updates", d) + return latest, package_names + +def create_distro_packages_list(distro_check_dir, d): + import shutil -def create_distro_packages_list(distro_check_dir): pkglst_dir = os.path.join(distro_check_dir, "package_lists") - if not os.path.isdir (pkglst_dir): - os.makedirs(pkglst_dir) - # first clear old stuff - for file in os.listdir(pkglst_dir): - os.unlink(os.path.join(pkglst_dir, file)) - - per_distro_functions = [ - ["Debian", get_latest_released_debian_source_package_list], - ["Ubuntu", get_latest_released_ubuntu_source_package_list], - ["Fedora", get_latest_released_fedora_source_package_list], - ["OpenSuSE", get_latest_released_opensuse_source_package_list], - ["Mandriva", get_latest_released_mandriva_source_package_list], - ["Meego", get_latest_released_meego_source_package_list] - ] - - from datetime import datetime - begin = datetime.now() - for distro in per_distro_functions: - name = distro[0] - release, package_list = distro[1]() + bb.utils.remove(pkglst_dir, True) + bb.utils.mkdirhier(pkglst_dir) + + per_distro_functions = ( + ("Debian", get_latest_released_debian_source_package_list), + ("Ubuntu", get_latest_released_ubuntu_source_package_list), + ("Fedora", get_latest_released_fedora_source_package_list), + ("openSUSE", get_latest_released_opensuse_source_package_list), + ("Clear", get_latest_released_clear_source_package_list), + ) + + for name, fetcher_func in per_distro_functions: + try: + release, package_list = fetcher_func(d) + except Exception as e: + bb.warn("Cannot fetch packages for %s: %s" % (name, e)) bb.note("Distro: %s, Latest Release: %s, # src packages: %d" % (name, release, len(package_list))) + if len(package_list) == 0: + bb.error("Didn't fetch any packages for %s %s" % (name, release)) + package_list_file = os.path.join(pkglst_dir, name + "-" + release) - f = open(package_list_file, "w+b") - for pkg in package_list: - f.write(pkg + "\n") - f.close() - end = datetime.now() - delta = end - begin - bb.note("package_list generatiosn took this much time: %d seconds" % delta.seconds) + with open(package_list_file, 'w') as f: + for pkg in sorted(package_list): + f.write(pkg + "\n") -def update_distro_data(distro_check_dir, datetime): +def update_distro_data(distro_check_dir, datetime, d): """ - If distro packages list data is old then rebuild it. - The operations has to be protected by a lock so that - only one thread performes it at a time. + If distro packages list data is old then rebuild it. + The operations has to be protected by a lock so that + only one thread performes it at a time. """ if not os.path.isdir (distro_check_dir): try: @@ -250,83 +195,71 @@ def update_distro_data(distro_check_dir, datetime): import fcntl try: if not os.path.exists(datetime_file): - open(datetime_file, 'w+b').close() # touch the file so that the next open won't fail + open(datetime_file, 'w+').close() # touch the file so that the next open won't fail - f = open(datetime_file, "r+b") + f = open(datetime_file, "r+") fcntl.lockf(f, fcntl.LOCK_EX) saved_datetime = f.read() if saved_datetime[0:8] != datetime[0:8]: bb.note("The build datetime did not match: saved:%s current:%s" % (saved_datetime, datetime)) bb.note("Regenerating distro package lists") - create_distro_packages_list(distro_check_dir) + create_distro_packages_list(distro_check_dir, d) f.seek(0) f.write(datetime) - except OSError: - raise Exception('Unable to read/write this file: %s' % (datetime_file)) + except OSError as e: + raise Exception('Unable to open timestamp: %s' % e) finally: fcntl.lockf(f, fcntl.LOCK_UN) f.close() - + def compare_in_distro_packages_list(distro_check_dir, d): if not os.path.isdir(distro_check_dir): raise Exception("compare_in_distro_packages_list: invalid distro_check_dir passed") - + localdata = bb.data.createCopy(d) pkglst_dir = os.path.join(distro_check_dir, "package_lists") matching_distros = [] - pn = d.getVar('PN', True) - recipe_name = d.getVar('PN', True) + pn = recipe_name = d.getVar('PN') bb.note("Checking: %s" % pn) - trim_dict = dict({"-native":"-native", "-cross":"-cross", "-initial":"-initial"}) - if pn.find("-native") != -1: pnstripped = pn.split("-native") - localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) - bb.data.update_data(localdata) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) recipe_name = pnstripped[0] if pn.startswith("nativesdk-"): pnstripped = pn.split("nativesdk-") - localdata.setVar('OVERRIDES', "pn-" + pnstripped[1] + ":" + d.getVar('OVERRIDES', True)) - bb.data.update_data(localdata) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[1] + ":" + d.getVar('OVERRIDES')) recipe_name = pnstripped[1] if pn.find("-cross") != -1: pnstripped = pn.split("-cross") - localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) - bb.data.update_data(localdata) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) recipe_name = pnstripped[0] if pn.find("-initial") != -1: pnstripped = pn.split("-initial") - localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) - bb.data.update_data(localdata) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) recipe_name = pnstripped[0] bb.note("Recipe: %s" % recipe_name) - tmp = localdata.getVar('DISTRO_PN_ALIAS', True) distro_exceptions = dict({"OE-Core":'OE-Core', "OpenedHand":'OpenedHand', "Intel":'Intel', "Upstream":'Upstream', "Windriver":'Windriver', "OSPDT":'OSPDT Approved', "Poky":'poky'}) - - if tmp: - list = tmp.split(' ') - for str in list: - if str and str.find("=") == -1 and distro_exceptions[str]: - matching_distros.append(str) + tmp = localdata.getVar('DISTRO_PN_ALIAS') or "" + for str in tmp.split(): + if str and str.find("=") == -1 and distro_exceptions[str]: + matching_distros.append(str) distro_pn_aliases = {} - if tmp: - list = tmp.split(' ') - for str in list: - if str.find("=") != -1: - (dist, pn_alias) = str.split('=') - distro_pn_aliases[dist.strip().lower()] = pn_alias.strip() - + for str in tmp.split(): + if "=" in str: + (dist, pn_alias) = str.split('=') + distro_pn_aliases[dist.strip().lower()] = pn_alias.strip() + for file in os.listdir(pkglst_dir): (distro, distro_release) = file.split("-") - f = open(os.path.join(pkglst_dir, file), "rb") + f = open(os.path.join(pkglst_dir, file), "r") for line in f: (pkg, section) = line.split(":") if distro.lower() in distro_pn_aliases: @@ -339,38 +272,34 @@ def compare_in_distro_packages_list(distro_check_dir, d): break f.close() - - if tmp != None: - list = tmp.split(' ') - for item in list: - matching_distros.append(item) + for item in tmp.split(): + matching_distros.append(item) bb.note("Matching: %s" % matching_distros) return matching_distros def create_log_file(d, logname): - import subprocess - logpath = d.getVar('LOG_DIR', True) + logpath = d.getVar('LOG_DIR') bb.utils.mkdirhier(logpath) logfn, logsuffix = os.path.splitext(logname) - logfile = os.path.join(logpath, "%s.%s%s" % (logfn, d.getVar('DATETIME', True), logsuffix)) + logfile = os.path.join(logpath, "%s.%s%s" % (logfn, d.getVar('DATETIME'), logsuffix)) if not os.path.exists(logfile): slogfile = os.path.join(logpath, logname) if os.path.exists(slogfile): os.remove(slogfile) - subprocess.call("touch %s" % logfile, shell=True) + open(logfile, 'w+').close() os.symlink(logfile, slogfile) d.setVar('LOG_FILE', logfile) return logfile def save_distro_check_result(result, datetime, result_file, d): - pn = d.getVar('PN', True) - logdir = d.getVar('LOG_DIR', True) + pn = d.getVar('PN') + logdir = d.getVar('LOG_DIR') if not logdir: bb.error("LOG_DIR variable is not defined, can't write the distro_check results") return - if not os.path.isdir(logdir): - os.makedirs(logdir) + bb.utils.mkdirhier(logdir) + line = pn for i in result: line = line + "," + i |