diff options
| -rw-r--r-- | lib/bb/fetch2/__init__.py | 4 | ||||
| -rw-r--r-- | lib/bb/fetch2/git.py | 52 | ||||
| -rw-r--r-- | lib/bb/fetch2/wget.py | 28 | ||||
| -rw-r--r-- | lib/bb/progress.py | 31 |
4 files changed, 107 insertions, 8 deletions
diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py index b6fcaaad5..a27512cc8 100644 --- a/lib/bb/fetch2/__init__.py +++ b/lib/bb/fetch2/__init__.py @@ -779,7 +779,7 @@ def localpath(url, d): fetcher = bb.fetch2.Fetch([url], d) return fetcher.localpath(url) -def runfetchcmd(cmd, d, quiet=False, cleanup=None): +def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None): """ Run cmd returning the command output Raise an error if interrupted or cmd fails @@ -821,7 +821,7 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None): error_message = "" try: - (output, errors) = bb.process.run(cmd, shell=True, stderr=subprocess.PIPE) + (output, errors) = bb.process.run(cmd, log=log, shell=True, stderr=subprocess.PIPE) success = True except bb.process.NotFoundError as e: error_message = "Fetch command %s" % (e.command) diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py index 59827e304..4e2dcec0d 100644 --- a/lib/bb/fetch2/git.py +++ b/lib/bb/fetch2/git.py @@ -71,11 +71,53 @@ import os import re import bb import errno +import bb.progress from bb import data from bb.fetch2 import FetchMethod from bb.fetch2 import runfetchcmd from bb.fetch2 import logger + +class GitProgressHandler(bb.progress.LineFilterProgressHandler): + """Extract progress information from git output""" + def __init__(self, d): + self._buffer = '' + self._count = 0 + super(GitProgressHandler, self).__init__(d) + # Send an initial progress event so the bar gets shown + self._fire_progress(-1) + + def write(self, string): + self._buffer += string + stages = ['Counting objects', 'Compressing objects', 'Receiving objects', 'Resolving deltas'] + stage_weights = [0.2, 0.05, 0.5, 0.25] + stagenum = 0 + for i, stage in reversed(list(enumerate(stages))): + if stage in self._buffer: + stagenum = i + self._buffer = '' + break + self._status = stages[stagenum] + percs = re.findall(r'(\d+)%', string) + if percs: + progress = int(round((int(percs[-1]) * stage_weights[stagenum]) + (sum(stage_weights[:stagenum]) * 100))) + rates = re.findall(r'([\d.]+ [a-zA-Z]*/s+)', string) + if rates: + rate = rates[-1] + else: + rate = None + self.update(progress, rate) + else: + if stagenum == 0: + percs = re.findall(r': (\d+)', string) + if percs: + count = int(percs[-1]) + if count > self._count: + self._count = count + self._fire_progress(-count) + super(GitProgressHandler, self).write(string) + + class Git(FetchMethod): """Class to fetch a module or modules from git repositories""" def init(self, d): @@ -196,10 +238,11 @@ class Git(FetchMethod): # We do this since git will use a "-l" option automatically for local urls where possible if repourl.startswith("file://"): repourl = repourl[7:] - clone_cmd = "%s clone --bare --mirror %s %s" % (ud.basecmd, repourl, ud.clonedir) + clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, repourl, ud.clonedir) if ud.proto.lower() != 'file': bb.fetch2.check_network_access(d, clone_cmd) - runfetchcmd(clone_cmd, d) + progresshandler = GitProgressHandler(d) + runfetchcmd(clone_cmd, d, log=progresshandler) os.chdir(ud.clonedir) # Update the checkout if needed @@ -214,10 +257,11 @@ class Git(FetchMethod): logger.debug(1, "No Origin") runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, repourl), d) - fetch_cmd = "%s fetch -f --prune %s refs/*:refs/*" % (ud.basecmd, repourl) + fetch_cmd = "LANG=C %s fetch -f --prune --progress %s refs/*:refs/*" % (ud.basecmd, repourl) if ud.proto.lower() != 'file': bb.fetch2.check_network_access(d, fetch_cmd, ud.url) - runfetchcmd(fetch_cmd, d) + progresshandler = GitProgressHandler(d) + runfetchcmd(fetch_cmd, d, log=progresshandler) runfetchcmd("%s prune-packed" % ud.basecmd, d) runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d) try: diff --git a/lib/bb/fetch2/wget.py b/lib/bb/fetch2/wget.py index 6cb22aeee..6b60d9b16 100644 --- a/lib/bb/fetch2/wget.py +++ b/lib/bb/fetch2/wget.py @@ -31,6 +31,7 @@ import subprocess import os import logging import bb +import bb.progress import urllib.request, urllib.parse, urllib.error from bb import data from bb.fetch2 import FetchMethod @@ -41,6 +42,27 @@ from bb.utils import export_proxies from bs4 import BeautifulSoup from bs4 import SoupStrainer +class WgetProgressHandler(bb.progress.LineFilterProgressHandler): + """ + Extract progress information from wget output. + Note: relies on --progress=dot (with -v or without -q/-nv) being + specified on the wget command line. + """ + def __init__(self, d): + super(WgetProgressHandler, self).__init__(d) + # Send an initial progress event so the bar gets shown + self._fire_progress(0) + + def writeline(self, line): + percs = re.findall(r'(\d+)%\s+([\d.]+[A-Z])', line) + if percs: + progress = int(percs[-1][0]) + rate = percs[-1][1] + '/s' + self.update(progress, rate) + return False + return True + + class Wget(FetchMethod): """Class to fetch urls via 'wget'""" def supports(self, ud, d): @@ -66,13 +88,15 @@ class Wget(FetchMethod): if not ud.localfile: ud.localfile = data.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", "."), d) - self.basecmd = d.getVar("FETCHCMD_wget", True) or "/usr/bin/env wget -t 2 -T 30 -nv --passive-ftp --no-check-certificate" + self.basecmd = d.getVar("FETCHCMD_wget", True) or "/usr/bin/env wget -t 2 -T 30 --passive-ftp --no-check-certificate" def _runwget(self, ud, d, command, quiet): + progresshandler = WgetProgressHandler(d) + logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command)) bb.fetch2.check_network_access(d, command) - runfetchcmd(command, d, quiet) + runfetchcmd(command + ' --progress=dot -v', d, quiet, log=progresshandler) def download(self, ud, d): """Fetch urls""" diff --git a/lib/bb/progress.py b/lib/bb/progress.py index ee6b9536b..343b18f8c 100644 --- a/lib/bb/progress.py +++ b/lib/bb/progress.py @@ -58,6 +58,37 @@ class ProgressHandler(object): self._lastevent = ts self._progress = progress +class LineFilterProgressHandler(ProgressHandler): + """ + A ProgressHandler variant that provides the ability to filter out + the lines if they contain progress information. Additionally, it + filters out anything before the last line feed on a line. This can + be used to keep the logs clean of output that we've only enabled for + getting progress, assuming that that can be done on a per-line + basis. + """ + def __init__(self, d, outfile=None): + self._linebuffer = '' + super(LineFilterProgressHandler, self).__init__(d, outfile) + + def write(self, string): + self._linebuffer += string + while True: + breakpos = self._linebuffer.find('\n') + 1 + if breakpos == 0: + break + line = self._linebuffer[:breakpos] + self._linebuffer = self._linebuffer[breakpos:] + # Drop any line feeds and anything that precedes them + lbreakpos = line.rfind('\r') + 1 + if lbreakpos: + line = line[lbreakpos:] + if self.writeline(line): + super(LineFilterProgressHandler, self).write(line) + + def writeline(self, line): + return True + class BasicProgressHandler(ProgressHandler): def __init__(self, d, regex=r'(\d+)%', outfile=None): super(BasicProgressHandler, self).__init__(d, outfile) |