aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRoss Burton <ross.burton@intel.com>2017-07-24 22:42:01 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2017-07-25 15:49:36 +0100
commit91d4ca93df092cf86ab84faaa94cc66ff9f43057 (patch)
tree5e7af6868290d37db17d7ded76c39e5bcf887f0e
parentb7514340cd6a2753eb217b059229bb279c3849ec (diff)
downloadbitbake-contrib-91d4ca93df092cf86ab84faaa94cc66ff9f43057.tar.gz
fetch/wget: mitigate a wget race condition when listing FTP directories
When wget is fetching a listing for a directory over FTP it writes to a temporary file called .listing in the current directory. If there are many such operations happening in parallel - for example during 'bitbake world -c checkpkg' - then up to BB_NUMBER_THREADS instances of wget will be racing to write to, read, and delete the same file. This results in various failures such as the file disappearing before wget has processed it or the file changing contents, which causes checkpkg to randomly fail. Mitigate the race condition by creating a temporary directory to run wget in when doing directory listings. [ YOCTO #11828 ] Signed-off-by: Ross Burton <ross.burton@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--lib/bb/fetch2/wget.py23
1 files changed, 11 insertions, 12 deletions
diff --git a/lib/bb/fetch2/wget.py b/lib/bb/fetch2/wget.py
index 208ee9bdd..8ee9769d3 100644
--- a/lib/bb/fetch2/wget.py
+++ b/lib/bb/fetch2/wget.py
@@ -90,13 +90,13 @@ class Wget(FetchMethod):
self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 30 --passive-ftp --no-check-certificate"
- def _runwget(self, ud, d, command, quiet):
+ def _runwget(self, ud, d, command, quiet, workdir=None):
progresshandler = WgetProgressHandler(d)
logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command))
bb.fetch2.check_network_access(d, command, ud.url)
- runfetchcmd(command + ' --progress=dot -v', d, quiet, log=progresshandler)
+ runfetchcmd(command + ' --progress=dot -v', d, quiet, log=progresshandler, workdir=workdir)
def download(self, ud, d):
"""Fetch urls"""
@@ -422,17 +422,16 @@ class Wget(FetchMethod):
Run fetch checkstatus to get directory information
"""
f = tempfile.NamedTemporaryFile()
+ with tempfile.TemporaryDirectory(prefix="wget-index-") as workdir, tempfile.NamedTemporaryFile(dir=workdir, prefix="wget-listing-") as f:
+ agent = "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.12) Gecko/20101027 Ubuntu/9.10 (karmic) Firefox/3.6.12"
+ fetchcmd = self.basecmd
+ fetchcmd += " -O " + f.name + " --user-agent='" + agent + "' '" + uri + "'"
+ try:
+ self._runwget(ud, d, fetchcmd, True, workdir=workdir)
+ fetchresult = f.read()
+ except bb.fetch2.BBFetchException:
+ fetchresult = ""
- agent = "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.12) Gecko/20101027 Ubuntu/9.10 (karmic) Firefox/3.6.12"
- fetchcmd = self.basecmd
- fetchcmd += " -O " + f.name + " --user-agent='" + agent + "' '" + uri + "'"
- try:
- self._runwget(ud, d, fetchcmd, True)
- fetchresult = f.read()
- except bb.fetch2.BBFetchException:
- fetchresult = ""
-
- f.close()
return fetchresult
def _check_latest_version(self, url, package, package_regex, current_version, ud, d):