aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2012-06-22 12:55:55 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2012-06-25 14:15:17 +0100
commit361fb71e907aa84c28ecec79fefc6ca39c39172f (patch)
treeb1f5f0db7463a22b6132d80a2d05450b1cb952c3
parentdb65080a6199baecc5c422294a4c4a9fd12dc29e (diff)
downloadbitbake-361fb71e907aa84c28ecec79fefc6ca39c39172f.tar.gz
bitbake-361fb71e907aa84c28ecec79fefc6ca39c39172f.tar.bz2
bitbake-361fb71e907aa84c28ecec79fefc6ca39c39172f.zip
process: Improve _logged_communicate buffering
There are two problems with the _logged_communicate that are both caused as a result of buffering I/O issues: 1) log truncation when python fails 2) While a bitbake task is running it is impossible to see what is going on if it is only writing a small incremental log that is smaller than the buffer, or you get only a partial log, up until the task exists. It is worse in the case that stderr and stdout are separate file handles, because previous code blocks on the read of stdout and then stderr, serially. The right approach is simply to use select() to determine if there is data available and also flush the log before exiting. This is based on a patch from Jason Wessel <jason.wessel@windriver.com> with some changes to flush upon exit, abstract the non blocking file descriptor setup and drop the buffer size parameter. Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--lib/bb/build.py3
-rw-r--r--lib/bb/process.py46
2 files changed, 35 insertions, 14 deletions
diff --git a/lib/bb/build.py b/lib/bb/build.py
index 2cb7a930..bea29260 100644
--- a/lib/bb/build.py
+++ b/lib/bb/build.py
@@ -135,7 +135,8 @@ class LogTee(object):
def __repr__(self):
return '<LogTee {0}>'.format(self.name)
-
+ def flush(self):
+ self.outfile.flush()
def exec_func(func, d, dirs = None):
"""Execute an BB 'function'"""
diff --git a/lib/bb/process.py b/lib/bb/process.py
index b74cb180..05b51725 100644
--- a/lib/bb/process.py
+++ b/lib/bb/process.py
@@ -1,6 +1,8 @@
import logging
import signal
import subprocess
+import errno
+import select
logger = logging.getLogger('BitBake.Process')
@@ -68,20 +70,38 @@ def _logged_communicate(pipe, log, input):
pipe.stdin.write(input)
pipe.stdin.close()
- bufsize = 512
outdata, errdata = [], []
- while pipe.poll() is None:
- if pipe.stdout is not None:
- data = pipe.stdout.read(bufsize)
- if data is not None:
- outdata.append(data)
- log.write(data)
-
- if pipe.stderr is not None:
- data = pipe.stderr.read(bufsize)
- if data is not None:
- errdata.append(data)
- log.write(data)
+ rin = []
+
+ if pipe.stdout is not None:
+ bb.utils.nonblockingfd(pipe.stdout.fileno())
+ rin.append(pipe.stdout)
+ if pipe.stderr is not None:
+ bb.utils.nonblockingfd(pipe.stderr.fileno())
+ rin.append(pipe.stderr)
+
+ try:
+ while pipe.poll() is None:
+ rlist = rin
+ try:
+ r,w,e = select.select (rlist, [], [])
+ except OSError, e:
+ if e.errno != errno.EINTR:
+ raise
+
+ if pipe.stdout in r:
+ data = pipe.stdout.read()
+ if data is not None:
+ outdata.append(data)
+ log.write(data)
+
+ if pipe.stderr in r:
+ data = pipe.stderr.read()
+ if data is not None:
+ errdata.append(data)
+ log.write(data)
+ finally:
+ log.flush()
return ''.join(outdata), ''.join(errdata)
def run(cmd, input=None, log=None, **options):