summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2018-12-28 16:40:33 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-01-08 11:15:58 +0000
commitae79868861568d673a70472e85a4bde9e2d84a8f (patch)
treeb0bff3cdacb40b63626c433f52ab594fbffddf23
parentb2d39fc37fcf3c81a562ec1ef4f8b4c1a493fc57 (diff)
downloadbitbake-contrib-ae79868861568d673a70472e85a4bde9e2d84a8f.tar.gz
bitbake-contrib-ae79868861568d673a70472e85a4bde9e2d84a8f.tar.bz2
bitbake-contrib-ae79868861568d673a70472e85a4bde9e2d84a8f.zip
cooker: Split recipes to parse amongst threads ahead of time
We have two choices, split the recipes amongst the parsing threads in blocks ahead of time, or have a queue which parsers pull from when idle. The optimum approach depends on how similar the pieces are. For the single recipe reparse case, there is currently a significant wait for the feeder thread to start (around 0.25s in a 2s command). Its possible splitting into blocks in advance may be unluckly for some other workloads but experimentally it seems to work better overall for me at least. Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--lib/bb/cooker.py17
1 files changed, 6 insertions, 11 deletions
diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index d1d2868d6f..e6b8d880ae 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -1941,11 +1941,8 @@ class Parser(multiprocessing.Process):
result = pending.pop()
else:
try:
- job = self.jobs.get(timeout=0.25)
- except queue.Empty:
- continue
-
- if job is None:
+ job = self.jobs.pop()
+ except IndexError:
break
result = self.parse(*job)
@@ -2032,12 +2029,12 @@ class CookerParser(object):
self.parser_quit = multiprocessing.Queue(maxsize=self.num_processes)
self.result_queue = multiprocessing.Queue()
- self.jobs = multiprocessing.Queue()
- for j in self.willparse:
- self.jobs.put(j)
+ def chunkify(lst,n):
+ return [lst[i::n] for i in range(n)]
+ self.jobs = chunkify(self.willparse, self.num_processes)
for i in range(0, self.num_processes):
- parser = Parser(self.jobs, self.result_queue, self.parser_quit, init, self.cooker.configuration.profile)
+ parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, init, self.cooker.configuration.profile)
parser.start()
self.process_names.append(parser.name)
self.processes.append(parser)
@@ -2065,8 +2062,6 @@ class CookerParser(object):
for process in self.processes:
self.parser_quit.put(None)
- self.jobs.cancel_join_thread()
-
for process in self.processes:
if force:
process.join(.1)