From ae79868861568d673a70472e85a4bde9e2d84a8f Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Fri, 28 Dec 2018 16:40:33 +0000 Subject: cooker: Split recipes to parse amongst threads ahead of time We have two choices, split the recipes amongst the parsing threads in blocks ahead of time, or have a queue which parsers pull from when idle. The optimum approach depends on how similar the pieces are. For the single recipe reparse case, there is currently a significant wait for the feeder thread to start (around 0.25s in a 2s command). Its possible splitting into blocks in advance may be unluckly for some other workloads but experimentally it seems to work better overall for me at least. Signed-off-by: Richard Purdie --- lib/bb/cooker.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py index d1d2868d6..e6b8d880a 100644 --- a/lib/bb/cooker.py +++ b/lib/bb/cooker.py @@ -1941,11 +1941,8 @@ class Parser(multiprocessing.Process): result = pending.pop() else: try: - job = self.jobs.get(timeout=0.25) - except queue.Empty: - continue - - if job is None: + job = self.jobs.pop() + except IndexError: break result = self.parse(*job) @@ -2032,12 +2029,12 @@ class CookerParser(object): self.parser_quit = multiprocessing.Queue(maxsize=self.num_processes) self.result_queue = multiprocessing.Queue() - self.jobs = multiprocessing.Queue() - for j in self.willparse: - self.jobs.put(j) + def chunkify(lst,n): + return [lst[i::n] for i in range(n)] + self.jobs = chunkify(self.willparse, self.num_processes) for i in range(0, self.num_processes): - parser = Parser(self.jobs, self.result_queue, self.parser_quit, init, self.cooker.configuration.profile) + parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, init, self.cooker.configuration.profile) parser.start() self.process_names.append(parser.name) self.processes.append(parser) @@ -2065,8 +2062,6 @@ class CookerParser(object): for process in self.processes: self.parser_quit.put(None) - self.jobs.cancel_join_thread() - for process in self.processes: if force: process.join(.1) -- cgit 1.2.3-korg