From 358b5b02d5de1ab0f98104c4ec4953e46999b9a5 Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Mon, 20 Feb 2023 09:30:44 +0000 Subject: server/process: Improve idle thread exception handling If the inotifier code has an exception, bitbake currently hangs. Catch any exception and exit if seen. Also check the idle thread is alive and exit if it disappears. This should stop bitbake hanging if such a situation arises in future such as this example: 3323260 21:48:31.554468 Running command ['getVariable', 'BBINCLUDELOGS'] Exception in thread Thread-1 (idle_thread): Traceback (most recent call last): File "/usr/lib64/python3.10/threading.py", line 1016, in _bootstrap_inner self.run() File "/usr/lib64/python3.10/threading.py", line 953, in run self._target(*self._args, **self._kwargs) File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/bitbake/lib/bb/server/process.py", line 408, in idle_thread self.cooker.process_inotify_updates() File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/bitbake/lib/bb/cooker.py", line 256, in process_inotify_updates n.read_events() File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/bitbake/lib/pyinotify.py", line 1207, in read_events if fcntl.ioctl(self._fd, termios.FIONREAD, buf_, 1) == -1: OSError: [Errno 9] Bad file descriptor 3323260 21:48:32.206995 Command Completed (socket: True) Signed-off-by: Richard Purdie --- lib/bb/server/process.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/bb/server/process.py b/lib/bb/server/process.py index 916ee0a0e..db417c842 100644 --- a/lib/bb/server/process.py +++ b/lib/bb/server/process.py @@ -405,7 +405,11 @@ class ProcessServer(): nextsleep = 0.1 fds = [] - self.cooker.process_inotify_updates() + try: + self.cooker.process_inotify_updates() + except Exception as exc: + serverlog("Exception %s in inofify updates broke the idle_thread, exiting" % traceback.format_exc()) + self.quit = True with bb.utils.lock_timeout(self._idlefuncsLock): items = list(self._idlefuns.items()) @@ -473,6 +477,10 @@ class ProcessServer(): if not self.idle: self.idle = threading.Thread(target=self.idle_thread) self.idle.start() + elif self.idle and not self.idle.is_alive(): + serverlog("Idle thread terminated, main thread exiting too") + bb.error("Idle thread terminated, main thread exiting too") + self.quit = True if nextsleep is not None: if self.xmlrpc: -- cgit 1.2.3-korg