process: Further improve robustness against server shutdown

Currently, if an exception occurs in an event handler, the server shuts
down but the UI simply hangs. This happens in two places, firstly waiting
for events and secondly, sending events to a server which no longer exists.

The latter does time out, the former does not. These patches improve
both code sections to check if the main server process is alive and if not,
trigger things to shut down gracefully. This avoids the timeout in the
command sending case too.

This resolves various cases where the UI would simply hang indefintely.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 14 insertions, 3 deletions

diff --git a/lib/bb/server/process.py b/lib/bb/server/process.py
index 84f3a7541..4bfff45f0 100644
--- a/lib/bb/server/process.py
+++ b/lib/bb/server/process.py
@@ -38,14 +38,18 @@ from . import BitBakeBaseServer, BitBakeBaseServerConnection, BaseImplServer
 logger = logging.getLogger('BitBake')
 
 class ServerCommunicator():
-    def __init__(self, connection, event_handle):
+    def __init__(self, connection, event_handle, server):
         self.connection = connection
         self.event_handle = event_handle
+        self.server = server
 
     def runCommand(self, command):
         # @todo try/except
         self.connection.send(command)
 
+        if not self.server.is_alive():
+            raise SystemExit
+
         while True:
             # don't let the user ctrl-c while we're waiting for a response
             try:
@@ -160,7 +164,7 @@ class BitBakeProcessServerConnection(BitBakeBaseServerConnection):
         self.procserver = serverImpl
         self.ui_channel = ui_channel
         self.event_queue = event_queue
-        self.connection = ServerCommunicator(self.ui_channel, self.procserver.event_handle)
+        self.connection = ServerCommunicator(self.ui_channel, self.procserver.event_handle, self.procserver)
         self.events = self.event_queue
 
     def sigterm_terminate(self):
@@ -199,14 +203,20 @@ class ProcessEventQueue(multiprocessing.queues.Queue):
 
     def waitEvent(self, timeout):
         if self.exit:
-            raise KeyboardInterrupt()
+            raise SystemExit
         try:
+            if not self.server.is_alive():
+                self.setexit()
+                return None
             return self.get(True, timeout)
         except Empty:
             return None
 
     def getEvent(self):
         try:
+            if not self.server.is_alive():
+                self.setexit()
+                return None
             return self.get(False)
         except Empty:
             return None
@@ -221,6 +231,7 @@ class BitBakeServer(BitBakeBaseServer):
         self.ui_channel, self.server_channel = Pipe()
         self.event_queue = ProcessEventQueue(0)
         self.serverImpl = ProcessServer(self.server_channel, self.event_queue, None)
+        self.event_queue.server = self.serverImpl
 
     def detach(self):
         self.serverImpl.start()