Kill processes in parallel when going offroad (#20469)

* Kill processes in parallel when going offroad

* smaller diff

* handle edge case

* more readable

* set variable after actually sending signal

* newline

selfdrive/manager/process.py

@@ -68,6 +68,7 @@ class ManagerProcess(ABC):
   last_watchdog_time = 0
   watchdog_max_dt = None
   watchdog_seen = False
+  shutting_down = False
 
   @abstractmethod
   def prepare(self):
@@ -101,15 +102,19 @@ class ManagerProcess(ABC):
     else:
       self.watchdog_seen = True
 
-  def stop(self, retry=True):
+  def stop(self, retry=True, block=True):
     if self.proc is None:
       return
 
-    cloudlog.info(f"killing {self.name}")
-
     if self.proc.exitcode is None:
-      sig = signal.SIGKILL if self.sigkill else signal.SIGINT
-      self.signal(sig)
+      if not self.shutting_down:
+        cloudlog.info(f"killing {self.name}")
+        sig = signal.SIGKILL if self.sigkill else signal.SIGINT
+        self.signal(sig)
+        self.shutting_down = True
+
+        if not block:
+          return
 
       join_process(self.proc, 5)
 
@@ -134,6 +139,7 @@ class ManagerProcess(ABC):
     cloudlog.info(f"{self.name} is dead with {ret}")
 
     if self.proc.exitcode is not None:
+      self.shutting_down = False
       self.proc = None
 
     return ret
@@ -175,6 +181,10 @@ class NativeProcess(ManagerProcess):
     pass
 
   def start(self):
+    # In case we only tried a non blocking stop we need to stop it before restarting
+    if self.shutting_down:
+        self.stop()
+
     if self.proc is not None:
       return
 
@@ -183,6 +193,7 @@ class NativeProcess(ManagerProcess):
     self.proc = Process(name=self.name, target=nativelauncher, args=(self.cmdline, cwd))
     self.proc.start()
     self.watchdog_seen = False
+    self.shutting_down = False
 
 
 class PythonProcess(ManagerProcess):
@@ -202,6 +213,10 @@ class PythonProcess(ManagerProcess):
       importlib.import_module(self.module)
 
   def start(self):
+    # In case we only tried a non blocking stop we need to stop it before restarting
+    if self.shutting_down:
+        self.stop()
+
     if self.proc is not None:
       return
 
@@ -209,6 +224,7 @@ class PythonProcess(ManagerProcess):
     self.proc = Process(name=self.name, target=launcher, args=(self.module,))
     self.proc.start()
     self.watchdog_seen = False
+    self.shutting_down = False
 
 
 class DaemonProcess(ManagerProcess):
@@ -248,7 +264,7 @@ class DaemonProcess(ManagerProcess):
 
     params.put(self.param_name, str(proc.pid))
 
-  def stop(self, retry=True):
+  def stop(self, retry=True, block=True):
     pass
 
 
@@ -256,12 +272,11 @@ def ensure_running(procs, started, driverview=False, not_run=None):
   if not_run is None:
     not_run = []
 
-  # TODO: can we do this in parallel?
   for p in procs:
     if p.name in not_run:
-      p.stop()
+      p.stop(block=False)
     elif not p.enabled:
-      p.stop()
+      p.stop(block=False)
     elif p.persistent:
       p.start()
     elif p.driverview and driverview:
@@ -269,7 +284,7 @@ def ensure_running(procs, started, driverview=False, not_run=None):
     elif started:
       p.start()
     else:
-      p.stop()
+      p.stop(block=False)
 
     p.check_watchdog(started)