merge(runtime): integrate process and task state fixes

artemis/runtime/__init__.py

@@ -28,6 +28,7 @@ from artemis.runtime.adb_endpoint import (
     current_adb_endpoint,
 )
 from artemis.runtime.device_pool import DevicePool, DeviceStatus, device_pool
+from artemis.runtime.process_probe import pid_is_alive
 from artemis.runtime.daemon_client import (
     ensure_daemon_running,
     get_daemon_session,
@@ -78,6 +79,7 @@ __all__ = [
     "get_server_status",
     "is_daemon_running",
     "is_port_in_use",
+    "pid_is_alive",
     "ProcessSupervisor",
     "process_supervisor",
     "read_server_info",

artemis/runtime/awake_lease.py

@@ -28,6 +28,7 @@ import time
 import uuid
 
 from artemis.config.paths import get_temp_dir
+from artemis.runtime.process_probe import pid_is_alive
 from artemis.utils.logger import get_logger
 
 logger = get_logger(__name__)
@@ -77,16 +78,11 @@ class ScreenAwakeLease:
     @staticmethod
     def _owner_is_alive(payload: dict) -> bool:
         try:
-            import psutil
-
             pid = int(payload["pid"])
-            process = psutil.Process(pid)
-            if not process.is_running():
-                return False
-            created_at = float(payload.get("process_created_at", 0.0))
-            return created_at <= 0 or abs(process.create_time() - created_at) < 1.0
-        except Exception:
+            created_at = float(payload.get("process_created_at", 0.0) or 0.0)
+        except (KeyError, TypeError, ValueError):
             return False
+        return pid_is_alive(pid, created_at if created_at > 0 else None)
 
     @staticmethod
     def _read_payload(path: Path) -> dict | None:

artemis/runtime/device_lock.py

@@ -27,6 +27,7 @@ from typing import Any
 import uuid
 
 from artemis.config.paths import get_temp_dir
+from artemis.runtime.process_probe import pid_is_alive
 
 
 class DeviceBusyError(RuntimeError):
@@ -202,22 +203,22 @@ class DeviceExecutionLock:
     def _owner_is_alive(owner: DeviceLockOwner) -> bool:
         if owner.pid <= 0:
             return False
-        try:
-            import psutil
+        if owner.process_created_at <= 0:
+            # Legacy owner payloads without a create time cannot use PID-reuse
+            # protection; require an Artemis-looking process before trusting
+            # the record (a recycled PID must not keep a lock alive forever).
+            try:
+                import psutil
 
-            process = psutil.Process(owner.pid)
-            if not process.is_running():
-                return False
-            if owner.process_created_at <= 0:
-                try:
-                    name = process.name().lower()
-                    cmdline = " ".join(process.cmdline()).lower()
-                    return any(k in name or k in cmdline for k in ("python", "artemis", "pytest"))
-                except Exception:
+                process = psutil.Process(owner.pid)
+                if not process.is_running():
                     return False
-            return abs(process.create_time() - owner.process_created_at) < 1.0
-        except Exception:
-            return False
+                name = process.name().lower()
+                cmdline = " ".join(process.cmdline()).lower()
+                return any(k in name or k in cmdline for k in ("python", "artemis", "pytest"))
+            except Exception:
+                return False
+        return pid_is_alive(owner.pid, owner.process_created_at)
 
     @staticmethod
     def _read_owner(path: Path) -> DeviceLockOwner | None:
@@ -268,6 +269,108 @@ class DeviceExecutionLock:
             payload["lock_scope"] = str(lock_scope)
         return payload
 
+    @classmethod
+    def _ticket_targets_device(
+        cls,
+        ticket_owner: DeviceLockOwner,
+        *,
+        target_lock_id: str,
+        target_scope: str | None,
+        include_pending: bool,
+    ) -> bool:
+        """Return whether a queue ticket belongs to the given device's queue.
+
+        A concrete ticket matches only its own device. A ticket whose device id
+        is still ``pending``/``any`` is a submission placeholder that no worker
+        has claimed yet; with ``include_pending=True`` it matches any device
+        with a compatible lock scope (used by listings, and by
+        ``_build_device_queue`` which then narrows the match down to a single
+        parked device), while ``include_pending=False`` excludes it.
+        """
+        ticket_dev = cls._normalize_device_id(ticket_owner.device_id)
+        ticket_lock_id = cls._normalize_lock_id(
+            ticket_owner.device_id,
+            ticket_owner.lock_scope,
+        )
+        if ticket_lock_id == target_lock_id:
+            return True
+        if not include_pending:
+            return False
+        same_scope = (
+            not ticket_owner.lock_scope
+            or not target_scope
+            or ticket_owner.lock_scope == target_scope
+        )
+        return ticket_dev in ("pending", "any") and same_scope
+
+    @classmethod
+    def _build_device_queue(
+        cls,
+        wait_files: list[Path],
+        *,
+        target_lock_id: str,
+        target_scope: str | None,
+    ) -> list[Path]:
+        """Return the FIFO wait-ticket queue that gates ``target_lock_id``.
+
+        Concrete tickets belong to exactly their own device's queue. An
+        unclaimed ``pending``/``any`` ticket is *parked* on at most one device:
+        pending tickets, taken in FIFO order, occupy the idle scope-compatible
+        candidate devices in sorted lock-id order. Historically such a ticket
+        was counted into every device's queue, so a single unclaimed
+        head-of-queue ticket blocked all devices at once. Parking preserves
+        submission order on the one device the ticket is presumed to claim
+        while leaving every other idle device schedulable.
+
+        The pending submission itself cannot starve: its ticket file keeps its
+        original timestamp, so once a worker claims a concrete device the
+        ticket sorts ahead of every younger ticket for that device.
+        """
+        entries: list[tuple[Path, str | None]] = []
+        known_lock_ids: set[str] = set()
+        for wait_path in wait_files:
+            owner = cls._read_owner(wait_path)
+            if owner is None:
+                continue
+            if cls._normalize_device_id(owner.device_id) in ("pending", "any"):
+                if cls._ticket_targets_device(
+                    owner,
+                    target_lock_id=target_lock_id,
+                    target_scope=target_scope,
+                    include_pending=True,
+                ):
+                    entries.append((wait_path, None))
+                continue
+            same_scope = (
+                not owner.lock_scope
+                or not target_scope
+                or owner.lock_scope == target_scope
+            )
+            if not same_scope:
+                continue
+            lock_id = cls._normalize_lock_id(owner.device_id, owner.lock_scope)
+            known_lock_ids.add(lock_id)
+            entries.append((wait_path, lock_id))
+
+        lock_dir = get_temp_dir("device-locks")
+        idle_candidates = sorted(
+            lock_id
+            for lock_id in known_lock_ids
+            if not (lock_dir / f"artemis-device-{lock_id}.lock").exists()
+        )
+        parked: dict[Path, str] = {}
+        next_candidate = 0
+        for wait_path, lock_id in entries:
+            if lock_id is None and next_candidate < len(idle_candidates):
+                parked[wait_path] = idle_candidates[next_candidate]
+                next_candidate += 1
+
+        return [
+            wait_path
+            for wait_path, lock_id in entries
+            if lock_id == target_lock_id or parked.get(wait_path) == target_lock_id
+        ]
+
     @classmethod
     def reserve(
         cls,
@@ -547,27 +650,15 @@ class DeviceExecutionLock:
                     )
                     is_eligible = is_reentrant or bool(all_wait_files and all_wait_files[0] == self._queue_path)
                 else:
-                    # In multi-device parallel mode, per-device FIFO applies
-                    target_dev = self.clean_device_id
-                    device_queue: list[Path] = []
-                    for wait_path in all_wait_files:
-                        ticket_owner = self._read_owner(wait_path)
-                        if ticket_owner is None:
-                            continue
-                        ticket_dev = self._normalize_device_id(ticket_owner.device_id)
-                        ticket_lock_id = self._normalize_lock_id(
-                            ticket_owner.device_id,
-                            ticket_owner.lock_scope,
-                        )
-                        same_scope = (
-                            not ticket_owner.lock_scope
-                            or not self.lock_scope
-                            or ticket_owner.lock_scope == self.lock_scope
-                        )
-                        if ticket_lock_id == target_dev or (
-                            ticket_dev in ("pending", "any") and same_scope
-                        ):
-                            device_queue.append(wait_path)
+                    # In multi-device parallel mode, per-device FIFO applies.
+                    # Unclaimed pending/any tickets are parked on at most one
+                    # idle device instead of gating the head of every device's
+                    # queue (see _build_device_queue).
+                    device_queue = self._build_device_queue(
+                        all_wait_files,
+                        target_lock_id=self.clean_device_id,
+                        target_scope=self.lock_scope,
+                    )
                     is_eligible = bool(device_queue and device_queue[0] == self._queue_path)
 
                 if is_eligible and self._try_acquire_owner_lock():
@@ -729,15 +820,11 @@ class DeviceExecutionLock:
             if owner.token in active_tokens:
                 continue
 
-            target_dev = cls._normalize_device_id(owner.device_id)
-            target_lock_id = cls._normalize_lock_id(owner.device_id, owner.lock_scope)
-            same_scope = (
-                not target_scope
-                or not owner.lock_scope
-                or owner.lock_scope == target_scope
-            )
-            if clean_target is not None and target_lock_id != clean_target and not (
-                target_dev in ("pending", "any") and same_scope
+            if clean_target is not None and not cls._ticket_targets_device(
+                owner,
+                target_lock_id=clean_target,
+                target_scope=target_scope,
+                include_pending=True,
             ):
                 continue
 

artemis/runtime/process_probe.py

@@ -0,0 +1,98 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Single source of truth for PID liveness probing across the Artemis codebase.
+
+Semantics (decided by the maintainers):
+
+- psutil only. No bare ``os.kill(pid, 0)`` fallback: on Windows, CPython
+  implements ``os.kill(pid, 0)`` for non-CTRL signals via
+  ``OpenProcess``/``TerminateProcess`` handle probing, which is both unreliable
+  for dead pids and semantically surprising.
+- ``psutil.NoSuchProcess`` -> dead. Zombie processes -> dead.
+- ``psutil.AccessDenied`` / ``PermissionError`` -> **alive**. When liveness
+  cannot be determined the probe defaults to "alive": it is always safer to
+  skip reaping a resource than to reap one that is still owned.
+- ``created_at`` (a ``psutil``-style process create time) enables PID-reuse
+  protection: when provided and the probed process's create time differs by
+  one second or more, the original process is dead and its PID was recycled.
+- Any other unexpected error -> alive (default-alive), with a warning log.
+"""
+
+from __future__ import annotations
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def pid_is_alive(pid: int, created_at: float | None = None) -> bool:
+    """Return whether the process identified by ``pid`` is still alive.
+
+    Args:
+        pid: The process id to probe. Non-positive or unparsable pids are dead.
+        created_at: Optional expected process create time (as reported by
+            ``psutil.Process.create_time()``). When given and > 0, a create-time
+            mismatch of >= 1 second means the PID was recycled -> dead.
+
+    Returns:
+        True when the process is alive or liveness cannot be determined
+        (default-alive); False when the process is definitively gone, a
+        zombie, or its PID has been recycled by another process.
+    """
+    try:
+        pid = int(pid)
+    except (TypeError, ValueError):
+        return False
+    if pid <= 0:
+        return False
+
+    try:
+        import psutil
+    except Exception as exc:  # pragma: no cover - psutil is a hard dependency
+        logger.warning(f"psutil unavailable while probing pid {pid}: {exc}; assuming alive")
+        return True
+
+    try:
+        process = psutil.Process(pid)
+        if not process.is_running():
+            return False
+
+        try:
+            if process.status() == psutil.STATUS_ZOMBIE:
+                return False
+        except psutil.NoSuchProcess:
+            return False
+        except (psutil.AccessDenied, PermissionError):
+            # Cannot inspect the status; keep the default-alive verdict.
+            pass
+
+        if created_at is not None and created_at > 0:
+            try:
+                if abs(process.create_time() - float(created_at)) >= 1.0:
+                    # Same PID, different birth time: the PID was recycled.
+                    return False
+            except psutil.NoSuchProcess:
+                return False
+            except (psutil.AccessDenied, PermissionError):
+                pass
+
+        return True
+    except psutil.NoSuchProcess:
+        return False
+    except (psutil.AccessDenied, PermissionError):
+        return True
+    except Exception as exc:
+        logger.warning(f"Could not determine liveness of pid {pid}: {exc}; assuming alive")
+        return True

artemis/runtime/server_lifecycle.py

@@ -168,31 +168,9 @@ def _reconcile_orphaned_sessions() -> int:
 
 def _any_pid_alive(pids: list[int]) -> bool:
     """Return whether any target process is still running (zombies excluded)."""
-    if not pids:
-        return False
-    try:
-        import psutil
+    from artemis.runtime.process_probe import pid_is_alive
 
-        for pid in pids:
-            try:
-                process = psutil.Process(pid)
-                if process.is_running() and process.status() != psutil.STATUS_ZOMBIE:
-                    return True
-            except psutil.NoSuchProcess:
-                continue
-            except psutil.AccessDenied:
-                return True
-        return False
-    except Exception:
-        return any(_pid_exists_portably(pid) for pid in pids)
-
-
-def _pid_exists_portably(pid: int) -> bool:
-    try:
-        os.kill(pid, 0)
-        return True
-    except OSError:
-        return False
+    return any(pid_is_alive(pid) for pid in pids)
 
 
 def find_server_pids(port: int = 8000) -> list[int]:
@@ -210,17 +188,10 @@ def find_server_pids(port: int = 8000) -> list[int]:
     if info and info.get("port") == port:
         saved_pid = info.get("pid")
         if saved_pid and isinstance(saved_pid, int):
-            try:
-                import psutil
+            from artemis.runtime.process_probe import pid_is_alive
 
-                if psutil.pid_exists(saved_pid):
-                    discovered.add(saved_pid)
-            except Exception:
-                try:
-                    os.kill(saved_pid, 0)
-                    discovered.add(saved_pid)
-                except OSError:
-                    pass
+            if pid_is_alive(saved_pid):
+                discovered.add(saved_pid)
 
     # 2. Check port listeners with platform-native tools
     # 2a. lsof (macOS & Linux)