merge(runtime): reconcile task state and device cleanup

tests/unit/runtime/test_device_lock.py

@@ -185,6 +185,78 @@ def test_submission_reservations_preserve_order_before_workers_start():
     thread.join(timeout=2.0)
 
 
+def test_pending_reservation_at_queue_head_does_not_block_other_devices():
+    """An unclaimed pending submission must not gate every device's queue head.
+
+    Regression: a reservation whose device is still "pending" used to be
+    counted into every device's FIFO queue, so one unclaimed head-of-queue
+    ticket blocked all devices at once. It must now park on at most one idle
+    candidate device (the first in sorted order) and leave the others
+    schedulable.
+    """
+    pending_ticket = DeviceExecutionLock.reserve("unclaimed submission")
+    # device-alpha is a known idle device (a queued submission targets it), so
+    # the pending ticket parks there; device-beta must stay schedulable.
+    alpha_ticket = DeviceExecutionLock.reserve(
+        "task for device alpha", device_id="device-alpha"
+    )
+    try:
+        beta = DeviceExecutionLock("device-beta", "task for device beta")
+        # Must acquire immediately even though the older pending ticket is at
+        # the head of the global queue directory.
+        beta.acquire(blocking=False)
+        beta.release()
+    finally:
+        DeviceExecutionLock.cancel_reservation(pending_ticket)
+        DeviceExecutionLock.cancel_reservation(alpha_ticket)
+
+
+def test_pending_reservation_is_served_with_original_fifo_position_after_claim():
+    """A pending reservation is not starved: once its worker claims a device it
+    keeps its original submission-order position in that device's queue."""
+    holder = DeviceExecutionLock("emulator-5554", "current holder")
+    holder.acquire()
+
+    # Submitted while the device is busy; the reservation stays "pending".
+    first_ticket = DeviceExecutionLock.reserve("first submitted task")
+
+    claimed = DeviceExecutionLock("emulator-5554", "claimed pending task", first_ticket)
+    youngest = DeviceExecutionLock("emulator-5554", "youngest task")
+    order = []
+    claimed_acquired = threading.Event()
+    youngest_acquired = threading.Event()
+
+    def run_claimed():
+        claimed.acquire()
+        order.append("claimed")
+        claimed_acquired.set()
+
+    def run_youngest():
+        youngest.acquire()
+        order.append("youngest")
+        youngest_acquired.set()
+
+    t1 = threading.Thread(target=run_claimed)
+    t2 = threading.Thread(target=run_youngest)
+    t1.start()
+    t2.start()
+    time.sleep(0.2)
+    assert not claimed_acquired.is_set()
+    assert not youngest_acquired.is_set()
+
+    holder.release()
+    # The claimed pending ticket kept its original (oldest) FIFO timestamp and
+    # must win the device before the younger concrete ticket.
+    assert claimed_acquired.wait(timeout=2.0)
+    assert not youngest_acquired.is_set()
+    claimed.release()
+    assert youngest_acquired.wait(timeout=2.0)
+    youngest.release()
+    t1.join(timeout=2.0)
+    t2.join(timeout=2.0)
+    assert order == ["claimed", "youngest"]
+
+
 def test_cancelled_reservation_cannot_execute():
     ticket = DeviceExecutionLock.reserve("cancelled task")
     assert DeviceExecutionLock.cancel_reservation(ticket)

tests/unit/runtime/test_process_probe.py

@@ -0,0 +1,65 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unit tests for the unified PID liveness probe."""
+
+import os
+from unittest.mock import MagicMock, patch
+
+import psutil
+
+from artemis.runtime.process_probe import pid_is_alive
+
+
+def test_own_pid_is_alive():
+    assert pid_is_alive(os.getpid()) is True
+
+
+def test_own_pid_with_matching_create_time_is_alive():
+    created_at = psutil.Process(os.getpid()).create_time()
+    assert pid_is_alive(os.getpid(), created_at) is True
+
+
+def test_recycled_pid_with_mismatched_create_time_is_dead():
+    assert pid_is_alive(os.getpid(), 1.0) is False
+
+
+def test_invalid_pids_are_dead():
+    assert pid_is_alive(0) is False
+    assert pid_is_alive(-5) is False
+    assert pid_is_alive(None) is False  # type: ignore[arg-type]
+    assert pid_is_alive("not-a-pid") is False  # type: ignore[arg-type]
+
+
+def test_no_such_process_is_dead():
+    with patch("psutil.Process", side_effect=psutil.NoSuchProcess(424242)):
+        assert pid_is_alive(424242) is False
+
+
+def test_zombie_process_is_dead():
+    proc = MagicMock()
+    proc.is_running.return_value = True
+    proc.status.return_value = psutil.STATUS_ZOMBIE
+    with patch("psutil.Process", return_value=proc):
+        assert pid_is_alive(1234) is False
+
+
+def test_access_denied_defaults_to_alive():
+    with patch("psutil.Process", side_effect=psutil.AccessDenied(1234)):
+        assert pid_is_alive(1234) is True
+
+
+def test_unexpected_error_defaults_to_alive():
+    with patch("psutil.Process", side_effect=RuntimeError("boom")):
+        assert pid_is_alive(1234) is True