feat: enable remote SDK and resilient device orchestration

tests/unit/runtime/test_daemon_client.py

@@ -53,13 +53,27 @@ def test_is_daemon_running_failure():
         assert is_daemon_running() is False
 
 
-def test_spawn_daemon():
+def test_spawn_daemon(tmp_path):
+    import sys
+
     mock_proc = MagicMock()
     mock_proc.pid = 54321
-    with patch("subprocess.Popen", return_value=mock_proc) as mock_popen:
-        proc = spawn_daemon(host="127.0.0.1", port=8000)
+    with (
+        patch("subprocess.Popen", return_value=mock_proc) as mock_popen,
+        patch(
+            "artemis.runtime.daemon_client.daemon_log_path",
+            return_value=tmp_path / "daemon-8000.log",
+        ),
+    ):
+        proc = spawn_daemon(host="127.0.0.1", port=9123)
         assert proc is mock_proc
         assert mock_popen.called
+        # The daemon must launch via `python -m`, never through the console-script
+        # shim (a resident artemis.exe would block uv sync reinstalls on Windows).
+        cmd = mock_popen.call_args.args[0]
+        assert cmd[0] == sys.executable
+        assert cmd[1:3] == ["-m", "apps.admin_console.server"]
+        assert cmd[3:] == ["--host", "127.0.0.1", "--port", "9123"]
 
 
 def test_ensure_daemon_running_when_already_active():

tests/unit/runtime/test_device_lock.py

@@ -67,6 +67,33 @@ def test_device_lock_recovers_stale_owner(monkeypatch):
     assert not lock.path.exists()
 
 
+def test_same_serial_on_different_adb_endpoints_has_independent_locks():
+    first = DeviceExecutionLock(
+        "emulator-5554",
+        "first endpoint",
+        lock_scope="tcp:127.0.0.1:5037",
+    )
+    second = DeviceExecutionLock(
+        "emulator-5554",
+        "second endpoint",
+        lock_scope="tcp:127.0.0.1:5038",
+    )
+
+    first.acquire()
+    second.acquire()
+    try:
+        assert first.path != second.path
+        owners = DeviceExecutionLock.get_active_owners()
+        assert len(owners) == 2
+        assert {owner.lock_scope for owner in owners.values()} == {
+            "tcp:127.0.0.1:5037",
+            "tcp:127.0.0.1:5038",
+        }
+    finally:
+        second.release()
+        first.release()
+
+
 def test_device_lock_release_does_not_remove_replacement_owner():
     lock = DeviceExecutionLock("emulator-5554", "original task")
     lock.acquire()

tests/unit/runtime/test_device_pool.py

@@ -12,11 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import asyncio
+import importlib
+from types import SimpleNamespace
+
 import pytest
 
 from artemis.runtime.device_lock import DeviceExecutionLock
 from artemis.runtime.device_pool import DevicePool, DeviceStatus
 
+device_pool_module = importlib.import_module("artemis.runtime.device_pool")
+
 
 @pytest.fixture(autouse=True)
 def isolated_lock_directory(tmp_path, monkeypatch):
@@ -83,3 +89,188 @@ def test_device_pool_list_devices_and_busy_status(monkeypatch):
         assert pool.select_device("emulator-5554") == "emulator-5554"
     finally:
         lock.release()
+
+
+RAW_DEVICE = ("emulator-5554", "device", "Emulator", "sdk")
+
+
+def _install_fake_clock(monkeypatch):
+    clock = {"now": 1000.0}
+    monkeypatch.setattr(
+        device_pool_module,
+        "time",
+        SimpleNamespace(monotonic=lambda: clock["now"]),
+    )
+    return clock
+
+
+def test_enumeration_snapshot_collapses_repeat_queries(monkeypatch):
+    pool = DevicePool()
+    calls = {"count": 0}
+
+    def fake_query(timeout=None):
+        calls["count"] += 1
+        return [RAW_DEVICE]
+
+    monkeypatch.setattr(pool, "_query_adb_devices_sync", fake_query)
+
+    first = pool.list_devices()
+    second = pool.list_devices()
+    assert calls["count"] == 1
+    assert [d.serial for d in first] == [d.serial for d in second] == ["emulator-5554"]
+
+
+def test_enumeration_failure_serves_last_known_snapshot(monkeypatch):
+    clock = _install_fake_clock(monkeypatch)
+    pool = DevicePool()
+    responses = iter([[RAW_DEVICE], None, None])
+    monkeypatch.setattr(pool, "_query_adb_devices_sync", lambda timeout=None: next(responses))
+
+    assert [d.serial for d in pool.list_devices()] == ["emulator-5554"]
+
+    # Past the fresh TTL but inside the stale-on-error window, a failed query
+    # keeps answering with the last successful snapshot.
+    clock["now"] += pool.CACHE_TTL + 1.0
+    assert [d.serial for d in pool.list_devices()] == ["emulator-5554"]
+
+    # Once the stale window closes, persistent failure is reported honestly.
+    clock["now"] += pool.STALE_ON_ERROR_TTL + 1.0
+    assert pool.list_devices() == []
+
+
+def test_query_timeout_widens_until_first_success(monkeypatch):
+    pool = DevicePool()
+    assert pool._current_query_timeout() == pool.COLD_QUERY_TIMEOUT
+    monkeypatch.setattr(pool, "_query_adb_devices_sync", lambda timeout=None: [RAW_DEVICE])
+    pool.list_devices()
+    assert pool._current_query_timeout() == pool.HOT_QUERY_TIMEOUT
+
+
+def test_async_enumeration_single_flight(monkeypatch):
+    pool = DevicePool()
+    calls = {"count": 0}
+
+    async def fake_query(timeout=None):
+        calls["count"] += 1
+        await asyncio.sleep(0.05)
+        return [RAW_DEVICE]
+
+    monkeypatch.setattr(pool, "_query_adb_devices_async", fake_query)
+
+    async def run():
+        results = await asyncio.gather(
+            pool.list_devices_async(),
+            pool.list_devices_async(),
+            pool.list_devices_async(),
+        )
+        return results
+
+    results = asyncio.run(run())
+    assert calls["count"] == 1
+    for devices in results:
+        assert [d.serial for d in devices] == ["emulator-5554"]
+
+
+def test_warm_up_waits_for_device_handshake(monkeypatch):
+    pool = DevicePool()
+    monkeypatch.setattr(pool, "_resolve_adb", lambda: "adb")
+    started = {"count": 0}
+
+    async def fake_start_server(timeout):
+        started["count"] += 1
+
+    responses = iter([[], [("emulator-5554", "offline", None, None)], [RAW_DEVICE]])
+
+    async def fake_query(timeout=None):
+        return next(responses)
+
+    monkeypatch.setattr(pool, "_start_adb_server", fake_start_server)
+    monkeypatch.setattr(pool, "_query_adb_devices_async", fake_query)
+
+    warmed = asyncio.run(pool.warm_up_async(settle_timeout=2.0, poll_interval=0.01))
+    assert warmed is True
+    assert started["count"] == 1
+    # The settled enumeration is cached for the first post-startup listings.
+    assert pool._cached_raw == [RAW_DEVICE]
+
+
+def test_warm_up_accepts_empty_enumeration_at_deadline(monkeypatch):
+    pool = DevicePool()
+    monkeypatch.setattr(pool, "_resolve_adb", lambda: "adb")
+
+    async def fake_start_server(timeout):
+        pass
+
+    async def fake_query(timeout=None):
+        return []
+
+    monkeypatch.setattr(pool, "_start_adb_server", fake_start_server)
+    monkeypatch.setattr(pool, "_query_adb_devices_async", fake_query)
+
+    warmed = asyncio.run(pool.warm_up_async(settle_timeout=0.0))
+    assert warmed is True
+    assert pool._current_query_timeout() == pool.HOT_QUERY_TIMEOUT
+
+
+def test_warm_up_without_adb_returns_false(monkeypatch):
+    pool = DevicePool()
+    monkeypatch.setattr(pool, "_resolve_adb", lambda: None)
+    assert asyncio.run(pool.warm_up_async(settle_timeout=0.0)) is False
+
+
+def test_try_list_devices_distinguishes_failure_from_empty(monkeypatch):
+    pool = DevicePool()
+    monkeypatch.setattr(pool, "_query_adb_devices_sync", lambda timeout=None: None)
+    assert pool.try_list_devices() is None
+    assert pool.list_devices() == []
+
+    fresh_pool = DevicePool()
+    monkeypatch.setattr(fresh_pool, "_query_adb_devices_sync", lambda timeout=None: [])
+    assert fresh_pool.try_list_devices() == []
+
+
+def test_validate_explicit_serial_fails_open_on_indeterminate_enumeration(monkeypatch):
+    pool = DevicePool()
+    monkeypatch.setattr(pool, "_query_adb_devices_sync", lambda timeout=None: None)
+    assert pool.validate_explicit_serial("pixel-10") is None
+
+    empty_pool = DevicePool()
+    monkeypatch.setattr(empty_pool, "_query_adb_devices_sync", lambda timeout=None: [])
+    assert empty_pool.validate_explicit_serial("pixel-10") is None
+
+
+def test_validate_explicit_serial_rejects_only_on_authoritative_enumeration(monkeypatch):
+    pool = DevicePool()
+    monkeypatch.setattr(
+        pool,
+        "_query_adb_devices_sync",
+        lambda timeout=None: [
+            ("pixel-10", "device", None, None),
+            ("pixel-11", "unauthorized", None, None),
+        ],
+    )
+
+    assert pool.validate_explicit_serial("pixel-10") is None
+    assert "not ready" in pool.validate_explicit_serial("pixel-11")
+    missing = pool.validate_explicit_serial("pixel-12")
+    assert "not connected" in missing
+    assert "pixel-10" in missing
+
+
+def test_validate_explicit_serial_async_matches_sync(monkeypatch):
+    pool = DevicePool()
+
+    async def fake_query(timeout=None):
+        return [("pixel-10", "device", None, None)]
+
+    monkeypatch.setattr(pool, "_query_adb_devices_async", fake_query)
+
+    async def run():
+        return (
+            await pool.validate_explicit_serial_async("pixel-10"),
+            await pool.validate_explicit_serial_async("pixel-12"),
+        )
+
+    ok, missing = asyncio.run(run())
+    assert ok is None
+    assert "not connected" in missing

tests/unit/runtime/test_server_process_lifecycle.py

@@ -14,6 +14,7 @@
 
 """Unit tests for Artemis Server Lifecycle Management and API Endpoints."""
 
+import asyncio
 import json
 import os
 from unittest.mock import MagicMock, patch
@@ -21,13 +22,15 @@ from unittest.mock import MagicMock, patch
 import pytest
 from httpx import ASGITransport, AsyncClient
 
-from apps.admin_console.server import app
+from apps.admin_console import server as server_module
+from apps.admin_console.server import LIFECYCLE_TOKEN, app
 from artemis.runtime.server_lifecycle import (
     clear_server_info,
     find_server_pids,
     get_server_status,
     is_port_in_use,
     read_server_info,
+    request_graceful_shutdown,
     stop_server,
     write_server_info,
 )
@@ -56,8 +59,12 @@ def test_write_read_clear_server_info(tmp_path, monkeypatch):
     assert info["host"] == "127.0.0.1"
     assert "started_at" in info
 
-    # Clear info
-    clear_server_info()
+    # A stop targeting another port must not erase this server's metadata.
+    clear_server_info(port=9999)
+    assert fake_info_file.exists()
+
+    # Clear matching info
+    clear_server_info(port=8888)
     assert not fake_info_file.exists()
     assert read_server_info() is None
 
@@ -121,6 +128,55 @@ def test_stop_server_when_not_running(monkeypatch):
     assert "No active server" in msg
 
 
+def test_server_main_parses_non_default_bind_address(monkeypatch):
+    run_server = MagicMock()
+    monkeypatch.setattr(server_module, "run_ui_server", run_server)
+
+    server_module.main(["--host", "127.0.0.1", "--port", "9123"])
+
+    run_server.assert_called_once_with(host="127.0.0.1", port=9123)
+
+
+def test_request_graceful_shutdown_uses_metadata_token(monkeypatch):
+    response = MagicMock(status=202)
+    response.__enter__.return_value = response
+    monkeypatch.setattr(
+        "artemis.runtime.server_lifecycle.read_server_info",
+        lambda: {"port": 9123, "lifecycle_token": "local-secret"},
+    )
+    urlopen = MagicMock(return_value=response)
+    monkeypatch.setattr("artemis.runtime.server_lifecycle.urllib.request.urlopen", urlopen)
+
+    assert request_graceful_shutdown(9123) is True
+    request = urlopen.call_args.args[0]
+    assert request.full_url == "http://127.0.0.1:9123/api/system/shutdown"
+    assert request.get_header("X-artemis-lifecycle-token") == "local-secret"
+
+
+def test_stop_server_prefers_graceful_shutdown(monkeypatch):
+    monkeypatch.setattr("artemis.runtime.server_lifecycle.find_server_pids", lambda port: [12345])
+    monkeypatch.setattr(
+        "artemis.runtime.server_lifecycle.request_graceful_shutdown", lambda port, timeout: True
+    )
+    monkeypatch.setattr("artemis.runtime.server_lifecycle.is_port_in_use", lambda port: False)
+    monkeypatch.setattr("artemis.runtime.server_lifecycle._any_pid_alive", lambda pids: False)
+    terminate = MagicMock()
+    monkeypatch.setattr(
+        "artemis.runtime.server_lifecycle.ProcessSupervisor.terminate_tree", terminate
+    )
+    monkeypatch.setattr(
+        "artemis.runtime.server_lifecycle.DeviceExecutionLock.cleanup_stale_locks", lambda: 0
+    )
+    monkeypatch.setattr("artemis.runtime.server_lifecycle.clear_server_info", lambda **kwargs: None)
+
+    success, msg, stopped = stop_server(9123)
+
+    assert success is True
+    assert stopped == [12345]
+    assert "gracefully" in msg
+    terminate.assert_not_called()
+
+
 def test_stop_server_terminates_tree(monkeypatch):
     """Verify stop_server calls ProcessSupervisor.terminate_tree and returns stopped pids."""
     terminated_pids = []
@@ -134,6 +190,9 @@ def test_stop_server_terminates_tree(monkeypatch):
         return False
 
     monkeypatch.setattr("artemis.runtime.server_lifecycle.is_port_in_use", mock_in_use)
+    monkeypatch.setattr(
+        "artemis.runtime.server_lifecycle.request_graceful_shutdown", lambda port, timeout: False
+    )
     monkeypatch.setattr(
         "artemis.runtime.supervisor.ProcessSupervisor.terminate_tree",
         lambda pid, timeout_seconds=4.0: terminated_pids.append(pid),
@@ -142,19 +201,25 @@ def test_stop_server_terminates_tree(monkeypatch):
         "artemis.runtime.device_lock.DeviceExecutionLock.cleanup_stale_locks",
         lambda: 1,
     )
+    reconcile = MagicMock(return_value=1)
+    monkeypatch.setattr(
+        "artemis.runtime.server_lifecycle._reconcile_orphaned_sessions", reconcile
+    )
 
     success, msg, stopped = stop_server(8000)
     assert success is True
     assert stopped == [12345]
     assert 12345 in terminated_pids
     assert "stopped" in msg.lower()
+    assert "Reconciled 1 interrupted session" in msg
+    reconcile.assert_called_once_with()
 
 
 @pytest.mark.asyncio
 async def test_api_server_status():
     """Verify GET /api/system/server-status returns server runtime metadata."""
     async with AsyncClient(
-        transport=ASGITransport(app=app), base_url="http://testserver"
+        transport=ASGITransport(app=app), base_url="http://localhost"
     ) as client:
         response = await client.get("/api/system/server-status")
         assert response.status_code == 200
@@ -172,7 +237,7 @@ async def test_api_restart_endpoint():
         mock_thread.return_value = mock_instance
 
         async with AsyncClient(
-            transport=ASGITransport(app=app), base_url="http://testserver"
+            transport=ASGITransport(app=app), base_url="http://localhost"
         ) as client:
             response = await client.post("/api/system/restart")
             assert response.status_code == 200
@@ -185,17 +250,23 @@ async def test_api_restart_endpoint():
 
 @pytest.mark.asyncio
 async def test_api_shutdown_endpoint():
-    """Verify POST /api/system/shutdown accepts request and schedules shutdown."""
-    with patch("threading.Thread") as mock_thread:
-        mock_instance = MagicMock()
-        mock_thread.return_value = mock_instance
+    """Verify shutdown is authenticated and drives Uvicorn's graceful exit flag."""
+    fake_server = MagicMock(should_exit=False)
+    app.state.uvicorn_server = fake_server
 
-        async with AsyncClient(
-            transport=ASGITransport(app=app), base_url="http://testserver"
-        ) as client:
-            response = await client.post("/api/system/shutdown")
-            assert response.status_code == 200
-            data = response.json()
-            assert data["status"] == "shutting_down"
-            mock_thread.assert_called_once()
-            mock_instance.start.assert_called_once()
+    async with AsyncClient(
+        transport=ASGITransport(app=app), base_url="http://localhost"
+    ) as client:
+        denied = await client.post("/api/system/shutdown")
+        assert denied.status_code == 403
+
+        response = await client.post(
+            "/api/system/shutdown",
+            headers={"X-Artemis-Lifecycle-Token": LIFECYCLE_TOKEN},
+        )
+        assert response.status_code == 202
+        assert response.json()["status"] == "shutting_down"
+        await asyncio.sleep(0.1)
+
+    assert fake_server.should_exit is True
+    app.state.uvicorn_server = None