fix(runtime): recover awake ownership and batch model failures

tests/unit/admin_console/test_automation.py

@@ -417,13 +417,21 @@ def test_batch_deletion_preserves_shared_screenshot_until_last_task(client, isol
     assert not image_file.exists()
 
 
-def test_restart_never_repeats_started_work(isolated):
+@pytest.mark.parametrize("model_paused", [False, True])
+def test_restart_never_repeats_started_work(isolated, model_paused):
     batch = service.create_batch(payload())
     service.batch_action(batch["id"], "resume")
+    if model_paused:
+        isolated.put(
+            "batch",
+            batch["id"],
+            {**isolated.get("batch", batch["id"]), "status": "paused", "pause_source": "model"},
+        )
     job = isolated.jobs(batch["id"])[0]
     isolated.save_job({**job, "status": "queued"})
     service.recover_batches()
     assert isolated.get("batch", batch["id"])["status"] == "paused"
+    assert isolated.get("batch", batch["id"])["pause_source"] == "recovery"
     assert [j["status"] for j in isolated.jobs(batch["id"])] == ["interrupted", "pending"]
     retry = service.batch_action(batch["id"], "retry_failed")
     assert retry["status"] == "draft"
@@ -603,6 +611,106 @@ async def test_device_outage_pauses_without_consuming_pending_case(isolated, mon
     assert all(job["status"] == "pending" for job in isolated.jobs(batch["id"]))
 
 
+def test_stop_all_overrides_model_auto_resume(isolated, monkeypatch):
+    from apps.admin_console.services import task_queue_service as queue
+
+    batch = service.create_batch(payload())
+    isolated.put("batch", batch["id"], {**batch, "status": "paused", "pause_source": "model"})
+    monkeypatch.setattr("apps.admin_console.services.automation_store.store", isolated)
+    monkeypatch.setattr(queue, "state", Mock(queue_items=[], active_connections={}))
+    for method in (
+        "_terminate_all_device_owners",
+        "_kill_all_local_runs",
+        "_clear_pause_file",
+        "ensure_worker_running",
+    ):
+        monkeypatch.setattr(queue.TaskQueueService, method, Mock())
+    assert queue.TaskQueueService._stop_all_tasks()
+    saved = isolated.get("batch", batch["id"])
+    assert saved["pause_source"] == "user"
+    assert "等待手动继续" in saved["reason"]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("manual_pause", [False, True])
+@pytest.mark.parametrize("stop_on_failure", [False, True])
+async def test_model_pause_recovery_preserves_user_pause_and_failure_policy(
+    isolated, monkeypatch, manual_pause, stop_on_failure
+):
+    from apps.admin_console import server
+    from artemis.runtime.device_pool import DeviceStatus
+
+    batch = service.create_batch(payload(stop_on_failure=stop_on_failure))
+    service.batch_action(batch["id"], "resume")
+    first, second = isolated.jobs(batch["id"])
+    isolated.save_job({**first, "status": "running", "started_at": time.time()})
+    pause_state = type("State", (), {"is_paused": True, "active_runs": {}})()
+    monkeypatch.setattr(service, "state", pause_state)
+    enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]})
+    monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue)
+    monkeypatch.setattr(
+        "artemis.runtime.device_pool.try_list_devices_async",
+        AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]),
+    )
+    monkeypatch.setattr(reports, "export_batch", Mock())
+    await service.tick_batches()
+    assert isolated.get("batch", batch["id"])["pause_source"] == "model"
+    if manual_pause:
+        service.batch_action(batch["id"], "pause")
+        # Another paused-model tick must not overwrite the explicit user pause.
+        await service.tick_batches()
+        assert isolated.get("batch", batch["id"])["pause_source"] == "user"
+    isolated.save_job({**first, "status": "failed"})
+    pause_state.is_paused = False
+    await service.tick_batches()
+    assert isolated.job(first["session_id"])["status"] == "failed"
+    expected = "pending" if manual_pause else "skipped" if stop_on_failure else "queued"
+    assert isolated.job(second["session_id"])["status"] == expected
+    assert enqueue.await_count == (0 if manual_pause or stop_on_failure else 1)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("code", ["initialization_failed", "execution_failed"])
+async def test_failure_reason_retained_and_only_initialization_blocks_remaining_batch(
+    isolated, monkeypatch, code
+):
+    from apps.admin_console import server
+    from artemis.runtime import task_result
+    from artemis.runtime.device_pool import DeviceStatus
+
+    batch = service.create_batch(payload())
+    service.batch_action(batch["id"], "resume")
+    first, second = isolated.jobs(batch["id"])
+    isolated.save_job({**first, "status": "running"})
+    monkeypatch.setattr(task_result, "TRACES_PATH", reports.TRACES_PATH)
+    task_result.save_failure_result(
+        first["session_id"], "PermissionError: awake lock denied", code=code
+    )
+    monkeypatch.setattr(reports, "export_session", Mock(side_effect=OSError("export unavailable")))
+    monkeypatch.setattr(service, "state", type("State", (), {"is_paused": False})())
+    enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]})
+    monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue)
+    monkeypatch.setattr(
+        "artemis.runtime.device_pool.try_list_devices_async",
+        AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]),
+    )
+
+    await service.on_terminal({"session_id": first["session_id"]}, "failed")
+    await service.tick_batches()
+    saved = isolated.job(first["session_id"])
+    assert saved["error"] == "PermissionError: awake lock denied"
+    assert saved["report_error"] == "export unavailable"
+    if code == "initialization_failed":
+        assert isolated.get("batch", batch["id"])["pause_source"] == "initialization"
+        assert isolated.job(second["session_id"])["status"] == "pending"
+        enqueue.assert_not_awaited()
+        service.batch_action(batch["id"], "resume")
+        await service.tick_batches()
+    assert enqueue.await_count == 1
+    assert enqueue.call_args.kwargs["session_id"] == second["session_id"]
+    assert isolated.job(first["session_id"])["status"] == "failed"
+
+
 @pytest.mark.asyncio
 async def test_fail_fast_includes_skipped_reports(isolated, monkeypatch):
     batch = service.create_batch(payload(stop_on_failure=True))

tests/unit/agents/test_flash_runner_ledger.py

@@ -429,6 +429,44 @@ def _report(status="completed", tc_id="tc-report", **extra):
 _INITIAL_OBSERVATION = ("shot0.png", b"IMG0", "[1] Settings")
 
 
+@pytest.mark.asyncio
+async def test_generation_failure_uses_fallback_without_repeating_device_actions(mock_context):
+    from artemis.llm.response_guard import ResponseIntegrityError
+    from artemis.services import llm as gateway
+
+    click = AIMessage(
+        content="tap", tool_calls=[{"name": "click", "args": {"index": 1}, "id": "click"}]
+    )
+    with patch("artemis.controllers.unified_controller.get_driver"):
+        runner = _make_runner(mock_context, [])
+    primary = runner._init_llm.return_value
+    fallback = Mock()
+    calls = []
+
+    async def invoke(model, tools, messages):
+        calls.append(model)
+        if len(calls) == 1:
+            assert gateway._FALLBACK_AVAILABLE.get()
+            raise ResponseIntegrityError("missing completed response")
+        if model is fallback:
+            assert not gateway._FALLBACK_AVAILABLE.get()
+            return click
+        return _report()
+
+    runner._invoke_model = AsyncMock(side_effect=invoke)
+    with (
+        patch(
+            "artemis.agents.flash.runner.capture_screenshot_and_parse_ui",
+            AsyncMock(return_value=_INITIAL_OBSERVATION),
+        ),
+        patch("artemis.agents.flash.runner.get_llm", return_value=fallback),
+    ):
+        result = await runner.run(State(initial_goal="Open Settings"))
+    assert result["status"] == "completed"
+    assert calls == [primary, fallback, primary]
+    assert runner.executor.execute.await_count == 1
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize("fallback_succeeds", [True, False])
 async def test_failed_actions_have_finite_fallback_and_restore_primary(

tests/unit/agents/test_operator_recovery.py

@@ -82,6 +82,31 @@ async def run_loop(operator, monkeypatch, responses):
     return result, messages, invoke
 
 
+@pytest.mark.asyncio
+async def test_generation_failure_hands_over_before_pausing(operator, monkeypatch):
+    from artemis.llm.response_guard import ResponseIntegrityError
+    from artemis.services import llm as gateway
+
+    node, state = operator
+    primary, fallback = Mock(), Mock()
+    contexts = []
+
+    async def invoke(model, messages):
+        contexts.append(gateway._FALLBACK_AVAILABLE.get())
+        if model is primary.bind_tools.return_value:
+            raise ResponseIntegrityError("missing completed response")
+        assert model is fallback.bind_tools.return_value
+        return click()
+
+    monkeypatch.setattr("artemis.agents.operator.operator.acomplete", invoke)
+    monkeypatch.setattr("artemis.agents.operator.operator.get_llm", Mock(return_value=fallback))
+    messages = []
+    result = await node._invoke_llm_loop(primary, messages, [], [], state)
+    assert contexts == [True, False]
+    assert len(result[0]) == 1
+    assert len([m for m in messages if isinstance(m, AIMessage)]) == 1
+
+
 @pytest.mark.asyncio
 async def test_prose_retry_keeps_context_and_recovers_once(operator, monkeypatch):
     prose = AIMessage(content="I will click the model menu.")

tests/unit/runtime/test_awake_service.py

@@ -24,6 +24,66 @@ def completed(stdout="", returncode=0, stderr=""):
     return MagicMock(returncode=returncode, stdout=stdout, stderr=stderr)
 
 
+@pytest.mark.parametrize("stage", ["directory", "acquire"])
+def test_inaccessible_awake_lock_skips_actions_and_can_recover(stage, monkeypatch, tmp_path):
+    owner = FileLock(tmp_path / "recovered.lock", thread_local=False)
+    denied_owner = MagicMock()
+    denied_owner.acquire.side_effect = PermissionError("awake lock access denied")
+    factory = MagicMock(
+        side_effect=[
+            PermissionError("awake directory denied") if stage == "directory" else denied_owner,
+            owner,
+        ]
+    )
+    configure = MagicMock(return_value=AWAKE_STRATEGY_USB)
+    monkeypatch.setattr("artemis.runtime.awake_service._owner_lock", factory)
+    monkeypatch.setattr("artemis.runtime.awake_service._configure_usb_stay_awake", configure)
+    service = ScreenAwakeService()
+    try:
+        assert service.ensure_device("device") is None
+        configure.assert_not_called()
+        assert service.device_ids == ()
+        assert service.ensure_device("device") == AWAKE_STRATEGY_USB
+        configure.assert_called_once_with("device")
+    finally:
+        service.shutdown()
+
+
+def test_awake_monitor_continues_to_other_devices_after_access_denied(monkeypatch, tmp_path):
+    def owner(device):
+        if device == "denied":
+            raise PermissionError("denied")
+        return FileLock(tmp_path / f"{device}.lock", thread_local=False)
+
+    monkeypatch.setattr("artemis.runtime.awake_service._owner_lock", owner)
+    configure = MagicMock(return_value=AWAKE_STRATEGY_USB)
+    monkeypatch.setattr("artemis.runtime.awake_service._configure_usb_stay_awake", configure)
+    service = ScreenAwakeService()
+    try:
+        service._reconcile_connected_devices({"denied", "healthy"})
+        service._reconcile_connected_devices({"denied", "healthy"})
+        assert service.device_ids == ("healthy",)
+        configure.assert_called_once_with("healthy")
+    finally:
+        service.shutdown()
+
+
+@patch(
+    "artemis.runtime.awake_service.ScreenAwakeLease",
+    side_effect=PermissionError("legacy directory denied"),
+)
+@patch("artemis.runtime.awake_service.subprocess.run")
+def test_optional_legacy_directory_failure_does_not_abort_usb_policy(mock_run, _lease):
+    mock_run.side_effect = [
+        completed(),
+        completed(),
+        completed(),
+        completed("2"),
+        completed("mStayOn=true"),
+    ]
+    assert _configure_usb_stay_awake("device") == AWAKE_STRATEGY_USB
+
+
 @patch(
     "artemis.runtime.awake_service._configure_usb_stay_awake", return_value=AWAKE_STRATEGY_HEARTBEAT
 )

tests/unit/sdk/test_agent_trace_finalization.py

@@ -64,7 +64,10 @@ async def test_device_context_uses_adb_size_without_starting_ui_client():
 
 
 @pytest.mark.asyncio
-async def test_task_that_never_acquires_queue_does_not_create_trace_session():
+async def test_task_that_never_acquires_queue_does_not_create_trace_session(tmp_path, monkeypatch):
+    from artemis.runtime import task_result
+
+    monkeypatch.setattr(task_result, "TRACES_PATH", tmp_path)
     agent = Agent()
     agent._initialized = True
     agent._device_context = DeviceContext(
@@ -86,6 +89,13 @@ async def test_task_that_never_acquires_queue_does_not_create_trace_session():
             await agent.run_task(goal="queued task", profile="flash")
 
     agent._prepare_tracing.assert_not_called()
+    outcomes = list(tmp_path.glob("*/run_outcome.json"))
+    assert len(outcomes) == 1
+    import json
+
+    outcome = json.loads(outcomes[0].read_text(encoding="utf-8"))
+    assert outcome["code"] == "initialization_failed"
+    assert "queue cancelled" in outcome["error"]
 
 
 @pytest.mark.asyncio

tests/unit/test_cli.py

@@ -163,6 +163,35 @@ def test_cli_trace_help():
     assert "view" in result.output
 
 
+@pytest.mark.asyncio
+async def test_worker_initialization_failure_is_persisted_and_cleaned(monkeypatch):
+    from unittest.mock import AsyncMock, MagicMock
+    import artemis.interfaces.cli.commands.run as run_module
+    from artemis.runtime import task_result
+
+    monkeypatch.setenv("ARTEMIS_SESSION_ID", "test-session")
+    monkeypatch.setenv("ARTEMIS_TASK_INGRESS", "test")
+    fake_agent = MagicMock()
+    fake_agent.init = AsyncMock(side_effect=PermissionError("device initialization denied"))
+    fake_agent.clean = AsyncMock()
+    fake_agent.run_task = AsyncMock()
+    monkeypatch.setattr(run_module, "initialize_llm_config", MagicMock())
+    monkeypatch.setattr(run_module, "AgentProfile", MagicMock())
+    monkeypatch.setattr(run_module, "Builders", MagicMock())
+    monkeypatch.setattr(run_module, "Agent", MagicMock(return_value=fake_agent))
+    save = MagicMock()
+    monkeypatch.setattr(task_result, "save_failure_result", save)
+    with pytest.raises(PermissionError, match="initialization denied"):
+        await run_module.execute_task(
+            "inspect", execution_target="computer", session_id="test-session"
+        )
+    save.assert_called_once_with(
+        "test-session", "device initialization denied", code="initialization_failed"
+    )
+    fake_agent.clean.assert_awaited_once()
+    fake_agent.run_task.assert_not_awaited()
+
+
 def test_cli_mcp_help():
     """Verify 'artemis mcp --help' lists server options."""
     result = runner.invoke(app, ["mcp", "--help"])

tests/unit/test_llm_pause.py

@@ -6,6 +6,40 @@ import pytest
 from artemis.services import llm
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("replace_pause", [False, True])
+async def test_cancelled_wait_clears_only_its_own_pause(tmp_path, monkeypatch, replace_pause):
+    pause = tmp_path / "pause"
+    pause.write_text("old request")
+    entered = asyncio.Event()
+
+    async def wait(_seconds):
+        entered.set()
+        await asyncio.Event().wait()
+
+    monkeypatch.setattr(llm.asyncio, "sleep", wait)
+    task = asyncio.create_task(llm._wait_for_resume(pause))
+    await entered.wait()
+    if replace_pause:
+        pause.write_text("a newer request owns this pause")
+    task.cancel()
+    with pytest.raises(asyncio.CancelledError):
+        await task
+    assert pause.exists() is replace_pause
+
+
+@pytest.mark.asyncio
+async def test_pause_deadline_clears_file(tmp_path, monkeypatch):
+    from unittest.mock import AsyncMock
+
+    pause = tmp_path / "pause"
+    pause.write_text("failed request")
+    monkeypatch.setattr(llm.settings, "LLM_PAUSE_TIMEOUT_SECONDS", 1)
+    monkeypatch.setattr(llm.asyncio, "sleep", AsyncMock())
+    assert await llm._wait_for_resume(pause) is False
+    assert not pause.exists()
+
+
 @pytest.mark.asyncio
 async def test_timeout_wrapper_cancels_llm_call_when_caller_is_cancelled():
     started = asyncio.Event()