fix(runtime): recover awake ownership and batch model failures

tests/unit/admin_console/test_automation.py

@@ -417,13 +417,21 @@ def test_batch_deletion_preserves_shared_screenshot_until_last_task(client, isol
     assert not image_file.exists()
 
 
-def test_restart_never_repeats_started_work(isolated):
+@pytest.mark.parametrize("model_paused", [False, True])
+def test_restart_never_repeats_started_work(isolated, model_paused):
     batch = service.create_batch(payload())
     service.batch_action(batch["id"], "resume")
+    if model_paused:
+        isolated.put(
+            "batch",
+            batch["id"],
+            {**isolated.get("batch", batch["id"]), "status": "paused", "pause_source": "model"},
+        )
     job = isolated.jobs(batch["id"])[0]
     isolated.save_job({**job, "status": "queued"})
     service.recover_batches()
     assert isolated.get("batch", batch["id"])["status"] == "paused"
+    assert isolated.get("batch", batch["id"])["pause_source"] == "recovery"
     assert [j["status"] for j in isolated.jobs(batch["id"])] == ["interrupted", "pending"]
     retry = service.batch_action(batch["id"], "retry_failed")
     assert retry["status"] == "draft"
@@ -603,6 +611,106 @@ async def test_device_outage_pauses_without_consuming_pending_case(isolated, mon
     assert all(job["status"] == "pending" for job in isolated.jobs(batch["id"]))
 
 
+def test_stop_all_overrides_model_auto_resume(isolated, monkeypatch):
+    from apps.admin_console.services import task_queue_service as queue
+
+    batch = service.create_batch(payload())
+    isolated.put("batch", batch["id"], {**batch, "status": "paused", "pause_source": "model"})
+    monkeypatch.setattr("apps.admin_console.services.automation_store.store", isolated)
+    monkeypatch.setattr(queue, "state", Mock(queue_items=[], active_connections={}))
+    for method in (
+        "_terminate_all_device_owners",
+        "_kill_all_local_runs",
+        "_clear_pause_file",
+        "ensure_worker_running",
+    ):
+        monkeypatch.setattr(queue.TaskQueueService, method, Mock())
+    assert queue.TaskQueueService._stop_all_tasks()
+    saved = isolated.get("batch", batch["id"])
+    assert saved["pause_source"] == "user"
+    assert "等待手动继续" in saved["reason"]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("manual_pause", [False, True])
+@pytest.mark.parametrize("stop_on_failure", [False, True])
+async def test_model_pause_recovery_preserves_user_pause_and_failure_policy(
+    isolated, monkeypatch, manual_pause, stop_on_failure
+):
+    from apps.admin_console import server
+    from artemis.runtime.device_pool import DeviceStatus
+
+    batch = service.create_batch(payload(stop_on_failure=stop_on_failure))
+    service.batch_action(batch["id"], "resume")
+    first, second = isolated.jobs(batch["id"])
+    isolated.save_job({**first, "status": "running", "started_at": time.time()})
+    pause_state = type("State", (), {"is_paused": True, "active_runs": {}})()
+    monkeypatch.setattr(service, "state", pause_state)
+    enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]})
+    monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue)
+    monkeypatch.setattr(
+        "artemis.runtime.device_pool.try_list_devices_async",
+        AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]),
+    )
+    monkeypatch.setattr(reports, "export_batch", Mock())
+    await service.tick_batches()
+    assert isolated.get("batch", batch["id"])["pause_source"] == "model"
+    if manual_pause:
+        service.batch_action(batch["id"], "pause")
+        # Another paused-model tick must not overwrite the explicit user pause.
+        await service.tick_batches()
+        assert isolated.get("batch", batch["id"])["pause_source"] == "user"
+    isolated.save_job({**first, "status": "failed"})
+    pause_state.is_paused = False
+    await service.tick_batches()
+    assert isolated.job(first["session_id"])["status"] == "failed"
+    expected = "pending" if manual_pause else "skipped" if stop_on_failure else "queued"
+    assert isolated.job(second["session_id"])["status"] == expected
+    assert enqueue.await_count == (0 if manual_pause or stop_on_failure else 1)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("code", ["initialization_failed", "execution_failed"])
+async def test_failure_reason_retained_and_only_initialization_blocks_remaining_batch(
+    isolated, monkeypatch, code
+):
+    from apps.admin_console import server
+    from artemis.runtime import task_result
+    from artemis.runtime.device_pool import DeviceStatus
+
+    batch = service.create_batch(payload())
+    service.batch_action(batch["id"], "resume")
+    first, second = isolated.jobs(batch["id"])
+    isolated.save_job({**first, "status": "running"})
+    monkeypatch.setattr(task_result, "TRACES_PATH", reports.TRACES_PATH)
+    task_result.save_failure_result(
+        first["session_id"], "PermissionError: awake lock denied", code=code
+    )
+    monkeypatch.setattr(reports, "export_session", Mock(side_effect=OSError("export unavailable")))
+    monkeypatch.setattr(service, "state", type("State", (), {"is_paused": False})())
+    enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]})
+    monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue)
+    monkeypatch.setattr(
+        "artemis.runtime.device_pool.try_list_devices_async",
+        AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]),
+    )
+
+    await service.on_terminal({"session_id": first["session_id"]}, "failed")
+    await service.tick_batches()
+    saved = isolated.job(first["session_id"])
+    assert saved["error"] == "PermissionError: awake lock denied"
+    assert saved["report_error"] == "export unavailable"
+    if code == "initialization_failed":
+        assert isolated.get("batch", batch["id"])["pause_source"] == "initialization"
+        assert isolated.job(second["session_id"])["status"] == "pending"
+        enqueue.assert_not_awaited()
+        service.batch_action(batch["id"], "resume")
+        await service.tick_batches()
+    assert enqueue.await_count == 1
+    assert enqueue.call_args.kwargs["session_id"] == second["session_id"]
+    assert isolated.job(first["session_id"])["status"] == "failed"
+
+
 @pytest.mark.asyncio
 async def test_fail_fast_includes_skipped_reports(isolated, monkeypatch):
     batch = service.create_batch(payload(stop_on_failure=True))