fix: preserve live tasks during daemon recovery

This commit is contained in:
chenmingxuan committed 2026-10-08 17:33:48 +08:00
1 parent 079d7b1e28
commit 0355af7833
11 files changed
+327 -17

No files matched your search

+110
View File
@@ -300,6 +300,116 @@ def test_restart_never_repeats_started_work(isolated):
assert isolated.jobs(retry["id"])[0]["session_id"] != job["session_id"]
@pytest.mark.parametrize("liveness_source", ["pid", "device_lock"])
def test_recovery_preserves_live_runner_and_batch(isolated, monkeypatch, liveness_source):
import os
from types import SimpleNamespace
from artemis.runtime import DeviceExecutionLock
batch = service.create_batch(payload())
service.batch_action(batch["id"], "resume")
job = isolated.jobs(batch["id"])[0]
isolated.save_job({**job, "status": "running"})
if liveness_source == "pid":
with sqlite3.connect(isolated.path) as conn:
conn.execute(
"INSERT INTO sessions(session_id,initial_goal,start_time,status,device_info,pid) "
"VALUES (?,?,?,?,?,?)",
(job["session_id"], "goal", 1, "running", "{}", os.getpid()),
)
monkeypatch.setattr(
DeviceExecutionLock,
"get_active_owners",
lambda: (
{"device": SimpleNamespace(session_id=job["session_id"])}
if liveness_source == "device_lock"
else {}
),
)
service.recover_batches()
assert isolated.job(job["session_id"])["status"] == "running"
assert not isolated.job(job["session_id"]).get("error")
assert isolated.get("batch", batch["id"])["status"] == "running"
@pytest.mark.asyncio
async def test_natural_completion_resolves_old_recovery_error_in_job_report_and_history(
client, isolated
):
job = isolated.jobs(service.create_batch(payload())["id"])[0]
message = "服务中断;未完成任务需手动重跑"
isolated.save_job({**job, "status": "interrupted", "error": message})
await service.on_terminal({"session_id": job["session_id"]}, "completed")
saved = isolated.job(job["session_id"])
assert saved["status"] == "completed"
assert not saved.get("error")
assert saved["recovery_events"][0]["message"] == message
assert reports.session_report(job["session_id"])["error"] == ""
assert not client.get("/api/automation/history").json()["items"][0].get("error")
def test_recovery_repairs_completed_legacy_warning_but_preserves_real_failure(isolated):
batch = service.create_batch(payload())
first, second = isolated.jobs(batch["id"])
isolated.save_job({**first, "status": "completed", "error": "服务中断,需手动重跑"})
isolated.save_job({**second, "status": "failed", "error": "任务执行超时(30 秒)"})
assert reports.session_report(first["session_id"])["error"] == ""
service.recover_batches()
assert not isolated.job(first["session_id"]).get("error")
assert "服务中断" in isolated.job(first["session_id"])["recovery_events"][0]["message"]
assert "超时" in isolated.job(second["session_id"])["error"]
@pytest.mark.asyncio
async def test_surviving_worker_is_finalized_when_it_exits_without_resubmission(
isolated, monkeypatch
):
import os
from artemis.runtime import DeviceExecutionLock
batch = service.create_batch(payload())
service.batch_action(batch["id"], "resume")
job = isolated.jobs(batch["id"])[0]
isolated.save_job({**job, "status": "running"})
monkeypatch.setattr(DeviceExecutionLock, "get_active_owners", lambda: {})
with sqlite3.connect(isolated.path) as conn:
conn.execute(
"INSERT INTO sessions(session_id,initial_goal,start_time,status,device_info,pid) "
"VALUES (?,?,?,?,?,?)",
(job["session_id"], "goal", time.time(), "running", "{}", os.getpid()),
)
service.recover_batches()
await service.reconcile_recovered_jobs()
assert isolated.job(job["session_id"])["status"] == "running"
with sqlite3.connect(isolated.path) as conn:
conn.execute(
"UPDATE sessions SET pid=NULL,status='completed' WHERE session_id=?",
(job["session_id"],),
)
await service.reconcile_recovered_jobs()
saved = isolated.job(job["session_id"])
assert saved["status"] == "completed"
assert saved["report_ready"]
assert not saved.get("recovery_waiting")
assert not saved.get("error")
assert len(isolated.jobs(batch["id"])) == 2
assert isolated.jobs(batch["id"])[1]["status"] == "pending"
@pytest.mark.asyncio
async def test_surviving_worker_that_dies_without_result_is_interrupted(isolated, monkeypatch):
from artemis.runtime import DeviceExecutionLock
job = isolated.jobs(service.create_batch(payload())["id"])[0]
isolated.save_job({**job, "status": "running", "recovery_waiting": True})
monkeypatch.setattr(DeviceExecutionLock, "get_active_owners", lambda: {})
await service.reconcile_recovered_jobs()
saved = isolated.job(job["session_id"])
assert saved["status"] == "interrupted"
assert "服务中断" in saved["error"]
assert not saved.get("recovery_waiting")
@pytest.mark.asyncio
async def test_batch_admission_idempotent_and_bound_device(isolated, monkeypatch):
from apps.admin_console import server