fix: preserve live tasks during daemon recovery
This commit is contained in:
1 parent
079d7b1e28
commit
0355af7833
11 files changed
+327
-17
No files matched your search
@@ -300,6 +300,116 @@ def test_restart_never_repeats_started_work(isolated):
|
||||
assert isolated.jobs(retry["id"])[0]["session_id"] != job["session_id"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("liveness_source", ["pid", "device_lock"])
|
||||
def test_recovery_preserves_live_runner_and_batch(isolated, monkeypatch, liveness_source):
|
||||
import os
|
||||
from types import SimpleNamespace
|
||||
from artemis.runtime import DeviceExecutionLock
|
||||
|
||||
batch = service.create_batch(payload())
|
||||
service.batch_action(batch["id"], "resume")
|
||||
job = isolated.jobs(batch["id"])[0]
|
||||
isolated.save_job({**job, "status": "running"})
|
||||
if liveness_source == "pid":
|
||||
with sqlite3.connect(isolated.path) as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO sessions(session_id,initial_goal,start_time,status,device_info,pid) "
|
||||
"VALUES (?,?,?,?,?,?)",
|
||||
(job["session_id"], "goal", 1, "running", "{}", os.getpid()),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
DeviceExecutionLock,
|
||||
"get_active_owners",
|
||||
lambda: (
|
||||
{"device": SimpleNamespace(session_id=job["session_id"])}
|
||||
if liveness_source == "device_lock"
|
||||
else {}
|
||||
),
|
||||
)
|
||||
service.recover_batches()
|
||||
assert isolated.job(job["session_id"])["status"] == "running"
|
||||
assert not isolated.job(job["session_id"]).get("error")
|
||||
assert isolated.get("batch", batch["id"])["status"] == "running"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_natural_completion_resolves_old_recovery_error_in_job_report_and_history(
|
||||
client, isolated
|
||||
):
|
||||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||||
message = "服务中断;未完成任务需手动重跑"
|
||||
isolated.save_job({**job, "status": "interrupted", "error": message})
|
||||
await service.on_terminal({"session_id": job["session_id"]}, "completed")
|
||||
saved = isolated.job(job["session_id"])
|
||||
assert saved["status"] == "completed"
|
||||
assert not saved.get("error")
|
||||
assert saved["recovery_events"][0]["message"] == message
|
||||
assert reports.session_report(job["session_id"])["error"] == ""
|
||||
assert not client.get("/api/automation/history").json()["items"][0].get("error")
|
||||
|
||||
|
||||
def test_recovery_repairs_completed_legacy_warning_but_preserves_real_failure(isolated):
|
||||
batch = service.create_batch(payload())
|
||||
first, second = isolated.jobs(batch["id"])
|
||||
isolated.save_job({**first, "status": "completed", "error": "服务中断,需手动重跑"})
|
||||
isolated.save_job({**second, "status": "failed", "error": "任务执行超时(30 秒)"})
|
||||
assert reports.session_report(first["session_id"])["error"] == ""
|
||||
service.recover_batches()
|
||||
assert not isolated.job(first["session_id"]).get("error")
|
||||
assert "服务中断" in isolated.job(first["session_id"])["recovery_events"][0]["message"]
|
||||
assert "超时" in isolated.job(second["session_id"])["error"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_surviving_worker_is_finalized_when_it_exits_without_resubmission(
|
||||
isolated, monkeypatch
|
||||
):
|
||||
import os
|
||||
from artemis.runtime import DeviceExecutionLock
|
||||
|
||||
batch = service.create_batch(payload())
|
||||
service.batch_action(batch["id"], "resume")
|
||||
job = isolated.jobs(batch["id"])[0]
|
||||
isolated.save_job({**job, "status": "running"})
|
||||
monkeypatch.setattr(DeviceExecutionLock, "get_active_owners", lambda: {})
|
||||
with sqlite3.connect(isolated.path) as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO sessions(session_id,initial_goal,start_time,status,device_info,pid) "
|
||||
"VALUES (?,?,?,?,?,?)",
|
||||
(job["session_id"], "goal", time.time(), "running", "{}", os.getpid()),
|
||||
)
|
||||
service.recover_batches()
|
||||
await service.reconcile_recovered_jobs()
|
||||
assert isolated.job(job["session_id"])["status"] == "running"
|
||||
with sqlite3.connect(isolated.path) as conn:
|
||||
conn.execute(
|
||||
"UPDATE sessions SET pid=NULL,status='completed' WHERE session_id=?",
|
||||
(job["session_id"],),
|
||||
)
|
||||
await service.reconcile_recovered_jobs()
|
||||
saved = isolated.job(job["session_id"])
|
||||
assert saved["status"] == "completed"
|
||||
assert saved["report_ready"]
|
||||
assert not saved.get("recovery_waiting")
|
||||
assert not saved.get("error")
|
||||
assert len(isolated.jobs(batch["id"])) == 2
|
||||
assert isolated.jobs(batch["id"])[1]["status"] == "pending"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_surviving_worker_that_dies_without_result_is_interrupted(isolated, monkeypatch):
|
||||
from artemis.runtime import DeviceExecutionLock
|
||||
|
||||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||||
isolated.save_job({**job, "status": "running", "recovery_waiting": True})
|
||||
monkeypatch.setattr(DeviceExecutionLock, "get_active_owners", lambda: {})
|
||||
await service.reconcile_recovered_jobs()
|
||||
saved = isolated.job(job["session_id"])
|
||||
assert saved["status"] == "interrupted"
|
||||
assert "服务中断" in saved["error"]
|
||||
assert not saved.get("recovery_waiting")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_batch_admission_idempotent_and_bound_device(isolated, monkeypatch):
|
||||
from apps.admin_console import server
|
||||
|
||||
Reference in new issue
Block a user