fix: preserve live tasks during daemon recovery
This commit is contained in:
1 parent
079d7b1e28
commit
0355af7833
11 files changed
+327
-17
No files matched your search
@@ -300,6 +300,116 @@ def test_restart_never_repeats_started_work(isolated):
|
||||
assert isolated.jobs(retry["id"])[0]["session_id"] != job["session_id"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("liveness_source", ["pid", "device_lock"])
|
||||
def test_recovery_preserves_live_runner_and_batch(isolated, monkeypatch, liveness_source):
|
||||
import os
|
||||
from types import SimpleNamespace
|
||||
from artemis.runtime import DeviceExecutionLock
|
||||
|
||||
batch = service.create_batch(payload())
|
||||
service.batch_action(batch["id"], "resume")
|
||||
job = isolated.jobs(batch["id"])[0]
|
||||
isolated.save_job({**job, "status": "running"})
|
||||
if liveness_source == "pid":
|
||||
with sqlite3.connect(isolated.path) as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO sessions(session_id,initial_goal,start_time,status,device_info,pid) "
|
||||
"VALUES (?,?,?,?,?,?)",
|
||||
(job["session_id"], "goal", 1, "running", "{}", os.getpid()),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
DeviceExecutionLock,
|
||||
"get_active_owners",
|
||||
lambda: (
|
||||
{"device": SimpleNamespace(session_id=job["session_id"])}
|
||||
if liveness_source == "device_lock"
|
||||
else {}
|
||||
),
|
||||
)
|
||||
service.recover_batches()
|
||||
assert isolated.job(job["session_id"])["status"] == "running"
|
||||
assert not isolated.job(job["session_id"]).get("error")
|
||||
assert isolated.get("batch", batch["id"])["status"] == "running"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_natural_completion_resolves_old_recovery_error_in_job_report_and_history(
|
||||
client, isolated
|
||||
):
|
||||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||||
message = "服务中断;未完成任务需手动重跑"
|
||||
isolated.save_job({**job, "status": "interrupted", "error": message})
|
||||
await service.on_terminal({"session_id": job["session_id"]}, "completed")
|
||||
saved = isolated.job(job["session_id"])
|
||||
assert saved["status"] == "completed"
|
||||
assert not saved.get("error")
|
||||
assert saved["recovery_events"][0]["message"] == message
|
||||
assert reports.session_report(job["session_id"])["error"] == ""
|
||||
assert not client.get("/api/automation/history").json()["items"][0].get("error")
|
||||
|
||||
|
||||
def test_recovery_repairs_completed_legacy_warning_but_preserves_real_failure(isolated):
|
||||
batch = service.create_batch(payload())
|
||||
first, second = isolated.jobs(batch["id"])
|
||||
isolated.save_job({**first, "status": "completed", "error": "服务中断,需手动重跑"})
|
||||
isolated.save_job({**second, "status": "failed", "error": "任务执行超时(30 秒)"})
|
||||
assert reports.session_report(first["session_id"])["error"] == ""
|
||||
service.recover_batches()
|
||||
assert not isolated.job(first["session_id"]).get("error")
|
||||
assert "服务中断" in isolated.job(first["session_id"])["recovery_events"][0]["message"]
|
||||
assert "超时" in isolated.job(second["session_id"])["error"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_surviving_worker_is_finalized_when_it_exits_without_resubmission(
|
||||
isolated, monkeypatch
|
||||
):
|
||||
import os
|
||||
from artemis.runtime import DeviceExecutionLock
|
||||
|
||||
batch = service.create_batch(payload())
|
||||
service.batch_action(batch["id"], "resume")
|
||||
job = isolated.jobs(batch["id"])[0]
|
||||
isolated.save_job({**job, "status": "running"})
|
||||
monkeypatch.setattr(DeviceExecutionLock, "get_active_owners", lambda: {})
|
||||
with sqlite3.connect(isolated.path) as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO sessions(session_id,initial_goal,start_time,status,device_info,pid) "
|
||||
"VALUES (?,?,?,?,?,?)",
|
||||
(job["session_id"], "goal", time.time(), "running", "{}", os.getpid()),
|
||||
)
|
||||
service.recover_batches()
|
||||
await service.reconcile_recovered_jobs()
|
||||
assert isolated.job(job["session_id"])["status"] == "running"
|
||||
with sqlite3.connect(isolated.path) as conn:
|
||||
conn.execute(
|
||||
"UPDATE sessions SET pid=NULL,status='completed' WHERE session_id=?",
|
||||
(job["session_id"],),
|
||||
)
|
||||
await service.reconcile_recovered_jobs()
|
||||
saved = isolated.job(job["session_id"])
|
||||
assert saved["status"] == "completed"
|
||||
assert saved["report_ready"]
|
||||
assert not saved.get("recovery_waiting")
|
||||
assert not saved.get("error")
|
||||
assert len(isolated.jobs(batch["id"])) == 2
|
||||
assert isolated.jobs(batch["id"])[1]["status"] == "pending"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_surviving_worker_that_dies_without_result_is_interrupted(isolated, monkeypatch):
|
||||
from artemis.runtime import DeviceExecutionLock
|
||||
|
||||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||||
isolated.save_job({**job, "status": "running", "recovery_waiting": True})
|
||||
monkeypatch.setattr(DeviceExecutionLock, "get_active_owners", lambda: {})
|
||||
await service.reconcile_recovered_jobs()
|
||||
saved = isolated.job(job["session_id"])
|
||||
assert saved["status"] == "interrupted"
|
||||
assert "服务中断" in saved["error"]
|
||||
assert not saved.get("recovery_waiting")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_batch_admission_idempotent_and_bound_device(isolated, monkeypatch):
|
||||
from apps.admin_console import server
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
import asyncio
|
||||
from contextlib import suppress
|
||||
import signal
|
||||
import socket
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
@@ -23,10 +24,55 @@ import uvicorn
|
||||
from apps.admin_console.core.state import state
|
||||
from apps.admin_console.routers.tasks import stream_events
|
||||
from apps.admin_console.server import ArtemisUvicornServer, app, on_shutdown
|
||||
from apps.admin_console import server as server_module
|
||||
|
||||
WINDOWS_FORCE_SIGNAL = getattr(signal, "SIGBREAK", signal.SIGTERM)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reload", [False, True])
|
||||
def test_occupied_port_never_initializes_or_overwrites_server_metadata(
|
||||
tmp_path, monkeypatch, reload
|
||||
):
|
||||
from artemis.runtime import server_lifecycle
|
||||
|
||||
metadata = tmp_path / "server.json"
|
||||
metadata.write_text('{"pid":123,"lifecycle_token":"original"}', encoding="utf-8")
|
||||
monkeypatch.setattr(server_lifecycle, "get_server_info_file", lambda: metadata)
|
||||
before = metadata.read_bytes()
|
||||
with (
|
||||
socket.create_server(("127.0.0.1", 0)) as original,
|
||||
patch.object(server_module.ArtemisUvicornServer, "run") as run,
|
||||
patch.object(server_module, "write_server_info") as publish,
|
||||
):
|
||||
with pytest.raises(SystemExit, match="1"):
|
||||
server_module.run_ui_server("127.0.0.1", original.getsockname()[1], reload=reload)
|
||||
run.assert_not_called()
|
||||
publish.assert_not_called()
|
||||
assert metadata.read_bytes() == before
|
||||
|
||||
|
||||
def test_server_holds_port_before_running_lifespan(monkeypatch):
|
||||
monkeypatch.setattr(state, "port", 0, raising=False)
|
||||
monkeypatch.setattr(state, "host", "127.0.0.1", raising=False)
|
||||
observed = []
|
||||
|
||||
def run(_server, sockets):
|
||||
listener = sockets[0]
|
||||
port = listener.getsockname()[1]
|
||||
observed.append(listener)
|
||||
assert state.port == port
|
||||
with pytest.raises(OSError), socket.create_server(("127.0.0.1", port)):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(server_module.ArtemisUvicornServer, "run", run)
|
||||
monkeypatch.setattr(server_module, "clear_server_info", lambda **kwargs: None)
|
||||
with patch.object(server_module, "write_server_info") as publish:
|
||||
server_module.run_ui_server("127.0.0.1", 0)
|
||||
publish.assert_not_called() # Publication belongs to the later lifespan.
|
||||
assert len(observed) == 1
|
||||
assert observed[0].fileno() == -1
|
||||
|
||||
|
||||
def _pending_queue_get_tasks():
|
||||
return {
|
||||
task
|
||||
|
||||
Reference in new issue
Block a user