fix: preserve live tasks during daemon recovery

apps/admin_console/services/automation_service.py

@@ -8,7 +8,7 @@ import time
 from uuid import UUID
 
 from apps.admin_console.core.state import state
-from apps.admin_console.services.automation_store import store, new_id
+from apps.admin_console.services.automation_store import RECOVERY_ERRORS, store, new_id
 from apps.admin_console.services.task_import import case_goal, case_title
 from artemis.config.settings import settings
 from artemis.runtime.task_limits import task_timeout
@@ -42,9 +42,46 @@ def register_submission(item: dict, case: dict | None = None, formats=None, **ex
     )
 
 
+def resolve_recovery_error(job: dict) -> bool:
+    """Archive a superseded recovery warning without erasing execution failures."""
+    if job.get("error") not in RECOVERY_ERRORS:
+        return False
+    job.setdefault("recovery_events", []).append(
+        {"message": job.pop("error"), "resolved_at": time.time()}
+    )
+    return True
+
+
 def recover_batches():
     """Recovery never silently repeats a potentially side-effecting interrupted task."""
+    from apps.admin_console.database.repositories.session_repository import SessionRepository
+    from artemis.runtime import DeviceExecutionLock
+
+    repository = SessionRepository(store.path)
+    live_sessions = {
+        str(owner.session_id)
+        for owner in DeviceExecutionLock.get_active_owners().values()
+        if owner.session_id
+    }
+    live_sessions.update(
+        str(row["session_id"])
+        for row in repository.get_all_sessions()
+        if row.get("status") not in TERMINAL and repository.process_is_alive(row.get("pid"))
+    )
+    live_sessions.update(
+        str(item["session_id"])
+        for item in state.queue_items
+        if isinstance(item, dict) and item.get("session_id") and item.get("status") not in TERMINAL
+    )
     for job in store.jobs():
+        if job["status"] == "completed" and resolve_recovery_error(job):
+            store.save_job(job)
+        if str(job["session_id"]) in live_sessions:
+            if job["status"] not in TERMINAL and str(job["session_id"]) not in state.active_runs:
+                job["recovery_waiting"] = True
+                store.save_job(job)
+            logger.info("Recovery preserved live session %s", job["session_id"])
+            continue
         if job["status"] in {"running", "queued", "finalizing", "task_paused", "resuming"}:
             job.update(
                 status=job.get("terminal_status", "interrupted"),
@@ -52,6 +89,9 @@ def recover_batches():
                 or ("" if job.get("terminal_status") else "服务中断；未完成任务需手动重跑"),
                 ended_at=job.get("ended_at") or time.time(),
             )
+            if job["status"] == "completed":
+                resolve_recovery_error(job)
+            logger.info("Recovery resolved session %s to %s", job["session_id"], job["status"])
             store.save_job(job)
     for batch in store.records("batch"):
         if has_legacy_replay(batch["id"]):
@@ -64,6 +104,8 @@ def recover_batches():
                 store.put("batch", batch["id"], batch)
             continue
         if batch["status"] in {"running", "pausing", "paused"}:
+            if any(str(job["session_id"]) in live_sessions for job in store.jobs(batch["id"])):
+                continue
             batch.update(
                 status="paused", reason="服务已重启；请检查中断任务，点击继续执行未开始任务"
             )
@@ -88,6 +130,9 @@ async def on_terminal(item: dict, status: str):
         raise RuntimeError("任务元数据写入失败")
     if job.get("timed_out"):
         status = "failed"
+    job.pop("recovery_waiting", None)
+    if status == "completed":
+        resolve_recovery_error(job)
     job.update(status="finalizing", terminal_status=status, ended_at=time.time())
     store.save_job(job)
     try:
@@ -117,11 +162,37 @@ async def on_terminal(item: dict, status: str):
     store.save_job(job)
 
 
+async def reconcile_recovered_jobs():
+    """Finalize surviving workers once they exit; never resubmit their goals."""
+    from apps.admin_console.database.repositories.session_repository import SessionRepository
+    from artemis.runtime import DeviceExecutionLock
+    from artemis.runtime.task_result import read_worker_outcome
+
+    waiting = [job for job in store.jobs() if job.get("recovery_waiting")]
+    if not waiting:
+        return
+    repository = SessionRepository(store.path)
+    owners = {str(owner.session_id) for owner in DeviceExecutionLock.get_active_owners().values()}
+    for job in waiting:
+        sid = str(job["session_id"])
+        row = repository.get_session_by_id(sid) or {}
+        if sid in owners or repository.process_is_alive(row.get("pid")):
+            continue
+        outcome = read_worker_outcome(sid)
+        status = job.get("terminal_status") or outcome.get("task_status") or row.get("status")
+        if status not in TERMINAL:
+            status = "interrupted"
+            job["error"] = job.get("error") or "服务中断；未完成任务需手动重跑"
+            store.save_job(job)
+        await on_terminal({"session_id": sid}, status)
+
+
 async def tick_batches():
     from apps.admin_console.server import task_queue_service
     from artemis.runtime import device_pool
     from apps.admin_console.services.report_export import export_batch
 
+    await reconcile_recovered_jobs()
     for batch in store.records("batch"):
         if batch["status"] not in {"running", "paused"}:
             continue

apps/admin_console/services/automation_store.py

@@ -12,6 +12,12 @@ import uuid
 from artemis.config import DB_PATH
 
 
+RECOVERY_ERRORS = {
+    "服务中断；未完成任务需手动重跑",
+    "服务中断，需手动重跑",
+}
+
+
 def identifier(value: str) -> str:
     if not re.fullmatch(r"[A-Za-z0-9_-]{1,100}", value):
         raise ValueError("无效的记录编号")

apps/admin_console/services/report_export.py

@@ -88,6 +88,10 @@ def failure_reason(directory: Path, status: str, recorded: str | None) -> str:
         )
         return safe[:500]
 
+    from apps.admin_console.services.automation_store import RECOVERY_ERRORS
+
+    if status == "completed" and recorded in RECOVERY_ERRORS:
+        return ""
     if recorded:
         return redact(recorded)
     if status not in {"failed", "interrupted"}: