fix: preserve live tasks during daemon recovery

apps/admin_console/server.py

@@ -24,6 +24,7 @@ import os
 from pathlib import Path
 import secrets
 import signal
+import socket
 import sys
 from types import FrameType
 
@@ -421,6 +422,14 @@ queue_goals = state.queue_goals
 class ArtemisUvicornServer(uvicorn.Server):
     """Notify application streams before Uvicorn waits for them to close."""
 
+    async def startup(self, sockets: list[socket.socket] | None = None) -> None:
+        # Reload workers start in a fresh interpreter; restore the actual bind
+        # address there before lifespan publishes runtime metadata.
+        state.host = self.config.host
+        state.port = self.config.port
+        app.state.uvicorn_server = self
+        await super().startup(sockets=sockets)
+
     @staticmethod
     def _task_is_active() -> bool:
         """Check task state without doing process I/O from a signal handler."""
@@ -470,31 +479,40 @@ class ArtemisUvicornServer(uvicorn.Server):
 
 def run_ui_server(host: str, port: int, reload: bool = False) -> None:
     """Run the UI server with bounded, signal-aware graceful shutdown."""
+    # Reserve a listening socket before publishing metadata or running lifespan.
+    # Uvicorn normally runs application startup *before* binding its port; a
+    # duplicate daemon would otherwise recover jobs owned by the real server.
+    family = socket.AF_INET6 if ":" in host else socket.AF_INET
+    try:
+        listener = socket.create_server((host, port), family=family)
+    except OSError as exc:
+        print(f"无法启动服务：{host}:{port} 端口不可用（{exc}）。现有服务和任务保持不变。")
+        raise SystemExit(1) from exc
+    listener.set_inheritable(True)
+    port = listener.getsockname()[1]
     state.host = host
     state.port = port
-    write_server_info(port=port, host=host, lifecycle_token=LIFECYCLE_TOKEN)
 
     try:
-        if reload:
-            uvicorn.run(
-                "apps.admin_console.server:app",
-                host=host,
-                port=port,
-                reload=True,
-                timeout_graceful_shutdown=5,
-            )
-            return
-
         config = uvicorn.Config(
-            app,
+            "apps.admin_console.server:app" if reload else app,
             host=host,
             port=port,
+            reload=reload,
             timeout_graceful_shutdown=5,
         )
+        if reload:
+            from uvicorn.supervisors import ChangeReload
+
+            server = ArtemisUvicornServer(config)
+            ChangeReload(config, target=server.run, sockets=[listener]).run()
+            return
+
         server = ArtemisUvicornServer(config)
         app.state.uvicorn_server = server
-        server.run()
+        server.run(sockets=[listener])
     finally:
+        listener.close()
         app.state.uvicorn_server = None
         clear_server_info(port=port, lifecycle_token=LIFECYCLE_TOKEN)
 

apps/admin_console/services/automation_service.py

@@ -8,7 +8,7 @@ import time
 from uuid import UUID
 
 from apps.admin_console.core.state import state
-from apps.admin_console.services.automation_store import store, new_id
+from apps.admin_console.services.automation_store import RECOVERY_ERRORS, store, new_id
 from apps.admin_console.services.task_import import case_goal, case_title
 from artemis.config.settings import settings
 from artemis.runtime.task_limits import task_timeout
@@ -42,9 +42,46 @@ def register_submission(item: dict, case: dict | None = None, formats=None, **ex
     )
 
 
+def resolve_recovery_error(job: dict) -> bool:
+    """Archive a superseded recovery warning without erasing execution failures."""
+    if job.get("error") not in RECOVERY_ERRORS:
+        return False
+    job.setdefault("recovery_events", []).append(
+        {"message": job.pop("error"), "resolved_at": time.time()}
+    )
+    return True
+
+
 def recover_batches():
     """Recovery never silently repeats a potentially side-effecting interrupted task."""
+    from apps.admin_console.database.repositories.session_repository import SessionRepository
+    from artemis.runtime import DeviceExecutionLock
+
+    repository = SessionRepository(store.path)
+    live_sessions = {
+        str(owner.session_id)
+        for owner in DeviceExecutionLock.get_active_owners().values()
+        if owner.session_id
+    }
+    live_sessions.update(
+        str(row["session_id"])
+        for row in repository.get_all_sessions()
+        if row.get("status") not in TERMINAL and repository.process_is_alive(row.get("pid"))
+    )
+    live_sessions.update(
+        str(item["session_id"])
+        for item in state.queue_items
+        if isinstance(item, dict) and item.get("session_id") and item.get("status") not in TERMINAL
+    )
     for job in store.jobs():
+        if job["status"] == "completed" and resolve_recovery_error(job):
+            store.save_job(job)
+        if str(job["session_id"]) in live_sessions:
+            if job["status"] not in TERMINAL and str(job["session_id"]) not in state.active_runs:
+                job["recovery_waiting"] = True
+                store.save_job(job)
+            logger.info("Recovery preserved live session %s", job["session_id"])
+            continue
         if job["status"] in {"running", "queued", "finalizing", "task_paused", "resuming"}:
             job.update(
                 status=job.get("terminal_status", "interrupted"),
@@ -52,6 +89,9 @@ def recover_batches():
                 or ("" if job.get("terminal_status") else "服务中断；未完成任务需手动重跑"),
                 ended_at=job.get("ended_at") or time.time(),
             )
+            if job["status"] == "completed":
+                resolve_recovery_error(job)
+            logger.info("Recovery resolved session %s to %s", job["session_id"], job["status"])
             store.save_job(job)
     for batch in store.records("batch"):
         if has_legacy_replay(batch["id"]):
@@ -64,6 +104,8 @@ def recover_batches():
                 store.put("batch", batch["id"], batch)
             continue
         if batch["status"] in {"running", "pausing", "paused"}:
+            if any(str(job["session_id"]) in live_sessions for job in store.jobs(batch["id"])):
+                continue
             batch.update(
                 status="paused", reason="服务已重启；请检查中断任务，点击继续执行未开始任务"
             )
@@ -88,6 +130,9 @@ async def on_terminal(item: dict, status: str):
         raise RuntimeError("任务元数据写入失败")
     if job.get("timed_out"):
         status = "failed"
+    job.pop("recovery_waiting", None)
+    if status == "completed":
+        resolve_recovery_error(job)
     job.update(status="finalizing", terminal_status=status, ended_at=time.time())
     store.save_job(job)
     try:
@@ -117,11 +162,37 @@ async def on_terminal(item: dict, status: str):
     store.save_job(job)
 
 
+async def reconcile_recovered_jobs():
+    """Finalize surviving workers once they exit; never resubmit their goals."""
+    from apps.admin_console.database.repositories.session_repository import SessionRepository
+    from artemis.runtime import DeviceExecutionLock
+    from artemis.runtime.task_result import read_worker_outcome
+
+    waiting = [job for job in store.jobs() if job.get("recovery_waiting")]
+    if not waiting:
+        return
+    repository = SessionRepository(store.path)
+    owners = {str(owner.session_id) for owner in DeviceExecutionLock.get_active_owners().values()}
+    for job in waiting:
+        sid = str(job["session_id"])
+        row = repository.get_session_by_id(sid) or {}
+        if sid in owners or repository.process_is_alive(row.get("pid")):
+            continue
+        outcome = read_worker_outcome(sid)
+        status = job.get("terminal_status") or outcome.get("task_status") or row.get("status")
+        if status not in TERMINAL:
+            status = "interrupted"
+            job["error"] = job.get("error") or "服务中断；未完成任务需手动重跑"
+            store.save_job(job)
+        await on_terminal({"session_id": sid}, status)
+
+
 async def tick_batches():
     from apps.admin_console.server import task_queue_service
     from artemis.runtime import device_pool
     from apps.admin_console.services.report_export import export_batch
 
+    await reconcile_recovered_jobs()
     for batch in store.records("batch"):
         if batch["status"] not in {"running", "paused"}:
             continue

apps/admin_console/services/automation_store.py

@@ -12,6 +12,12 @@ import uuid
 from artemis.config import DB_PATH
 
 
+RECOVERY_ERRORS = {
+    "服务中断；未完成任务需手动重跑",
+    "服务中断，需手动重跑",
+}
+
+
 def identifier(value: str) -> str:
     if not re.fullmatch(r"[A-Za-z0-9_-]{1,100}", value):
         raise ValueError("无效的记录编号")

apps/admin_console/services/report_export.py

@@ -88,6 +88,10 @@ def failure_reason(directory: Path, status: str, recorded: str | None) -> str:
         )
         return safe[:500]
 
+    from apps.admin_console.services.automation_store import RECOVERY_ERRORS
+
+    if status == "completed" and recorded in RECOVERY_ERRORS:
+        return ""
     if recorded:
         return redact(recorded)
     if status not in {"failed", "interrupted"}: