feat: enable full AI automation and safe startup updates

apps/admin_console/services/automation_service.py

@@ -0,0 +1,276 @@
+"""Persistent batch coordinator layered on the existing device queue."""
+
+import asyncio
+import logging
+import os
+import time
+
+from apps.admin_console.core.state import state
+from apps.admin_console.services.automation_store import store, new_id
+from apps.admin_console.services.task_import import case_goal
+
+logger = logging.getLogger(__name__)
+TERMINAL = {"completed", "failed", "cancelled", "skipped", "interrupted"}
+background_tasks: set[asyncio.Task] = set()
+
+
+def spawn(coro):
+    task = asyncio.create_task(coro)
+    background_tasks.add(task)
+    task.add_done_callback(background_tasks.discard)
+    return task
+
+
+def register_submission(item: dict, case: dict | None = None, formats=None, **extra):
+    sid = str(item["session_id"])
+    if store.job(sid):
+        return
+    store.save_job(
+        {
+            "session_id": sid,
+            "case": case or {"name": item.get("goal", sid)[:120], "goal": item.get("goal", sid)},
+            "status": "queued",
+            "created_at": time.time(),
+            "device_serial": item.get("device_serial"),
+            "formats": formats if formats is not None else store.preferences()["formats"],
+            **extra,
+        }
+    )
+
+
+def recover_batches():
+    """Recovery never silently repeats a potentially side-effecting interrupted task."""
+    for job in store.jobs():
+        if job["status"] in {"running", "queued", "finalizing"}:
+            job.update(
+                status=job.get("terminal_status", "interrupted"),
+                error=job.get("error")
+                or ("" if job.get("terminal_status") else "服务中断；未完成任务需手动重跑"),
+                ended_at=job.get("ended_at") or time.time(),
+            )
+            store.save_job(job)
+    for batch in store.records("batch"):
+        if has_legacy_replay(batch["id"]):
+            for job in store.jobs(batch["id"]):
+                if job["status"] not in TERMINAL:
+                    job.update(status="cancelled", error="操作回放功能已移除", ended_at=time.time())
+                    store.save_job(job)
+            if batch["status"] not in TERMINAL:
+                batch.update(status="cancelled", reason="操作回放功能已移除；保留历史供查看")
+                store.put("batch", batch["id"], batch)
+            continue
+        if batch["status"] in {"running", "pausing", "paused"}:
+            batch.update(
+                status="paused", reason="服务已重启；请检查中断任务，点击继续执行未开始任务"
+            )
+            for job in store.jobs(batch["id"]):
+                if job["status"] in {"running", "queued", "finalizing"}:
+                    job.update(
+                        status="interrupted", error="服务中断，需手动重跑", ended_at=time.time()
+                    )
+                    store.save_job(job)
+            store.put("batch", batch["id"], batch)
+
+
+async def on_terminal(item: dict, status: str):
+    from apps.admin_console.services.report_export import export_session, session_report
+
+    sid = str(item["session_id"])
+    job = store.job(sid)
+    if not job:
+        register_submission(item)
+        job = store.job(sid)
+    if job is None:
+        raise RuntimeError("任务元数据写入失败")
+    job.update(status="finalizing", ended_at=time.time())
+    if job.get("timed_out"):
+        status = "failed"
+    store.save_job(job)
+    try:
+        job["status"] = status
+        report = await asyncio.to_thread(session_report, sid)
+        if report["verdict"] == "验证失败" or report["outcome"].get("task_status") in {
+            "blocked",
+            "partial",
+            "failed",
+        }:
+            job["status"] = "failed"
+            job["error"] = "任务执行结束，但业务检查未通过"
+        # Persist terminal classification before export, but the coordinator waits for finalization.
+        store.save_job({**job, "status": "finalizing", "terminal_status": job["status"]})
+        await asyncio.to_thread(export_session, sid, job["formats"])
+        job["report_ready"] = True
+    except Exception as exc:
+        logger.exception("Report finalization failed for %s", sid)
+        job["report_error"] = str(exc)[:500]
+    store.save_job(job)
+
+
+async def tick_batches():
+    from apps.admin_console.server import task_queue_service
+    from artemis.runtime import device_pool
+    from apps.admin_console.services.report_export import export_batch
+
+    for batch in store.records("batch"):
+        if batch["status"] not in {"running", "paused"}:
+            continue
+        jobs = store.jobs(batch["id"])
+        active = [j for j in jobs if j["status"] in {"running", "queued", "finalizing"}]
+        if active:
+            if state.is_paused:
+                batch.update(status="paused", reason="模型请求已暂停，请先恢复模型连接再继续批次")
+                store.put("batch", batch["id"], batch)
+            for job in active:
+                limit = int(batch.get("timeout_seconds", 1800))
+                if (
+                    job["status"] != "finalizing"
+                    and not job.get("timed_out")
+                    and time.time() - job.get("started_at", time.time()) > limit
+                ):
+                    job.update(error="任务达到批次配置的执行时限", timed_out=True)
+                    store.save_job(job)
+                    task_queue_service.stop_tasks(session_id=job["session_id"])
+                    batch.update(status="paused", reason="任务超时，已暂停剩余任务；请确认设备状态")
+                    store.put("batch", batch["id"], batch)
+            continue
+        if batch["status"] != "running":
+            continue
+        pending = [j for j in jobs if j["status"] == "pending"]
+        failed = [j for j in jobs if j["status"] in {"failed", "cancelled", "interrupted"}]
+        if batch.get("stop_on_failure") and failed and pending:
+            for job in pending:
+                job.update(status="skipped", error="本批次设置为失败即停止", ended_at=time.time())
+                store.save_job(job)
+            pending = []
+        if not pending:
+            batch.update(status="completed", ended_at=time.time())
+            try:
+                await asyncio.to_thread(export_batch, batch["id"])
+                batch["report_ready"] = True
+            except Exception as exc:
+                logger.exception("Batch report generation failed")
+                batch["report_error"] = str(exc)[:500]
+            store.put("batch", batch["id"], batch)
+            continue
+        if state.is_paused:
+            batch.update(status="paused", reason="模型请求已暂停，请先恢复模型连接再继续批次")
+            store.put("batch", batch["id"], batch)
+            continue
+        serial = batch.get("device_serial")
+        devices = await device_pool.try_list_devices_async()
+        rejection = (
+            None
+            if serial and any(d.serial == serial and d.state == "device" for d in devices or [])
+            else "无法确认所选设备在线且已授权，请恢复设备连接后继续批次"
+        )
+        if rejection:
+            batch.update(status="paused", reason=str(rejection))
+            store.put("batch", batch["id"], batch)
+            continue
+        if (store.get("batch", batch["id"]) or {}).get("status") != "running":
+            continue
+        job = pending[0]
+        job.update(status="queued", started_at=time.time())
+        store.save_job(job)
+        try:
+            response = await task_queue_service.enqueue_tasks(
+                [case_goal(job["case"])],
+                profile=batch.get("profile", "flash"),
+                expected_output="用简体中文报告实际结果与证据。明确区分执行完成、验证通过、验证失败和前置条件未满足。"
+                + job["case"].get("expected", ""),
+                enable_outputter=True,
+                device_serial=serial,
+                session_id=job["session_id"],
+                ingress="batch",
+            )
+            if response.get("status") == "rejected" or not response.get("tasks"):
+                raise ValueError(response.get("error") or "队列拒绝接收任务")
+            bound = response["tasks"][0].get("device_serial")
+            if bound and not serial:
+                batch["device_serial"] = bound
+                store.put("batch", batch["id"], batch)
+        except Exception as exc:
+            job.update(status="pending", error=str(exc)[:500])
+            store.save_job(job)
+            batch.update(status="paused", reason=str(exc)[:500])
+            store.put("batch", batch["id"], batch)
+
+
+async def batch_loop():
+    recover_batches()
+    while True:
+        try:
+            await tick_batches()
+        except Exception:
+            logger.exception("Batch coordinator failed; persisted work is retained")
+        await asyncio.sleep(1)
+
+
+def has_legacy_replay(batch_id: str) -> bool:
+    return any(
+        j.get("workflow_id") or j.get("execution_mode") == "strict" or j.get("adaptive_source")
+        for j in store.jobs(batch_id)
+    )
+
+
+def create_batch(payload: dict) -> dict:
+    bid = new_id()
+    batch = {
+        "id": bid,
+        "name": payload["name"],
+        "status": "draft",
+        "created_at": time.time(),
+        "formats": payload.get("formats") or store.preferences()["formats"],
+        "profile": payload.get("profile", "flash"),
+        "device_serial": payload.get("device_serial"),
+        "timeout_seconds": payload.get("timeout_seconds")
+        or int(os.getenv("ARTEMIS_BATCH_TASK_TIMEOUT", "1800")),
+        "stop_on_failure": payload.get("stop_on_failure", False),
+        "request_id": payload.get("request_id"),
+    }
+    jobs = []
+    for number, case in enumerate(payload["cases"], 1):
+        jobs.append(
+            {
+                "session_id": new_id(),
+                "batch_id": bid,
+                "position": number,
+                "case": case,
+                "status": "pending",
+                "formats": batch["formats"],
+                "created_at": time.time(),
+                "device_serial": batch["device_serial"],
+            }
+        )
+    return store.create_batch(batch, jobs, payload)
+
+
+def batch_action(batch_id: str, action: str) -> dict:
+    batch = store.get("batch", batch_id)
+    if not batch:
+        raise ValueError("批次不存在")
+    if has_legacy_replay(batch_id):
+        raise ValueError("操作回放功能已移除；历史记录仅供查看")
+    if action == "pause":
+        if batch["status"] not in {"running", "paused"}:
+            raise ValueError("只能暂停正在执行的批次")
+        batch.update(status="paused", reason="用户暂停；当前任务继续，后续任务等待")
+    elif action == "resume":
+        if batch["status"] not in {"draft", "paused"}:
+            raise ValueError("只能开始草稿或继续已暂停批次")
+        batch.update(status="running", reason="")
+    elif action == "retry_failed":
+        if batch["status"] == "running":
+            raise ValueError("请先暂停批次")
+        failed = [
+            j for j in store.jobs(batch_id) if j["status"] in {"failed", "cancelled", "interrupted"}
+        ]
+        if not failed:
+            raise ValueError("没有可重跑的失败或中断任务")
+        revised = create_batch(
+            {**batch, "name": batch["name"] + " · 失败项重跑", "cases": [j["case"] for j in failed]}
+        )
+        return revised
+    else:
+        raise ValueError("不支持的批次操作")
+    return store.put("batch", batch_id, batch)