feat: enable full AI automation and safe startup updates

This commit is contained in:
chenmingxuan committed 2026-09-22 23:50:03 +08:00
1 parent 2ddec53954
commit 7b1a0755e2
62 files changed
+3123 -240

No files matched your search

+202
View File
@@ -0,0 +1,202 @@
"""Reports, validated spreadsheet batches and indexed history."""
import asyncio
import os
from typing import Literal
from fastapi import APIRouter, HTTPException, Query, UploadFile
from fastapi.responses import FileResponse, Response
from pydantic import BaseModel, Field
from apps.admin_console.services.automation_store import identifier, store
from apps.admin_console.services import automation_service as service
from apps.admin_console.services import report_export, task_import
router = APIRouter(prefix="/api/automation", tags=["automation"])
class ReportPreferences(BaseModel):
formats: list[Literal["docx", "xlsx"]] = Field(
default=["docx", "xlsx"], min_length=1, max_length=2
)
class BatchRequest(ReportPreferences):
request_id: str | None = Field(default=None, pattern=r"^[A-Za-z0-9_-]{1,100}$")
name: str = Field(min_length=1, max_length=200)
cases: list[task_import.TaskCase] = Field(min_length=1, max_length=task_import.MAX_ROWS)
device_serial: str = Field(min_length=1, max_length=200)
profile: Literal["flash", "pro"] = "flash"
stop_on_failure: bool = False
timeout_seconds: int = Field(
default_factory=lambda: int(os.getenv("ARTEMIS_BATCH_TASK_TIMEOUT", "1800")),
ge=30,
le=86400,
)
def require(kind, key):
try:
value = store.get(kind, identifier(key))
except ValueError as exc:
raise HTTPException(400, str(exc)) from exc
if value is None:
raise HTTPException(404, "记录不存在")
return value
@router.get("/preferences")
def preferences():
return {
**store.preferences(),
"batch_timeout_seconds": int(os.getenv("ARTEMIS_BATCH_TASK_TIMEOUT", "1800")),
}
@router.put("/preferences")
def save_preferences(value: ReportPreferences):
return store.put("settings", "reports", value.model_dump())
@router.get("/template")
def template():
return Response(
task_import.template(),
media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
headers={"Content-Disposition": 'attachment; filename="artemis-tasks.xlsx"'},
)
@router.post("/import/preview")
async def preview(file: UploadFile):
data = await file.read(task_import.MAX_BYTES + 1)
try:
return await asyncio.to_thread(task_import.parse_workbook, data)
except (ValueError, OSError) as exc:
raise HTTPException(400, str(exc)) from exc
@router.post("/batches")
def create_batch(value: BatchRequest):
try:
payload = value.model_dump()
if "formats" not in value.model_fields_set:
payload["formats"] = store.preferences()["formats"]
return service.create_batch(payload)
except ValueError as exc:
raise HTTPException(409, str(exc)) from exc
@router.get("/batches")
def batches():
return [
{
**b,
"counts": {
s: sum(j["status"] == s for j in store.jobs(b["id"]))
for s in (
"pending",
"queued",
"running",
"completed",
"failed",
"interrupted",
"skipped",
"cancelled",
)
},
}
for b in store.records("batch")
]
@router.get("/batches/by-request/{request_id}")
def batch_by_request(request_id: str):
record = require("batch_request", request_id)
return batch_detail(record["batch_id"])
@router.get("/batches/{batch_id}")
def batch_detail(batch_id: str):
return {**require("batch", batch_id), "jobs": store.jobs(batch_id)}
@router.post("/batches/{batch_id}/{action}")
def batch_action(batch_id: str, action: Literal["pause", "resume", "retry_failed"]):
require("batch", batch_id)
try:
return service.batch_action(batch_id, action)
except ValueError as exc:
raise HTTPException(409, str(exc)) from exc
@router.get("/batches/{batch_id}/archive")
async def batch_archive(batch_id: str):
require("batch", batch_id)
path = await asyncio.to_thread(report_export.export_batch, batch_id)
return FileResponse(path, filename=f"批次报告-{batch_id[:8]}.zip")
@router.get("/reports/{session_id}")
def report(session_id: str):
try:
return report_export.session_report(session_id)
except ValueError as exc:
raise HTTPException(404, str(exc)) from exc
@router.get("/reports/{session_id}/{fmt}")
async def download_report(session_id: str, fmt: Literal["docx", "xlsx"]):
try:
paths = await asyncio.to_thread(report_export.export_session, session_id, [fmt])
return FileResponse(paths[0], filename=f"任务报告-{session_id[:8]}.{fmt}")
except ValueError as exc:
raise HTTPException(404, str(exc)) from exc
@router.get("/history")
def history(
q: str = Query(default="", max_length=200),
status: str = "",
start: float | None = None,
end: float | None = None,
page: int = Query(default=1, ge=1),
size: int = Query(default=30, ge=1, le=100),
):
# SQL filtering/pagination searches the entire persisted history, not a UI cache.
query = """WITH history AS (
SELECT s.session_id, s.initial_goal, s.start_time, s.end_time,
COALESCE(j.status,s.status) status, COALESCE(json_extract(j.payload,'$.case.name'),s.initial_goal) name,
COALESCE(json_extract(j.payload,'$.case.tags'),'') tags
FROM sessions s LEFT JOIN automation_jobs j ON s.session_id=j.session_id
UNION ALL
SELECT j.session_id,json_extract(j.payload,'$.case.goal'),
COALESCE(json_extract(j.payload,'$.started_at'),json_extract(j.payload,'$.created_at')),
json_extract(j.payload,'$.ended_at'),j.status,json_extract(j.payload,'$.case.name'),
COALESCE(json_extract(j.payload,'$.case.tags'),'')
FROM automation_jobs j WHERE NOT EXISTS (SELECT 1 FROM sessions s WHERE s.session_id=j.session_id)
) """
where = " WHERE status IN ('completed','success','failed','cancelled','interrupted','skipped')"
args = []
if q:
where += " AND (name LIKE ? ESCAPE '\\' OR initial_goal LIKE ? ESCAPE '\\' OR session_id LIKE ? ESCAPE '\\' OR tags LIKE ? ESCAPE '\\')"
escaped = "%" + q.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%"
args.extend([escaped] * 4)
for condition, value in [
("status = ?", status or None),
("start_time >= ?", start),
("start_time < ?", end),
]:
if value is not None:
where += " AND " + condition
args.append(value)
with store.connection() as conn:
total = conn.execute(query + "SELECT count(*) FROM history" + where, args).fetchone()[0]
rows = conn.execute(
query
+ "SELECT * FROM history"
+ where
+ " ORDER BY start_time DESC,session_id LIMIT ? OFFSET ?",
[*args, size, (page - 1) * size],
).fetchall()
return {"items": [dict(r) for r in rows], "total": total, "page": page, "size": size}
+6
View File
@@ -216,6 +216,9 @@ async def cleanup_history_endpoint():
storage = StorageManager(DB_PATH, TRACES_PATH)
storage.clear_all_data()
from apps.admin_console.services.automation_store import store
store.delete_history()
return {
"status": "success",
"message": "History cleaned up successfully",
@@ -231,6 +234,9 @@ async def delete_session_endpoint(session_id: str):
storage = StorageManager(DB_PATH, TRACES_PATH)
storage.delete_session(UUID(session_id))
from apps.admin_console.services.automation_store import store
store.delete_history(session_id)
return {
"status": "success",
"message": f"Session {session_id} deleted successfully",
+7 -1
View File
@@ -165,7 +165,7 @@ async def run_task(request: RunRequest):
if verified_serial and not request.device_serial:
target_serial = verified_serial
return await task_queue_service.enqueue_tasks(
response = await task_queue_service.enqueue_tasks(
incoming_goals,
profile=request.profile or "flash",
expected_output=request.expected_output,
@@ -180,6 +180,12 @@ async def run_task(request: RunRequest):
conversation_id=request.conversation_id,
)
from apps.admin_console.services.automation_service import register_submission
for item in response.get("tasks", []):
register_submission(item)
return response
@router.get("/api/run/defaults")
async def get_run_defaults():
+14
View File
@@ -159,12 +159,19 @@ async def on_startup():
await ipc_service.start_server()
state.worker_task = asyncio.create_task(task_queue_service.queue_worker())
from apps.admin_console.services.automation_service import batch_loop
app.state.batch_worker = asyncio.create_task(batch_loop())
async def on_shutdown():
"""Stop task and IPC children before the UI server exits."""
state.is_shutting_down = True
state.shutdown_event.set()
batch_worker = getattr(app.state, "batch_worker", None)
if batch_worker:
batch_worker.cancel()
await asyncio.gather(batch_worker, return_exceptions=True)
task_queue_service._broadcast_event("server_shutdown", {"status": "stopping"})
owned_session_ids = {
str(item["session_id"])
@@ -207,6 +214,10 @@ async def on_shutdown():
for session_id in owned_session_ids:
session_repo.update_session_status(session_id, "cancelled")
from apps.admin_console.services.automation_service import background_tasks
if background_tasks:
await asyncio.gather(*list(background_tasks), return_exceptions=True)
await ipc_service.stop_server()
state.ipc_subscribers.clear()
await asyncio.to_thread(shutdown_awake_service)
@@ -224,6 +235,9 @@ app.include_router(steps.router)
app.include_router(tasks.router)
app.include_router(replay.router)
app.include_router(system.router)
from apps.admin_console.routers.automation import router as automation_router
app.include_router(automation_router)
# Mount cloud gateway router for Frappe / Cloud integration if present
try:
@@ -0,0 +1,276 @@
"""Persistent batch coordinator layered on the existing device queue."""
import asyncio
import logging
import os
import time
from apps.admin_console.core.state import state
from apps.admin_console.services.automation_store import store, new_id
from apps.admin_console.services.task_import import case_goal
logger = logging.getLogger(__name__)
TERMINAL = {"completed", "failed", "cancelled", "skipped", "interrupted"}
background_tasks: set[asyncio.Task] = set()
def spawn(coro):
task = asyncio.create_task(coro)
background_tasks.add(task)
task.add_done_callback(background_tasks.discard)
return task
def register_submission(item: dict, case: dict | None = None, formats=None, **extra):
sid = str(item["session_id"])
if store.job(sid):
return
store.save_job(
{
"session_id": sid,
"case": case or {"name": item.get("goal", sid)[:120], "goal": item.get("goal", sid)},
"status": "queued",
"created_at": time.time(),
"device_serial": item.get("device_serial"),
"formats": formats if formats is not None else store.preferences()["formats"],
**extra,
}
)
def recover_batches():
"""Recovery never silently repeats a potentially side-effecting interrupted task."""
for job in store.jobs():
if job["status"] in {"running", "queued", "finalizing"}:
job.update(
status=job.get("terminal_status", "interrupted"),
error=job.get("error")
or ("" if job.get("terminal_status") else "服务中断;未完成任务需手动重跑"),
ended_at=job.get("ended_at") or time.time(),
)
store.save_job(job)
for batch in store.records("batch"):
if has_legacy_replay(batch["id"]):
for job in store.jobs(batch["id"]):
if job["status"] not in TERMINAL:
job.update(status="cancelled", error="操作回放功能已移除", ended_at=time.time())
store.save_job(job)
if batch["status"] not in TERMINAL:
batch.update(status="cancelled", reason="操作回放功能已移除;保留历史供查看")
store.put("batch", batch["id"], batch)
continue
if batch["status"] in {"running", "pausing", "paused"}:
batch.update(
status="paused", reason="服务已重启;请检查中断任务,点击继续执行未开始任务"
)
for job in store.jobs(batch["id"]):
if job["status"] in {"running", "queued", "finalizing"}:
job.update(
status="interrupted", error="服务中断,需手动重跑", ended_at=time.time()
)
store.save_job(job)
store.put("batch", batch["id"], batch)
async def on_terminal(item: dict, status: str):
from apps.admin_console.services.report_export import export_session, session_report
sid = str(item["session_id"])
job = store.job(sid)
if not job:
register_submission(item)
job = store.job(sid)
if job is None:
raise RuntimeError("任务元数据写入失败")
job.update(status="finalizing", ended_at=time.time())
if job.get("timed_out"):
status = "failed"
store.save_job(job)
try:
job["status"] = status
report = await asyncio.to_thread(session_report, sid)
if report["verdict"] == "验证失败" or report["outcome"].get("task_status") in {
"blocked",
"partial",
"failed",
}:
job["status"] = "failed"
job["error"] = "任务执行结束,但业务检查未通过"
# Persist terminal classification before export, but the coordinator waits for finalization.
store.save_job({**job, "status": "finalizing", "terminal_status": job["status"]})
await asyncio.to_thread(export_session, sid, job["formats"])
job["report_ready"] = True
except Exception as exc:
logger.exception("Report finalization failed for %s", sid)
job["report_error"] = str(exc)[:500]
store.save_job(job)
async def tick_batches():
from apps.admin_console.server import task_queue_service
from artemis.runtime import device_pool
from apps.admin_console.services.report_export import export_batch
for batch in store.records("batch"):
if batch["status"] not in {"running", "paused"}:
continue
jobs = store.jobs(batch["id"])
active = [j for j in jobs if j["status"] in {"running", "queued", "finalizing"}]
if active:
if state.is_paused:
batch.update(status="paused", reason="模型请求已暂停,请先恢复模型连接再继续批次")
store.put("batch", batch["id"], batch)
for job in active:
limit = int(batch.get("timeout_seconds", 1800))
if (
job["status"] != "finalizing"
and not job.get("timed_out")
and time.time() - job.get("started_at", time.time()) > limit
):
job.update(error="任务达到批次配置的执行时限", timed_out=True)
store.save_job(job)
task_queue_service.stop_tasks(session_id=job["session_id"])
batch.update(status="paused", reason="任务超时,已暂停剩余任务;请确认设备状态")
store.put("batch", batch["id"], batch)
continue
if batch["status"] != "running":
continue
pending = [j for j in jobs if j["status"] == "pending"]
failed = [j for j in jobs if j["status"] in {"failed", "cancelled", "interrupted"}]
if batch.get("stop_on_failure") and failed and pending:
for job in pending:
job.update(status="skipped", error="本批次设置为失败即停止", ended_at=time.time())
store.save_job(job)
pending = []
if not pending:
batch.update(status="completed", ended_at=time.time())
try:
await asyncio.to_thread(export_batch, batch["id"])
batch["report_ready"] = True
except Exception as exc:
logger.exception("Batch report generation failed")
batch["report_error"] = str(exc)[:500]
store.put("batch", batch["id"], batch)
continue
if state.is_paused:
batch.update(status="paused", reason="模型请求已暂停,请先恢复模型连接再继续批次")
store.put("batch", batch["id"], batch)
continue
serial = batch.get("device_serial")
devices = await device_pool.try_list_devices_async()
rejection = (
None
if serial and any(d.serial == serial and d.state == "device" for d in devices or [])
else "无法确认所选设备在线且已授权,请恢复设备连接后继续批次"
)
if rejection:
batch.update(status="paused", reason=str(rejection))
store.put("batch", batch["id"], batch)
continue
if (store.get("batch", batch["id"]) or {}).get("status") != "running":
continue
job = pending[0]
job.update(status="queued", started_at=time.time())
store.save_job(job)
try:
response = await task_queue_service.enqueue_tasks(
[case_goal(job["case"])],
profile=batch.get("profile", "flash"),
expected_output="用简体中文报告实际结果与证据。明确区分执行完成、验证通过、验证失败和前置条件未满足。"
+ job["case"].get("expected", ""),
enable_outputter=True,
device_serial=serial,
session_id=job["session_id"],
ingress="batch",
)
if response.get("status") == "rejected" or not response.get("tasks"):
raise ValueError(response.get("error") or "队列拒绝接收任务")
bound = response["tasks"][0].get("device_serial")
if bound and not serial:
batch["device_serial"] = bound
store.put("batch", batch["id"], batch)
except Exception as exc:
job.update(status="pending", error=str(exc)[:500])
store.save_job(job)
batch.update(status="paused", reason=str(exc)[:500])
store.put("batch", batch["id"], batch)
async def batch_loop():
recover_batches()
while True:
try:
await tick_batches()
except Exception:
logger.exception("Batch coordinator failed; persisted work is retained")
await asyncio.sleep(1)
def has_legacy_replay(batch_id: str) -> bool:
return any(
j.get("workflow_id") or j.get("execution_mode") == "strict" or j.get("adaptive_source")
for j in store.jobs(batch_id)
)
def create_batch(payload: dict) -> dict:
bid = new_id()
batch = {
"id": bid,
"name": payload["name"],
"status": "draft",
"created_at": time.time(),
"formats": payload.get("formats") or store.preferences()["formats"],
"profile": payload.get("profile", "flash"),
"device_serial": payload.get("device_serial"),
"timeout_seconds": payload.get("timeout_seconds")
or int(os.getenv("ARTEMIS_BATCH_TASK_TIMEOUT", "1800")),
"stop_on_failure": payload.get("stop_on_failure", False),
"request_id": payload.get("request_id"),
}
jobs = []
for number, case in enumerate(payload["cases"], 1):
jobs.append(
{
"session_id": new_id(),
"batch_id": bid,
"position": number,
"case": case,
"status": "pending",
"formats": batch["formats"],
"created_at": time.time(),
"device_serial": batch["device_serial"],
}
)
return store.create_batch(batch, jobs, payload)
def batch_action(batch_id: str, action: str) -> dict:
batch = store.get("batch", batch_id)
if not batch:
raise ValueError("批次不存在")
if has_legacy_replay(batch_id):
raise ValueError("操作回放功能已移除;历史记录仅供查看")
if action == "pause":
if batch["status"] not in {"running", "paused"}:
raise ValueError("只能暂停正在执行的批次")
batch.update(status="paused", reason="用户暂停;当前任务继续,后续任务等待")
elif action == "resume":
if batch["status"] not in {"draft", "paused"}:
raise ValueError("只能开始草稿或继续已暂停批次")
batch.update(status="running", reason="")
elif action == "retry_failed":
if batch["status"] == "running":
raise ValueError("请先暂停批次")
failed = [
j for j in store.jobs(batch_id) if j["status"] in {"failed", "cancelled", "interrupted"}
]
if not failed:
raise ValueError("没有可重跑的失败或中断任务")
revised = create_batch(
{**batch, "name": batch["name"] + " · 失败项重跑", "cases": [j["case"] for j in failed]}
)
return revised
else:
raise ValueError("不支持的批次操作")
return store.put("batch", batch_id, batch)
@@ -0,0 +1,177 @@
"""Durable batch, execution and versioned workflow metadata; no device side effects."""
from contextlib import contextmanager
import json
import hashlib
from pathlib import Path
import re
import sqlite3
import time
import uuid
from artemis.config import DB_PATH
def identifier(value: str) -> str:
if not re.fullmatch(r"[A-Za-z0-9_-]{1,100}", value):
raise ValueError("无效的记录编号")
return value
def new_id() -> str:
return str(uuid.uuid4())
class AutomationStore:
def __init__(self, path: Path = DB_PATH):
self.path = Path(path)
@contextmanager
def connection(self):
self.path.parent.mkdir(parents=True, exist_ok=True)
conn = sqlite3.connect(self.path, timeout=15)
try:
conn.row_factory = sqlite3.Row
conn.executescript("""
CREATE TABLE IF NOT EXISTS automation_records (
kind TEXT NOT NULL, id TEXT NOT NULL, payload TEXT NOT NULL,
updated_at REAL NOT NULL, PRIMARY KEY(kind,id));
CREATE TABLE IF NOT EXISTS automation_jobs (
session_id TEXT PRIMARY KEY, batch_id TEXT, position INTEGER NOT NULL,
status TEXT NOT NULL, payload TEXT NOT NULL, updated_at REAL NOT NULL);
CREATE INDEX IF NOT EXISTS automation_batch_jobs ON automation_jobs(batch_id,position);
""")
yield conn
finally:
conn.close()
def get(self, kind: str, key: str) -> dict | None:
with self.connection() as conn:
row = conn.execute(
"SELECT payload FROM automation_records WHERE kind=? AND id=?", (kind, key)
).fetchone()
return json.loads(row[0]) if row else None
def put(self, kind: str, key: str, payload: dict) -> dict:
with self.connection() as conn:
conn.execute(
"INSERT INTO automation_records VALUES (?,?,?,?) ON CONFLICT(kind,id) DO UPDATE SET payload=excluded.payload,updated_at=excluded.updated_at",
(kind, key, json.dumps(payload, ensure_ascii=False), time.time()),
)
conn.commit()
return payload
def records(self, kind: str, limit: int = 500) -> list[dict]:
with self.connection() as conn:
rows = conn.execute(
"SELECT payload FROM automation_records WHERE kind=? ORDER BY updated_at DESC LIMIT ?",
(kind, limit),
).fetchall()
return [json.loads(row[0]) for row in rows]
def job(self, session_id: str) -> dict | None:
with self.connection() as conn:
row = conn.execute(
"SELECT payload,status FROM automation_jobs WHERE session_id=?", (session_id,)
).fetchone()
return {**json.loads(row[0]), "status": row[1]} if row else None
def save_job(self, payload: dict) -> dict:
with self.connection() as conn:
conn.execute(
"INSERT INTO automation_jobs VALUES (?,?,?,?,?,?) ON CONFLICT(session_id) DO UPDATE SET payload=excluded.payload,status=excluded.status,updated_at=excluded.updated_at",
(
payload["session_id"],
payload.get("batch_id"),
payload.get("position", 0),
payload["status"],
json.dumps(payload, ensure_ascii=False),
time.time(),
),
)
conn.commit()
return payload
def jobs(self, batch_id: str | None = None) -> list[dict]:
with self.connection() as conn:
rows = (
conn.execute(
"SELECT payload,status FROM automation_jobs WHERE batch_id=? ORDER BY position",
(batch_id,),
).fetchall()
if batch_id
else conn.execute(
"SELECT payload,status FROM automation_jobs ORDER BY updated_at DESC LIMIT 10000"
).fetchall()
)
return [{**json.loads(row[0]), "status": row[1]} for row in rows]
def preferences(self) -> dict:
return self.get("settings", "reports") or {"formats": ["docx", "xlsx"]}
def create_batch(self, batch: dict, jobs: list[dict], payload: dict) -> dict:
"""Commit a batch and all its jobs atomically; repeated request IDs never duplicate work."""
request_id = payload.get("request_id")
fingerprint = hashlib.sha256(
json.dumps(payload, sort_keys=True, ensure_ascii=False).encode()
).hexdigest()
with self.connection() as conn:
conn.execute("BEGIN IMMEDIATE")
if request_id:
identifier(request_id)
row = conn.execute(
"SELECT payload FROM automation_records WHERE kind='batch_request' AND id=?",
(request_id,),
).fetchone()
if row:
previous = json.loads(row[0])
if previous["fingerprint"] != fingerprint:
raise ValueError("请求编号已用于不同内容,请使用新的 request_id")
existing = conn.execute(
"SELECT payload FROM automation_records WHERE kind='batch' AND id=?",
(previous["batch_id"],),
).fetchone()
if not existing:
raise ValueError("该请求对应的历史批次已删除,不能自动重新执行")
return json.loads(existing[0])
now = time.time()
conn.execute(
"INSERT INTO automation_records VALUES (?,?,?,?)",
("batch", batch["id"], json.dumps(batch, ensure_ascii=False), now),
)
for job in jobs:
conn.execute(
"INSERT INTO automation_jobs VALUES (?,?,?,?,?,?)",
(
job["session_id"],
batch["id"],
job["position"],
job["status"],
json.dumps(job, ensure_ascii=False),
now,
),
)
if request_id:
conn.execute(
"INSERT INTO automation_records VALUES (?,?,?,?)",
(
"batch_request",
request_id,
json.dumps({"batch_id": batch["id"], "fingerprint": fingerprint}),
now,
),
)
conn.commit()
return batch
def delete_history(self, session_id: str | None = None):
with self.connection() as conn:
if session_id:
conn.execute("DELETE FROM automation_jobs WHERE session_id=?", (session_id,))
else:
conn.execute("DELETE FROM automation_jobs")
conn.execute("DELETE FROM automation_records WHERE kind='batch'")
conn.commit()
store = AutomationStore()
@@ -4,6 +4,7 @@ import os
import tempfile
import threading
from pathlib import Path
from typing import Literal
from urllib.parse import urlsplit
from pydantic import BaseModel, Field, SecretStr, field_validator
@@ -18,8 +19,12 @@ class ModelSettingsUpdate(BaseModel):
base_url: str = "https://api.openai.com/v1"
model: str = Field(min_length=1, max_length=200)
api_key: SecretStr | None = None # Omitted/blank keeps the existing key.
timeout_seconds: float = Field(default=120, gt=0, le=3600)
max_retries: int = Field(default=0, ge=0, le=10)
timeout_seconds: float = Field(default=300, gt=0, le=3600)
max_retries: int = Field(default=5, ge=0, le=10)
retry_interval_seconds: float = Field(default=10, ge=0, le=300)
reasoning_effort: Literal["default", "none", "minimal", "low", "medium", "high", "xhigh"] = (
"high"
)
@field_validator("base_url")
@classmethod
@@ -57,6 +62,9 @@ def get_model_settings() -> dict:
"api_key_configured": bool(settings.OPENAI_API_KEY),
"timeout_seconds": settings.OPENAI_TIMEOUT_SECONDS,
"max_retries": settings.OPENAI_MAX_RETRIES,
"retry_interval_seconds": settings.OPENAI_RETRY_INTERVAL_SECONDS,
"reasoning_effort": settings.OPENAI_REASONING_EFFORT,
"model_presets": ["gpt-5.6-luna", "mimo-v2.6-flash"],
"env_path": str(get_env_file()),
}
@@ -69,6 +77,8 @@ def save_model_settings(update: ModelSettingsUpdate) -> dict:
"OPENAI_MODEL": update.model,
"OPENAI_TIMEOUT_SECONDS": str(update.timeout_seconds),
"OPENAI_MAX_RETRIES": str(update.max_retries),
"OPENAI_RETRY_INTERVAL_SECONDS": str(update.retry_interval_seconds),
"OPENAI_REASONING_EFFORT": update.reasoning_effort,
}
if update.api_key and update.api_key.get_secret_value().strip():
values["OPENAI_API_KEY"] = update.api_key.get_secret_value().strip()
@@ -77,6 +87,8 @@ def save_model_settings(update: ModelSettingsUpdate) -> dict:
settings.OPENAI_MODEL = update.model
settings.OPENAI_TIMEOUT_SECONDS = update.timeout_seconds
settings.OPENAI_MAX_RETRIES = update.max_retries
settings.OPENAI_RETRY_INTERVAL_SECONDS = update.retry_interval_seconds
settings.OPENAI_REASONING_EFFORT = update.reasoning_effort
if "OPENAI_API_KEY" in values:
settings.OPENAI_API_KEY = SecretStr(values["OPENAI_API_KEY"])
from artemis.llm.router import ModelFactory
@@ -116,7 +128,7 @@ def persist_environment(values: dict[str, str]) -> None:
class CapabilitySettings(BaseModel):
video_enabled: bool = True
video_enabled: bool = False
outputter_enabled: bool = True
step_summarizer_enabled: bool = True
@@ -152,7 +164,7 @@ async def test_model_settings(update: ModelSettingsUpdate) -> dict:
api_base=update.base_url,
timeout_seconds=update.timeout_seconds,
max_retries=0,
max_tokens=64,
reasoning_effort=None if update.reasoning_effort == "default" else update.reasoning_effort,
)
model = ModelFactory.create_model(endpoint)
result = await model.ainvoke([HumanMessage(content="Reply with OK only.")])
@@ -0,0 +1,328 @@
"""Evidence-backed Word/Excel artifacts, independent of the language model runtime."""
from datetime import datetime
from typing import cast
from docx.styles.style import ParagraphStyle
import json
from pathlib import Path
import re
import threading
import zipfile
from docx import Document
from docx.shared import Inches, Pt
from docx.oxml.ns import qn
from openpyxl import Workbook
from openpyxl.styles import Font, PatternFill, Alignment
from artemis.config import IMAGES_DIR, TRACES_PATH, WORKSPACE_ROOT
from apps.admin_console.database.repositories.session_repository import SessionRepository
from apps.admin_console.database.repositories.step_repository import StepRepository
from apps.admin_console.services.automation_store import store, identifier
EXPORT_ROOT = WORKSPACE_ROOT / "artifacts" / "reports"
EXPORT_LOCK = threading.RLock()
STATUS = {
"completed": "执行完成",
"success": "执行完成",
"failed": "执行失败",
"cancelled": "已取消",
"interrupted": "执行中断",
"skipped": "已跳过",
"pending": "待执行",
"queued": "已排队",
"running": "执行中",
}
def read_json(path: Path, fallback=None):
try:
return json.loads(path.read_text(encoding="utf-8"))
except (OSError, ValueError):
return fallback
def text(value) -> str:
raw = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, default=str)
return re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f]", "", raw)
def verdict(outcome: dict | None) -> str:
tests = (outcome or {}).get("tests") or {}
if tests.get("failed", 0):
return "验证失败"
if tests.get("unchecked", 0) or tests.get("inconclusive", 0):
return "验证不完整"
if tests.get("passed", 0):
return "验证通过"
return "未验证(执行完成不等于测试通过)"
def session_report(session_id: str, db_path=None) -> dict:
identifier(session_id)
meta = store.job(session_id) or {}
repo = SessionRepository(db_path) if db_path else SessionRepository()
session = repo.get_session_by_id(session_id) or {}
if not session and not meta:
raise ValueError("任务记录不存在")
steps = StepRepository(db_path).get_session_steps(session_id) if session else []
steps = [s for s in steps if s.get("step_id") != "pre-planning"]
directory = TRACES_PATH / session_id
outcome = read_json(directory / "run_outcome.json", {})
content = ""
for name in ("output.md", "output.txt", "output.json"):
path = directory / "notes" / name
if path.is_file():
content = path.read_text(encoding="utf-8")
break
result = read_json(directory / "result.json")
if not content and result is not None:
content = text(result)
for step in reversed(steps):
action = step.get("action_taken") or {}
if isinstance(action, dict) and action.get("action") == "report_task_status":
args = action.get("args") or {}
if not content:
content = args.get("explanation") or ""
if not outcome:
outcome = {"task_status": args.get("status"), "tests": {}}
break
if not content:
content = "本任务没有保存最终结论。以下内容仅来自已记录的执行状态和证据。"
case = meta.get("case") or {}
return {
"id": session_id,
"name": case.get("name") or session.get("initial_goal", session_id)[:120],
"goal": case.get("goal") or session.get("initial_goal", ""),
"expected": case.get("expected", ""),
"status": meta.get("terminal_status")
or meta.get("status")
or session.get("status", "unknown"),
"verdict": verdict(outcome),
"outcome": outcome,
"content": content,
"steps": steps,
"start": session.get("start_time") or meta.get("started_at"),
"end": session.get("end_time") or meta.get("ended_at"),
"error": meta.get("error", ""),
"device": meta.get("device_serial") or text(session.get("device_info") or ""),
"tags": case.get("tags", ""),
}
def time_text(value) -> str:
return (
datetime.fromtimestamp(float(value)).astimezone().isoformat(timespec="seconds")
if value
else "—"
)
def metadata(report: dict) -> list[tuple]:
return [
("任务编号", report["id"]),
("任务目标", report["goal"]),
("预期结果", report["expected"] or "未指定"),
("执行状态", STATUS.get(report["status"], report["status"])),
("验证结论", report["verdict"]),
("开始时间", time_text(report["start"])),
("结束时间", time_text(report["end"])),
("设备", report["device"]),
("异常原因", report["error"] or "无独立异常记录"),
]
def image_path(name: str | None) -> Path | None:
if not name:
return None
root = Path(IMAGES_DIR).resolve()
path = (root / (name if Path(name).suffix else name + ".jpg")).resolve()
return (
path
if path.is_relative_to(root)
and path.is_file()
and path.suffix.lower() in {".jpg", ".png", ".jpeg"}
else None
)
def markdown_blocks(content: str):
lines = content.splitlines()
index = 0
while index < len(lines):
line = lines[index]
if (
"|" in line
and index + 1 < len(lines)
and re.fullmatch(r"[\s|:\-]+", lines[index + 1])
and "-" in lines[index + 1]
):
rows = [[c.strip() for c in line.strip().strip("|").split("|")]]
index += 2
while index < len(lines) and "|" in lines[index] and lines[index].strip():
rows.append([c.strip() for c in lines[index].strip().strip("|").split("|")])
index += 1
yield "table", rows
else:
yield "text", line
index += 1
def word_report(report: dict, path: Path):
doc = Document()
style = cast(ParagraphStyle, doc.styles["Normal"])
style.font.name = "Microsoft YaHei"
style.font.size = Pt(10)
style.element.rPr.rFonts.set(qn("w:eastAsia"), "微软雅黑")
doc.add_heading(report["name"], 0)
table = doc.add_table(rows=0, cols=2)
table.columns[0].width = Inches(1.3)
table.columns[1].width = Inches(5.2)
table.style = "Light Shading Accent 1"
for label, value in metadata(report):
cells = table.add_row().cells
cells[0].text, cells[1].text = label, text(value)
cells[0].width, cells[1].width = Inches(1.3), Inches(5.2)
doc.add_heading("结论与任务专属结果", 1)
for kind, value in markdown_blocks(text(report["content"])):
if kind == "table":
table = doc.add_table(rows=0, cols=max(map(len, value)))
table.style = "Light Shading Accent 1"
for row in value:
cells = table.add_row().cells
for cell, item in zip(cells, row):
cell.text = item
elif isinstance(value, str) and value.startswith("#"):
doc.add_heading(value.lstrip("# "), min(len(value) - len(value.lstrip("#")), 3))
else:
doc.add_paragraph(str(value))
doc.add_heading("执行步骤与证据", 1)
for step in report["steps"]:
doc.add_paragraph(
f"步骤 {step['step_number']} · {time_text(step.get('timestamp'))} · {step.get('summary') or '已记录操作'}"
)
evidence = report["steps"][:1] + report["steps"][-3:]
seen = set()
for step in evidence:
name = step.get("post_image_name") or step.get("pre_image_name")
photo = image_path(name)
if photo and name not in seen:
seen.add(name)
doc.add_paragraph(f"证据:步骤 {step['step_number']} / {name}")
doc.add_picture(str(photo), width=Inches(2.6))
doc.add_paragraph(f"追溯编号:{report['id']}。未观察、未检查的项目不计为验证通过。")
doc.save(str(path))
def sheet_rows(ws, rows):
for row in rows:
ws.append([text(value)[:32700] if value is not None else "" for value in row])
for cell in ws[ws.max_row]:
# Always store text, including leading '=': never export executable formulas.
cell.data_type = "s"
cell.alignment = Alignment(wrap_text=True, vertical="top")
ws.freeze_panes = "A2"
if ws.max_row:
ws.auto_filter.ref = ws.dimensions
for cell in ws[1]:
cell.font = Font(bold=True, color="FFFFFF")
cell.fill = PatternFill("solid", fgColor="245C78")
for column in ws.columns:
ws.column_dimensions[column[0].column_letter].width = 32
def excel_report(report: dict, path: Path):
wb = Workbook()
assert wb.active is not None
wb.active.title = "任务结果"
sheet_rows(wb.active, [("项目", "内容"), *metadata(report)])
sheet_rows(
wb.create_sheet("结论"),
[("最终结论",), *[(line,) for line in report["content"].splitlines()]],
)
for number, (kind, rows) in enumerate(markdown_blocks(report["content"])):
if kind == "table":
sheet_rows(wb.create_sheet(f"结果表{number + 1}"), rows)
sheet_rows(
wb.create_sheet("执行证据"),
[
("步骤", "时间", "说明", "结果", "截图引用"),
*[
(
s["step_number"],
time_text(s.get("timestamp")),
s.get("summary"),
text(s.get("last_execution_result")),
s.get("post_image_name") or s.get("pre_image_name"),
)
for s in report["steps"]
],
],
)
wb.save(path)
def export_session(session_id: str, formats: list[str] | None = None) -> list[Path]:
with EXPORT_LOCK:
report = session_report(session_id)
directory = EXPORT_ROOT / "sessions" / identifier(session_id)
directory.mkdir(parents=True, exist_ok=True)
outputs = []
for fmt in formats if formats is not None else store.preferences()["formats"]:
if fmt not in {"docx", "xlsx"}:
raise ValueError("不支持的报告格式")
path = directory / f"任务报告.{fmt}"
temporary = directory / f"生成中.{fmt}"
(word_report if fmt == "docx" else excel_report)(report, temporary)
temporary.replace(path)
outputs.append(path)
return outputs
def export_batch(batch_id: str) -> Path:
with EXPORT_LOCK:
batch = store.get("batch", identifier(batch_id))
if not batch:
raise ValueError("批次不存在")
directory = EXPORT_ROOT / "batches" / batch_id
directory.mkdir(parents=True, exist_ok=True)
formats = batch["formats"]
records = [session_report(job["session_id"]) for job in store.jobs(batch_id)]
rows = [("任务编号", "任务名称", "执行状态", "验证结论", "异常原因")]
rows += [
(r["id"], r["name"], STATUS.get(r["status"], r["status"]), r["verdict"], r["error"])
for r in records
]
outputs = []
if "xlsx" in formats:
wb = Workbook()
assert wb.active is not None
wb.active.title = "批次结果"
sheet_rows(wb.active, rows)
path = directory / "批次结果.xlsx"
wb.save(path)
outputs.append(path)
if "docx" in formats:
doc = Document()
doc.add_heading(batch["name"], 0)
doc.add_paragraph(
f"批次编号:{batch_id};共 {len(records)} 个任务。验证通过与执行完成分别统计。"
)
table = doc.add_table(rows=0, cols=5)
table.style = "Light Shading Accent 1"
for row in rows:
for cell, value in zip(table.add_row().cells, row):
cell.text = str(value)
path = directory / "批次汇总.docx"
doc.save(str(path))
outputs.append(path)
target = directory / "批次报告.zip"
temporary = directory / "批次报告.tmp"
with zipfile.ZipFile(temporary, "w", zipfile.ZIP_DEFLATED) as archive:
for path in outputs:
archive.write(path, path.name)
for number, report in enumerate(records, 1):
for path in export_session(report["id"], formats):
archive.write(path, f"任务报告/{number:04d}_{report['id']}/{path.name}")
temporary.replace(target)
return target
+127
View File
@@ -0,0 +1,127 @@
"""Versioned Excel task template and bounded, non-executing import validation."""
from io import BytesIO
import json
import os
import zipfile
from openpyxl import Workbook, load_workbook
from openpyxl.utils import get_column_letter
from openpyxl.styles import Font, PatternFill, Alignment
from pydantic import BaseModel, Field, ConfigDict
HEADERS = {
"任务名称": "name",
"任务目标": "goal",
"预期结果": "expected",
"目标应用": "app",
"前置条件": "preconditions",
"输入参数(JSON)": "parameters",
"标签": "tags",
}
MAX_ROWS = int(os.getenv("ARTEMIS_IMPORT_MAX_ROWS", "1000"))
MAX_BYTES = int(os.getenv("ARTEMIS_IMPORT_MAX_BYTES", "10485760"))
class TaskCase(BaseModel):
model_config = ConfigDict(extra="forbid")
name: str = Field(min_length=1, max_length=200)
goal: str = Field(min_length=1, max_length=10000)
expected: str = Field(default="", max_length=10000)
app: str = Field(default="", max_length=200)
preconditions: str = Field(default="", max_length=10000)
parameters: dict[str, str | int | float | bool] = Field(default_factory=dict)
tags: str = Field(default="", max_length=500)
def template() -> bytes:
wb = Workbook()
ws = wb.active
assert ws is not None
ws.title = "任务"
ws.append(list(HEADERS))
ws.freeze_panes = "A2"
ws.auto_filter.ref = "A1:G1"
for cell in ws[1]:
cell.font = Font(bold=True, color="FFFFFF")
cell.fill = PatternFill("solid", fgColor="245C78")
ws.column_dimensions[get_column_letter(int(cell.column or 1))].width = 32
help_sheet = wb.create_sheet("填写说明")
for line in [
"模板版本:1",
"每行一个独立任务;任务名称、任务目标必填。",
"预期结果决定要验证的业务条件。未填写时只报告任务完成情况。",
"前置条件描述实际用例起点;不会统一重启或清空应用数据。",
'输入参数使用 JSON 对象,例如 {"关键词":"天气"};目标可引用 {{关键词}}。',
"目标应用可填名称或包名;空白时由任务决定。",
"不要填写 Excel 公式、宏或外部链接。导入仅预览,点击开始后执行。",
]:
help_sheet.append([line])
help_sheet.column_dimensions["A"].width = 105
for row in help_sheet:
row[0].alignment = Alignment(wrap_text=True)
stream = BytesIO()
wb.save(stream)
return stream.getvalue()
def parse_workbook(data: bytes) -> dict:
if len(data) > MAX_BYTES:
raise ValueError("文件过大,请拆分批次后导入")
try:
with zipfile.ZipFile(BytesIO(data)) as archive:
if (
len(archive.infolist()) > 2000
or sum(i.file_size for i in archive.infolist()) > MAX_BYTES * 10
):
raise ValueError("Excel 解压体积超过限制")
if any(
"vbaproject" in i.filename.lower() or "externallinks" in i.filename.lower()
for i in archive.infolist()
):
raise ValueError("不支持宏或外部链接")
wb = load_workbook(BytesIO(data), read_only=True, data_only=False, keep_links=False)
except (zipfile.BadZipFile, KeyError, OSError) as exc:
raise ValueError("请上传有效的 .xlsx 模板文件") from exc
try:
if "任务" not in wb.sheetnames:
raise ValueError("缺少“任务”工作表,请使用下载的模板")
rows = iter(wb["任务"].iter_rows())
headers = [str(cell.value or "").strip() for cell in next(rows, [])]
if headers != list(HEADERS):
raise ValueError("列名或顺序与模板不一致,请重新下载模板")
tasks, errors = [], []
for number, row in enumerate(rows, 2):
if number > MAX_ROWS + 1:
raise ValueError(f"单批最多 {MAX_ROWS} 行任务")
if not any(cell.value is not None for cell in row):
continue
try:
if any(cell.data_type == "f" for cell in row):
raise ValueError("不允许公式,请粘贴为纯文本")
values = [str(cell.value or "").strip() for cell in row[: len(HEADERS)]]
item = dict(zip(HEADERS.values(), values))
item["parameters"] = json.loads(item.get("parameters") or "{}")
case = TaskCase.model_validate(item)
tasks.append({"row": number, **case.model_dump()})
except (ValueError, TypeError) as exc:
errors.append({"row": number, "message": str(exc)[:500]})
if not tasks and not errors:
errors.append({"row": 2, "message": "没有任务,请填写模板后重新导入"})
return {"version": 1, "tasks": tasks, "errors": errors, "valid": bool(tasks) and not errors}
finally:
wb.close()
def case_goal(case: dict) -> str:
goal = case["goal"]
for key, value in case.get("parameters", {}).items():
goal = goal.replace("{{" + key + "}}", str(value))
return (
f"任务名称:{case['name']}\n任务目标:{goal}\n目标应用:{case.get('app') or '按目标判断'}\n"
f"前置条件:{case.get('preconditions') or '根据实际用例判断合适起点'}\n"
"先观察手机并恢复到本用例需要的起始位置,记录准备过程。不要固定重启或清空数据。"
"如果无法建立前置条件,明确报告前置条件未满足,不继续正文操作。\n"
f"预期结果:{case.get('expected') or '仅确认任务完成情况,不宣称测试通过'}"
)
@@ -466,6 +466,12 @@ class TaskQueueService:
"""Mark the task as running and announce the launch to subscribers."""
task_item["status"] = "running"
task_item["start_time"] = time.time()
from apps.admin_console.services.automation_store import store
job = store.job(str(sess_id)) if sess_id else None
if job:
job.update(status="running", started_at=time.time())
store.save_job(job)
# A fresh launch clears a stale stop request left over for this run
# key from a previous task. Manual stops are tracked per run in
@@ -559,6 +565,12 @@ class TaskQueueService:
if device_serial:
cmd.extend(["--device-serial", str(device_serial)])
env["ADB_DEVICE_SERIAL"] = str(device_serial)
from apps.admin_console.services.automation_store import store
from artemis.config import TRACES_PATH
job = store.job(str(sess_id)) if sess_id else None
if job:
env["RESULTS_OUTPUT_PATH"] = str(TRACES_PATH / str(sess_id) / "result.json")
return cmd, env
@classmethod
@@ -768,6 +780,11 @@ class TaskQueueService:
manual_stop: bool,
) -> None:
"""Broadcast session_ended and dispatch the external completion notification."""
from apps.admin_console.services.automation_store import store
from apps.admin_console.services.automation_service import spawn, on_terminal
if sess_id and store.job(str(sess_id)):
spawn(on_terminal(task_item, new_status))
state.active_connections.pop(sess_id, None)
cls._broadcast_event(
"session_ended",
@@ -938,7 +955,7 @@ class TaskQueueService:
}
# 2. Debounce duplicate rapid submissions (e.g. UI double-click or network retry within 1s)
if len(goals) == 1:
if len(goals) == 1 and not session_id:
first_goal = goals[0]
recent_duplicate = next(
(
@@ -1212,10 +1229,19 @@ class TaskQueueService:
@classmethod
def _stop_all_tasks(cls) -> bool:
"""Terminate all active device owners and clear pending queue submissions."""
from apps.admin_console.services.automation_store import store
from apps.admin_console.services.automation_service import on_terminal, spawn
for batch in store.records("batch"):
if batch["status"] == "running":
batch.update(status="paused", reason="已停止全部任务,等待手动继续")
store.put("batch", batch["id"], batch)
# 1. Cancel local queue reservations
for item in state.queue_items:
if isinstance(item, dict) and item.get("status") != "running":
DeviceExecutionLock.cancel_reservation(item.get("queue_ticket"))
if store.job(str(item.get("session_id"))):
spawn(on_terminal(item, "cancelled"))
state.clear_queue()
# 2. Terminate all active owners across all devices
@@ -1534,6 +1560,12 @@ class TaskQueueService:
),
None,
)
if stopped_item and stopped_item.get("status") != "running":
from apps.admin_console.services.automation_store import store
from apps.admin_console.services.automation_service import on_terminal, spawn
if store.job(str(stopped_session_id)):
spawn(on_terminal(stopped_item, "cancelled"))
if stopped_item and not reservation_cancelled:
DeviceExecutionLock.cancel_reservation(stopped_item.get("queue_ticket"))
state.queue_items = [