fix: preserve live tasks during daemon recovery
This commit is contained in:
1 parent
079d7b1e28
commit
0355af7833
11 files changed
+326
-16
No files matched your search
@@ -50,6 +50,8 @@ ARTEMIS_CHECKPOINT_TIMEOUT_SECONDS=180
|
|||||||
ARTEMIS_SETTLEMENT_TIMEOUT_SECONDS=120
|
ARTEMIS_SETTLEMENT_TIMEOUT_SECONDS=120
|
||||||
ARTEMIS_CREDENTIAL_TIMEOUT_SECONDS=12
|
ARTEMIS_CREDENTIAL_TIMEOUT_SECONDS=12
|
||||||
ARTEMIS_MODEL_CACHE_TTL_SECONDS=10
|
ARTEMIS_MODEL_CACHE_TTL_SECONDS=10
|
||||||
|
# 守护服务轻量探测超时;探测超时且端口仍占用时,不重复启动服务。
|
||||||
|
ARTEMIS_DAEMON_PROBE_TIMEOUT_SECONDS=2
|
||||||
# 模型重试耗尽后的暂停等待上限(秒);0 表示无限等待
|
# 模型重试耗尽后的暂停等待上限(秒);0 表示无限等待
|
||||||
LLM_PAUSE_TIMEOUT_SECONDS=900
|
LLM_PAUSE_TIMEOUT_SECONDS=900
|
||||||
|
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ import os
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import secrets
|
import secrets
|
||||||
import signal
|
import signal
|
||||||
|
import socket
|
||||||
import sys
|
import sys
|
||||||
from types import FrameType
|
from types import FrameType
|
||||||
|
|
||||||
@@ -421,6 +422,14 @@ queue_goals = state.queue_goals
|
|||||||
class ArtemisUvicornServer(uvicorn.Server):
|
class ArtemisUvicornServer(uvicorn.Server):
|
||||||
"""Notify application streams before Uvicorn waits for them to close."""
|
"""Notify application streams before Uvicorn waits for them to close."""
|
||||||
|
|
||||||
|
async def startup(self, sockets: list[socket.socket] | None = None) -> None:
|
||||||
|
# Reload workers start in a fresh interpreter; restore the actual bind
|
||||||
|
# address there before lifespan publishes runtime metadata.
|
||||||
|
state.host = self.config.host
|
||||||
|
state.port = self.config.port
|
||||||
|
app.state.uvicorn_server = self
|
||||||
|
await super().startup(sockets=sockets)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _task_is_active() -> bool:
|
def _task_is_active() -> bool:
|
||||||
"""Check task state without doing process I/O from a signal handler."""
|
"""Check task state without doing process I/O from a signal handler."""
|
||||||
@@ -470,31 +479,40 @@ class ArtemisUvicornServer(uvicorn.Server):
|
|||||||
|
|
||||||
def run_ui_server(host: str, port: int, reload: bool = False) -> None:
|
def run_ui_server(host: str, port: int, reload: bool = False) -> None:
|
||||||
"""Run the UI server with bounded, signal-aware graceful shutdown."""
|
"""Run the UI server with bounded, signal-aware graceful shutdown."""
|
||||||
|
# Reserve a listening socket before publishing metadata or running lifespan.
|
||||||
|
# Uvicorn normally runs application startup *before* binding its port; a
|
||||||
|
# duplicate daemon would otherwise recover jobs owned by the real server.
|
||||||
|
family = socket.AF_INET6 if ":" in host else socket.AF_INET
|
||||||
|
try:
|
||||||
|
listener = socket.create_server((host, port), family=family)
|
||||||
|
except OSError as exc:
|
||||||
|
print(f"无法启动服务:{host}:{port} 端口不可用({exc})。现有服务和任务保持不变。")
|
||||||
|
raise SystemExit(1) from exc
|
||||||
|
listener.set_inheritable(True)
|
||||||
|
port = listener.getsockname()[1]
|
||||||
state.host = host
|
state.host = host
|
||||||
state.port = port
|
state.port = port
|
||||||
write_server_info(port=port, host=host, lifecycle_token=LIFECYCLE_TOKEN)
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if reload:
|
config = uvicorn.Config(
|
||||||
uvicorn.run(
|
"apps.admin_console.server:app" if reload else app,
|
||||||
"apps.admin_console.server:app",
|
|
||||||
host=host,
|
host=host,
|
||||||
port=port,
|
port=port,
|
||||||
reload=True,
|
reload=reload,
|
||||||
timeout_graceful_shutdown=5,
|
timeout_graceful_shutdown=5,
|
||||||
)
|
)
|
||||||
|
if reload:
|
||||||
|
from uvicorn.supervisors import ChangeReload
|
||||||
|
|
||||||
|
server = ArtemisUvicornServer(config)
|
||||||
|
ChangeReload(config, target=server.run, sockets=[listener]).run()
|
||||||
return
|
return
|
||||||
|
|
||||||
config = uvicorn.Config(
|
|
||||||
app,
|
|
||||||
host=host,
|
|
||||||
port=port,
|
|
||||||
timeout_graceful_shutdown=5,
|
|
||||||
)
|
|
||||||
server = ArtemisUvicornServer(config)
|
server = ArtemisUvicornServer(config)
|
||||||
app.state.uvicorn_server = server
|
app.state.uvicorn_server = server
|
||||||
server.run()
|
server.run(sockets=[listener])
|
||||||
finally:
|
finally:
|
||||||
|
listener.close()
|
||||||
app.state.uvicorn_server = None
|
app.state.uvicorn_server = None
|
||||||
clear_server_info(port=port, lifecycle_token=LIFECYCLE_TOKEN)
|
clear_server_info(port=port, lifecycle_token=LIFECYCLE_TOKEN)
|
||||||
|
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import time
|
|||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
from apps.admin_console.core.state import state
|
from apps.admin_console.core.state import state
|
||||||
from apps.admin_console.services.automation_store import store, new_id
|
from apps.admin_console.services.automation_store import RECOVERY_ERRORS, store, new_id
|
||||||
from apps.admin_console.services.task_import import case_goal, case_title
|
from apps.admin_console.services.task_import import case_goal, case_title
|
||||||
from artemis.config.settings import settings
|
from artemis.config.settings import settings
|
||||||
from artemis.runtime.task_limits import task_timeout
|
from artemis.runtime.task_limits import task_timeout
|
||||||
@@ -42,9 +42,46 @@ def register_submission(item: dict, case: dict | None = None, formats=None, **ex
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_recovery_error(job: dict) -> bool:
|
||||||
|
"""Archive a superseded recovery warning without erasing execution failures."""
|
||||||
|
if job.get("error") not in RECOVERY_ERRORS:
|
||||||
|
return False
|
||||||
|
job.setdefault("recovery_events", []).append(
|
||||||
|
{"message": job.pop("error"), "resolved_at": time.time()}
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
def recover_batches():
|
def recover_batches():
|
||||||
"""Recovery never silently repeats a potentially side-effecting interrupted task."""
|
"""Recovery never silently repeats a potentially side-effecting interrupted task."""
|
||||||
|
from apps.admin_console.database.repositories.session_repository import SessionRepository
|
||||||
|
from artemis.runtime import DeviceExecutionLock
|
||||||
|
|
||||||
|
repository = SessionRepository(store.path)
|
||||||
|
live_sessions = {
|
||||||
|
str(owner.session_id)
|
||||||
|
for owner in DeviceExecutionLock.get_active_owners().values()
|
||||||
|
if owner.session_id
|
||||||
|
}
|
||||||
|
live_sessions.update(
|
||||||
|
str(row["session_id"])
|
||||||
|
for row in repository.get_all_sessions()
|
||||||
|
if row.get("status") not in TERMINAL and repository.process_is_alive(row.get("pid"))
|
||||||
|
)
|
||||||
|
live_sessions.update(
|
||||||
|
str(item["session_id"])
|
||||||
|
for item in state.queue_items
|
||||||
|
if isinstance(item, dict) and item.get("session_id") and item.get("status") not in TERMINAL
|
||||||
|
)
|
||||||
for job in store.jobs():
|
for job in store.jobs():
|
||||||
|
if job["status"] == "completed" and resolve_recovery_error(job):
|
||||||
|
store.save_job(job)
|
||||||
|
if str(job["session_id"]) in live_sessions:
|
||||||
|
if job["status"] not in TERMINAL and str(job["session_id"]) not in state.active_runs:
|
||||||
|
job["recovery_waiting"] = True
|
||||||
|
store.save_job(job)
|
||||||
|
logger.info("Recovery preserved live session %s", job["session_id"])
|
||||||
|
continue
|
||||||
if job["status"] in {"running", "queued", "finalizing", "task_paused", "resuming"}:
|
if job["status"] in {"running", "queued", "finalizing", "task_paused", "resuming"}:
|
||||||
job.update(
|
job.update(
|
||||||
status=job.get("terminal_status", "interrupted"),
|
status=job.get("terminal_status", "interrupted"),
|
||||||
@@ -52,6 +89,9 @@ def recover_batches():
|
|||||||
or ("" if job.get("terminal_status") else "服务中断;未完成任务需手动重跑"),
|
or ("" if job.get("terminal_status") else "服务中断;未完成任务需手动重跑"),
|
||||||
ended_at=job.get("ended_at") or time.time(),
|
ended_at=job.get("ended_at") or time.time(),
|
||||||
)
|
)
|
||||||
|
if job["status"] == "completed":
|
||||||
|
resolve_recovery_error(job)
|
||||||
|
logger.info("Recovery resolved session %s to %s", job["session_id"], job["status"])
|
||||||
store.save_job(job)
|
store.save_job(job)
|
||||||
for batch in store.records("batch"):
|
for batch in store.records("batch"):
|
||||||
if has_legacy_replay(batch["id"]):
|
if has_legacy_replay(batch["id"]):
|
||||||
@@ -64,6 +104,8 @@ def recover_batches():
|
|||||||
store.put("batch", batch["id"], batch)
|
store.put("batch", batch["id"], batch)
|
||||||
continue
|
continue
|
||||||
if batch["status"] in {"running", "pausing", "paused"}:
|
if batch["status"] in {"running", "pausing", "paused"}:
|
||||||
|
if any(str(job["session_id"]) in live_sessions for job in store.jobs(batch["id"])):
|
||||||
|
continue
|
||||||
batch.update(
|
batch.update(
|
||||||
status="paused", reason="服务已重启;请检查中断任务,点击继续执行未开始任务"
|
status="paused", reason="服务已重启;请检查中断任务,点击继续执行未开始任务"
|
||||||
)
|
)
|
||||||
@@ -88,6 +130,9 @@ async def on_terminal(item: dict, status: str):
|
|||||||
raise RuntimeError("任务元数据写入失败")
|
raise RuntimeError("任务元数据写入失败")
|
||||||
if job.get("timed_out"):
|
if job.get("timed_out"):
|
||||||
status = "failed"
|
status = "failed"
|
||||||
|
job.pop("recovery_waiting", None)
|
||||||
|
if status == "completed":
|
||||||
|
resolve_recovery_error(job)
|
||||||
job.update(status="finalizing", terminal_status=status, ended_at=time.time())
|
job.update(status="finalizing", terminal_status=status, ended_at=time.time())
|
||||||
store.save_job(job)
|
store.save_job(job)
|
||||||
try:
|
try:
|
||||||
@@ -117,11 +162,37 @@ async def on_terminal(item: dict, status: str):
|
|||||||
store.save_job(job)
|
store.save_job(job)
|
||||||
|
|
||||||
|
|
||||||
|
async def reconcile_recovered_jobs():
|
||||||
|
"""Finalize surviving workers once they exit; never resubmit their goals."""
|
||||||
|
from apps.admin_console.database.repositories.session_repository import SessionRepository
|
||||||
|
from artemis.runtime import DeviceExecutionLock
|
||||||
|
from artemis.runtime.task_result import read_worker_outcome
|
||||||
|
|
||||||
|
waiting = [job for job in store.jobs() if job.get("recovery_waiting")]
|
||||||
|
if not waiting:
|
||||||
|
return
|
||||||
|
repository = SessionRepository(store.path)
|
||||||
|
owners = {str(owner.session_id) for owner in DeviceExecutionLock.get_active_owners().values()}
|
||||||
|
for job in waiting:
|
||||||
|
sid = str(job["session_id"])
|
||||||
|
row = repository.get_session_by_id(sid) or {}
|
||||||
|
if sid in owners or repository.process_is_alive(row.get("pid")):
|
||||||
|
continue
|
||||||
|
outcome = read_worker_outcome(sid)
|
||||||
|
status = job.get("terminal_status") or outcome.get("task_status") or row.get("status")
|
||||||
|
if status not in TERMINAL:
|
||||||
|
status = "interrupted"
|
||||||
|
job["error"] = job.get("error") or "服务中断;未完成任务需手动重跑"
|
||||||
|
store.save_job(job)
|
||||||
|
await on_terminal({"session_id": sid}, status)
|
||||||
|
|
||||||
|
|
||||||
async def tick_batches():
|
async def tick_batches():
|
||||||
from apps.admin_console.server import task_queue_service
|
from apps.admin_console.server import task_queue_service
|
||||||
from artemis.runtime import device_pool
|
from artemis.runtime import device_pool
|
||||||
from apps.admin_console.services.report_export import export_batch
|
from apps.admin_console.services.report_export import export_batch
|
||||||
|
|
||||||
|
await reconcile_recovered_jobs()
|
||||||
for batch in store.records("batch"):
|
for batch in store.records("batch"):
|
||||||
if batch["status"] not in {"running", "paused"}:
|
if batch["status"] not in {"running", "paused"}:
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -12,6 +12,12 @@ import uuid
|
|||||||
from artemis.config import DB_PATH
|
from artemis.config import DB_PATH
|
||||||
|
|
||||||
|
|
||||||
|
RECOVERY_ERRORS = {
|
||||||
|
"服务中断;未完成任务需手动重跑",
|
||||||
|
"服务中断,需手动重跑",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def identifier(value: str) -> str:
|
def identifier(value: str) -> str:
|
||||||
if not re.fullmatch(r"[A-Za-z0-9_-]{1,100}", value):
|
if not re.fullmatch(r"[A-Za-z0-9_-]{1,100}", value):
|
||||||
raise ValueError("无效的记录编号")
|
raise ValueError("无效的记录编号")
|
||||||
|
|||||||
@@ -88,6 +88,10 @@ def failure_reason(directory: Path, status: str, recorded: str | None) -> str:
|
|||||||
)
|
)
|
||||||
return safe[:500]
|
return safe[:500]
|
||||||
|
|
||||||
|
from apps.admin_console.services.automation_store import RECOVERY_ERRORS
|
||||||
|
|
||||||
|
if status == "completed" and recorded in RECOVERY_ERRORS:
|
||||||
|
return ""
|
||||||
if recorded:
|
if recorded:
|
||||||
return redact(recorded)
|
return redact(recorded)
|
||||||
if status not in {"failed", "interrupted"}:
|
if status not in {"failed", "interrupted"}:
|
||||||
|
|||||||
@@ -108,6 +108,7 @@ class Settings(BaseSettings):
|
|||||||
OPENAI_REASONING_EFFORT: str = "high"
|
OPENAI_REASONING_EFFORT: str = "high"
|
||||||
ARTEMIS_CREDENTIAL_TIMEOUT_SECONDS: float = Field(default=12.0, gt=0)
|
ARTEMIS_CREDENTIAL_TIMEOUT_SECONDS: float = Field(default=12.0, gt=0)
|
||||||
ARTEMIS_MODEL_CACHE_TTL_SECONDS: float = Field(default=10.0, ge=0)
|
ARTEMIS_MODEL_CACHE_TTL_SECONDS: float = Field(default=10.0, ge=0)
|
||||||
|
ARTEMIS_DAEMON_PROBE_TIMEOUT_SECONDS: float = Field(default=2.0, gt=0)
|
||||||
ARTEMIS_VIDEO_TIMEOUT_SECONDS: float = Field(default=120.0, gt=0)
|
ARTEMIS_VIDEO_TIMEOUT_SECONDS: float = Field(default=120.0, gt=0)
|
||||||
ARTEMIS_VIDEO_MAX_RETRIES: int = Field(default=1, ge=0)
|
ARTEMIS_VIDEO_MAX_RETRIES: int = Field(default=1, ge=0)
|
||||||
ARTEMIS_VIDEO_CHUNK_SECONDS: float = Field(default=60.0, gt=0)
|
ARTEMIS_VIDEO_CHUNK_SECONDS: float = Field(default=60.0, gt=0)
|
||||||
|
|||||||
@@ -28,6 +28,9 @@ import urllib.error
|
|||||||
import urllib.request
|
import urllib.request
|
||||||
|
|
||||||
from artemis.config.paths import ROOT_DIR
|
from artemis.config.paths import ROOT_DIR
|
||||||
|
from artemis.config.settings import settings
|
||||||
|
from artemis.runtime.process_probe import pid_is_alive
|
||||||
|
from artemis.runtime.server_lifecycle import is_port_in_use, read_server_info
|
||||||
from artemis.utils.logger import get_logger
|
from artemis.utils.logger import get_logger
|
||||||
|
|
||||||
logger = get_logger(__name__)
|
logger = get_logger(__name__)
|
||||||
@@ -44,10 +47,11 @@ def is_standalone_forced() -> bool:
|
|||||||
def is_daemon_running(
|
def is_daemon_running(
|
||||||
host: str = DEFAULT_DAEMON_HOST,
|
host: str = DEFAULT_DAEMON_HOST,
|
||||||
port: int = DEFAULT_DAEMON_PORT,
|
port: int = DEFAULT_DAEMON_PORT,
|
||||||
timeout: float = 0.3,
|
timeout: float | None = None,
|
||||||
) -> bool:
|
) -> bool:
|
||||||
"""Fast probe to check if the Artemis Daemon HTTP server is responding."""
|
"""Fast probe to check if the Artemis Daemon HTTP server is responding."""
|
||||||
url = f"http://{host}:{port}/api/system/readiness"
|
timeout = timeout if timeout is not None else settings.ARTEMIS_DAEMON_PROBE_TIMEOUT_SECONDS
|
||||||
|
url = f"http://{host}:{port}/api/system/server-status"
|
||||||
try:
|
try:
|
||||||
req = urllib.request.Request(url, headers={"User-Agent": "Artemis-Daemon-Probe"})
|
req = urllib.request.Request(url, headers={"User-Agent": "Artemis-Daemon-Probe"})
|
||||||
with urllib.request.urlopen(req, timeout=timeout) as response:
|
with urllib.request.urlopen(req, timeout=timeout) as response:
|
||||||
@@ -190,6 +194,15 @@ def ensure_daemon_running(
|
|||||||
if is_daemon_running(host, port):
|
if is_daemon_running(host, port):
|
||||||
return True, f"http://{host}:{port}"
|
return True, f"http://{host}:{port}"
|
||||||
|
|
||||||
|
if is_port_in_use(port, host):
|
||||||
|
# A slow/busy HTTP loop is not a stopped daemon. Only trust persisted
|
||||||
|
# metadata belonging to a live process on the requested endpoint.
|
||||||
|
info = read_server_info() or {}
|
||||||
|
if info.get("port") == port and info.get("host") == host and pid_is_alive(info.get("pid")):
|
||||||
|
return True, f"http://{host}:{port}"
|
||||||
|
logger.warning(f"Port {host}:{port} is occupied; refusing to spawn another daemon")
|
||||||
|
return False, None
|
||||||
|
|
||||||
# Auto-spawn Daemon in background
|
# Auto-spawn Daemon in background
|
||||||
spawn_daemon(host, port)
|
spawn_daemon(host, port)
|
||||||
|
|
||||||
|
|||||||
@@ -22,7 +22,7 @@
|
|||||||
|
|
||||||
同一批次按顺序派发。默认失败后继续且不自动重试,也可以选择失败即停止。当前任务因设备异常或超时失败后,默认继续后续用例;设置失败即停止时才停止后续派发。暂停批次只停止后续派发。单个任务可在安全边界暂停并释放设备;批次下一项可继续,暂停的任务待提交指导或点击继续后在设备队列末尾恢复。运行中提交指导会中断当前模型思考或只读诊断,重新观察画面;已经开始的设备动作会安全结束。网页流程会显示指导的提交与领取状态,仍应检查后续动作是否符合指导。暂停等待和重新排队时间不计入该任务执行时限。
|
同一批次按顺序派发。默认失败后继续且不自动重试,也可以选择失败即停止。当前任务因设备异常或超时失败后,默认继续后续用例;设置失败即停止时才停止后续派发。暂停批次只停止后续派发。单个任务可在安全边界暂停并释放设备;批次下一项可继续,暂停的任务待提交指导或点击继续后在设备队列末尾恢复。运行中提交指导会中断当前模型思考或只读诊断,重新观察画面;已经开始的设备动作会安全结束。网页流程会显示指导的提交与领取状态,仍应检查后续动作是否符合指导。暂停等待和重新排队时间不计入该任务执行时限。
|
||||||
|
|
||||||
服务重启后保留批次,等待用户继续未开始的用例。正在执行或暂停的任务标记中断,不自动重做;“复制失败项为新批次”保留原记录并产生新任务编号。
|
服务启动前先取得监听端口,重复启动不会修改现有任务或服务信息。恢复时核对原任务进程和设备锁:仍存活的任务继续等待其真实结果,退出后补齐报告;已经退出且没有终态的任务标记中断,不自动重做,批次等待用户继续未开始的用例。“复制失败项为新批次”保留原记录并产生新任务编号。任务正常完成后,已失效的“服务中断”提示归档为恢复事件,不再显示为当前失败原因;启动时也会纠正已有完成记录中的此类旧提示。
|
||||||
|
|
||||||
每个任务结束后生成所选格式报告。批次 ZIP 包含汇总、每项任务的报告和已保存的有序步骤截图;已开启录屏且文件存在时,还会按任务收录录屏片段。失败、跳过和中断任务也会列入。暂停时下载的是当前快照。录屏与 AI 视频分析分别配置,录屏默认关闭;仅开启分析时,录制内容用于本次分析,结束后不保留下载文件。暂停期间不录制其他任务的设备画面。
|
每个任务结束后生成所选格式报告。批次 ZIP 包含汇总、每项任务的报告和已保存的有序步骤截图;已开启录屏且文件存在时,还会按任务收录录屏片段。失败、跳过和中断任务也会列入。暂停时下载的是当前快照。录屏与 AI 视频分析分别配置,录屏默认关闭;仅开启分析时,录制内容用于本次分析,结束后不保留下载文件。暂停期间不录制其他任务的设备画面。
|
||||||
|
|
||||||
@@ -36,6 +36,8 @@
|
|||||||
|
|
||||||
`.env` 可配置 `ARTEMIS_TASK_TIMEOUT_SECONDS`(单任务总执行时限,默认 1800 秒)、`ARTEMIS_IMPORT_MAX_ROWS`(默认 1000 行)和 `ARTEMIS_IMPORT_MAX_BYTES`(默认 10 MiB)。修改后重启服务。网页的单批次超时设置优先于环境默认值,每行 JSON 的 `timeout_seconds` 优先于批次。单任务、批次和 MCP 均受总执行时限保护,不计排队和用户暂停时间;即使执行进程卡住,也由父进程终止并保存失败原因。
|
`.env` 可配置 `ARTEMIS_TASK_TIMEOUT_SECONDS`(单任务总执行时限,默认 1800 秒)、`ARTEMIS_IMPORT_MAX_ROWS`(默认 1000 行)和 `ARTEMIS_IMPORT_MAX_BYTES`(默认 10 MiB)。修改后重启服务。网页的单批次超时设置优先于环境默认值,每行 JSON 的 `timeout_seconds` 优先于批次。单任务、批次和 MCP 均受总执行时限保护,不计排队和用户暂停时间;即使执行进程卡住,也由父进程终止并保存失败原因。
|
||||||
|
|
||||||
|
守护服务通过轻量状态接口探测,`ARTEMIS_DAEMON_PROBE_TIMEOUT_SECONDS` 默认 2 秒。短暂超时且端口仍被存活的原服务占用时复用原服务,不重复启动;未知进程占用端口时明确拒绝覆盖。
|
||||||
|
|
||||||
网页“功能配置 → 执行保护”及 MCP `mobile_config(section="capabilities")` 可配置总时限、设备观测时限(默认 60 秒)、设备观测次数(默认 3 次)、ADB 命令超时(默认 10 秒)、连续动作失败次数(默认 3 次)。任务 JSON 可用 `observation_timeout_seconds`、`observation_max_attempts`、`device_command_timeout_seconds`、`action_failure_attempts` 覆盖。截图失效或收到占位图时先有限重取,耗尽后结束任务,不把旧画面发给模型;有效截图即使没有 XML 也可按坐标定位。连续动作失败时先刷新画面再尝试兜底,兜底耗尽后结束,不循环重提用例。
|
网页“功能配置 → 执行保护”及 MCP `mobile_config(section="capabilities")` 可配置总时限、设备观测时限(默认 60 秒)、设备观测次数(默认 3 次)、ADB 命令超时(默认 10 秒)、连续动作失败次数(默认 3 次)。任务 JSON 可用 `observation_timeout_seconds`、`observation_max_attempts`、`device_command_timeout_seconds`、`action_failure_attempts` 覆盖。截图失效或收到占位图时先有限重取,耗尽后结束任务,不把旧画面发给模型;有效截图即使没有 XML 也可按坐标定位。连续动作失败时先刷新画面再尝试兜底,兜底耗尽后结束,不循环重提用例。
|
||||||
|
|
||||||
批次、任务元数据和报告偏好保存在现有 SQLite 数据库的 `automation_records`、`automation_jobs` 表。全局“清空任务历史”会清除批次与执行元数据;它与上文的单批次删除不同,已导出的文件会保留。
|
批次、任务元数据和报告偏好保存在现有 SQLite 数据库的 `automation_records`、`automation_jobs` 表。全局“清空任务历史”会清除批次与执行元数据;它与上文的单批次删除不同,已导出的文件会保留。
|
||||||
|
|||||||
@@ -300,6 +300,116 @@ def test_restart_never_repeats_started_work(isolated):
|
|||||||
assert isolated.jobs(retry["id"])[0]["session_id"] != job["session_id"]
|
assert isolated.jobs(retry["id"])[0]["session_id"] != job["session_id"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("liveness_source", ["pid", "device_lock"])
|
||||||
|
def test_recovery_preserves_live_runner_and_batch(isolated, monkeypatch, liveness_source):
|
||||||
|
import os
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from artemis.runtime import DeviceExecutionLock
|
||||||
|
|
||||||
|
batch = service.create_batch(payload())
|
||||||
|
service.batch_action(batch["id"], "resume")
|
||||||
|
job = isolated.jobs(batch["id"])[0]
|
||||||
|
isolated.save_job({**job, "status": "running"})
|
||||||
|
if liveness_source == "pid":
|
||||||
|
with sqlite3.connect(isolated.path) as conn:
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO sessions(session_id,initial_goal,start_time,status,device_info,pid) "
|
||||||
|
"VALUES (?,?,?,?,?,?)",
|
||||||
|
(job["session_id"], "goal", 1, "running", "{}", os.getpid()),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
DeviceExecutionLock,
|
||||||
|
"get_active_owners",
|
||||||
|
lambda: (
|
||||||
|
{"device": SimpleNamespace(session_id=job["session_id"])}
|
||||||
|
if liveness_source == "device_lock"
|
||||||
|
else {}
|
||||||
|
),
|
||||||
|
)
|
||||||
|
service.recover_batches()
|
||||||
|
assert isolated.job(job["session_id"])["status"] == "running"
|
||||||
|
assert not isolated.job(job["session_id"]).get("error")
|
||||||
|
assert isolated.get("batch", batch["id"])["status"] == "running"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_natural_completion_resolves_old_recovery_error_in_job_report_and_history(
|
||||||
|
client, isolated
|
||||||
|
):
|
||||||
|
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||||||
|
message = "服务中断;未完成任务需手动重跑"
|
||||||
|
isolated.save_job({**job, "status": "interrupted", "error": message})
|
||||||
|
await service.on_terminal({"session_id": job["session_id"]}, "completed")
|
||||||
|
saved = isolated.job(job["session_id"])
|
||||||
|
assert saved["status"] == "completed"
|
||||||
|
assert not saved.get("error")
|
||||||
|
assert saved["recovery_events"][0]["message"] == message
|
||||||
|
assert reports.session_report(job["session_id"])["error"] == ""
|
||||||
|
assert not client.get("/api/automation/history").json()["items"][0].get("error")
|
||||||
|
|
||||||
|
|
||||||
|
def test_recovery_repairs_completed_legacy_warning_but_preserves_real_failure(isolated):
|
||||||
|
batch = service.create_batch(payload())
|
||||||
|
first, second = isolated.jobs(batch["id"])
|
||||||
|
isolated.save_job({**first, "status": "completed", "error": "服务中断,需手动重跑"})
|
||||||
|
isolated.save_job({**second, "status": "failed", "error": "任务执行超时(30 秒)"})
|
||||||
|
assert reports.session_report(first["session_id"])["error"] == ""
|
||||||
|
service.recover_batches()
|
||||||
|
assert not isolated.job(first["session_id"]).get("error")
|
||||||
|
assert "服务中断" in isolated.job(first["session_id"])["recovery_events"][0]["message"]
|
||||||
|
assert "超时" in isolated.job(second["session_id"])["error"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_surviving_worker_is_finalized_when_it_exits_without_resubmission(
|
||||||
|
isolated, monkeypatch
|
||||||
|
):
|
||||||
|
import os
|
||||||
|
from artemis.runtime import DeviceExecutionLock
|
||||||
|
|
||||||
|
batch = service.create_batch(payload())
|
||||||
|
service.batch_action(batch["id"], "resume")
|
||||||
|
job = isolated.jobs(batch["id"])[0]
|
||||||
|
isolated.save_job({**job, "status": "running"})
|
||||||
|
monkeypatch.setattr(DeviceExecutionLock, "get_active_owners", lambda: {})
|
||||||
|
with sqlite3.connect(isolated.path) as conn:
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO sessions(session_id,initial_goal,start_time,status,device_info,pid) "
|
||||||
|
"VALUES (?,?,?,?,?,?)",
|
||||||
|
(job["session_id"], "goal", time.time(), "running", "{}", os.getpid()),
|
||||||
|
)
|
||||||
|
service.recover_batches()
|
||||||
|
await service.reconcile_recovered_jobs()
|
||||||
|
assert isolated.job(job["session_id"])["status"] == "running"
|
||||||
|
with sqlite3.connect(isolated.path) as conn:
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE sessions SET pid=NULL,status='completed' WHERE session_id=?",
|
||||||
|
(job["session_id"],),
|
||||||
|
)
|
||||||
|
await service.reconcile_recovered_jobs()
|
||||||
|
saved = isolated.job(job["session_id"])
|
||||||
|
assert saved["status"] == "completed"
|
||||||
|
assert saved["report_ready"]
|
||||||
|
assert not saved.get("recovery_waiting")
|
||||||
|
assert not saved.get("error")
|
||||||
|
assert len(isolated.jobs(batch["id"])) == 2
|
||||||
|
assert isolated.jobs(batch["id"])[1]["status"] == "pending"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_surviving_worker_that_dies_without_result_is_interrupted(isolated, monkeypatch):
|
||||||
|
from artemis.runtime import DeviceExecutionLock
|
||||||
|
|
||||||
|
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||||||
|
isolated.save_job({**job, "status": "running", "recovery_waiting": True})
|
||||||
|
monkeypatch.setattr(DeviceExecutionLock, "get_active_owners", lambda: {})
|
||||||
|
await service.reconcile_recovered_jobs()
|
||||||
|
saved = isolated.job(job["session_id"])
|
||||||
|
assert saved["status"] == "interrupted"
|
||||||
|
assert "服务中断" in saved["error"]
|
||||||
|
assert not saved.get("recovery_waiting")
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_batch_admission_idempotent_and_bound_device(isolated, monkeypatch):
|
async def test_batch_admission_idempotent_and_bound_device(isolated, monkeypatch):
|
||||||
from apps.admin_console import server
|
from apps.admin_console import server
|
||||||
|
|||||||
@@ -15,6 +15,7 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
from contextlib import suppress
|
from contextlib import suppress
|
||||||
import signal
|
import signal
|
||||||
|
import socket
|
||||||
from unittest.mock import AsyncMock, MagicMock, patch
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
@@ -23,10 +24,55 @@ import uvicorn
|
|||||||
from apps.admin_console.core.state import state
|
from apps.admin_console.core.state import state
|
||||||
from apps.admin_console.routers.tasks import stream_events
|
from apps.admin_console.routers.tasks import stream_events
|
||||||
from apps.admin_console.server import ArtemisUvicornServer, app, on_shutdown
|
from apps.admin_console.server import ArtemisUvicornServer, app, on_shutdown
|
||||||
|
from apps.admin_console import server as server_module
|
||||||
|
|
||||||
WINDOWS_FORCE_SIGNAL = getattr(signal, "SIGBREAK", signal.SIGTERM)
|
WINDOWS_FORCE_SIGNAL = getattr(signal, "SIGBREAK", signal.SIGTERM)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("reload", [False, True])
|
||||||
|
def test_occupied_port_never_initializes_or_overwrites_server_metadata(
|
||||||
|
tmp_path, monkeypatch, reload
|
||||||
|
):
|
||||||
|
from artemis.runtime import server_lifecycle
|
||||||
|
|
||||||
|
metadata = tmp_path / "server.json"
|
||||||
|
metadata.write_text('{"pid":123,"lifecycle_token":"original"}', encoding="utf-8")
|
||||||
|
monkeypatch.setattr(server_lifecycle, "get_server_info_file", lambda: metadata)
|
||||||
|
before = metadata.read_bytes()
|
||||||
|
with (
|
||||||
|
socket.create_server(("127.0.0.1", 0)) as original,
|
||||||
|
patch.object(server_module.ArtemisUvicornServer, "run") as run,
|
||||||
|
patch.object(server_module, "write_server_info") as publish,
|
||||||
|
):
|
||||||
|
with pytest.raises(SystemExit, match="1"):
|
||||||
|
server_module.run_ui_server("127.0.0.1", original.getsockname()[1], reload=reload)
|
||||||
|
run.assert_not_called()
|
||||||
|
publish.assert_not_called()
|
||||||
|
assert metadata.read_bytes() == before
|
||||||
|
|
||||||
|
|
||||||
|
def test_server_holds_port_before_running_lifespan(monkeypatch):
|
||||||
|
monkeypatch.setattr(state, "port", 0, raising=False)
|
||||||
|
monkeypatch.setattr(state, "host", "127.0.0.1", raising=False)
|
||||||
|
observed = []
|
||||||
|
|
||||||
|
def run(_server, sockets):
|
||||||
|
listener = sockets[0]
|
||||||
|
port = listener.getsockname()[1]
|
||||||
|
observed.append(listener)
|
||||||
|
assert state.port == port
|
||||||
|
with pytest.raises(OSError), socket.create_server(("127.0.0.1", port)):
|
||||||
|
pass
|
||||||
|
|
||||||
|
monkeypatch.setattr(server_module.ArtemisUvicornServer, "run", run)
|
||||||
|
monkeypatch.setattr(server_module, "clear_server_info", lambda **kwargs: None)
|
||||||
|
with patch.object(server_module, "write_server_info") as publish:
|
||||||
|
server_module.run_ui_server("127.0.0.1", 0)
|
||||||
|
publish.assert_not_called() # Publication belongs to the later lifespan.
|
||||||
|
assert len(observed) == 1
|
||||||
|
assert observed[0].fileno() == -1
|
||||||
|
|
||||||
|
|
||||||
def _pending_queue_get_tasks():
|
def _pending_queue_get_tasks():
|
||||||
return {
|
return {
|
||||||
task
|
task
|
||||||
|
|||||||
@@ -93,6 +93,7 @@ def test_ensure_daemon_running_standalone_override(monkeypatch):
|
|||||||
def test_ensure_daemon_running_auto_spawns():
|
def test_ensure_daemon_running_auto_spawns():
|
||||||
with (
|
with (
|
||||||
patch("artemis.runtime.daemon_client.is_daemon_running", side_effect=[False, True]),
|
patch("artemis.runtime.daemon_client.is_daemon_running", side_effect=[False, True]),
|
||||||
|
patch("artemis.runtime.daemon_client.is_port_in_use", return_value=False),
|
||||||
patch("artemis.runtime.daemon_client.spawn_daemon") as mock_spawn,
|
patch("artemis.runtime.daemon_client.spawn_daemon") as mock_spawn,
|
||||||
):
|
):
|
||||||
ok, base_url = ensure_daemon_running(timeout=1.0, wait_ready=True)
|
ok, base_url = ensure_daemon_running(timeout=1.0, wait_ready=True)
|
||||||
@@ -101,6 +102,42 @@ def test_ensure_daemon_running_auto_spawns():
|
|||||||
mock_spawn.assert_called_once()
|
mock_spawn.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_busy_live_daemon_is_reused_without_spawning():
|
||||||
|
with (
|
||||||
|
patch("artemis.runtime.daemon_client.is_daemon_running", return_value=False),
|
||||||
|
patch("artemis.runtime.daemon_client.is_port_in_use", return_value=True),
|
||||||
|
patch(
|
||||||
|
"artemis.runtime.daemon_client.read_server_info",
|
||||||
|
return_value={"host": "127.0.0.1", "port": 8000, "pid": 12345},
|
||||||
|
),
|
||||||
|
patch("artemis.runtime.daemon_client.pid_is_alive", return_value=True),
|
||||||
|
patch("artemis.runtime.daemon_client.spawn_daemon") as spawn,
|
||||||
|
):
|
||||||
|
assert ensure_daemon_running() == (True, "http://127.0.0.1:8000")
|
||||||
|
spawn.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_unknown_port_owner_is_not_overwritten():
|
||||||
|
with (
|
||||||
|
patch("artemis.runtime.daemon_client.is_daemon_running", return_value=False),
|
||||||
|
patch("artemis.runtime.daemon_client.is_port_in_use", return_value=True),
|
||||||
|
patch("artemis.runtime.daemon_client.read_server_info", return_value=None),
|
||||||
|
patch("artemis.runtime.daemon_client.spawn_daemon") as spawn,
|
||||||
|
):
|
||||||
|
assert ensure_daemon_running() == (False, None)
|
||||||
|
spawn.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_liveness_probe_uses_lightweight_endpoint_and_configured_timeout(monkeypatch):
|
||||||
|
from artemis.config.settings import settings
|
||||||
|
|
||||||
|
monkeypatch.setattr(settings, "ARTEMIS_DAEMON_PROBE_TIMEOUT_SECONDS", 7.0)
|
||||||
|
with patch("urllib.request.urlopen", return_value=_daemon_ok_response(b"{}")) as probe:
|
||||||
|
assert is_daemon_running()
|
||||||
|
assert probe.call_args.args[0].full_url.endswith("/api/system/server-status")
|
||||||
|
assert probe.call_args.kwargs["timeout"] == 7.0
|
||||||
|
|
||||||
|
|
||||||
def test_submit_task_to_daemon():
|
def test_submit_task_to_daemon():
|
||||||
mock_resp = MagicMock()
|
mock_resp = MagicMock()
|
||||||
mock_resp.status = 200
|
mock_resp.status = 200
|
||||||
|
|||||||
Reference in new issue
Block a user