fix: preserve live tasks during daemon recovery

This commit is contained in:
chenmingxuan committed 2026-10-08 17:33:48 +08:00
1 parent 079d7b1e28
commit 0355af7833
11 files changed
+326 -16

No files matched your search

+2
View File
@@ -50,6 +50,8 @@ ARTEMIS_CHECKPOINT_TIMEOUT_SECONDS=180
ARTEMIS_SETTLEMENT_TIMEOUT_SECONDS=120 ARTEMIS_SETTLEMENT_TIMEOUT_SECONDS=120
ARTEMIS_CREDENTIAL_TIMEOUT_SECONDS=12 ARTEMIS_CREDENTIAL_TIMEOUT_SECONDS=12
ARTEMIS_MODEL_CACHE_TTL_SECONDS=10 ARTEMIS_MODEL_CACHE_TTL_SECONDS=10
# 守护服务轻量探测超时;探测超时且端口仍占用时,不重复启动服务。
ARTEMIS_DAEMON_PROBE_TIMEOUT_SECONDS=2
# 模型重试耗尽后的暂停等待上限(秒);0 表示无限等待 # 模型重试耗尽后的暂停等待上限(秒);0 表示无限等待
LLM_PAUSE_TIMEOUT_SECONDS=900 LLM_PAUSE_TIMEOUT_SECONDS=900
+30 -12
View File
@@ -24,6 +24,7 @@ import os
from pathlib import Path from pathlib import Path
import secrets import secrets
import signal import signal
import socket
import sys import sys
from types import FrameType from types import FrameType
@@ -421,6 +422,14 @@ queue_goals = state.queue_goals
class ArtemisUvicornServer(uvicorn.Server): class ArtemisUvicornServer(uvicorn.Server):
"""Notify application streams before Uvicorn waits for them to close.""" """Notify application streams before Uvicorn waits for them to close."""
async def startup(self, sockets: list[socket.socket] | None = None) -> None:
# Reload workers start in a fresh interpreter; restore the actual bind
# address there before lifespan publishes runtime metadata.
state.host = self.config.host
state.port = self.config.port
app.state.uvicorn_server = self
await super().startup(sockets=sockets)
@staticmethod @staticmethod
def _task_is_active() -> bool: def _task_is_active() -> bool:
"""Check task state without doing process I/O from a signal handler.""" """Check task state without doing process I/O from a signal handler."""
@@ -470,31 +479,40 @@ class ArtemisUvicornServer(uvicorn.Server):
def run_ui_server(host: str, port: int, reload: bool = False) -> None: def run_ui_server(host: str, port: int, reload: bool = False) -> None:
"""Run the UI server with bounded, signal-aware graceful shutdown.""" """Run the UI server with bounded, signal-aware graceful shutdown."""
# Reserve a listening socket before publishing metadata or running lifespan.
# Uvicorn normally runs application startup *before* binding its port; a
# duplicate daemon would otherwise recover jobs owned by the real server.
family = socket.AF_INET6 if ":" in host else socket.AF_INET
try:
listener = socket.create_server((host, port), family=family)
except OSError as exc:
print(f"无法启动服务:{host}:{port} 端口不可用({exc})。现有服务和任务保持不变。")
raise SystemExit(1) from exc
listener.set_inheritable(True)
port = listener.getsockname()[1]
state.host = host state.host = host
state.port = port state.port = port
write_server_info(port=port, host=host, lifecycle_token=LIFECYCLE_TOKEN)
try: try:
if reload: config = uvicorn.Config(
uvicorn.run( "apps.admin_console.server:app" if reload else app,
"apps.admin_console.server:app",
host=host, host=host,
port=port, port=port,
reload=True, reload=reload,
timeout_graceful_shutdown=5, timeout_graceful_shutdown=5,
) )
if reload:
from uvicorn.supervisors import ChangeReload
server = ArtemisUvicornServer(config)
ChangeReload(config, target=server.run, sockets=[listener]).run()
return return
config = uvicorn.Config(
app,
host=host,
port=port,
timeout_graceful_shutdown=5,
)
server = ArtemisUvicornServer(config) server = ArtemisUvicornServer(config)
app.state.uvicorn_server = server app.state.uvicorn_server = server
server.run() server.run(sockets=[listener])
finally: finally:
listener.close()
app.state.uvicorn_server = None app.state.uvicorn_server = None
clear_server_info(port=port, lifecycle_token=LIFECYCLE_TOKEN) clear_server_info(port=port, lifecycle_token=LIFECYCLE_TOKEN)
@@ -8,7 +8,7 @@ import time
from uuid import UUID from uuid import UUID
from apps.admin_console.core.state import state from apps.admin_console.core.state import state
from apps.admin_console.services.automation_store import store, new_id from apps.admin_console.services.automation_store import RECOVERY_ERRORS, store, new_id
from apps.admin_console.services.task_import import case_goal, case_title from apps.admin_console.services.task_import import case_goal, case_title
from artemis.config.settings import settings from artemis.config.settings import settings
from artemis.runtime.task_limits import task_timeout from artemis.runtime.task_limits import task_timeout
@@ -42,9 +42,46 @@ def register_submission(item: dict, case: dict | None = None, formats=None, **ex
) )
def resolve_recovery_error(job: dict) -> bool:
"""Archive a superseded recovery warning without erasing execution failures."""
if job.get("error") not in RECOVERY_ERRORS:
return False
job.setdefault("recovery_events", []).append(
{"message": job.pop("error"), "resolved_at": time.time()}
)
return True
def recover_batches(): def recover_batches():
"""Recovery never silently repeats a potentially side-effecting interrupted task.""" """Recovery never silently repeats a potentially side-effecting interrupted task."""
from apps.admin_console.database.repositories.session_repository import SessionRepository
from artemis.runtime import DeviceExecutionLock
repository = SessionRepository(store.path)
live_sessions = {
str(owner.session_id)
for owner in DeviceExecutionLock.get_active_owners().values()
if owner.session_id
}
live_sessions.update(
str(row["session_id"])
for row in repository.get_all_sessions()
if row.get("status") not in TERMINAL and repository.process_is_alive(row.get("pid"))
)
live_sessions.update(
str(item["session_id"])
for item in state.queue_items
if isinstance(item, dict) and item.get("session_id") and item.get("status") not in TERMINAL
)
for job in store.jobs(): for job in store.jobs():
if job["status"] == "completed" and resolve_recovery_error(job):
store.save_job(job)
if str(job["session_id"]) in live_sessions:
if job["status"] not in TERMINAL and str(job["session_id"]) not in state.active_runs:
job["recovery_waiting"] = True
store.save_job(job)
logger.info("Recovery preserved live session %s", job["session_id"])
continue
if job["status"] in {"running", "queued", "finalizing", "task_paused", "resuming"}: if job["status"] in {"running", "queued", "finalizing", "task_paused", "resuming"}:
job.update( job.update(
status=job.get("terminal_status", "interrupted"), status=job.get("terminal_status", "interrupted"),
@@ -52,6 +89,9 @@ def recover_batches():
or ("" if job.get("terminal_status") else "服务中断;未完成任务需手动重跑"), or ("" if job.get("terminal_status") else "服务中断;未完成任务需手动重跑"),
ended_at=job.get("ended_at") or time.time(), ended_at=job.get("ended_at") or time.time(),
) )
if job["status"] == "completed":
resolve_recovery_error(job)
logger.info("Recovery resolved session %s to %s", job["session_id"], job["status"])
store.save_job(job) store.save_job(job)
for batch in store.records("batch"): for batch in store.records("batch"):
if has_legacy_replay(batch["id"]): if has_legacy_replay(batch["id"]):
@@ -64,6 +104,8 @@ def recover_batches():
store.put("batch", batch["id"], batch) store.put("batch", batch["id"], batch)
continue continue
if batch["status"] in {"running", "pausing", "paused"}: if batch["status"] in {"running", "pausing", "paused"}:
if any(str(job["session_id"]) in live_sessions for job in store.jobs(batch["id"])):
continue
batch.update( batch.update(
status="paused", reason="服务已重启;请检查中断任务,点击继续执行未开始任务" status="paused", reason="服务已重启;请检查中断任务,点击继续执行未开始任务"
) )
@@ -88,6 +130,9 @@ async def on_terminal(item: dict, status: str):
raise RuntimeError("任务元数据写入失败") raise RuntimeError("任务元数据写入失败")
if job.get("timed_out"): if job.get("timed_out"):
status = "failed" status = "failed"
job.pop("recovery_waiting", None)
if status == "completed":
resolve_recovery_error(job)
job.update(status="finalizing", terminal_status=status, ended_at=time.time()) job.update(status="finalizing", terminal_status=status, ended_at=time.time())
store.save_job(job) store.save_job(job)
try: try:
@@ -117,11 +162,37 @@ async def on_terminal(item: dict, status: str):
store.save_job(job) store.save_job(job)
async def reconcile_recovered_jobs():
"""Finalize surviving workers once they exit; never resubmit their goals."""
from apps.admin_console.database.repositories.session_repository import SessionRepository
from artemis.runtime import DeviceExecutionLock
from artemis.runtime.task_result import read_worker_outcome
waiting = [job for job in store.jobs() if job.get("recovery_waiting")]
if not waiting:
return
repository = SessionRepository(store.path)
owners = {str(owner.session_id) for owner in DeviceExecutionLock.get_active_owners().values()}
for job in waiting:
sid = str(job["session_id"])
row = repository.get_session_by_id(sid) or {}
if sid in owners or repository.process_is_alive(row.get("pid")):
continue
outcome = read_worker_outcome(sid)
status = job.get("terminal_status") or outcome.get("task_status") or row.get("status")
if status not in TERMINAL:
status = "interrupted"
job["error"] = job.get("error") or "服务中断;未完成任务需手动重跑"
store.save_job(job)
await on_terminal({"session_id": sid}, status)
async def tick_batches(): async def tick_batches():
from apps.admin_console.server import task_queue_service from apps.admin_console.server import task_queue_service
from artemis.runtime import device_pool from artemis.runtime import device_pool
from apps.admin_console.services.report_export import export_batch from apps.admin_console.services.report_export import export_batch
await reconcile_recovered_jobs()
for batch in store.records("batch"): for batch in store.records("batch"):
if batch["status"] not in {"running", "paused"}: if batch["status"] not in {"running", "paused"}:
continue continue
@@ -12,6 +12,12 @@ import uuid
from artemis.config import DB_PATH from artemis.config import DB_PATH
RECOVERY_ERRORS = {
"服务中断;未完成任务需手动重跑",
"服务中断,需手动重跑",
}
def identifier(value: str) -> str: def identifier(value: str) -> str:
if not re.fullmatch(r"[A-Za-z0-9_-]{1,100}", value): if not re.fullmatch(r"[A-Za-z0-9_-]{1,100}", value):
raise ValueError("无效的记录编号") raise ValueError("无效的记录编号")
@@ -88,6 +88,10 @@ def failure_reason(directory: Path, status: str, recorded: str | None) -> str:
) )
return safe[:500] return safe[:500]
from apps.admin_console.services.automation_store import RECOVERY_ERRORS
if status == "completed" and recorded in RECOVERY_ERRORS:
return ""
if recorded: if recorded:
return redact(recorded) return redact(recorded)
if status not in {"failed", "interrupted"}: if status not in {"failed", "interrupted"}:
+1
View File
@@ -108,6 +108,7 @@ class Settings(BaseSettings):
OPENAI_REASONING_EFFORT: str = "high" OPENAI_REASONING_EFFORT: str = "high"
ARTEMIS_CREDENTIAL_TIMEOUT_SECONDS: float = Field(default=12.0, gt=0) ARTEMIS_CREDENTIAL_TIMEOUT_SECONDS: float = Field(default=12.0, gt=0)
ARTEMIS_MODEL_CACHE_TTL_SECONDS: float = Field(default=10.0, ge=0) ARTEMIS_MODEL_CACHE_TTL_SECONDS: float = Field(default=10.0, ge=0)
ARTEMIS_DAEMON_PROBE_TIMEOUT_SECONDS: float = Field(default=2.0, gt=0)
ARTEMIS_VIDEO_TIMEOUT_SECONDS: float = Field(default=120.0, gt=0) ARTEMIS_VIDEO_TIMEOUT_SECONDS: float = Field(default=120.0, gt=0)
ARTEMIS_VIDEO_MAX_RETRIES: int = Field(default=1, ge=0) ARTEMIS_VIDEO_MAX_RETRIES: int = Field(default=1, ge=0)
ARTEMIS_VIDEO_CHUNK_SECONDS: float = Field(default=60.0, gt=0) ARTEMIS_VIDEO_CHUNK_SECONDS: float = Field(default=60.0, gt=0)
+15 -2
View File
@@ -28,6 +28,9 @@ import urllib.error
import urllib.request import urllib.request
from artemis.config.paths import ROOT_DIR from artemis.config.paths import ROOT_DIR
from artemis.config.settings import settings
from artemis.runtime.process_probe import pid_is_alive
from artemis.runtime.server_lifecycle import is_port_in_use, read_server_info
from artemis.utils.logger import get_logger from artemis.utils.logger import get_logger
logger = get_logger(__name__) logger = get_logger(__name__)
@@ -44,10 +47,11 @@ def is_standalone_forced() -> bool:
def is_daemon_running( def is_daemon_running(
host: str = DEFAULT_DAEMON_HOST, host: str = DEFAULT_DAEMON_HOST,
port: int = DEFAULT_DAEMON_PORT, port: int = DEFAULT_DAEMON_PORT,
timeout: float = 0.3, timeout: float | None = None,
) -> bool: ) -> bool:
"""Fast probe to check if the Artemis Daemon HTTP server is responding.""" """Fast probe to check if the Artemis Daemon HTTP server is responding."""
url = f"http://{host}:{port}/api/system/readiness" timeout = timeout if timeout is not None else settings.ARTEMIS_DAEMON_PROBE_TIMEOUT_SECONDS
url = f"http://{host}:{port}/api/system/server-status"
try: try:
req = urllib.request.Request(url, headers={"User-Agent": "Artemis-Daemon-Probe"}) req = urllib.request.Request(url, headers={"User-Agent": "Artemis-Daemon-Probe"})
with urllib.request.urlopen(req, timeout=timeout) as response: with urllib.request.urlopen(req, timeout=timeout) as response:
@@ -190,6 +194,15 @@ def ensure_daemon_running(
if is_daemon_running(host, port): if is_daemon_running(host, port):
return True, f"http://{host}:{port}" return True, f"http://{host}:{port}"
if is_port_in_use(port, host):
# A slow/busy HTTP loop is not a stopped daemon. Only trust persisted
# metadata belonging to a live process on the requested endpoint.
info = read_server_info() or {}
if info.get("port") == port and info.get("host") == host and pid_is_alive(info.get("pid")):
return True, f"http://{host}:{port}"
logger.warning(f"Port {host}:{port} is occupied; refusing to spawn another daemon")
return False, None
# Auto-spawn Daemon in background # Auto-spawn Daemon in background
spawn_daemon(host, port) spawn_daemon(host, port)
+3 -1
View File
@@ -22,7 +22,7 @@
同一批次按顺序派发。默认失败后继续且不自动重试,也可以选择失败即停止。当前任务因设备异常或超时失败后,默认继续后续用例;设置失败即停止时才停止后续派发。暂停批次只停止后续派发。单个任务可在安全边界暂停并释放设备;批次下一项可继续,暂停的任务待提交指导或点击继续后在设备队列末尾恢复。运行中提交指导会中断当前模型思考或只读诊断,重新观察画面;已经开始的设备动作会安全结束。网页流程会显示指导的提交与领取状态,仍应检查后续动作是否符合指导。暂停等待和重新排队时间不计入该任务执行时限。 同一批次按顺序派发。默认失败后继续且不自动重试,也可以选择失败即停止。当前任务因设备异常或超时失败后,默认继续后续用例;设置失败即停止时才停止后续派发。暂停批次只停止后续派发。单个任务可在安全边界暂停并释放设备;批次下一项可继续,暂停的任务待提交指导或点击继续后在设备队列末尾恢复。运行中提交指导会中断当前模型思考或只读诊断,重新观察画面;已经开始的设备动作会安全结束。网页流程会显示指导的提交与领取状态,仍应检查后续动作是否符合指导。暂停等待和重新排队时间不计入该任务执行时限。
服务重启后保留批次,等待用户继续未开始的用例。正在执行或暂停的任务标记中断,不自动重做;“复制失败项为新批次”保留原记录并产生新任务编号。 服务启动前先取得监听端口,重复启动不会修改现有任务或服务信息。恢复时核对原任务进程和设备锁:仍存活的任务继续等待其真实结果,退出后补齐报告;已经退出且没有终态的任务标记中断,不自动重做,批次等待用户继续未开始的用例。“复制失败项为新批次”保留原记录并产生新任务编号。任务正常完成后,已失效的“服务中断”提示归档为恢复事件,不再显示为当前失败原因;启动时也会纠正已有完成记录中的此类旧提示。
每个任务结束后生成所选格式报告。批次 ZIP 包含汇总、每项任务的报告和已保存的有序步骤截图;已开启录屏且文件存在时,还会按任务收录录屏片段。失败、跳过和中断任务也会列入。暂停时下载的是当前快照。录屏与 AI 视频分析分别配置,录屏默认关闭;仅开启分析时,录制内容用于本次分析,结束后不保留下载文件。暂停期间不录制其他任务的设备画面。 每个任务结束后生成所选格式报告。批次 ZIP 包含汇总、每项任务的报告和已保存的有序步骤截图;已开启录屏且文件存在时,还会按任务收录录屏片段。失败、跳过和中断任务也会列入。暂停时下载的是当前快照。录屏与 AI 视频分析分别配置,录屏默认关闭;仅开启分析时,录制内容用于本次分析,结束后不保留下载文件。暂停期间不录制其他任务的设备画面。
@@ -36,6 +36,8 @@
`.env` 可配置 `ARTEMIS_TASK_TIMEOUT_SECONDS`(单任务总执行时限,默认 1800 秒)、`ARTEMIS_IMPORT_MAX_ROWS`(默认 1000 行)和 `ARTEMIS_IMPORT_MAX_BYTES`(默认 10 MiB)。修改后重启服务。网页的单批次超时设置优先于环境默认值,每行 JSON 的 `timeout_seconds` 优先于批次。单任务、批次和 MCP 均受总执行时限保护,不计排队和用户暂停时间;即使执行进程卡住,也由父进程终止并保存失败原因。 `.env` 可配置 `ARTEMIS_TASK_TIMEOUT_SECONDS`(单任务总执行时限,默认 1800 秒)、`ARTEMIS_IMPORT_MAX_ROWS`(默认 1000 行)和 `ARTEMIS_IMPORT_MAX_BYTES`(默认 10 MiB)。修改后重启服务。网页的单批次超时设置优先于环境默认值,每行 JSON 的 `timeout_seconds` 优先于批次。单任务、批次和 MCP 均受总执行时限保护,不计排队和用户暂停时间;即使执行进程卡住,也由父进程终止并保存失败原因。
守护服务通过轻量状态接口探测,`ARTEMIS_DAEMON_PROBE_TIMEOUT_SECONDS` 默认 2 秒。短暂超时且端口仍被存活的原服务占用时复用原服务,不重复启动;未知进程占用端口时明确拒绝覆盖。
网页“功能配置 → 执行保护”及 MCP `mobile_config(section="capabilities")` 可配置总时限、设备观测时限(默认 60 秒)、设备观测次数(默认 3 次)、ADB 命令超时(默认 10 秒)、连续动作失败次数(默认 3 次)。任务 JSON 可用 `observation_timeout_seconds`、`observation_max_attempts`、`device_command_timeout_seconds`、`action_failure_attempts` 覆盖。截图失效或收到占位图时先有限重取,耗尽后结束任务,不把旧画面发给模型;有效截图即使没有 XML 也可按坐标定位。连续动作失败时先刷新画面再尝试兜底,兜底耗尽后结束,不循环重提用例。 网页“功能配置 → 执行保护”及 MCP `mobile_config(section="capabilities")` 可配置总时限、设备观测时限(默认 60 秒)、设备观测次数(默认 3 次)、ADB 命令超时(默认 10 秒)、连续动作失败次数(默认 3 次)。任务 JSON 可用 `observation_timeout_seconds`、`observation_max_attempts`、`device_command_timeout_seconds`、`action_failure_attempts` 覆盖。截图失效或收到占位图时先有限重取,耗尽后结束任务,不把旧画面发给模型;有效截图即使没有 XML 也可按坐标定位。连续动作失败时先刷新画面再尝试兜底,兜底耗尽后结束,不循环重提用例。
批次、任务元数据和报告偏好保存在现有 SQLite 数据库的 `automation_records`、`automation_jobs` 表。全局“清空任务历史”会清除批次与执行元数据;它与上文的单批次删除不同,已导出的文件会保留。 批次、任务元数据和报告偏好保存在现有 SQLite 数据库的 `automation_records`、`automation_jobs` 表。全局“清空任务历史”会清除批次与执行元数据;它与上文的单批次删除不同,已导出的文件会保留。
+110
View File
@@ -300,6 +300,116 @@ def test_restart_never_repeats_started_work(isolated):
assert isolated.jobs(retry["id"])[0]["session_id"] != job["session_id"] assert isolated.jobs(retry["id"])[0]["session_id"] != job["session_id"]
@pytest.mark.parametrize("liveness_source", ["pid", "device_lock"])
def test_recovery_preserves_live_runner_and_batch(isolated, monkeypatch, liveness_source):
import os
from types import SimpleNamespace
from artemis.runtime import DeviceExecutionLock
batch = service.create_batch(payload())
service.batch_action(batch["id"], "resume")
job = isolated.jobs(batch["id"])[0]
isolated.save_job({**job, "status": "running"})
if liveness_source == "pid":
with sqlite3.connect(isolated.path) as conn:
conn.execute(
"INSERT INTO sessions(session_id,initial_goal,start_time,status,device_info,pid) "
"VALUES (?,?,?,?,?,?)",
(job["session_id"], "goal", 1, "running", "{}", os.getpid()),
)
monkeypatch.setattr(
DeviceExecutionLock,
"get_active_owners",
lambda: (
{"device": SimpleNamespace(session_id=job["session_id"])}
if liveness_source == "device_lock"
else {}
),
)
service.recover_batches()
assert isolated.job(job["session_id"])["status"] == "running"
assert not isolated.job(job["session_id"]).get("error")
assert isolated.get("batch", batch["id"])["status"] == "running"
@pytest.mark.asyncio
async def test_natural_completion_resolves_old_recovery_error_in_job_report_and_history(
client, isolated
):
job = isolated.jobs(service.create_batch(payload())["id"])[0]
message = "服务中断;未完成任务需手动重跑"
isolated.save_job({**job, "status": "interrupted", "error": message})
await service.on_terminal({"session_id": job["session_id"]}, "completed")
saved = isolated.job(job["session_id"])
assert saved["status"] == "completed"
assert not saved.get("error")
assert saved["recovery_events"][0]["message"] == message
assert reports.session_report(job["session_id"])["error"] == ""
assert not client.get("/api/automation/history").json()["items"][0].get("error")
def test_recovery_repairs_completed_legacy_warning_but_preserves_real_failure(isolated):
batch = service.create_batch(payload())
first, second = isolated.jobs(batch["id"])
isolated.save_job({**first, "status": "completed", "error": "服务中断,需手动重跑"})
isolated.save_job({**second, "status": "failed", "error": "任务执行超时(30 秒)"})
assert reports.session_report(first["session_id"])["error"] == ""
service.recover_batches()
assert not isolated.job(first["session_id"]).get("error")
assert "服务中断" in isolated.job(first["session_id"])["recovery_events"][0]["message"]
assert "超时" in isolated.job(second["session_id"])["error"]
@pytest.mark.asyncio
async def test_surviving_worker_is_finalized_when_it_exits_without_resubmission(
isolated, monkeypatch
):
import os
from artemis.runtime import DeviceExecutionLock
batch = service.create_batch(payload())
service.batch_action(batch["id"], "resume")
job = isolated.jobs(batch["id"])[0]
isolated.save_job({**job, "status": "running"})
monkeypatch.setattr(DeviceExecutionLock, "get_active_owners", lambda: {})
with sqlite3.connect(isolated.path) as conn:
conn.execute(
"INSERT INTO sessions(session_id,initial_goal,start_time,status,device_info,pid) "
"VALUES (?,?,?,?,?,?)",
(job["session_id"], "goal", time.time(), "running", "{}", os.getpid()),
)
service.recover_batches()
await service.reconcile_recovered_jobs()
assert isolated.job(job["session_id"])["status"] == "running"
with sqlite3.connect(isolated.path) as conn:
conn.execute(
"UPDATE sessions SET pid=NULL,status='completed' WHERE session_id=?",
(job["session_id"],),
)
await service.reconcile_recovered_jobs()
saved = isolated.job(job["session_id"])
assert saved["status"] == "completed"
assert saved["report_ready"]
assert not saved.get("recovery_waiting")
assert not saved.get("error")
assert len(isolated.jobs(batch["id"])) == 2
assert isolated.jobs(batch["id"])[1]["status"] == "pending"
@pytest.mark.asyncio
async def test_surviving_worker_that_dies_without_result_is_interrupted(isolated, monkeypatch):
from artemis.runtime import DeviceExecutionLock
job = isolated.jobs(service.create_batch(payload())["id"])[0]
isolated.save_job({**job, "status": "running", "recovery_waiting": True})
monkeypatch.setattr(DeviceExecutionLock, "get_active_owners", lambda: {})
await service.reconcile_recovered_jobs()
saved = isolated.job(job["session_id"])
assert saved["status"] == "interrupted"
assert "服务中断" in saved["error"]
assert not saved.get("recovery_waiting")
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_batch_admission_idempotent_and_bound_device(isolated, monkeypatch): async def test_batch_admission_idempotent_and_bound_device(isolated, monkeypatch):
from apps.admin_console import server from apps.admin_console import server
@@ -15,6 +15,7 @@
import asyncio import asyncio
from contextlib import suppress from contextlib import suppress
import signal import signal
import socket
from unittest.mock import AsyncMock, MagicMock, patch from unittest.mock import AsyncMock, MagicMock, patch
import pytest import pytest
@@ -23,10 +24,55 @@ import uvicorn
from apps.admin_console.core.state import state from apps.admin_console.core.state import state
from apps.admin_console.routers.tasks import stream_events from apps.admin_console.routers.tasks import stream_events
from apps.admin_console.server import ArtemisUvicornServer, app, on_shutdown from apps.admin_console.server import ArtemisUvicornServer, app, on_shutdown
from apps.admin_console import server as server_module
WINDOWS_FORCE_SIGNAL = getattr(signal, "SIGBREAK", signal.SIGTERM) WINDOWS_FORCE_SIGNAL = getattr(signal, "SIGBREAK", signal.SIGTERM)
@pytest.mark.parametrize("reload", [False, True])
def test_occupied_port_never_initializes_or_overwrites_server_metadata(
tmp_path, monkeypatch, reload
):
from artemis.runtime import server_lifecycle
metadata = tmp_path / "server.json"
metadata.write_text('{"pid":123,"lifecycle_token":"original"}', encoding="utf-8")
monkeypatch.setattr(server_lifecycle, "get_server_info_file", lambda: metadata)
before = metadata.read_bytes()
with (
socket.create_server(("127.0.0.1", 0)) as original,
patch.object(server_module.ArtemisUvicornServer, "run") as run,
patch.object(server_module, "write_server_info") as publish,
):
with pytest.raises(SystemExit, match="1"):
server_module.run_ui_server("127.0.0.1", original.getsockname()[1], reload=reload)
run.assert_not_called()
publish.assert_not_called()
assert metadata.read_bytes() == before
def test_server_holds_port_before_running_lifespan(monkeypatch):
monkeypatch.setattr(state, "port", 0, raising=False)
monkeypatch.setattr(state, "host", "127.0.0.1", raising=False)
observed = []
def run(_server, sockets):
listener = sockets[0]
port = listener.getsockname()[1]
observed.append(listener)
assert state.port == port
with pytest.raises(OSError), socket.create_server(("127.0.0.1", port)):
pass
monkeypatch.setattr(server_module.ArtemisUvicornServer, "run", run)
monkeypatch.setattr(server_module, "clear_server_info", lambda **kwargs: None)
with patch.object(server_module, "write_server_info") as publish:
server_module.run_ui_server("127.0.0.1", 0)
publish.assert_not_called() # Publication belongs to the later lifespan.
assert len(observed) == 1
assert observed[0].fileno() == -1
def _pending_queue_get_tasks(): def _pending_queue_get_tasks():
return { return {
task task
+37
View File
@@ -93,6 +93,7 @@ def test_ensure_daemon_running_standalone_override(monkeypatch):
def test_ensure_daemon_running_auto_spawns(): def test_ensure_daemon_running_auto_spawns():
with ( with (
patch("artemis.runtime.daemon_client.is_daemon_running", side_effect=[False, True]), patch("artemis.runtime.daemon_client.is_daemon_running", side_effect=[False, True]),
patch("artemis.runtime.daemon_client.is_port_in_use", return_value=False),
patch("artemis.runtime.daemon_client.spawn_daemon") as mock_spawn, patch("artemis.runtime.daemon_client.spawn_daemon") as mock_spawn,
): ):
ok, base_url = ensure_daemon_running(timeout=1.0, wait_ready=True) ok, base_url = ensure_daemon_running(timeout=1.0, wait_ready=True)
@@ -101,6 +102,42 @@ def test_ensure_daemon_running_auto_spawns():
mock_spawn.assert_called_once() mock_spawn.assert_called_once()
def test_busy_live_daemon_is_reused_without_spawning():
with (
patch("artemis.runtime.daemon_client.is_daemon_running", return_value=False),
patch("artemis.runtime.daemon_client.is_port_in_use", return_value=True),
patch(
"artemis.runtime.daemon_client.read_server_info",
return_value={"host": "127.0.0.1", "port": 8000, "pid": 12345},
),
patch("artemis.runtime.daemon_client.pid_is_alive", return_value=True),
patch("artemis.runtime.daemon_client.spawn_daemon") as spawn,
):
assert ensure_daemon_running() == (True, "http://127.0.0.1:8000")
spawn.assert_not_called()
def test_unknown_port_owner_is_not_overwritten():
with (
patch("artemis.runtime.daemon_client.is_daemon_running", return_value=False),
patch("artemis.runtime.daemon_client.is_port_in_use", return_value=True),
patch("artemis.runtime.daemon_client.read_server_info", return_value=None),
patch("artemis.runtime.daemon_client.spawn_daemon") as spawn,
):
assert ensure_daemon_running() == (False, None)
spawn.assert_not_called()
def test_liveness_probe_uses_lightweight_endpoint_and_configured_timeout(monkeypatch):
from artemis.config.settings import settings
monkeypatch.setattr(settings, "ARTEMIS_DAEMON_PROBE_TIMEOUT_SECONDS", 7.0)
with patch("urllib.request.urlopen", return_value=_daemon_ok_response(b"{}")) as probe:
assert is_daemon_running()
assert probe.call_args.args[0].full_url.endswith("/api/system/server-status")
assert probe.call_args.kwargs["timeout"] == 7.0
def test_submit_task_to_daemon(): def test_submit_task_to_daemon():
mock_resp = MagicMock() mock_resp = MagicMock()
mock_resp.status = 200 mock_resp.status = 200