fix: bound task failures and release stalled workers

This commit is contained in:
chenmingxuan committed 2026-10-08 16:10:10 +08:00
1 parent 606ed94160
commit 1fdf7c0d1d
32 files changed
+1121 -93

No files matched your search

@@ -10,6 +10,8 @@ from uuid import UUID
from apps.admin_console.core.state import state
from apps.admin_console.services.automation_store import store, new_id
from apps.admin_console.services.task_import import case_goal, case_title
from artemis.config.settings import settings
from artemis.runtime.task_limits import task_timeout
logger = logging.getLogger(__name__)
TERMINAL = {"completed", "failed", "cancelled", "skipped", "interrupted"}
@@ -135,10 +137,11 @@ async def tick_batches():
# not consume this case's execution budget.
if job["status"] != "running":
continue
limit = int(
(job.get("case", {}).get("task_config") or {}).get("timeout_seconds")
or batch.get("timeout_seconds", 1800)
)
from artemis.runtime.task_result import read_worker_outcome
if read_worker_outcome(str(job["session_id"])):
continue # Execution has ended; the worker supervisor bounds cleanup.
limit = task_timeout({}, job, batch)
task_state = getattr(state, "active_runs", {}).get(str(job["session_id"])) or {}
excluded = float(task_state.get("non_execution_seconds") or 0)
if task_state.get("non_execution_started_at"):
@@ -151,7 +154,17 @@ async def tick_batches():
error=f"任务执行超时({limit} 秒,不含暂停/排队时间)", timed_out=True
)
store.save_job(job)
task_queue_service.stop_tasks(session_id=job["session_id"])
from artemis.runtime.task_result import save_failure_result
try:
await asyncio.to_thread(
save_failure_result,
str(job["session_id"]),
job["error"],
code="task_timeout",
)
finally:
task_queue_service.stop_tasks(session_id=job["session_id"])
# on_terminal records the failed case. The next pending
# case then runs normally unless stop_on_failure is set.
continue
@@ -255,8 +268,7 @@ def create_batch(payload: dict) -> dict:
"formats": payload.get("formats") or store.preferences()["formats"],
"profile": payload.get("profile", "flash"),
"device_serial": payload.get("device_serial"),
"timeout_seconds": payload.get("timeout_seconds")
or int(os.getenv("ARTEMIS_BATCH_TASK_TIMEOUT", "1800")),
"timeout_seconds": payload.get("timeout_seconds") or settings.ARTEMIS_TASK_TIMEOUT_SECONDS,
"stop_on_failure": payload.get("stop_on_failure", False),
"request_id": payload.get("request_id"),
}