fix: bound task failures and release stalled workers
This commit is contained in:
1 parent
606ed94160
commit
1fdf7c0d1d
32 files changed
+1121
-93
No files matched your search
@@ -10,6 +10,8 @@ from uuid import UUID
|
||||
from apps.admin_console.core.state import state
|
||||
from apps.admin_console.services.automation_store import store, new_id
|
||||
from apps.admin_console.services.task_import import case_goal, case_title
|
||||
from artemis.config.settings import settings
|
||||
from artemis.runtime.task_limits import task_timeout
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
TERMINAL = {"completed", "failed", "cancelled", "skipped", "interrupted"}
|
||||
@@ -135,10 +137,11 @@ async def tick_batches():
|
||||
# not consume this case's execution budget.
|
||||
if job["status"] != "running":
|
||||
continue
|
||||
limit = int(
|
||||
(job.get("case", {}).get("task_config") or {}).get("timeout_seconds")
|
||||
or batch.get("timeout_seconds", 1800)
|
||||
)
|
||||
from artemis.runtime.task_result import read_worker_outcome
|
||||
|
||||
if read_worker_outcome(str(job["session_id"])):
|
||||
continue # Execution has ended; the worker supervisor bounds cleanup.
|
||||
limit = task_timeout({}, job, batch)
|
||||
task_state = getattr(state, "active_runs", {}).get(str(job["session_id"])) or {}
|
||||
excluded = float(task_state.get("non_execution_seconds") or 0)
|
||||
if task_state.get("non_execution_started_at"):
|
||||
@@ -151,7 +154,17 @@ async def tick_batches():
|
||||
error=f"任务执行超时({limit} 秒,不含暂停/排队时间)", timed_out=True
|
||||
)
|
||||
store.save_job(job)
|
||||
task_queue_service.stop_tasks(session_id=job["session_id"])
|
||||
from artemis.runtime.task_result import save_failure_result
|
||||
|
||||
try:
|
||||
await asyncio.to_thread(
|
||||
save_failure_result,
|
||||
str(job["session_id"]),
|
||||
job["error"],
|
||||
code="task_timeout",
|
||||
)
|
||||
finally:
|
||||
task_queue_service.stop_tasks(session_id=job["session_id"])
|
||||
# on_terminal records the failed case. The next pending
|
||||
# case then runs normally unless stop_on_failure is set.
|
||||
continue
|
||||
@@ -255,8 +268,7 @@ def create_batch(payload: dict) -> dict:
|
||||
"formats": payload.get("formats") or store.preferences()["formats"],
|
||||
"profile": payload.get("profile", "flash"),
|
||||
"device_serial": payload.get("device_serial"),
|
||||
"timeout_seconds": payload.get("timeout_seconds")
|
||||
or int(os.getenv("ARTEMIS_BATCH_TASK_TIMEOUT", "1800")),
|
||||
"timeout_seconds": payload.get("timeout_seconds") or settings.ARTEMIS_TASK_TIMEOUT_SECONDS,
|
||||
"stop_on_failure": payload.get("stop_on_failure", False),
|
||||
"request_id": payload.get("request_id"),
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user