fix: bound task failures and release stalled workers

This commit is contained in:
chenmingxuan committed 2026-10-08 16:10:10 +08:00
1 parent 606ed94160
commit 1fdf7c0d1d
32 files changed
+1121 -93

No files matched your search

+2
View File
@@ -221,6 +221,7 @@ def submit_task_to_daemon(
conversation_id: str | None = None,
verification_level: str | None = None,
explorer_mode: str | None = None,
runtime_overrides: dict[str, Any] | None = None,
base_url: str | None = None,
timeout: float = 15.0,
) -> dict[str, Any] | None:
@@ -246,6 +247,7 @@ def submit_task_to_daemon(
"session_id": session_id,
"ingress": ingress,
"conversation_id": conversation_id,
"runtime_overrides": runtime_overrides,
}
try:
+30
View File
@@ -0,0 +1,30 @@
"""Execution budgets shared by supervised web and MCP tasks."""
from dataclasses import dataclass, field
import time
from artemis.config.settings import settings
def task_timeout(item: dict, job: dict | None = None, batch: dict | None = None) -> int:
options = item.get("runtime_overrides") or (job or {}).get("case", {}).get("task_config") or {}
return int(
options.get("timeout_seconds")
or (batch or {}).get("timeout_seconds")
or settings.ARTEMIS_TASK_TIMEOUT_SECONDS
)
@dataclass
class ExecutionBudget:
limit: float
elapsed: float = 0
_last: float = field(default_factory=time.monotonic)
_excluded: bool = False
def update(self, *, excluded: bool = False) -> float:
now = time.monotonic()
if not self._excluded:
self.elapsed += max(0, now - self._last)
self._last, self._excluded = now, excluded
return max(0, self.limit - self.elapsed)
+64
View File
@@ -0,0 +1,64 @@
"""Persist useful failure evidence even when a worker must be terminated."""
import json
import os
from pathlib import Path
import uuid
from artemis.config import TRACES_PATH
def read_worker_outcome(session_id: str) -> dict:
"""Read a worker's terminal signal without waiting for hung SDK threads."""
try:
data = json.loads(
(TRACES_PATH / session_id / "run_outcome.json").read_text(encoding="utf-8")
)
if (
isinstance(data, dict)
and data.get("worker_finalized") is True
and data.get("task_status")
in {
"failed",
"completed",
"cancelled",
"interrupted",
}
):
return data
except (OSError, ValueError):
pass # Atomic outcome is absent until the task finalizes.
return {}
def save_failure_result(session_id: str, error: str, *, code: str = "execution_failed") -> None:
if not session_id or not all(c.isalnum() or c in "-_" for c in session_id):
raise ValueError("无效的任务编号")
directory = TRACES_PATH / session_id
directory.mkdir(parents=True, exist_ok=True)
payload: dict = {}
try:
previous = json.loads((directory / "result.json").read_text(encoding="utf-8"))
if isinstance(previous, dict):
payload.update(previous)
except (OSError, ValueError):
pass # Missing/truncated results are expected after a killed worker.
payload.update(
{
"status": "failed",
"task_status": "failed",
"explanation": error,
"error": error,
"code": code,
"worker_finalized": True,
"tests": payload.get("tests") or {},
}
)
for name in ("result.json", "run_outcome.json"):
target = directory / name
temporary = target.with_name(f".{name}.{uuid.uuid4().hex}.tmp")
try:
temporary.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
os.replace(temporary, target)
finally:
temporary.unlink(missing_ok=True)