fix: bound task failures and release stalled workers
This commit is contained in:
1 parent
606ed94160
commit
1fdf7c0d1d
32 files changed
+1121
-93
No files matched your search
@@ -221,6 +221,7 @@ def submit_task_to_daemon(
|
||||
conversation_id: str | None = None,
|
||||
verification_level: str | None = None,
|
||||
explorer_mode: str | None = None,
|
||||
runtime_overrides: dict[str, Any] | None = None,
|
||||
base_url: str | None = None,
|
||||
timeout: float = 15.0,
|
||||
) -> dict[str, Any] | None:
|
||||
@@ -246,6 +247,7 @@ def submit_task_to_daemon(
|
||||
"session_id": session_id,
|
||||
"ingress": ingress,
|
||||
"conversation_id": conversation_id,
|
||||
"runtime_overrides": runtime_overrides,
|
||||
}
|
||||
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Execution budgets shared by supervised web and MCP tasks."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
import time
|
||||
|
||||
from artemis.config.settings import settings
|
||||
|
||||
|
||||
def task_timeout(item: dict, job: dict | None = None, batch: dict | None = None) -> int:
|
||||
options = item.get("runtime_overrides") or (job or {}).get("case", {}).get("task_config") or {}
|
||||
return int(
|
||||
options.get("timeout_seconds")
|
||||
or (batch or {}).get("timeout_seconds")
|
||||
or settings.ARTEMIS_TASK_TIMEOUT_SECONDS
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExecutionBudget:
|
||||
limit: float
|
||||
elapsed: float = 0
|
||||
_last: float = field(default_factory=time.monotonic)
|
||||
_excluded: bool = False
|
||||
|
||||
def update(self, *, excluded: bool = False) -> float:
|
||||
now = time.monotonic()
|
||||
if not self._excluded:
|
||||
self.elapsed += max(0, now - self._last)
|
||||
self._last, self._excluded = now, excluded
|
||||
return max(0, self.limit - self.elapsed)
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Persist useful failure evidence even when a worker must be terminated."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import uuid
|
||||
|
||||
from artemis.config import TRACES_PATH
|
||||
|
||||
|
||||
def read_worker_outcome(session_id: str) -> dict:
|
||||
"""Read a worker's terminal signal without waiting for hung SDK threads."""
|
||||
try:
|
||||
data = json.loads(
|
||||
(TRACES_PATH / session_id / "run_outcome.json").read_text(encoding="utf-8")
|
||||
)
|
||||
if (
|
||||
isinstance(data, dict)
|
||||
and data.get("worker_finalized") is True
|
||||
and data.get("task_status")
|
||||
in {
|
||||
"failed",
|
||||
"completed",
|
||||
"cancelled",
|
||||
"interrupted",
|
||||
}
|
||||
):
|
||||
return data
|
||||
except (OSError, ValueError):
|
||||
pass # Atomic outcome is absent until the task finalizes.
|
||||
return {}
|
||||
|
||||
|
||||
def save_failure_result(session_id: str, error: str, *, code: str = "execution_failed") -> None:
|
||||
if not session_id or not all(c.isalnum() or c in "-_" for c in session_id):
|
||||
raise ValueError("无效的任务编号")
|
||||
directory = TRACES_PATH / session_id
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
payload: dict = {}
|
||||
try:
|
||||
previous = json.loads((directory / "result.json").read_text(encoding="utf-8"))
|
||||
if isinstance(previous, dict):
|
||||
payload.update(previous)
|
||||
except (OSError, ValueError):
|
||||
pass # Missing/truncated results are expected after a killed worker.
|
||||
payload.update(
|
||||
{
|
||||
"status": "failed",
|
||||
"task_status": "failed",
|
||||
"explanation": error,
|
||||
"error": error,
|
||||
"code": code,
|
||||
"worker_finalized": True,
|
||||
"tests": payload.get("tests") or {},
|
||||
}
|
||||
)
|
||||
for name in ("result.json", "run_outcome.json"):
|
||||
target = directory / name
|
||||
temporary = target.with_name(f".{name}.{uuid.uuid4().hex}.tmp")
|
||||
try:
|
||||
temporary.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
|
||||
os.replace(temporary, target)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
Reference in new issue
Block a user