fix: bound task failures and release stalled workers

mcp_server/tools/task_runner.py

@@ -35,6 +35,10 @@ from artemis.runtime import (
     submit_task_to_daemon,
     trace_store,
 )
+from artemis.runtime.task_limits import ExecutionBudget, task_timeout
+from artemis.runtime.task_control import read_state
+from artemis.runtime.task_result import save_failure_result, read_worker_outcome
+from artemis.config.settings import settings
 
 # Seconds the spawned runner gets to finish its imports and open its log files
 # before the spawn is declared dead. Normal startup creates stdout.log within a
@@ -44,6 +48,54 @@ from artemis.runtime import (
 SPAWN_WATCHDOG_SECONDS = 60
 
 
+def _watch_execution(proc, trace_id: str, limit: int, cleanup_limit: float | None = None) -> None:
+    """Supervise standalone MCP workers without charging device queue or pause time."""
+    budget = ExecutionBudget(limit)
+    finalized_at = None
+    cleanup_limit = cleanup_limit or settings.ARTEMIS_WORKER_CLEANUP_TIMEOUT_SECONDS
+    while proc.poll() is None:
+        status = trace_store.read_status(trace_id) or {}
+        outcome = read_worker_outcome(trace_id)
+        if outcome and status.get("status") not in {
+            "completed",
+            "success",
+            "failed",
+            "cancelled",
+            "interrupted",
+        }:
+            try:
+                trace_store.update_trace_status(
+                    trace_id, outcome["task_status"], error=outcome.get("error")
+                )
+            except OSError:
+                logging.exception("Cannot persist terminal status for %s", trace_id)
+        if outcome or status.get("status") in {
+            "completed",
+            "success",
+            "failed",
+            "cancelled",
+            "interrupted",
+        }:
+            finalized_at = finalized_at if finalized_at is not None else time.monotonic()
+            if time.monotonic() - finalized_at >= cleanup_limit:
+                _kill_process_tree(proc.pid)
+                return
+            time.sleep(0.5)
+            continue
+        control_status = read_state(trace_id).get("status")
+        remaining = budget.update(excluded=control_status in {"queued", "task_paused", "resuming"})
+        if remaining <= 0:
+            error = f"任务执行超时（{limit} 秒，不含暂停/排队时间）"
+            _kill_process_tree(proc.pid)
+            try:
+                save_failure_result(trace_id, error, code="task_timeout")
+                trace_store.update_trace_status(trace_id, "failed", error=error)
+            except OSError:
+                logging.exception("Cannot persist timeout report for %s", trace_id)
+            return
+        time.sleep(0.5)
+
+
 def _kill_process_tree(pid: int) -> None:
     """Terminate a spawned runner and its children (the venv shim re-execs python)."""
     if sys.platform == "win32":
@@ -212,6 +264,7 @@ def mobile_run_task(
     device_serial: str | None = None,
     verification_level: str | None = None,
     explorer_mode: str | None = None,
+    runtime_overrides: dict[str, Any] | None = None,
 ) -> dict[str, Any]:
     """Starts an autonomous mobile UI automation subagent on a connected Android device.
 
@@ -277,7 +330,25 @@ def mobile_run_task(
           by the Operator: `"flash"` (1-shot detection, the default),
           `"pro"` (3-turn ReAct), `"ultra"` (deep pixel reasoning; slowest).
           Ignored for Flash.
+        runtime_overrides: Optional TaskOptions JSON for this task only. Supports
+          timeout_seconds (total execution, default 1800; excludes queue/pause),
+          observation_timeout_seconds (60), observation_max_attempts (3),
+          device_command_timeout_seconds (10), action_failure_attempts (3),
+          worker_cleanup_timeout_seconds (30),
+          model (API model name), reasoning_effort, request timeout/retries,
+          recording and video analysis. Unknown fields and credentials are rejected.
     """
+    if runtime_overrides:
+        from apps.admin_console.services.task_import import TaskOptions
+
+        runtime_overrides = TaskOptions.model_validate(runtime_overrides).model_dump(
+            exclude_none=True
+        )
+        model = runtime_overrides.get("profile") or model
+        device_serial = runtime_overrides.get("device_serial") or device_serial
+        locked_app_package = runtime_overrides.get("locked_app_package") or locked_app_package
+        verification_level = runtime_overrides.get("verification_level") or verification_level
+        explorer_mode = runtime_overrides.get("explorer_mode") or explorer_mode
     # 0. Validate and normalize model
     if model.lower() not in ("flash", "pro"):
         raise ValueError(f"Invalid model '{model}'. Must be either 'Flash' or 'Pro'.")
@@ -325,6 +396,7 @@ def mobile_run_task(
                     conversation_id=conversation_id,
                     verification_level=verification_level,
                     explorer_mode=explorer_mode,
+                    runtime_overrides=runtime_overrides,
                     base_url=base_url,
                 )
                 if resp and resp.get("status") == "rejected":
@@ -475,6 +547,26 @@ def mobile_run_task(
         env = os.environ.copy()
         env["ARTEMIS_SESSION_ID"] = trace_id
         env["ARTEMIS_TASK_INGRESS"] = "mcp"
+        env["ARTEMIS_TASK_WORKER"] = "1"
+        for option, variable in {
+            "model": "OPENAI_MODEL",
+            "reasoning_effort": "OPENAI_REASONING_EFFORT",
+            "request_timeout_seconds": "OPENAI_TIMEOUT_SECONDS",
+            "max_retries": "OPENAI_MAX_RETRIES",
+            "retry_interval_seconds": "OPENAI_RETRY_INTERVAL_SECONDS",
+            "record_video": "ARTEMIS_RECORDING_ENABLED",
+            "analyze_video": "ARTEMIS_VIDEO_ENABLED",
+            "observation_timeout_seconds": "ARTEMIS_OBSERVATION_TIMEOUT_SECONDS",
+            "observation_max_attempts": "ARTEMIS_OBSERVATION_MAX_ATTEMPTS",
+            "device_command_timeout_seconds": "ARTEMIS_DEVICE_COMMAND_TIMEOUT_SECONDS",
+            "action_failure_attempts": "ARTEMIS_ACTION_FAILURE_ATTEMPTS",
+        }.items():
+            if (runtime_overrides or {}).get(option) is not None:
+                env[variable] = (
+                    str(runtime_overrides[option]).lower()
+                    if isinstance(runtime_overrides[option], bool)
+                    else str(runtime_overrides[option])
+                )
         if device_serial:
             env["ADB_DEVICE_SERIAL"] = device_serial
             env["ARTEMIS_DEVICE_ID"] = device_serial
@@ -519,6 +611,17 @@ def mobile_run_task(
         # interpreter startup and must be killed instead of squatting on the
         # queue as 'running' forever.
         _start_spawn_watchdog(trace_id, proc.pid, queue_ticket, conversation_id)
+        threading.Thread(
+            target=_watch_execution,
+            args=(
+                proc,
+                trace_id,
+                task_timeout({"runtime_overrides": runtime_overrides}),
+                (runtime_overrides or {}).get("worker_cleanup_timeout_seconds"),
+            ),
+            daemon=True,
+            name=f"artemis-task-budget-{trace_id[:8]}",
+        ).start()
 
         trace_dir = trace_store.get_trace_dir(trace_id)
         stdout_log_path = os.path.join(trace_dir, "stdout.log")