fix: bound task failures and release stalled workers

apps/admin_console/services/task_queue_service.py

@@ -581,6 +581,10 @@ class TaskQueueService:
             "retry_interval_seconds": "OPENAI_RETRY_INTERVAL_SECONDS",
             "record_video": "ARTEMIS_RECORDING_ENABLED",
             "analyze_video": "ARTEMIS_VIDEO_ENABLED",
+            "observation_timeout_seconds": "ARTEMIS_OBSERVATION_TIMEOUT_SECONDS",
+            "observation_max_attempts": "ARTEMIS_OBSERVATION_MAX_ATTEMPTS",
+            "device_command_timeout_seconds": "ARTEMIS_DEVICE_COMMAND_TIMEOUT_SECONDS",
+            "action_failure_attempts": "ARTEMIS_ACTION_FAILURE_ATTEMPTS",
         }.items():
             if overrides.get(option) is not None:
                 env[variable] = (
@@ -630,6 +634,72 @@ class TaskQueueService:
             env["RESULTS_OUTPUT_PATH"] = str(TRACES_PATH / str(sess_id) / "result.json")
         return cmd, env
 
+    @classmethod
+    async def _await_worker_with_budget(cls, proc, task_item: dict, run_key: str) -> int:
+        from artemis.runtime.task_limits import ExecutionBudget, task_timeout
+        from artemis.config.settings import settings
+        from artemis.runtime.task_result import save_failure_result, read_worker_outcome
+        from artemis.runtime.task_control import read_state
+        from apps.admin_console.services.automation_store import store
+
+        sid = str(task_item.get("session_id") or run_key)
+        job = store.job(sid)
+        batch = store.get("batch", job["batch_id"]) if job and job.get("batch_id") else None
+        limit = task_timeout(task_item, job, batch)
+        budget = ExecutionBudget(limit)
+        cleanup_limit = (task_item.get("runtime_overrides") or {}).get(
+            "worker_cleanup_timeout_seconds"
+        ) or settings.ARTEMIS_WORKER_CLEANUP_TIMEOUT_SECONDS
+        finalized_at = None
+        waiter = asyncio.create_task(cls._wait_for_worker_process(proc))
+        try:
+            while True:
+                done, _ = await asyncio.wait({waiter}, timeout=0.25)
+                if waiter in done:
+                    return waiter.result()
+                outcome = read_worker_outcome(sid)
+                if outcome:
+                    finalized_at = finalized_at if finalized_at is not None else time.monotonic()
+                    if time.monotonic() - finalized_at >= cleanup_limit:
+                        logger.warning(
+                            "%s: task finalized but worker cleanup exceeded %s seconds; reclaiming process",
+                            sid,
+                            cleanup_limit,
+                        )
+                        await cls._terminate_worker_process(proc)
+                        return await waiter
+                    continue  # Final result is authoritative; only bound resource cleanup now.
+                cls._sync_control_states()
+                run = state.active_runs.get(run_key, {})
+                remaining = budget.update(
+                    excluded=bool(run.get("parked")) or read_state(sid).get("status") == "queued"
+                )
+                if remaining > 0:
+                    continue
+                error = f"任务执行超时（{limit} 秒，不含暂停/排队时间）"
+                task_item.update(timed_out=True, error=error)
+                logger.error("%s: %s", sid, error)
+                try:
+                    latest = store.job(sid)
+                    if latest:
+                        latest.update(timed_out=True, error=error)
+                        store.save_job(latest)
+                except Exception:
+                    logger.exception(
+                        "Cannot persist timeout for %s; terminating worker anyway", sid
+                    )
+                await cls._terminate_worker_process(proc)
+                returncode = await waiter
+                try:
+                    await asyncio.to_thread(save_failure_result, sid, error, code="task_timeout")
+                except OSError:
+                    logger.exception("Cannot save timeout report for %s", sid)
+                return returncode
+        finally:
+            if not waiter.done():
+                waiter.cancel()
+            await asyncio.gather(waiter, return_exceptions=True)
+
     @classmethod
     def _register_worker_run(
         cls,
@@ -721,6 +791,10 @@ class TaskQueueService:
             returncode,
             manual_stop,
         )
+        from apps.admin_console.services.automation_store import store
+
+        if (store.job(str(sess_id)) or {}).get("timed_out"):
+            new_status, should_persist = "failed", True
         if (
             state.is_shutting_down
             and not manual_stop
@@ -943,7 +1017,7 @@ class TaskQueueService:
             await cls._terminate_if_cancelled_during_launch(run_key, sess_id, proc)
 
             # 3. Await subprocess completion
-            returncode = await cls._wait_for_worker_process(proc)
+            returncode = await cls._await_worker_with_budget(proc, task_item, run_key)
             print(f"[QueueWorker] Task [{sess_id}] exited with returncode {returncode}")
 
             manual_stop = run_key in state.manually_stopped_run_ids or bool(
@@ -1141,6 +1215,12 @@ class TaskQueueService:
         forwarded to the worker as ``--verification-level`` / ``--explorer-pro-mode``;
         they are normalised here so the queue item and the CLI see one spelling.
         """
+        if runtime_overrides:
+            profile = runtime_overrides.get("profile") or profile
+            device_serial = runtime_overrides.get("device_serial") or device_serial
+            locked_app_package = runtime_overrides.get("locked_app_package") or locked_app_package
+            verification_level = runtime_overrides.get("verification_level") or verification_level
+            explorer_mode = runtime_overrides.get("explorer_mode") or explorer_mode
         verification_level = (
             str(verification_level).strip().lower() or None if verification_level else None
         )