Improve task resilience and LLM retry visibility

Persist and publish recoverable provider retries so temporary 503 and throttling events remain visible in live and historical task streams.

Aggregate repeated retries into an expandable action card, preserve terminal pause recovery, harden Windows worker isolation and output forwarding, and retain reliable Android hierarchy fallbacks with regression coverage.

apps/admin_console/database/repositories/step_repository.py

@@ -177,11 +177,22 @@ class StepRepository:
             return None
 
         if trace_type == "llm_call":
-            return (
-                {"error": self._clean_value(payload_obj.get("error"))}
-                if "error" in payload_obj
-                else None
+            allowed_keys = (
+                "error",
+                "delay",
+                "attempt",
+                "max_retries",
+                "provider",
+                "source",
+                "recoverable",
+                "pause",
             )
+            cleaned = {
+                key: self._clean_value(payload_obj.get(key))
+                for key in allowed_keys
+                if key in payload_obj
+            }
+            return cleaned or None
 
         args = payload_obj.get("args", payload_obj)
         cleaned_args = (
@@ -291,7 +302,7 @@ class StepRepository:
                         " t1.payload, t2.name as agent_name FROM traces t1 LEFT"
                         " JOIN traces t2 ON t1.parent_trace_id = t2.trace_id WHERE"
                         " t1.step_id = ? AND (t1.type = 'tool' OR t1.type = 'action' OR (t1.type ="
-                        " 'llm_call' AND t1.status = 'failed') OR (t1.type = 'log' AND"
+                        " 'llm_call' AND t1.status IN ('failed', 'retrying')) OR (t1.type = 'log' AND"
                         " t1.payload LIKE '%LLM Error:%Pausing execution%' AND NOT EXISTS"
                         " (SELECT 1 FROM traces t3 WHERE t3.session_id = t1.session_id AND"
                         " t3.type = 'llm_call' AND t3.status = 'failed' AND"
@@ -363,7 +374,7 @@ class StepRepository:
                 " t2.name as agent_name FROM traces t1 LEFT JOIN traces t2 ON"
                 " t1.parent_trace_id = t2.trace_id WHERE t1.session_id = ? AND"
                 " (t1.step_id IS NULL OR t1.step_id = '') AND (t1.type = 'tool' OR"
-                " (t1.type = 'llm_call' AND t1.status = 'failed') OR (t1.type = 'log'"
+                " (t1.type = 'llm_call' AND t1.status IN ('failed', 'retrying')) OR (t1.type = 'log'"
                 " AND t1.payload LIKE '%LLM Error:%Pausing execution%' AND NOT EXISTS"
                 " (SELECT 1 FROM traces t3 WHERE t3.session_id = t1.session_id AND"
                 " t3.type = 'llm_call' AND t3.status = 'failed' AND"

apps/admin_console/services/task_queue_service.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import asyncio
+import codecs
 from datetime import datetime
 import os
 import shutil
@@ -78,17 +79,66 @@ class TaskQueueService:
         ]
 
     @staticmethod
-    def _subprocess_creation_kwargs() -> dict[str, int]:
-        """Keep task workers out of the UI server's Windows console group.
+    def _subprocess_creation_kwargs() -> dict[str, Any]:
+        """Isolate task workers from the UI server's Windows console.
 
-        Without a new process group, a console control event intended for the
-        uvicorn server is also delivered to a task that is still importing
-        Artemis, causing STATUS_CONTROL_C_EXIT (0xC000013A).
+        A new process group alone is insufficient on Windows: the worker still
+        shares the parent's console, so a CTRL_C_EVENT generated anywhere in
+        that console can reach the UI server. CREATE_NO_WINDOW removes that
+        shared console boundary. Output is captured and forwarded explicitly
+        so detached workers remain visible in the server terminal.
         """
         if sys.platform == "win32":
-            return {"creationflags": subprocess.CREATE_NEW_PROCESS_GROUP}
+            return {
+                "creationflags": (
+                    subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.CREATE_NO_WINDOW
+                ),
+                "stdout": asyncio.subprocess.PIPE,
+                "stderr": asyncio.subprocess.STDOUT,
+            }
         return {}
 
+    @staticmethod
+    async def _forward_worker_output(stream: asyncio.StreamReader | None) -> None:
+        """Forward a detached Windows worker's combined output without corrupting UTF-8."""
+        if stream is None:
+            return
+
+        decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")
+        while True:
+            chunk = await stream.read(4096)
+            if not chunk:
+                break
+            text = decoder.decode(chunk)
+            if text:
+                sys.stdout.write(text)
+                sys.stdout.flush()
+
+        tail = decoder.decode(b"", final=True)
+        if tail:
+            sys.stdout.write(tail)
+            sys.stdout.flush()
+
+    @staticmethod
+    async def _finish_output_forwarder(output_task: asyncio.Task[None] | None) -> None:
+        """Drain final worker output without allowing inherited handles to stall the queue."""
+        if output_task is None:
+            return
+        try:
+            await asyncio.wait_for(asyncio.shield(output_task), timeout=2.0)
+        except asyncio.CancelledError:
+            if output_task.cancelled():
+                return
+            raise
+        except TimeoutError:
+            output_task.cancel()
+            try:
+                await output_task
+            except asyncio.CancelledError:
+                pass
+        except Exception as exc:
+            print(f"[QueueWorker] Failed to forward detached worker output: {exc}")
+
     @staticmethod
     async def _terminate_worker_process(proc: asyncio.subprocess.Process | None) -> None:
         """Stop a worker without changing the established POSIX behavior."""
@@ -125,6 +175,7 @@ class TaskQueueService:
             sess_id = None
             goal = None
             profile = "flash"
+            output_task: asyncio.Task[None] | None = None
             try:
                 if state.is_running:
                     await asyncio.sleep(0.2)
@@ -212,6 +263,8 @@ class TaskQueueService:
                 )
                 state.current_process = proc
                 task_item["pid"] = proc.pid
+                if sys.platform == "win32" and isinstance(proc.stdout, asyncio.StreamReader):
+                    output_task = asyncio.create_task(cls._forward_worker_output(proc.stdout))
 
                 if sess_id and (
                     str(sess_id) in getattr(state, "cancelled_session_ids", set())
@@ -255,6 +308,7 @@ class TaskQueueService:
                 print(f"[QueueWorker] Unexpected error executing task: {e}")
                 await asyncio.sleep(0.5)
             finally:
+                await cls._finish_output_forwarder(output_task)
                 # 5. Clean up completed task from queue and reset execution state
                 if sess_id:
                     cls._remove_task(sess_id)