Improve task resilience and LLM retry visibility

Persist and publish recoverable provider retries so temporary 503 and throttling events remain visible in live and historical task streams.

Aggregate repeated retries into an expandable action card, preserve terminal pause recovery, harden Windows worker isolation and output forwarding, and retain reliable Android hierarchy fallbacks with regression coverage.

tests/unit/admin_console/test_step_repository.py

@@ -43,3 +43,33 @@ def test_unrelated_log_is_not_presented_as_llm_failure():
 
     assert normalized["type"] == "log"
     assert normalized["status"] == "success"
+
+
+def test_retrying_llm_payload_keeps_user_visible_retry_details():
+    repository = StepRepository()
+    trace = {
+        "trace_id": "retry-1",
+        "type": "llm_call",
+        "name": "llm_retry",
+        "status": "retrying",
+        "payload": json.dumps(
+            {
+                "error": "503 UNAVAILABLE: high demand",
+                "delay": 1.18,
+                "provider": "google",
+                "source": "provider_sdk",
+                "recoverable": True,
+                "internal_noise": "do not expose",
+            }
+        ),
+    }
+
+    normalized = repository._normalize_display_trace(trace)
+
+    assert normalized["payload"] == {
+        "error": "503 UNAVAILABLE: high demand",
+        "delay": 1.18,
+        "provider": "google",
+        "source": "provider_sdk",
+        "recoverable": True,
+    }

tests/unit/admin_console/test_task_queue_service.py

@@ -14,6 +14,9 @@
 
 import asyncio
 import importlib
+import json
+import os
+import subprocess
 import sys
 from unittest.mock import AsyncMock, MagicMock, patch
 import pytest
@@ -203,9 +206,11 @@ async def test_queue_worker_cmd_construction():
         assert "--app-path" in cmd
         assert "/path/to/app.apk" in cmd
         if sys.platform == "win32":
-            import subprocess
-
-            assert executed_kwargs[0]["creationflags"] == subprocess.CREATE_NEW_PROCESS_GROUP
+            assert executed_kwargs[0]["creationflags"] == (
+                subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.CREATE_NO_WINDOW
+            )
+            assert executed_kwargs[0]["stdout"] == asyncio.subprocess.PIPE
+            assert executed_kwargs[0]["stderr"] == asyncio.subprocess.STDOUT
         else:
             assert "creationflags" not in executed_kwargs[0]
 
@@ -218,6 +223,40 @@ async def test_queue_worker_cmd_construction():
                 pass
 
 
+@pytest.mark.asyncio
+async def test_forward_worker_output_preserves_split_utf8(capsys):
+    stream = asyncio.StreamReader()
+    encoded = "worker 输出正常\n".encode()
+    stream.feed_data(encoded[:9])
+    stream.feed_data(encoded[9:])
+    stream.feed_eof()
+
+    await TaskQueueService._forward_worker_output(stream)
+
+    assert capsys.readouterr().out == "worker 输出正常\n"
+
+
+@pytest.mark.skipif(sys.platform != "win32", reason="Windows console isolation only")
+def test_windows_worker_has_no_inherited_console():
+    probe = r"""
+import ctypes
+import json
+
+buffer = (ctypes.c_uint32 * 32)()
+count = ctypes.windll.kernel32.GetConsoleProcessList(buffer, len(buffer))
+print(json.dumps(list(buffer[:min(count, len(buffer))])))
+"""
+    result = subprocess.run(
+        [sys.executable, "-c", probe],
+        check=True,
+        text=True,
+        **TaskQueueService._subprocess_creation_kwargs(),
+    )
+
+    attached_processes = json.loads(result.stdout)
+    assert os.getpid() not in attached_processes
+
+
 @pytest.mark.asyncio
 async def test_stop_tasks_dead_process_resets_state():
     state.queue_items = [

tests/unit/mcp/test_mcp_tools.py

@@ -17,7 +17,10 @@
 import inspect
 import shutil
 import tempfile
+from types import SimpleNamespace
 import uuid
+from unittest.mock import AsyncMock, MagicMock, patch
+
 import pytest
 
 from mcp_server.tools import (
@@ -106,6 +109,40 @@ def test_mobile_manage_task_inject_instruction(temp_trace_env):
     assert "Successfully injected" in ok_res["message"]
 
 
+@pytest.mark.asyncio
+async def test_mobile_get_device_state_hierarchy_without_ocr():
+    """XML hierarchy remains available when OCR is not configured."""
+    controller = MagicMock()
+    controller.ctx.device.device_width = 1080
+    controller.ctx.device.device_height = 2400
+    controller.get_screen_data = AsyncMock(
+        return_value=SimpleNamespace(
+            base64="ZHVtbXk=",
+            elements=[
+                {
+                    "class": "android.widget.Button",
+                    "text": "Start",
+                    "bounds": "[100,500][400,600]",
+                }
+            ],
+            width=1080,
+            height=2400,
+        )
+    )
+
+    with (
+        patch("mcp_server.tools.device_state._get_controller", return_value=controller),
+        patch("mcp_server.tools.device_state.is_ocr_configured", return_value=False),
+        patch(
+            "mcp_server.tools.device_state.perform_ocr", new_callable=AsyncMock
+        ) as perform_ocr_mock,
+    ):
+        result = await mobile_get_device_state(view_type="hierarchy")
+
+    assert "Start" in result
+    perform_ocr_mock.assert_not_awaited()
+
+
 @pytest.mark.asyncio
 async def test_mobile_inspect_trace_invalid_action():
     res = await mobile_inspect_trace(action="invalid_action", trace_id="trace-123")

tests/unit/test_drivers.py

@@ -14,14 +14,24 @@
 
 """Unit tests for Unified Base Device Driver and implementations."""
 
-import pytest
+import base64
+from types import SimpleNamespace
 from unittest.mock import MagicMock
 
+import pytest
+
 from artemis.drivers.base import KeyCode, ScreenData, SwipeDirection
 from artemis.drivers.mock.mock_driver import MockDeviceDriver
 from artemis.drivers.android.adb_driver import AndroidAdbDriver
 
 
+_ONE_PIXEL_PNG = base64.b64encode(
+    base64.b64decode(
+        "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
+    )
+).decode("ascii")
+
+
 @pytest.mark.asyncio
 async def test_mock_driver_actions():
     """Verify MockDeviceDriver records and performs all core actions."""
@@ -89,3 +99,70 @@ async def test_android_driver_with_mock_adb():
     # Test press key
     await driver.press_key(KeyCode.BACK)
     mock_adb_device.shell.assert_called_with("input keyevent 4")
+
+
+@pytest.mark.asyncio
+async def test_android_driver_preserves_elements_from_combined_screen_data():
+    """The combined uiautomator2 response is the authoritative live snapshot."""
+    mock_adb_client = MagicMock()
+    mock_ui_client = MagicMock()
+    expected_elements = [
+        {
+            "text": "",
+            "class": "android.widget.FrameLayout",
+            "bounds": "[0,0][1080,2400]",
+            "clickable": "false",
+            "focusable": "false",
+        },
+        {
+            "text": "Start",
+            "class": "android.widget.Button",
+            "bounds": "[10,20][100,80]",
+        }
+    ]
+    mock_ui_client.get_screen_data.return_value = SimpleNamespace(
+        base64=_ONE_PIXEL_PNG,
+        hierarchy_xml="<hierarchy />",
+        elements=expected_elements,
+        width=1080,
+        height=2400,
+    )
+
+    driver = AndroidAdbDriver(
+        device_id="emulator-5554",
+        adb_client=mock_adb_client,
+        ui_adb_client=mock_ui_client,
+    )
+
+    screen_data = await driver.get_screen_data(skip_settling=True)
+
+    assert screen_data.ui_elements == [expected_elements[1]]
+    assert screen_data.ui_hierarchy_xml == "<hierarchy />"
+    mock_ui_client.get_hierarchy.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_android_driver_parses_xml_hierarchy_fallback():
+    """XML-only clients remain supported without discarding their hierarchy."""
+    mock_adb_client = MagicMock()
+    mock_ui_client = MagicMock()
+    mock_ui_client.get_screen_data.return_value = SimpleNamespace(
+        base64=_ONE_PIXEL_PNG,
+        width=1080,
+        height=2400,
+    )
+    mock_ui_client.get_hierarchy.return_value = (
+        '<hierarchy><node text="Settings" class="android.widget.TextView" '
+        'bounds="[0,0][100,100]" /></hierarchy>'
+    )
+
+    driver = AndroidAdbDriver(
+        device_id="emulator-5554",
+        adb_client=mock_adb_client,
+        ui_adb_client=mock_ui_client,
+    )
+
+    screen_data = await driver.get_screen_data(skip_settling=True)
+
+    assert screen_data.ui_hierarchy_xml is not None
+    assert any(element.get("text") == "Settings" for element in screen_data.ui_elements)

tests/unit/test_llm_pause.py

@@ -1,8 +1,11 @@
+import logging
+
 from artemis.services import llm
 
 
 class _RecordingEngine:
     current_step_id = "step-7"
+    current_session_id = "session-1"
 
     def __init__(self):
         self.trace_kwargs = None
@@ -10,6 +13,7 @@ class _RecordingEngine:
 
     def record_trace(self, **kwargs):
         self.trace_kwargs = kwargs
+        return "trace-1"
 
     def _publish(self, event_type, payload):
         self.published = (event_type, payload)
@@ -35,3 +39,42 @@ def test_pause_error_is_persisted_and_published(monkeypatch, tmp_path):
     assert payload["error"] == "503 high demand"
     assert payload["step_id"] == "step-7"
     assert isinstance(payload["timestamp"], float)
+
+
+def test_provider_sdk_retry_is_persisted_and_published(monkeypatch):
+    engine = _RecordingEngine()
+    monkeypatch.setattr(llm, "_CURRENT_DATA_ENGINE", engine)
+    record = logging.LogRecord(
+        name="google_genai._api_client",
+        level=logging.INFO,
+        pathname=__file__,
+        lineno=1,
+        msg=(
+            "Retrying google.genai request in 1.18 seconds as it raised "
+            "ServerError: 503 UNAVAILABLE: high demand"
+        ),
+        args=(),
+        exc_info=None,
+    )
+
+    llm._ProviderRetryTelemetryHandler().emit(record)
+
+    assert engine.trace_kwargs == {
+        "type": "llm_call",
+        "name": "llm_retry",
+        "payload": {
+            "error": "ServerError: 503 UNAVAILABLE: high demand",
+            "delay": 1.18,
+            "provider": "google",
+            "source": "provider_sdk",
+            "recoverable": True,
+        },
+        "step_id": "step-7",
+        "parent_trace_id": None,
+        "status": "retrying",
+    }
+    event_type, payload = engine.published
+    assert event_type == "llm_retrying"
+    assert payload["trace_id"] == "trace-1"
+    assert payload["delay"] == 1.18
+    assert payload["error"].startswith("ServerError: 503")