Improve task resilience and LLM retry visibility

Persist and publish recoverable provider retries so temporary 503 and throttling events remain visible in live and historical task streams.

Aggregate repeated retries into an expandable action card, preserve terminal pause recovery, harden Windows worker isolation and output forwarding, and retain reliable Android hierarchy fallbacks with regression coverage.
This commit is contained in:
somew1nd committed 2026-08-20 00:37:53 -07:00
1 parent c1a4b49a21
commit 0f5fb1d32c
17 files changed
+946 -97

No files matched your search

@@ -43,3 +43,33 @@ def test_unrelated_log_is_not_presented_as_llm_failure():
assert normalized["type"] == "log"
assert normalized["status"] == "success"
def test_retrying_llm_payload_keeps_user_visible_retry_details():
repository = StepRepository()
trace = {
"trace_id": "retry-1",
"type": "llm_call",
"name": "llm_retry",
"status": "retrying",
"payload": json.dumps(
{
"error": "503 UNAVAILABLE: high demand",
"delay": 1.18,
"provider": "google",
"source": "provider_sdk",
"recoverable": True,
"internal_noise": "do not expose",
}
),
}
normalized = repository._normalize_display_trace(trace)
assert normalized["payload"] == {
"error": "503 UNAVAILABLE: high demand",
"delay": 1.18,
"provider": "google",
"source": "provider_sdk",
"recoverable": True,
}
@@ -14,6 +14,9 @@
import asyncio
import importlib
import json
import os
import subprocess
import sys
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@@ -203,9 +206,11 @@ async def test_queue_worker_cmd_construction():
assert "--app-path" in cmd
assert "/path/to/app.apk" in cmd
if sys.platform == "win32":
import subprocess
assert executed_kwargs[0]["creationflags"] == subprocess.CREATE_NEW_PROCESS_GROUP
assert executed_kwargs[0]["creationflags"] == (
subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.CREATE_NO_WINDOW
)
assert executed_kwargs[0]["stdout"] == asyncio.subprocess.PIPE
assert executed_kwargs[0]["stderr"] == asyncio.subprocess.STDOUT
else:
assert "creationflags" not in executed_kwargs[0]
@@ -218,6 +223,40 @@ async def test_queue_worker_cmd_construction():
pass
@pytest.mark.asyncio
async def test_forward_worker_output_preserves_split_utf8(capsys):
stream = asyncio.StreamReader()
encoded = "worker 输出正常\n".encode()
stream.feed_data(encoded[:9])
stream.feed_data(encoded[9:])
stream.feed_eof()
await TaskQueueService._forward_worker_output(stream)
assert capsys.readouterr().out == "worker 输出正常\n"
@pytest.mark.skipif(sys.platform != "win32", reason="Windows console isolation only")
def test_windows_worker_has_no_inherited_console():
probe = r"""
import ctypes
import json
buffer = (ctypes.c_uint32 * 32)()
count = ctypes.windll.kernel32.GetConsoleProcessList(buffer, len(buffer))
print(json.dumps(list(buffer[:min(count, len(buffer))])))
"""
result = subprocess.run(
[sys.executable, "-c", probe],
check=True,
text=True,
**TaskQueueService._subprocess_creation_kwargs(),
)
attached_processes = json.loads(result.stdout)
assert os.getpid() not in attached_processes
@pytest.mark.asyncio
async def test_stop_tasks_dead_process_resets_state():
state.queue_items = [