fix: bound task failures and release stalled workers

This commit is contained in:
chenmingxuan committed 2026-10-08 16:10:10 +08:00
1 parent 606ed94160
commit 1fdf7c0d1d
32 files changed
+1121 -93

No files matched your search

+27 -12
View File
@@ -15,6 +15,9 @@
"""Unit tests for Universal FlashRunner."""
import base64
from io import BytesIO
from PIL import Image
from artemis.mcp.observation import DeviceObservationError
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock, patch
import pytest
@@ -72,9 +75,18 @@ def test_flash_runner_tools_initialization(mock_context):
# --- Post-action screenshot fallback (failed action / unknown tool) ----------------
def _post_image():
buffer = BytesIO()
Image.new("RGB", (64, 64), "white").save(buffer, format="PNG")
return buffer.getvalue()
_POST_IMG = _post_image()
def _screen(elements):
return SimpleNamespace(
base64=base64.b64encode(b"POST-IMG").decode("ascii"),
base64=base64.b64encode(_POST_IMG).decode("ascii"),
elements=elements,
width=1080,
height=2400,
@@ -120,7 +132,7 @@ async def test_fallback_observation_renders_the_indexed_ui_list(mock_context, tm
exec_result, "click", frozenset({"click"}), state
)
assert post == b"POST-IMG"
assert post == _POST_IMG
assert isinstance(exec_result.ui_elements_text, str)
assert exec_result.ui_elements_text == "[1] Text: 'Settings' | Bounds: [37,50][962,112]"
assert "{'text'" not in exec_result.ui_elements_text
@@ -154,8 +166,8 @@ async def test_fallback_observation_is_skipped_for_non_action_tools(mock_context
@pytest.mark.asyncio
async def test_fallback_observation_keeps_the_executors_ui_list(mock_context, tmp_path):
"""An action that reported a UI list but no screenshot keeps its own list."""
async def test_fallback_observation_replaces_stale_executor_ui_list(mock_context, tmp_path):
"""Fresh screenshots and element lists must describe the same observation."""
with patch("artemis.controllers.unified_controller.get_driver"):
runner = FlashRunner(mock_context, goal="g")
runner.controller = Mock()
@@ -175,12 +187,13 @@ async def test_fallback_observation_keeps_the_executors_ui_list(mock_context, tm
exec_result, "click", frozenset({"click"}), None
)
assert post == b"POST-IMG"
assert exec_result.ui_elements_text == "[1] Text: 'Mine' | Bounds: [0,0][10,10]"
assert post == _POST_IMG
assert "Other" in exec_result.ui_elements_text
assert "Mine" not in exec_result.ui_elements_text
@pytest.mark.asyncio
async def test_fallback_observation_failure_is_contained(mock_context):
async def test_fallback_observation_failure_stops_after_bounded_recovery(mock_context):
with patch("artemis.controllers.unified_controller.get_driver"):
runner = FlashRunner(mock_context, goal="g")
runner.controller = Mock()
@@ -189,11 +202,13 @@ async def test_fallback_observation_failure_is_contained(mock_context):
state.latest_screenshot = "old.jpg"
exec_result = _failed_click()
post = await runner._capture_post_screenshot(exec_result, "click", frozenset({"click"}), state)
assert post is None
assert exec_result.ui_elements_text is None
assert state.latest_screenshot == "old.jpg"
with patch("artemis.mcp.observation.settings.ARTEMIS_OBSERVATION_MAX_ATTEMPTS", 3):
with pytest.raises(DeviceObservationError, match="adb gone"):
await runner._capture_post_screenshot(exec_result, "click", frozenset({"click"}), state)
assert runner.controller.get_screen_data.await_count == 3
assert state.latest_screenshot is None
assert state.indexed_elements == []
assert state.indexed_points == []
def test_flash_runner_executor_follows_the_flash_explorer_tier(mock_context):
+94 -1
View File
@@ -213,7 +213,8 @@ def test_observation_tail_has_pro_shape(mock_context):
texts = [b["text"] for b in later.content]
assert OBSERVATION_HEADER_RE.match(texts[0])
assert "objective" not in "".join(texts).lower()
assert texts[1:] == [
assert "当前没有可用的元素索引" in texts[1]
assert texts[2:] == [
"nudge",
"[INJECTED]",
'User instruction: "stop"',
@@ -408,6 +409,98 @@ def _report(status="completed", tc_id="tc-report", **extra):
_INITIAL_OBSERVATION = ("shot0.png", b"IMG0", "[1] Settings")
@pytest.mark.asyncio
@pytest.mark.parametrize("fallback_succeeds", [True, False])
async def test_failed_actions_have_finite_fallback_and_restore_primary(
mock_context, monkeypatch, fallback_succeeds
):
monkeypatch.setattr("artemis.agents.flash.runner.settings.ARTEMIS_ACTION_FAILURE_ATTEMPTS", 3)
monkeypatch.setattr(
"artemis.agents.flash.runner.settings.ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS", 3
)
action_count = 4 if fallback_succeeds else 6
responses = [
AIMessage(
content="try", tool_calls=[{"name": "click", "args": {"index": 1}, "id": f"tc-{i}"}]
)
for i in range(action_count)
]
if fallback_succeeds:
responses.append(_report())
with patch("artemis.controllers.unified_controller.get_driver"):
runner = _make_runner(mock_context, responses)
fallback = Mock()
count = 0
def execute(name, args, tc_id, state, **kwargs):
nonlocal count
count += 1
result = _exec_result(tc_id, name, b"current-screen")
if count <= 3 or not fallback_succeeds:
result.status = "error"
result.text_summary = "目标不存在"
return result
runner.executor.execute = AsyncMock(side_effect=execute)
with (
patch(
"artemis.agents.flash.runner.capture_screenshot_and_parse_ui",
AsyncMock(return_value=_INITIAL_OBSERVATION),
) as observe,
patch("artemis.agents.flash.runner.get_llm", return_value=fallback),
):
outcome = await runner.run(State(initial_goal="Open Settings"))
primary = runner._init_llm.return_value
assert all(call["llm"] is primary for call in runner.calls[:3])
assert runner.calls[3]["llm"] is fallback
assert observe.await_count == 2 # initial + fresh observation before fallback
if fallback_succeeds:
assert outcome["status"] == "completed"
assert runner.calls[-1]["llm"] is primary
else:
assert outcome["status"] == "failed"
assert "兜底模型连续 3" in outcome["explanation"]
assert "目标不存在" in outcome["explanation"]
assert len(runner.calls) == 6
@pytest.mark.asyncio
async def test_empty_current_ui_clears_previous_turn_element_list(mock_context):
response = AIMessage(
content="tap", tool_calls=[{"name": "click", "args": {"index": 1}, "id": "tc"}]
)
with patch("artemis.controllers.unified_controller.get_driver"):
runner = _make_runner(mock_context, [response, _report()])
runner.executor.execute = AsyncMock(
return_value=_exec_result("tc", "click", b"fresh-screen", ui="")
)
with patch(
"artemis.agents.flash.runner.capture_screenshot_and_parse_ui",
AsyncMock(return_value=_INITIAL_OBSERVATION),
):
await runner.run(State(initial_goal="Open Settings"))
tail = runner.calls[-1]["messages"][-1]
text = "\n".join(block.get("text", "") for block in tail.content)
assert "当前没有可用的元素索引" in text
assert "[1] Settings" not in text
@pytest.mark.asyncio
async def test_unavailable_screen_ends_before_any_model_call(mock_context):
from artemis.mcp.observation import DeviceObservationError
with patch("artemis.controllers.unified_controller.get_driver"):
runner = _make_runner(mock_context, [])
with patch(
"artemis.agents.flash.runner.capture_screenshot_and_parse_ui",
AsyncMock(side_effect=DeviceObservationError("设备断连")),
):
outcome = await runner.run(State(initial_goal="Open Settings"))
assert outcome["code"] == "device_observation_failed"
assert outcome["error"] == "设备断连"
runner._invoke_model.assert_not_awaited()
@pytest.mark.asyncio
async def test_run_builds_prompt_from_ledger_with_session_offsets(mock_context):
"""Turn 2 must see: the static system prefix, the committed turn 1 (its
+12 -1
View File
@@ -33,6 +33,17 @@ class DummyState:
self.operator_replan_reason = operator_replan_reason
import base64
from io import BytesIO
from PIL import Image
def _valid_screen_base64():
buffer = BytesIO()
Image.new("RGB", (64, 64), "white").save(buffer, format="PNG")
return base64.b64encode(buffer.getvalue()).decode("ascii")
@pytest.fixture
def mock_context():
ctx = Mock(spec=ArtemisContext)
@@ -41,7 +52,7 @@ def mock_context():
ctx.adb_client = Mock()
ctx.ui_adb_client = Mock()
mock_screen_data = Mock()
mock_screen_data.base64 = "YmFzZTY0"
mock_screen_data.base64 = _valid_screen_base64()
mock_screen_data.elements = []
mock_screen_data.width = 1080
mock_screen_data.height = 2400