fix: bound task failures and release stalled workers
This commit is contained in:
1 parent
606ed94160
commit
1fdf7c0d1d
32 files changed
+1121
-93
No files matched your search
@@ -15,6 +15,9 @@
|
||||
"""Unit tests for Universal FlashRunner."""
|
||||
|
||||
import base64
|
||||
from io import BytesIO
|
||||
from PIL import Image
|
||||
from artemis.mcp.observation import DeviceObservationError
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
import pytest
|
||||
@@ -72,9 +75,18 @@ def test_flash_runner_tools_initialization(mock_context):
|
||||
# --- Post-action screenshot fallback (failed action / unknown tool) ----------------
|
||||
|
||||
|
||||
def _post_image():
|
||||
buffer = BytesIO()
|
||||
Image.new("RGB", (64, 64), "white").save(buffer, format="PNG")
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
_POST_IMG = _post_image()
|
||||
|
||||
|
||||
def _screen(elements):
|
||||
return SimpleNamespace(
|
||||
base64=base64.b64encode(b"POST-IMG").decode("ascii"),
|
||||
base64=base64.b64encode(_POST_IMG).decode("ascii"),
|
||||
elements=elements,
|
||||
width=1080,
|
||||
height=2400,
|
||||
@@ -120,7 +132,7 @@ async def test_fallback_observation_renders_the_indexed_ui_list(mock_context, tm
|
||||
exec_result, "click", frozenset({"click"}), state
|
||||
)
|
||||
|
||||
assert post == b"POST-IMG"
|
||||
assert post == _POST_IMG
|
||||
assert isinstance(exec_result.ui_elements_text, str)
|
||||
assert exec_result.ui_elements_text == "[1] Text: 'Settings' | Bounds: [37,50][962,112]"
|
||||
assert "{'text'" not in exec_result.ui_elements_text
|
||||
@@ -154,8 +166,8 @@ async def test_fallback_observation_is_skipped_for_non_action_tools(mock_context
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fallback_observation_keeps_the_executors_ui_list(mock_context, tmp_path):
|
||||
"""An action that reported a UI list but no screenshot keeps its own list."""
|
||||
async def test_fallback_observation_replaces_stale_executor_ui_list(mock_context, tmp_path):
|
||||
"""Fresh screenshots and element lists must describe the same observation."""
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = FlashRunner(mock_context, goal="g")
|
||||
runner.controller = Mock()
|
||||
@@ -175,12 +187,13 @@ async def test_fallback_observation_keeps_the_executors_ui_list(mock_context, tm
|
||||
exec_result, "click", frozenset({"click"}), None
|
||||
)
|
||||
|
||||
assert post == b"POST-IMG"
|
||||
assert exec_result.ui_elements_text == "[1] Text: 'Mine' | Bounds: [0,0][10,10]"
|
||||
assert post == _POST_IMG
|
||||
assert "Other" in exec_result.ui_elements_text
|
||||
assert "Mine" not in exec_result.ui_elements_text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fallback_observation_failure_is_contained(mock_context):
|
||||
async def test_fallback_observation_failure_stops_after_bounded_recovery(mock_context):
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = FlashRunner(mock_context, goal="g")
|
||||
runner.controller = Mock()
|
||||
@@ -189,11 +202,13 @@ async def test_fallback_observation_failure_is_contained(mock_context):
|
||||
state.latest_screenshot = "old.jpg"
|
||||
exec_result = _failed_click()
|
||||
|
||||
post = await runner._capture_post_screenshot(exec_result, "click", frozenset({"click"}), state)
|
||||
|
||||
assert post is None
|
||||
assert exec_result.ui_elements_text is None
|
||||
assert state.latest_screenshot == "old.jpg"
|
||||
with patch("artemis.mcp.observation.settings.ARTEMIS_OBSERVATION_MAX_ATTEMPTS", 3):
|
||||
with pytest.raises(DeviceObservationError, match="adb gone"):
|
||||
await runner._capture_post_screenshot(exec_result, "click", frozenset({"click"}), state)
|
||||
assert runner.controller.get_screen_data.await_count == 3
|
||||
assert state.latest_screenshot is None
|
||||
assert state.indexed_elements == []
|
||||
assert state.indexed_points == []
|
||||
|
||||
|
||||
def test_flash_runner_executor_follows_the_flash_explorer_tier(mock_context):
|
||||
|
||||
@@ -213,7 +213,8 @@ def test_observation_tail_has_pro_shape(mock_context):
|
||||
texts = [b["text"] for b in later.content]
|
||||
assert OBSERVATION_HEADER_RE.match(texts[0])
|
||||
assert "objective" not in "".join(texts).lower()
|
||||
assert texts[1:] == [
|
||||
assert "当前没有可用的元素索引" in texts[1]
|
||||
assert texts[2:] == [
|
||||
"nudge",
|
||||
"[INJECTED]",
|
||||
'User instruction: "stop"',
|
||||
@@ -408,6 +409,98 @@ def _report(status="completed", tc_id="tc-report", **extra):
|
||||
_INITIAL_OBSERVATION = ("shot0.png", b"IMG0", "[1] Settings")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("fallback_succeeds", [True, False])
|
||||
async def test_failed_actions_have_finite_fallback_and_restore_primary(
|
||||
mock_context, monkeypatch, fallback_succeeds
|
||||
):
|
||||
monkeypatch.setattr("artemis.agents.flash.runner.settings.ARTEMIS_ACTION_FAILURE_ATTEMPTS", 3)
|
||||
monkeypatch.setattr(
|
||||
"artemis.agents.flash.runner.settings.ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS", 3
|
||||
)
|
||||
action_count = 4 if fallback_succeeds else 6
|
||||
responses = [
|
||||
AIMessage(
|
||||
content="try", tool_calls=[{"name": "click", "args": {"index": 1}, "id": f"tc-{i}"}]
|
||||
)
|
||||
for i in range(action_count)
|
||||
]
|
||||
if fallback_succeeds:
|
||||
responses.append(_report())
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = _make_runner(mock_context, responses)
|
||||
fallback = Mock()
|
||||
count = 0
|
||||
|
||||
def execute(name, args, tc_id, state, **kwargs):
|
||||
nonlocal count
|
||||
count += 1
|
||||
result = _exec_result(tc_id, name, b"current-screen")
|
||||
if count <= 3 or not fallback_succeeds:
|
||||
result.status = "error"
|
||||
result.text_summary = "目标不存在"
|
||||
return result
|
||||
|
||||
runner.executor.execute = AsyncMock(side_effect=execute)
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.flash.runner.capture_screenshot_and_parse_ui",
|
||||
AsyncMock(return_value=_INITIAL_OBSERVATION),
|
||||
) as observe,
|
||||
patch("artemis.agents.flash.runner.get_llm", return_value=fallback),
|
||||
):
|
||||
outcome = await runner.run(State(initial_goal="Open Settings"))
|
||||
primary = runner._init_llm.return_value
|
||||
assert all(call["llm"] is primary for call in runner.calls[:3])
|
||||
assert runner.calls[3]["llm"] is fallback
|
||||
assert observe.await_count == 2 # initial + fresh observation before fallback
|
||||
if fallback_succeeds:
|
||||
assert outcome["status"] == "completed"
|
||||
assert runner.calls[-1]["llm"] is primary
|
||||
else:
|
||||
assert outcome["status"] == "failed"
|
||||
assert "兜底模型连续 3" in outcome["explanation"]
|
||||
assert "目标不存在" in outcome["explanation"]
|
||||
assert len(runner.calls) == 6
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_current_ui_clears_previous_turn_element_list(mock_context):
|
||||
response = AIMessage(
|
||||
content="tap", tool_calls=[{"name": "click", "args": {"index": 1}, "id": "tc"}]
|
||||
)
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = _make_runner(mock_context, [response, _report()])
|
||||
runner.executor.execute = AsyncMock(
|
||||
return_value=_exec_result("tc", "click", b"fresh-screen", ui="")
|
||||
)
|
||||
with patch(
|
||||
"artemis.agents.flash.runner.capture_screenshot_and_parse_ui",
|
||||
AsyncMock(return_value=_INITIAL_OBSERVATION),
|
||||
):
|
||||
await runner.run(State(initial_goal="Open Settings"))
|
||||
tail = runner.calls[-1]["messages"][-1]
|
||||
text = "\n".join(block.get("text", "") for block in tail.content)
|
||||
assert "当前没有可用的元素索引" in text
|
||||
assert "[1] Settings" not in text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unavailable_screen_ends_before_any_model_call(mock_context):
|
||||
from artemis.mcp.observation import DeviceObservationError
|
||||
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = _make_runner(mock_context, [])
|
||||
with patch(
|
||||
"artemis.agents.flash.runner.capture_screenshot_and_parse_ui",
|
||||
AsyncMock(side_effect=DeviceObservationError("设备断连")),
|
||||
):
|
||||
outcome = await runner.run(State(initial_goal="Open Settings"))
|
||||
assert outcome["code"] == "device_observation_failed"
|
||||
assert outcome["error"] == "设备断连"
|
||||
runner._invoke_model.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_builds_prompt_from_ledger_with_session_offsets(mock_context):
|
||||
"""Turn 2 must see: the static system prefix, the committed turn 1 (its
|
||||
|
||||
@@ -33,6 +33,17 @@ class DummyState:
|
||||
self.operator_replan_reason = operator_replan_reason
|
||||
|
||||
|
||||
import base64
|
||||
from io import BytesIO
|
||||
from PIL import Image
|
||||
|
||||
|
||||
def _valid_screen_base64():
|
||||
buffer = BytesIO()
|
||||
Image.new("RGB", (64, 64), "white").save(buffer, format="PNG")
|
||||
return base64.b64encode(buffer.getvalue()).decode("ascii")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context():
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
@@ -41,7 +52,7 @@ def mock_context():
|
||||
ctx.adb_client = Mock()
|
||||
ctx.ui_adb_client = Mock()
|
||||
mock_screen_data = Mock()
|
||||
mock_screen_data.base64 = "YmFzZTY0"
|
||||
mock_screen_data.base64 = _valid_screen_base64()
|
||||
mock_screen_data.elements = []
|
||||
mock_screen_data.width = 1080
|
||||
mock_screen_data.height = 2400
|
||||
|
||||
Reference in new issue
Block a user