feat: improve task execution, verification and packaging
Add execution incidents, action bursts and shared Flash history. Expose verification and Explorer settings with streamed check results. Support graceful cancellation, recording recovery and bundled UI resources. Add CI quality checks and clean up comments and documentation.
This commit is contained in:
1 parent
5ccf06d677
commit
67248c887e
289 files changed
+15883
-6919
No files matched your search
@@ -67,7 +67,9 @@ def test_planner_blocks_with_checks_mount_generation_and_audit():
|
||||
assert "Never silently promise" in generation
|
||||
|
||||
audit = data["blocks"]["check_audit"]
|
||||
assert "REJECT" in audit
|
||||
# Advisory review: the audit flags weakened check standards, never blocks.
|
||||
assert "FLAG" in audit
|
||||
assert "REJECT" not in audit
|
||||
|
||||
|
||||
def _mock_ctx(tmp_path=None, plan: str | None = None, **setup_kwargs):
|
||||
@@ -108,7 +110,7 @@ async def test_explainer_renders_iff_plan_has_check_lines(tmp_path):
|
||||
await component(builder2, _state(), ctx2)
|
||||
joined = "\n".join(p for p in builder2.human_parts if isinstance(p, str))
|
||||
assert "check lines" in joined
|
||||
assert "independent Checker" in joined
|
||||
assert "never declare a check passed yourself" in joined
|
||||
assert "automatically restored" in joined
|
||||
|
||||
|
||||
@@ -148,13 +150,13 @@ async def test_main_template_diagnosis_trigger_gated_on_verification():
|
||||
with_checks = await _render_main_template(
|
||||
disable_checker=False, disable_planner_validation=True
|
||||
)
|
||||
assert "Ambiguous Validation Rejection or Verification Finding" in with_checks
|
||||
assert "Ambiguous Verification Finding" in with_checks
|
||||
|
||||
# Planner validation alone can also produce rejections.
|
||||
# Planner validation alone can also produce (advisory) findings.
|
||||
with_validation = await _render_main_template(
|
||||
disable_checker=True, disable_planner_validation=False
|
||||
)
|
||||
assert "Ambiguous Validation Rejection or Verification Finding" in with_validation
|
||||
assert "Ambiguous Verification Finding" in with_validation
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""The structured verdict request must never end on a model turn (Gemini
|
||||
rejects such requests with 400 "Requests ending with a model turn are not
|
||||
supported"), which is the natural state after the loop's tool-free reply."""
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage
|
||||
|
||||
from artemis.agents.checker.checker import CheckReport, _structured_report
|
||||
|
||||
|
||||
def _llm(report: CheckReport):
|
||||
structured = MagicMock()
|
||||
structured.ainvoke = AsyncMock(return_value=report)
|
||||
llm = MagicMock()
|
||||
llm.with_structured_output.return_value = structured
|
||||
return llm, structured
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_structured_report_appends_user_turn_after_model_reply():
|
||||
llm, structured = _llm(CheckReport(verdicts=[]))
|
||||
messages = [SystemMessage(content="s"), HumanMessage(content="h"), AIMessage(content="done")]
|
||||
|
||||
await _structured_report(llm, messages)
|
||||
|
||||
sent = structured.ainvoke.call_args.args[0]
|
||||
assert isinstance(sent[-1], HumanMessage)
|
||||
assert sent[-2] is messages[-1]
|
||||
assert len(messages) == 3 # caller's list untouched
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_structured_report_keeps_user_ending_conversation_as_is():
|
||||
llm, structured = _llm(CheckReport(verdicts=[]))
|
||||
messages = [SystemMessage(content="s"), HumanMessage(content="final iteration")]
|
||||
|
||||
await _structured_report(llm, messages)
|
||||
|
||||
assert structured.ainvoke.call_args.args[0] is messages
|
||||
@@ -0,0 +1,197 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""The Operator-facing execution incident block and the incident record."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.agents.operator.prompts import (
|
||||
EXECUTION_INCIDENT_MARKER,
|
||||
ExecutionIncidentPromptComponent,
|
||||
PromptBuilder,
|
||||
render_closed_incident,
|
||||
render_execution_incident,
|
||||
)
|
||||
from artemis.agents.validator.categories import ValidationErrorCategory
|
||||
from artemis.agents.validator.incidents import (
|
||||
KIND_EXEC_ERROR,
|
||||
KIND_SAFETY_NET,
|
||||
ExecutionIncident,
|
||||
open_incident,
|
||||
)
|
||||
|
||||
STEPS = [
|
||||
{
|
||||
"step_number": 11,
|
||||
"action_taken": [{"action": "click", "coordinates": [540, 1800], "target_text": "player"}],
|
||||
"last_execution_result": {"status": "success"},
|
||||
},
|
||||
{
|
||||
"step_number": 12,
|
||||
"action_taken": [{"action": "click", "target_text": "Skip"}],
|
||||
"last_execution_result": {"status": "failed"},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _disappeared(consecutive=1, **overrides):
|
||||
incident = open_incident(
|
||||
previous=None,
|
||||
kind=KIND_SAFETY_NET,
|
||||
category=ValidationErrorCategory.TARGET_DISAPPEARED,
|
||||
reason="Target element 'Skip' was not found on the screen.",
|
||||
action_item={
|
||||
"action": "tap",
|
||||
"coordinates": [950, 288],
|
||||
"normalized_coordinates": [880, 120],
|
||||
"target_text": "Skip",
|
||||
"attempts": ["Pre-execution validation failed: ..."],
|
||||
},
|
||||
action_description="Tapped 'Skip' at [950, 288]",
|
||||
step_number=12,
|
||||
)
|
||||
incident["consecutive_failures"] = consecutive
|
||||
incident.update(overrides)
|
||||
return incident
|
||||
|
||||
|
||||
# --- record -------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_open_incident_strips_internal_keys_and_starts_at_one():
|
||||
incident = _disappeared()
|
||||
assert incident["consecutive_failures"] == 1
|
||||
assert "attempts" not in incident["action"]
|
||||
assert incident["category"] == "target_disappeared"
|
||||
assert incident["burst_size"] == 1 and incident["action_index"] == 0
|
||||
|
||||
|
||||
def test_open_incident_continues_the_previous_count():
|
||||
previous = _disappeared(consecutive=2)
|
||||
incident = open_incident(
|
||||
previous=previous,
|
||||
kind=KIND_EXEC_ERROR,
|
||||
category=ValidationErrorCategory.GENERAL,
|
||||
reason="Error: tap rejected",
|
||||
action_item={"action": "tap"},
|
||||
action_description="Tapped element",
|
||||
)
|
||||
assert incident["consecutive_failures"] == 3
|
||||
assert ExecutionIncident.from_dict(incident).is_burst is False
|
||||
assert ExecutionIncident.from_dict(None) is None
|
||||
assert ExecutionIncident.from_dict({"kind": "safety_net", "unknown_key": 1}) is not None
|
||||
|
||||
|
||||
# --- rendering ----------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_render_disappeared_names_target_trigger_and_burst_exception():
|
||||
text = render_execution_incident(_disappeared(), STEPS)
|
||||
assert text.startswith(EXECUTION_INCIDENT_MARKER)
|
||||
assert "Opened at Step 12; consecutive failed turns: 1." in text
|
||||
assert "was NOT executed. The pre-execution safety net refused it" in text
|
||||
# Recorded target with normalized coordinates and text.
|
||||
assert 'normalized [880, 120] "Skip"' in text
|
||||
# The last successful action before the incident is the trigger candidate.
|
||||
assert "Your last successfully executed action was `Tapped 'player' at [540, 1800]`" in text
|
||||
assert "(Step 11)" in text
|
||||
# Facts and evidence only: the response protocol is stated once, in the
|
||||
# static system prompt, so the per-turn block never restates it.
|
||||
assert "How to respond" not in text
|
||||
assert "settle the original intent" not in text
|
||||
assert "e.g." not in text
|
||||
# No escalation ladder: the Operator keeps its freedom.
|
||||
assert "Escalation" not in text
|
||||
assert "ask_diagnoser" not in text
|
||||
|
||||
|
||||
def test_render_shifted_and_occupied_evidence():
|
||||
shifted = _disappeared(
|
||||
category="target_shifted",
|
||||
evidence={"new_location": [500, 640], "new_bounds": [400, 600, 600, 680]},
|
||||
)
|
||||
text = render_execution_incident(shifted, STEPS)
|
||||
assert "has moved at normalized [500, 640] (bounds [400, 600, 600, 680])" in text
|
||||
|
||||
occupied = _disappeared(category="target_occupied", evidence={"occupant": "'Sign Up' button"})
|
||||
text = render_execution_incident(occupied, STEPS)
|
||||
assert "now covered by 'Sign Up' button" in text
|
||||
|
||||
|
||||
def test_render_burst_abort_and_exec_error():
|
||||
burst = _disappeared(kind=KIND_EXEC_ERROR, category="general", burst_size=3, action_index=1)
|
||||
burst["reason"] = "Error: tap rejected"
|
||||
text = render_execution_incident(burst, STEPS)
|
||||
assert "action 2 of your 3-action fast burst" in text
|
||||
assert "The 1 action(s) after it were NOT executed" in text
|
||||
|
||||
plain = _disappeared(kind=KIND_EXEC_ERROR, category="general")
|
||||
text = render_execution_incident(plain, STEPS)
|
||||
assert "was dispatched, but the device/executor reported" in text
|
||||
|
||||
|
||||
def test_render_without_history_omits_the_trigger_hint():
|
||||
text = render_execution_incident(_disappeared(), [])
|
||||
assert "last successfully executed action" not in text
|
||||
assert 'Its recorded target was normalized [880, 120] "Skip".' in text
|
||||
|
||||
|
||||
# --- component ----------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_component_renders_only_while_an_incident_is_open():
|
||||
builder = PromptBuilder()
|
||||
ctx = SimpleNamespace(data_engine=None)
|
||||
await ExecutionIncidentPromptComponent()(
|
||||
builder, SimpleNamespace(open_incident=None), ctx, steps=STEPS
|
||||
)
|
||||
assert builder.human_parts == []
|
||||
|
||||
await ExecutionIncidentPromptComponent()(
|
||||
builder, SimpleNamespace(open_incident=_disappeared()), ctx, steps=STEPS
|
||||
)
|
||||
assert len(builder.human_parts) == 1
|
||||
# Its own text part: the transcript scrub keeps it verbatim across turns.
|
||||
assert builder.human_parts[0].startswith(EXECUTION_INCIDENT_MARKER)
|
||||
|
||||
|
||||
def test_render_closed_notice_settles_the_intent():
|
||||
closed = dict(_disappeared(), closed_at_step=15)
|
||||
text = render_closed_incident(closed)
|
||||
assert text.startswith("--- Execution Incident (CLOSED at Step 15) ---")
|
||||
assert "opened at Step 12 on `Tapped 'Skip' at [950, 288]`" in text
|
||||
assert "already served, still pending, or no longer needed" in text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_component_renders_closed_notice_once_no_incident_is_open():
|
||||
builder = PromptBuilder()
|
||||
ctx = SimpleNamespace(data_engine=None)
|
||||
state = SimpleNamespace(
|
||||
open_incident=None, last_closed_incident=dict(_disappeared(), closed_at_step=15)
|
||||
)
|
||||
await ExecutionIncidentPromptComponent()(builder, state, ctx, steps=STEPS)
|
||||
assert len(builder.human_parts) == 1
|
||||
assert builder.human_parts[0].startswith("--- Execution Incident (CLOSED at Step 15) ---")
|
||||
|
||||
# An open incident takes precedence over a stale closed notice.
|
||||
builder = PromptBuilder()
|
||||
state = SimpleNamespace(
|
||||
open_incident=_disappeared(), last_closed_incident=dict(_disappeared(), closed_at_step=15)
|
||||
)
|
||||
await ExecutionIncidentPromptComponent()(builder, state, ctx, steps=STEPS)
|
||||
assert builder.human_parts[0].startswith(EXECUTION_INCIDENT_MARKER)
|
||||
@@ -1041,25 +1041,26 @@ async def test_explorer_inspect_region():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_explorer_final_turn_tool_stripping():
|
||||
"""Test that on the final iteration (turn 10), Explorer strips other tools and injects warning."""
|
||||
"""On the final turn the Explorer strips other tools and injects the warning."""
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.device = MagicMock()
|
||||
mock_ctx.device.device_width = 1080
|
||||
mock_ctx.device.device_height = 2400
|
||||
mock_ctx.data_engine = MagicMock()
|
||||
mock_ctx.data_engine.base_dir = "/tmp/test_session"
|
||||
explorer = Explorer(mock_ctx)
|
||||
|
||||
mock_ctx.llm_config = MagicMock()
|
||||
mock_ctx.llm_config.explorer = MagicMock()
|
||||
mock_ctx.llm_config.explorer.model = "gemini-3.7-flash"
|
||||
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.latest_screenshot = "/tmp/test_screenshot.jpg"
|
||||
|
||||
# Set up generate_content mock to capture call arguments
|
||||
# A client already shared on the context selects the native engine and is
|
||||
# reused lazily by ``run`` (no ``genai.Client`` construction).
|
||||
mock_client = MagicMock()
|
||||
mock_ctx._genai_client = mock_client
|
||||
explorer = Explorer(mock_ctx)
|
||||
assert explorer.use_native_gemini is True
|
||||
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.latest_screenshot = "/tmp/test_screenshot.jpg"
|
||||
|
||||
# Turn 1: model returns search_xml_ocr
|
||||
fc_search = MagicMock()
|
||||
@@ -1094,14 +1095,14 @@ async def test_explorer_final_turn_tool_stripping():
|
||||
minimal_list="test list",
|
||||
)
|
||||
|
||||
# Total number of iterations is max_iterations = 3 under "pro" mode default
|
||||
# The default "pro" tier budgets 3 turns (tiers.EXPLORER_TIERS)
|
||||
assert mock_client.aio.models.generate_content.call_count == 3
|
||||
|
||||
# Check the final turn call (index 2) keyword args
|
||||
call_args_3 = mock_client.aio.models.generate_content.call_args_list[2]
|
||||
config = call_args_3.kwargs["config"]
|
||||
|
||||
# Tools in the 10th call should only contain "submit_answer"
|
||||
# Tools in the final call should only contain "submit_answer"
|
||||
tools = config.tools[0].function_declarations
|
||||
assert len(tools) == 1
|
||||
assert tools[0].name == "submit_answer"
|
||||
@@ -1120,9 +1121,16 @@ async def test_explorer_final_turn_tool_stripping():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_explorer_ask_perception_tool():
|
||||
"""The perception bundle answers from the in-memory screen index.
|
||||
|
||||
No Data Engine lookup is involved: the text search, the coordinate
|
||||
audit and the OCR inventory all read ``explorer.screen_index``.
|
||||
"""
|
||||
import numpy as np
|
||||
from unittest.mock import mock_open
|
||||
|
||||
from artemis.agents.explorer.screen_index import ScreenIndex
|
||||
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = MagicMock()
|
||||
mock_ctx.data_engine.base_dir = "/tmp/test_session"
|
||||
@@ -1144,37 +1152,24 @@ async def test_explorer_ask_perception_tool():
|
||||
"description": "Original complete screenshot",
|
||||
}
|
||||
}
|
||||
|
||||
# Mock search_ui_func
|
||||
mock_search_ui_res = {
|
||||
"matches": [
|
||||
explorer.screen_index = ScreenIndex.from_hierarchy(
|
||||
[
|
||||
{
|
||||
"matched_text": "Settings",
|
||||
"bounds": [540, 480, 540, 480],
|
||||
"type": "xml",
|
||||
"bounds": "[500,400][600,560]",
|
||||
"text": "Settings",
|
||||
"resource-id": "com.android.settings:id/btn_settings",
|
||||
"class": "android.widget.Button",
|
||||
"clickable": "true",
|
||||
},
|
||||
{
|
||||
"matched_text": "Sign In",
|
||||
"bounds": [100, 100, 200, 200],
|
||||
"type": "ocr",
|
||||
"bounds": "[100,100][200,200]",
|
||||
"class": "android.widget.FrameLayout [OCR]",
|
||||
"ocr_elements": [{"text": "Settings page", "bounds": "[100,100][200,200]"}],
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
# Mock storage record
|
||||
mock_record = MagicMock()
|
||||
mock_record.ui_tree = [
|
||||
{
|
||||
"bounds": "[500,400][600,560]",
|
||||
"text": "Settings Button",
|
||||
"resource-id": "com.android.settings:id/btn_settings",
|
||||
"class": "android.widget.Button",
|
||||
}
|
||||
]
|
||||
mock_record.ocr_result = []
|
||||
|
||||
mock_storage = MagicMock()
|
||||
mock_storage.get_image.return_value = mock_record
|
||||
],
|
||||
1080,
|
||||
2400,
|
||||
)
|
||||
|
||||
# Mock _run_object_detection
|
||||
mock_detector_res = {
|
||||
@@ -1186,18 +1181,7 @@ async def test_explorer_ask_perception_tool():
|
||||
mock_draw_dots = MagicMock()
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.explorer.explorer.search_ui_func",
|
||||
return_value=mock_search_ui_res,
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.explorer.explorer.StorageManager",
|
||||
return_value=mock_storage,
|
||||
),
|
||||
patch(
|
||||
"artemis.mcp.xml_search_server.StorageManager",
|
||||
return_value=mock_storage,
|
||||
),
|
||||
patch("artemis.agents.explorer.explorer.StorageManager") as storage_cls,
|
||||
patch(
|
||||
"artemis.agents.explorer.explorer._run_object_detection",
|
||||
return_value=mock_detector_res,
|
||||
@@ -1216,37 +1200,46 @@ async def test_explorer_ask_perception_tool():
|
||||
detect_queries=["profile icon"],
|
||||
)
|
||||
|
||||
assert "Text Search Results are:" in result["text"]
|
||||
assert "Coordinate Search Results are:" in result["text"]
|
||||
assert "Object Detection Results are:" in result["text"]
|
||||
storage_cls.assert_not_called()
|
||||
assert "Text Search Results are:" in result["text"]
|
||||
assert "Coordinate Search Results are:" in result["text"]
|
||||
assert "Object Detection Results are:" in result["text"]
|
||||
|
||||
# XML Text search should return label X1
|
||||
assert "[X1]" in result["text"]
|
||||
# OCR Text search should return label O2
|
||||
assert "[O2]" in result["text"]
|
||||
# Coordinate search should return label X3
|
||||
assert "[X3]" in result["text"]
|
||||
# Object detection should return label D4 (since global label idx is shared/contiguous)
|
||||
assert "[D4]" in result["text"]
|
||||
# UI-tree text match -> X1, OCR text match -> O2 (both from the index)
|
||||
assert "[X1] 'Settings' at [509,200]" in result["text"]
|
||||
assert "[O2] 'Settings page' at [138,62]" in result["text"]
|
||||
# Coordinate audit -> X3 with the innermost element's description
|
||||
assert (
|
||||
"Matched element at [550,200]: [X3] | 'Settings' class=android.widget.Button"
|
||||
" id=com.android.settings:id/btn_settings source=ui-tree interactive"
|
||||
) in result["text"]
|
||||
# Object detection -> D4 (the label counter is shared and contiguous)
|
||||
assert "[D4]" in result["text"]
|
||||
|
||||
assert len(result["image_paths"]) == 3
|
||||
# Element-backed labels are registered; detection points are not.
|
||||
assert explorer.label_registry["X1"].text == "Settings"
|
||||
assert explorer.label_registry["O2"].source == "ocr"
|
||||
assert explorer.label_registry["X3"] is explorer.label_registry["X1"]
|
||||
assert "D4" not in explorer.label_registry
|
||||
|
||||
# Verify call arguments of draw_dots
|
||||
assert mock_draw_dots.call_count == 3
|
||||
# First call (XML/OCR Text search): green color
|
||||
first_call = mock_draw_dots.call_args_list[0]
|
||||
assert first_call.kwargs.get("color") == "green"
|
||||
assert first_call.args[2] == ["X1", "O2"]
|
||||
assert len(result["image_paths"]) == 3
|
||||
|
||||
# Second call (XML Coordinate audit): blue color
|
||||
second_call = mock_draw_dots.call_args_list[1]
|
||||
assert second_call.kwargs.get("color") == "blue"
|
||||
assert second_call.args[2] == ["X3"]
|
||||
# Verify call arguments of draw_dots
|
||||
assert mock_draw_dots.call_count == 3
|
||||
# First call (XML/OCR Text search): green color
|
||||
first_call = mock_draw_dots.call_args_list[0]
|
||||
assert first_call.kwargs.get("color") == "green"
|
||||
assert first_call.args[2] == ["X1", "O2"]
|
||||
|
||||
# Third call (Object detection): red color
|
||||
third_call = mock_draw_dots.call_args_list[2]
|
||||
assert third_call.kwargs.get("color") == "red"
|
||||
assert third_call.args[2] == ["D4"]
|
||||
# Second call (XML Coordinate audit): blue color
|
||||
second_call = mock_draw_dots.call_args_list[1]
|
||||
assert second_call.kwargs.get("color") == "blue"
|
||||
assert second_call.args[2] == ["X3"]
|
||||
|
||||
# Third call (Object detection): red color
|
||||
third_call = mock_draw_dots.call_args_list[2]
|
||||
assert third_call.kwargs.get("color") == "red"
|
||||
assert third_call.args[2] == ["D4"]
|
||||
|
||||
|
||||
def test_explorer_prune_historical_images():
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Screen geometry shared by the Explorer and the ``ask_explorer`` tool."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.agents.explorer.geometry import (
|
||||
FALLBACK_SCREEN_SIZE,
|
||||
NORMALIZED_MAX,
|
||||
is_valid_norm_point,
|
||||
norm_to_pixel,
|
||||
pixel_to_norm,
|
||||
resolve_screen_size,
|
||||
)
|
||||
|
||||
|
||||
def _ctx(width=None, height=None, device=True):
|
||||
if not device:
|
||||
return SimpleNamespace(device=None)
|
||||
return SimpleNamespace(device=SimpleNamespace(device_width=width, device_height=height))
|
||||
|
||||
|
||||
def _state(raw):
|
||||
return SimpleNamespace(operator_raw_data=raw)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# resolve_screen_size
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_operator_observation_wins_over_the_device_context():
|
||||
ctx = _ctx(1080, 2400)
|
||||
state = _state({"width": 720, "height": 1600})
|
||||
assert resolve_screen_size(ctx, state) == (720, 1600)
|
||||
|
||||
|
||||
def test_device_context_is_used_when_the_observation_has_no_size():
|
||||
ctx = _ctx(1080, 2400)
|
||||
assert resolve_screen_size(ctx, _state(None)) == (1080, 2400)
|
||||
assert resolve_screen_size(ctx, _state({})) == (1080, 2400)
|
||||
assert resolve_screen_size(ctx, _state("not a dict")) == (1080, 2400)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw",
|
||||
[
|
||||
{"width": 0, "height": 1600},
|
||||
{"width": 720, "height": -1},
|
||||
{"width": "720", "height": 1600},
|
||||
{"width": 720.0, "height": 1600},
|
||||
{"width": None, "height": None},
|
||||
{"width": 720},
|
||||
],
|
||||
)
|
||||
def test_non_int_or_non_positive_observation_sizes_are_ignored(raw):
|
||||
ctx = _ctx(1080, 2400)
|
||||
assert resolve_screen_size(ctx, _state(raw)) == (1080, 2400)
|
||||
|
||||
|
||||
def test_fallback_when_neither_source_reports_a_size():
|
||||
assert resolve_screen_size(_ctx(device=False), _state(None)) == FALLBACK_SCREEN_SIZE
|
||||
assert resolve_screen_size(_ctx(None, None), _state(None)) == FALLBACK_SCREEN_SIZE
|
||||
assert resolve_screen_size(SimpleNamespace(), SimpleNamespace()) == FALLBACK_SCREEN_SIZE
|
||||
|
||||
|
||||
def test_device_dimensions_fall_back_independently():
|
||||
fb_w, fb_h = FALLBACK_SCREEN_SIZE
|
||||
assert resolve_screen_size(_ctx(1440, 0), _state(None)) == (1440, fb_h)
|
||||
assert resolve_screen_size(_ctx("1440", 3200), _state(None)) == (fb_w, 3200)
|
||||
assert resolve_screen_size(_ctx(None, 3200), _state(None)) == (fb_w, 3200)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# norm_to_pixel / pixel_to_norm
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_norm_to_pixel_scales_on_the_0_1000_grid():
|
||||
assert norm_to_pixel(500, 500, 1080, 2400) == (540, 1200)
|
||||
assert norm_to_pixel(0, 0, 1080, 2400) == (0, 0)
|
||||
assert norm_to_pixel(NORMALIZED_MAX, NORMALIZED_MAX, 1080, 2400) == (1080, 2400)
|
||||
assert norm_to_pixel(250, 750, 1080, 2400) == (270, 1800)
|
||||
|
||||
|
||||
def test_norm_to_pixel_clamps_out_of_range_inputs():
|
||||
assert norm_to_pixel(-50, -1, 1080, 2400) == (0, 0)
|
||||
assert norm_to_pixel(1500, 2000, 1080, 2400) == (1080, 2400)
|
||||
|
||||
|
||||
def test_pixel_to_norm_scales_back_to_the_0_1000_grid():
|
||||
assert pixel_to_norm(540, 1200, 1080, 2400) == (500, 500)
|
||||
assert pixel_to_norm(0, 0, 1080, 2400) == (0, 0)
|
||||
assert pixel_to_norm(1080, 2400, 1080, 2400) == (NORMALIZED_MAX, NORMALIZED_MAX)
|
||||
|
||||
|
||||
def test_pixel_to_norm_clamps_out_of_range_inputs():
|
||||
assert pixel_to_norm(-10, -10, 1080, 2400) == (0, 0)
|
||||
assert pixel_to_norm(5000, 9000, 1080, 2400) == (NORMALIZED_MAX, NORMALIZED_MAX)
|
||||
|
||||
|
||||
def test_conversions_round_trip_on_grid_points():
|
||||
for nx, ny in [(0, 0), (500, 500), (1000, 1000), (250, 750)]:
|
||||
px, py = norm_to_pixel(nx, ny, 1000, 2000)
|
||||
assert pixel_to_norm(px, py, 1000, 2000) == (nx, ny)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# is_valid_norm_point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"coords",
|
||||
[[0, 0], [500, 500], (1000, 1000), ["500", "250"], [12.7, 999.2]],
|
||||
)
|
||||
def test_valid_norm_points(coords):
|
||||
assert is_valid_norm_point(coords)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"coords",
|
||||
[
|
||||
None,
|
||||
"500,500",
|
||||
[500],
|
||||
[500, 500, 500],
|
||||
[-1, 0],
|
||||
[0, 1001],
|
||||
["x", 0],
|
||||
[None, 5],
|
||||
{"x": 1, "y": 2},
|
||||
],
|
||||
)
|
||||
def test_invalid_norm_points(coords):
|
||||
assert not is_valid_norm_point(coords)
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Regression tests for Explorer perception-tool failure paths."""
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.agents.explorer.perception_tools import PerceptionToolsMixin
|
||||
from artemis.agents.explorer.screen_index import ScreenIndex
|
||||
|
||||
|
||||
def _perception_host() -> PerceptionToolsMixin:
|
||||
host = PerceptionToolsMixin()
|
||||
host.ctx = MagicMock()
|
||||
host.global_label_idx = 1
|
||||
host.width = 1080
|
||||
host.height = 2400
|
||||
host.image_name = "screen-1"
|
||||
host.screenshot_path = "screen-1.jpg"
|
||||
host.image_pool = {
|
||||
"img_0": {
|
||||
"path": "screen-1.jpg",
|
||||
"transform": {
|
||||
"offset_x": 0.0,
|
||||
"offset_y": 0.0,
|
||||
"scale_x": 1.0,
|
||||
"scale_y": 1.0,
|
||||
},
|
||||
}
|
||||
}
|
||||
host.next_img_id = 1
|
||||
host.screen_index = ScreenIndex.empty(1080, 2400)
|
||||
host.label_registry = {}
|
||||
return host
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_ocr_list_reads_the_screen_index_without_storage() -> None:
|
||||
host = _perception_host()
|
||||
|
||||
with patch("artemis.agents.explorer.explorer.StorageManager") as storage_cls:
|
||||
result = await host.exec_get_ocr_list()
|
||||
|
||||
assert result == {
|
||||
"text": "No text elements detected on the screen.",
|
||||
"image_path": None,
|
||||
}
|
||||
storage_cls.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_image_processor_empty_outputs_returns_stable_error() -> None:
|
||||
host = _perception_host()
|
||||
processor = MagicMock()
|
||||
processor.run = AsyncMock(return_value={"outputs": [], "summary": "nothing found"})
|
||||
|
||||
with patch(
|
||||
"artemis.agents.explorer.perception_tools.ImageProcessor",
|
||||
return_value=processor,
|
||||
):
|
||||
result = await host.exec_ask_image_processor("crop", target_image_id="img_0")
|
||||
|
||||
assert result == {
|
||||
"text": "ImageProcessor error: no output images were produced.",
|
||||
"image_paths": [],
|
||||
}
|
||||
@@ -0,0 +1,463 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Explorer in-memory screen search, label registry, bounds enrichment and pre-pass."""
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.agents.explorer.explorer import Explorer
|
||||
from artemis.agents.explorer.screen_index import ScreenElement, ScreenIndex
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.state import State
|
||||
|
||||
FACADE = "artemis.agents.explorer.explorer"
|
||||
W, H = 1080, 2400
|
||||
|
||||
#: One "Settings" button, two "Send" buttons and two OCR-only texts.
|
||||
HIERARCHY = [
|
||||
{
|
||||
"text": "Settings",
|
||||
"bounds": "[500,400][600,560]",
|
||||
"class": "android.widget.Button",
|
||||
"resource-id": "app:id/settings",
|
||||
"clickable": "true",
|
||||
},
|
||||
{"text": "Send", "bounds": "[100,2000][300,2100]", "class": "android.widget.Button"},
|
||||
{"text": "Send", "bounds": "[700,2000][900,2100]", "class": "android.widget.Button"},
|
||||
{
|
||||
"bounds": "[0,0][1080,2400]",
|
||||
"class": "android.widget.FrameLayout",
|
||||
"ocr_elements": [
|
||||
{"text": "Settings page", "bounds": "[100,100][300,160]"},
|
||||
{"text": "Welcome", "bounds": "[100,200][400,260]"},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
#: "Settings" [500,400][600,560] on a 1080x2400 screen, in normalized units.
|
||||
SETTINGS_COORDS = [509, 200]
|
||||
SETTINGS_BOUNDS = [462, 166, 555, 233]
|
||||
|
||||
|
||||
def _context() -> MagicMock:
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.device = MagicMock()
|
||||
ctx.device.device_width = W
|
||||
ctx.device.device_height = H
|
||||
ctx.data_engine = MagicMock()
|
||||
ctx.data_engine.base_dir = None
|
||||
ctx.llm_config = MagicMock()
|
||||
ctx.llm_config.explorer = MagicMock()
|
||||
ctx.llm_config.explorer.model = "gemini-3.8-flash"
|
||||
ctx.agent_config = MagicMock()
|
||||
ctx.agent_config.denylisted_tools = {}
|
||||
return ctx
|
||||
|
||||
|
||||
def _state(screenshot: str, hierarchy=None) -> MagicMock:
|
||||
state = MagicMock(spec=State)
|
||||
state.latest_screenshot = screenshot
|
||||
state.latest_ui_hierarchy = HIERARCHY if hierarchy is None else hierarchy
|
||||
state.operator_raw_data = {}
|
||||
return state
|
||||
|
||||
|
||||
def _indexed_explorer(hierarchy=None) -> Explorer:
|
||||
"""An Explorer whose screen is already indexed (no ``run`` involved)."""
|
||||
explorer = Explorer(_context())
|
||||
explorer.width, explorer.height = W, H
|
||||
explorer.screenshot_path = "/tmp/screen.jpg"
|
||||
explorer.screen_index = ScreenIndex.from_hierarchy(
|
||||
HIERARCHY if hierarchy is None else hierarchy, W, H
|
||||
)
|
||||
return explorer
|
||||
|
||||
|
||||
def _storage(record=None) -> MagicMock:
|
||||
storage = MagicMock()
|
||||
storage.get_image.return_value = record
|
||||
return storage
|
||||
|
||||
|
||||
def _outcome(candidates=None, fallback: str = "") -> str:
|
||||
return json.dumps({"candidates": candidates or [], "fallback_message": fallback})
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def screenshot(tmp_path):
|
||||
path = tmp_path / "screen.jpg"
|
||||
path.write_bytes(b"fake-jpeg-bytes")
|
||||
return str(path)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def no_drawing():
|
||||
"""Keeps the annotation helpers away from the file system."""
|
||||
with (
|
||||
patch(f"{FACADE}.draw_dots") as draw_dots,
|
||||
patch("pathlib.Path.mkdir"),
|
||||
patch("glob.glob", return_value=[]),
|
||||
):
|
||||
yield draw_dots
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Screen index construction inside run()
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_index_is_built_from_state_hierarchy_when_record_is_missing(screenshot, no_drawing):
|
||||
explorer = Explorer(_context())
|
||||
explorer._run_flash = AsyncMock(return_value=_outcome(fallback="nothing"))
|
||||
|
||||
with (
|
||||
patch(f"{FACADE}.StorageManager", return_value=_storage(None)),
|
||||
patch(f"{FACADE}.perform_ocr", new_callable=AsyncMock) as perform_ocr,
|
||||
):
|
||||
await explorer.run("gear icon", "", screenshot, _state(screenshot), version="flash")
|
||||
text = (await explorer._search_ui_helper("Settings", prefix="X"))["text"]
|
||||
|
||||
perform_ocr.assert_not_awaited()
|
||||
assert len(explorer.screen_index) == 5
|
||||
assert "not found in Data Engine" not in text
|
||||
assert "[X1] 'Settings' at [509,200]" in text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_index_ignores_state_hierarchy_for_another_screenshot(screenshot):
|
||||
explorer = Explorer(_context())
|
||||
explorer._run_flash = AsyncMock(return_value=_outcome())
|
||||
state = _state("/tmp/other.jpg")
|
||||
|
||||
with patch(f"{FACADE}.StorageManager", return_value=_storage(None)):
|
||||
await explorer.run("Settings", "", screenshot, state, version="flash")
|
||||
|
||||
assert len(explorer.screen_index) == 0
|
||||
explorer._run_flash.assert_awaited_once_with("Settings", screenshot)
|
||||
assert (await explorer._search_ui_helper("Settings"))["text"] == (
|
||||
"No UI-tree data is available for this screen; rely on visual detection."
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Perception helpers answer from the index
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_ui_helper_merges_lenient_matches_without_duplicates(no_drawing):
|
||||
hierarchy = [*HIERARCHY, {"text": "Settled", "bounds": "[100,300][300,360]"}]
|
||||
explorer = _indexed_explorer(hierarchy)
|
||||
spy = MagicMock(wraps=explorer.screen_index.search_text)
|
||||
explorer.screen_index.search_text = spy
|
||||
|
||||
result = await explorer._search_ui_helper("Settings", prefix="X", color="green")
|
||||
|
||||
assert [call.args[1] for call in spy.call_args_list] == [0.7, 0.4]
|
||||
parts = result["text"].split(" | ")
|
||||
assert parts[0] == "[X1] 'Settings' at [509,200]"
|
||||
assert parts[1] == "[O2] 'Settings page' at [185,54]"
|
||||
# "Settled" (similarity ~0.53) only arrives through the lenient pass...
|
||||
assert any(part.startswith("[X") and "'Settled'" in part for part in parts)
|
||||
# ...and the strict hits are not listed a second time.
|
||||
assert sum("'Settings'" in part for part in parts) == 1
|
||||
assert explorer.label_registry["X1"].text == "Settings"
|
||||
assert explorer.label_registry["O2"].source == "ocr"
|
||||
assert no_drawing.call_args.kwargs["color"] == "green"
|
||||
assert result["image_path"] is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_ui_helper_skips_lenient_pass_with_enough_strict_hits(no_drawing):
|
||||
explorer = _indexed_explorer()
|
||||
strict = explorer.screen_index.search_text("Settings", 0.4) # 5 hits, all "strict"
|
||||
explorer.screen_index.search_text = MagicMock(return_value=strict)
|
||||
|
||||
await explorer._search_ui_helper("Settings", prefix="X")
|
||||
|
||||
explorer.screen_index.search_text.assert_called_once_with("Settings", 0.7)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_by_coords_helper_reports_innermost_first(no_drawing):
|
||||
explorer = _indexed_explorer(
|
||||
[
|
||||
{"text": "Card", "bounds": "[0,0][1080,1200]", "class": "android.widget.FrameLayout"},
|
||||
{"text": "Title", "bounds": "[500,400][600,560]", "class": "android.widget.TextView"},
|
||||
]
|
||||
)
|
||||
|
||||
hit = await explorer._search_by_coords_helper(550, 200, prefix="X", color="blue")
|
||||
miss = await explorer._search_by_coords_helper(10, 990, prefix="X", color="blue")
|
||||
|
||||
assert hit["text"] == (
|
||||
"Matched element at [550,200]: [X1]"
|
||||
" | 'Title' class=android.widget.TextView source=ui-tree"
|
||||
" | 'Card' class=android.widget.FrameLayout source=ui-tree"
|
||||
)
|
||||
assert explorer.label_registry["X1"].text == "Title"
|
||||
assert miss["text"] == "No elements found at [10,990]."
|
||||
assert "X2" not in explorer.label_registry
|
||||
# The blue dot is drawn for both probes at the pixel position.
|
||||
assert no_drawing.call_count == 2
|
||||
assert no_drawing.call_args_list[0].args[1] == [[594, 480]]
|
||||
assert no_drawing.call_args_list[0].args[2] == ["X1"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_ocr_list_reads_the_index(no_drawing):
|
||||
explorer = _indexed_explorer()
|
||||
|
||||
with patch(f"{FACADE}.StorageManager") as storage_cls:
|
||||
result = await explorer.exec_get_ocr_list()
|
||||
|
||||
storage_cls.assert_not_called()
|
||||
assert result["text"].splitlines() == [
|
||||
"[O1] 'Settings page' coords: [185,54]",
|
||||
"[O2] 'Welcome' coords: [231,95]",
|
||||
]
|
||||
assert result["image_path"] is not None
|
||||
assert {k: v.source for k, v in explorer.label_registry.items()} == {"O1": "ocr", "O2": "ocr"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_ocr_list_without_ocr_elements():
|
||||
explorer = _indexed_explorer([{"text": "Only XML", "bounds": "[0,0][100,100]"}])
|
||||
result = await explorer.exec_get_ocr_list()
|
||||
assert result == {"text": "No text elements detected on the screen.", "image_path": None}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Label registry from the initial numbered list
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_initial_list_registers_numbered_labels(no_drawing):
|
||||
explorer = _indexed_explorer()
|
||||
|
||||
minimal_list, _marked = explorer._annotate_initial_screenshot(
|
||||
HIERARCHY, "/tmp/screen.jpg", None, ""
|
||||
)
|
||||
|
||||
assert minimal_list.splitlines()[0].startswith("[1] Text: 'Settings'")
|
||||
assert sorted(explorer.label_registry) == ["1", "2", "3", "4", "5"]
|
||||
settings = explorer.label_registry["1"]
|
||||
assert settings.bounds == (500, 400, 600, 560)
|
||||
assert settings.source == "xml" and settings.interactive # taken from the index
|
||||
assert explorer.label_registry["4"].source == "ocr"
|
||||
assert explorer.label_registry["4"].text == "Settings page"
|
||||
assert explorer.global_label_idx == 6
|
||||
assert no_drawing.call_args.args[1][0] == [550, 480]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Submit enrichment
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_enrich_candidates_adds_normalized_bounds_only_when_label_and_coords_agree():
|
||||
explorer = _indexed_explorer()
|
||||
settings = explorer.screen_index.exact_matches("Settings")[0]
|
||||
explorer._register_label("X1", settings)
|
||||
|
||||
inside = {"label": "X1", "coords": SETTINGS_COORDS, "description": "gear"}
|
||||
# 614px is 14px right of the element, inside the 2% (21.6px) tolerance.
|
||||
near = {"label": "X1", "coords": [569, 200], "description": "gear"}
|
||||
far = {"label": "X1", "coords": [100, 900], "description": "gear"}
|
||||
unknown = {"label": "D7", "coords": SETTINGS_COORDS, "description": "gear"}
|
||||
invalid = {"label": "X1", "coords": [5000, 5], "description": "gear"}
|
||||
|
||||
enriched = explorer._enrich_candidates([inside, near, far, unknown, invalid, "junk"])
|
||||
|
||||
assert enriched[0] == {**inside, "bounds": SETTINGS_BOUNDS, "source": "xml"}
|
||||
assert enriched[1] == {**near, "bounds": SETTINGS_BOUNDS, "source": "xml"}
|
||||
assert enriched[2:] == [far, unknown, invalid, "junk"]
|
||||
assert "bounds" not in far and "bounds" not in inside # inputs are never mutated
|
||||
|
||||
|
||||
def test_enrich_candidates_withholds_degenerate_bounds():
|
||||
explorer = _indexed_explorer()
|
||||
explorer._register_label("X9", ScreenElement("dot", (10, 10, 11, 11), "xml"))
|
||||
cand = {"label": "X9", "coords": [9, 4]}
|
||||
assert explorer._enrich_candidates([cand]) == [cand]
|
||||
|
||||
|
||||
def test_native_submit_enriches_registered_candidates():
|
||||
explorer = _indexed_explorer()
|
||||
explorer._register_label("O2", explorer.screen_index.ocr_elements[0])
|
||||
call = MagicMock()
|
||||
call.name = "submit_answer"
|
||||
call.args = {
|
||||
"candidates": [
|
||||
{"label": "O2", "coords": [185, 54], "description": "header"},
|
||||
{"label": "D3", "coords": [900, 100], "description": "icon"},
|
||||
],
|
||||
"fallback_message": "",
|
||||
}
|
||||
turn_record = {"tool_calls": []}
|
||||
|
||||
raw = explorer._process_submit_answer([call], turn_record, [])
|
||||
|
||||
data = json.loads(raw)
|
||||
assert list(data) == ["candidates", "fallback_message"]
|
||||
assert data["candidates"][0] == {
|
||||
"label": "O2",
|
||||
"coords": [185, 54],
|
||||
"description": "header",
|
||||
"bounds": [92, 41, 277, 66],
|
||||
"source": "ocr",
|
||||
}
|
||||
assert data["candidates"][1] == call.args["candidates"][1]
|
||||
|
||||
|
||||
def test_universal_submit_enriches_registered_candidates():
|
||||
explorer = _indexed_explorer()
|
||||
explorer._register_label("1", explorer.screen_index.exact_matches("Settings")[0])
|
||||
tool_calls = [
|
||||
{
|
||||
"name": "submit_answer",
|
||||
"args": {
|
||||
"candidates": [
|
||||
{"label": "1", "coords": SETTINGS_COORDS, "description": "gear"},
|
||||
{"label": "1", "coords": [5, 5000], "description": "bad"},
|
||||
],
|
||||
"fallback_message": "note",
|
||||
},
|
||||
"id": "c1",
|
||||
}
|
||||
]
|
||||
|
||||
data = json.loads(explorer._universal_submit_outcome(tool_calls))
|
||||
|
||||
assert data["fallback_message"] == "note"
|
||||
assert data["candidates"] == [
|
||||
{
|
||||
"label": "1",
|
||||
"coords": SETTINGS_COORDS,
|
||||
"description": "gear",
|
||||
"bounds": SETTINGS_BOUNDS,
|
||||
"source": "xml",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Structural pre-pass
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version", ["flash", "pro"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_prepass_answers_unique_exact_match_without_the_engine(screenshot, version):
|
||||
explorer = Explorer(_context())
|
||||
explorer._run_flash = AsyncMock()
|
||||
explorer._run_loop = AsyncMock()
|
||||
|
||||
with patch(f"{FACADE}.StorageManager", return_value=_storage(None)):
|
||||
raw = await explorer.run(" settings ", "", screenshot, _state(screenshot), version=version)
|
||||
|
||||
explorer._run_flash.assert_not_called()
|
||||
explorer._run_loop.assert_not_called()
|
||||
assert json.loads(raw) == {
|
||||
"candidates": [
|
||||
{
|
||||
"label": "T1",
|
||||
"coords": SETTINGS_COORDS,
|
||||
"bounds": SETTINGS_BOUNDS,
|
||||
"source": "xml",
|
||||
"description": "Settings (exact text match)",
|
||||
}
|
||||
],
|
||||
"fallback_message": "",
|
||||
}
|
||||
assert explorer.label_registry["T1"].text == "Settings"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prepass_leaves_ambiguous_matches_to_the_engine(screenshot):
|
||||
explorer = Explorer(_context())
|
||||
engine = _outcome([{"label": "D1", "coords": [185, 854], "description": "Send"}])
|
||||
explorer._run_flash = AsyncMock(return_value=engine)
|
||||
|
||||
with patch(f"{FACADE}.StorageManager", return_value=_storage(None)):
|
||||
raw = await explorer.run("Send", "", screenshot, _state(screenshot), version="flash")
|
||||
|
||||
explorer._run_flash.assert_awaited_once_with("Send", screenshot)
|
||||
assert raw == engine
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prepass_without_pipe_or_match_passes_the_query_through(screenshot):
|
||||
explorer = Explorer(_context())
|
||||
engine = _outcome(fallback="Failed to detect: gear icon")
|
||||
explorer._run_flash = AsyncMock(return_value=engine)
|
||||
|
||||
with patch(f"{FACADE}.StorageManager", return_value=_storage(None)):
|
||||
raw = await explorer.run("gear icon", "", screenshot, _state(screenshot), version="flash")
|
||||
|
||||
explorer._run_flash.assert_awaited_once_with("gear icon", screenshot)
|
||||
assert raw == engine
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prepass_merges_partial_results_with_the_engine_outcome(screenshot):
|
||||
explorer = Explorer(_context())
|
||||
detected = {"label": "D1", "coords": [185, 854], "description": "Send"}
|
||||
explorer._run_loop = AsyncMock(return_value=_outcome([detected], fallback="two Send buttons"))
|
||||
|
||||
with patch(f"{FACADE}.StorageManager", return_value=_storage(None)):
|
||||
raw = await explorer.run(
|
||||
"Settings | Send | Welcome", "", screenshot, _state(screenshot), version="pro"
|
||||
)
|
||||
|
||||
# Only the unresolved part reaches the engine; "Welcome" is an OCR exact match.
|
||||
assert explorer._run_loop.await_args.args[1] == "Send"
|
||||
data = json.loads(raw)
|
||||
assert [c["label"] for c in data["candidates"]] == ["T1", "T2", "D1"]
|
||||
assert data["candidates"][1]["source"] == "ocr"
|
||||
assert data["candidates"][1]["description"] == "Welcome (exact text match)"
|
||||
assert data["candidates"][2] == detected
|
||||
assert data["fallback_message"] == "two Send buttons"
|
||||
|
||||
|
||||
def test_merge_prepass_keeps_structural_hits_over_an_unreadable_engine_answer():
|
||||
prepass = [{"label": "T1", "coords": [1, 2]}]
|
||||
merged = json.loads(Explorer._merge_prepass(prepass, "Error: max iterations"))
|
||||
assert merged == {"candidates": prepass, "fallback_message": "Error: max iterations"}
|
||||
assert Explorer._merge_prepass([], "raw") == "raw"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prepass_never_runs_on_the_fly_ocr_for_flash(screenshot):
|
||||
record = SimpleNamespace(ui_tree=HIERARCHY[:3], ocr_result=None)
|
||||
explorer = Explorer(_context())
|
||||
explorer._run_flash = AsyncMock(return_value=_outcome())
|
||||
explorer._run_loop = AsyncMock(return_value=_outcome())
|
||||
|
||||
with (
|
||||
patch(f"{FACADE}.StorageManager", return_value=_storage(record)),
|
||||
patch(f"{FACADE}.is_ocr_configured", return_value=True),
|
||||
patch(f"{FACADE}.perform_ocr", new_callable=AsyncMock, return_value=[]) as perform_ocr,
|
||||
):
|
||||
await explorer.run("gear icon", "", screenshot, _state(screenshot, []), version="flash")
|
||||
perform_ocr.assert_not_awaited()
|
||||
assert len(explorer.screen_index) == 3 # the record still feeds the index
|
||||
|
||||
await explorer.run("gear icon", "", screenshot, _state(screenshot, []), version="pro")
|
||||
perform_ocr.assert_awaited_once()
|
||||
@@ -0,0 +1,120 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for the Explorer's in-memory screen index."""
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.agents.explorer.screen_index import (
|
||||
ScreenElement,
|
||||
ScreenIndex,
|
||||
normalize_text,
|
||||
text_similarity,
|
||||
)
|
||||
|
||||
W, H = 1080, 2400
|
||||
|
||||
|
||||
def _node(text=None, bounds="[0,0][100,100]", desc=None, **attrs):
|
||||
node = {"bounds": bounds, "class": "android.widget.TextView"}
|
||||
if text is not None:
|
||||
node["text"] = text
|
||||
if desc is not None:
|
||||
node["content-desc"] = desc
|
||||
node.update(attrs)
|
||||
return node
|
||||
|
||||
|
||||
def _hierarchy():
|
||||
return [
|
||||
_node("Gmail", "[300,1600][500,1800]", clickable="true", **{"resource-id": "app:gmail"}),
|
||||
_node(None, "[0,0][1080,2400]"), # unlabeled root wrapper is skipped
|
||||
_node(desc="Settings", bounds="[900,50][1000,150]", clickable="true"),
|
||||
{
|
||||
"bounds": "[100,2100][900,2300]",
|
||||
"class": "android.widget.FrameLayout [OCR]",
|
||||
"ocr_elements": [{"text": "Search", "bounds": "[150,2150][350,2250]"}],
|
||||
},
|
||||
_node("Search", "[120,2120][380,2280]"), # overlaps the OCR "Search"
|
||||
_node("Offscreen", "[-500,-500][-400,-400]"),
|
||||
_node("Zero", "[10,10][10,10]"),
|
||||
]
|
||||
|
||||
|
||||
def test_normalize_and_similarity():
|
||||
assert normalize_text(" Send\n Message ") == "send message"
|
||||
assert text_similarity("gmail", "Gmail") == 1.0
|
||||
assert text_similarity("send", "Send message") >= 0.85
|
||||
assert text_similarity("ok", "Look here") < 0.85
|
||||
assert text_similarity("", "x") == 0.0
|
||||
|
||||
|
||||
def test_from_hierarchy_skips_invisible_and_unlabeled():
|
||||
index = ScreenIndex.from_hierarchy(_hierarchy(), W, H)
|
||||
texts = sorted(e.text for e in index.elements)
|
||||
assert texts == ["Gmail", "Search", "Search", "Settings"]
|
||||
assert len(index.ocr_elements) == 1
|
||||
gmail = next(e for e in index.elements if e.text == "Gmail")
|
||||
assert gmail.interactive and gmail.resource_id == "app:gmail" and gmail.source == "xml"
|
||||
assert gmail.center == (400, 1700)
|
||||
|
||||
|
||||
def test_from_hierarchy_tolerates_garbage():
|
||||
index = ScreenIndex.from_hierarchy([None, "x", {"bounds": "bad"}, {}], W, H)
|
||||
assert len(index) == 0
|
||||
assert ScreenIndex.empty(W, H).search_text("x") == []
|
||||
|
||||
|
||||
def test_search_text_orders_by_score_and_respects_threshold():
|
||||
index = ScreenIndex.from_hierarchy(_hierarchy(), W, H)
|
||||
matches = index.search_text("Gmai", threshold=0.6)
|
||||
assert matches and matches[0].element.text == "Gmail"
|
||||
assert all(m.score > 0.6 for m in matches)
|
||||
assert index.search_text("zzzz", threshold=0.6) == []
|
||||
assert len(index.search_text("Search", threshold=0.4, limit=1)) == 1
|
||||
|
||||
|
||||
def test_exact_matches_collapse_overlapping_duplicates_preferring_xml():
|
||||
index = ScreenIndex.from_hierarchy(_hierarchy(), W, H)
|
||||
hits = index.exact_matches(" search ")
|
||||
assert len(hits) == 1
|
||||
assert hits[0].source == "xml"
|
||||
assert index.exact_matches("settings")[0].text == "Settings"
|
||||
assert index.exact_matches("") == []
|
||||
assert index.exact_matches("nope") == []
|
||||
|
||||
|
||||
def test_elements_at_reports_innermost_then_ocr():
|
||||
elements = [
|
||||
ScreenElement("outer", (0, 0, 1000, 1000), "xml"),
|
||||
ScreenElement("middle", (100, 100, 900, 900), "xml", interactive=False),
|
||||
ScreenElement("button", (400, 400, 600, 600), "xml", interactive=True),
|
||||
ScreenElement("label", (450, 450, 550, 550), "ocr"),
|
||||
]
|
||||
index = ScreenIndex(elements, W, H)
|
||||
found = index.elements_at(500, 500)
|
||||
# innermost interactive node stops the UI-tree walk; OCR appended after
|
||||
assert [e.text for e in found] == ["button", "label"]
|
||||
found = index.elements_at(150, 150)
|
||||
assert [e.text for e in found] == ["middle", "outer"]
|
||||
assert index.elements_at(1050, 1050) == []
|
||||
|
||||
|
||||
def test_describe_and_overlap():
|
||||
a = ScreenElement("A", (0, 0, 100, 100), "xml", class_name="Btn", resource_id="id/a")
|
||||
b = ScreenElement("B", (50, 50, 150, 150), "ocr")
|
||||
assert "class=Btn" in a.describe() and "id=id/a" in a.describe()
|
||||
assert "source=ocr" in b.describe()
|
||||
assert a.overlap_ratio(b) == pytest.approx(0.25)
|
||||
assert a.overlap_ratio(ScreenElement("C", (200, 200, 300, 300), "xml")) == 0.0
|
||||
@@ -0,0 +1,555 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Explorer tier integration: engine selection, tool exposure, caching, prompts."""
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from google.genai import types
|
||||
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
|
||||
import pytest
|
||||
|
||||
from artemis.agents.explorer.explorer import Explorer
|
||||
from artemis.agents.explorer.tiers import EXPLORER_TIERS, PERCEPTION_TOOLS, get_tier
|
||||
from artemis.config import settings
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.state import State
|
||||
|
||||
FACADE = "artemis.agents.explorer.explorer"
|
||||
|
||||
|
||||
def _context(model: str = "gemini-3.8-flash", denylist: list[str] | None = None) -> MagicMock:
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.device = MagicMock()
|
||||
ctx.device.device_width = 1080
|
||||
ctx.device.device_height = 2400
|
||||
ctx.data_engine = MagicMock()
|
||||
ctx.data_engine.base_dir = None
|
||||
ctx.llm_config = MagicMock()
|
||||
ctx.llm_config.explorer = MagicMock()
|
||||
ctx.llm_config.explorer.model = model
|
||||
ctx.llm_config.explorer.temperature = 0.1
|
||||
ctx.llm_config.explorer.thinking_level = None
|
||||
ctx.llm_config.explorer.fallback = None
|
||||
ctx.agent_config = MagicMock()
|
||||
ctx.agent_config.denylisted_tools = {"explorer": denylist or []}
|
||||
ctx.agent_config.explorer = SimpleNamespace(caching=None)
|
||||
ctx.execution_setup = SimpleNamespace(explorer=SimpleNamespace(caching=None))
|
||||
return ctx
|
||||
|
||||
|
||||
def _state(screenshot: str) -> MagicMock:
|
||||
state = MagicMock(spec=State)
|
||||
state.latest_screenshot = screenshot
|
||||
state.latest_ui_hierarchy = []
|
||||
state.operator_raw_data = {}
|
||||
return state
|
||||
|
||||
|
||||
def _submit_response(coords=(500, 600)) -> MagicMock:
|
||||
call = MagicMock()
|
||||
call.name = "submit_answer"
|
||||
call.args = {
|
||||
"candidates": [{"label": "D1", "coords": list(coords), "description": "target"}],
|
||||
"fallback_message": "",
|
||||
}
|
||||
response = MagicMock()
|
||||
response.function_calls = [call]
|
||||
response.candidates = [MagicMock()]
|
||||
response.candidates[0].content = types.Content(
|
||||
role="model",
|
||||
parts=[types.Part.from_function_call(name="submit_answer", args=call.args)],
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def _native_client(responses) -> MagicMock:
|
||||
client = MagicMock()
|
||||
client.aio.models.generate_content = AsyncMock(side_effect=list(responses))
|
||||
cache = SimpleNamespace(name="cachedContents/abc")
|
||||
client.aio.caches.create = AsyncMock(return_value=cache)
|
||||
client.aio.caches.delete = AsyncMock()
|
||||
client.aio.files.upload = AsyncMock(
|
||||
return_value=SimpleNamespace(uri="files/1", mime_type="image/jpeg", name="files/1")
|
||||
)
|
||||
client.aio.files.delete = AsyncMock()
|
||||
return client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def screenshot(tmp_path):
|
||||
path = tmp_path / "screen.jpg"
|
||||
path.write_bytes(b"fake-jpeg-bytes")
|
||||
return str(path)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _isolated_env(monkeypatch):
|
||||
monkeypatch.delenv("ARTEMIS_EXPLORER_CACHING", raising=False)
|
||||
monkeypatch.setenv("ARTEMIS_USE_FILE_API", "false")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Tier -> turn budget / tool exposure
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_construction_creates_no_genai_client():
|
||||
with patch(f"{FACADE}.genai.Client") as client_cls:
|
||||
explorer = Explorer(_context())
|
||||
client_cls.assert_not_called()
|
||||
assert explorer.client is None
|
||||
assert explorer.tier is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("version", "max_turns", "expected_tools"),
|
||||
[
|
||||
("pro", 3, {"ask_perception_tool", "submit_answer"}),
|
||||
("ultra", 8, PERCEPTION_TOOLS | {"submit_answer"}),
|
||||
],
|
||||
)
|
||||
def test_tier_sets_turn_budget_and_exposed_tools(version, max_turns, expected_tools):
|
||||
explorer = Explorer(_context())
|
||||
with patch(f"{FACADE}.is_ocr_configured", return_value=True):
|
||||
explorer._apply_tier(get_tier(version))
|
||||
native_names = {t.name for t in explorer.get_exposed_tools()}
|
||||
universal_names = {t["function"]["name"] for t in explorer._universal_exposed_tools()}
|
||||
|
||||
assert explorer.max_turns == max_turns
|
||||
assert native_names == expected_tools
|
||||
assert universal_names == expected_tools
|
||||
assert explorer.denylisted_tools >= get_tier(version).hidden_tools
|
||||
|
||||
|
||||
def test_unconfigured_ocr_and_user_denylist_hide_tools_on_ultra():
|
||||
explorer = Explorer(_context(denylist=["inspect_region"]))
|
||||
with patch(f"{FACADE}.is_ocr_configured", return_value=False):
|
||||
explorer._apply_tier(get_tier("ultra"))
|
||||
names = {t.name for t in explorer.get_exposed_tools()}
|
||||
assert "get_ocr_list" not in names
|
||||
assert "inspect_region" not in names
|
||||
assert {"detect_objects", "ask_image_processor", "ask_perception_tool"} <= names
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Engine selection
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_gemini_model_without_google_key_uses_universal_engine():
|
||||
with patch.object(settings, "GOOGLE_API_KEY", None):
|
||||
explorer = Explorer(_context(model="gemini-3.8-flash"))
|
||||
assert explorer.use_native_gemini is False
|
||||
|
||||
|
||||
def test_gemini_model_with_shared_client_uses_native_engine():
|
||||
ctx = _context(model="google/gemini-3.8-pro")
|
||||
ctx._genai_client = MagicMock()
|
||||
with patch.object(settings, "GOOGLE_API_KEY", None):
|
||||
explorer = Explorer(ctx)
|
||||
assert explorer.use_native_gemini is True
|
||||
assert explorer.model_name == "gemini-3.8-pro"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_universal_run_without_google_key_never_touches_genai(screenshot):
|
||||
ctx = _context(model="claude-sonnet-4")
|
||||
response = AIMessage(
|
||||
content="",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "submit_answer",
|
||||
"args": {
|
||||
"candidates": [{"label": "1", "coords": [100, 200], "description": "ok"}],
|
||||
"fallback_message": "",
|
||||
},
|
||||
"id": "call_1",
|
||||
}
|
||||
],
|
||||
)
|
||||
llm = MagicMock()
|
||||
bound = MagicMock()
|
||||
bound.ainvoke = AsyncMock(return_value=response)
|
||||
llm.bind_tools.return_value = bound
|
||||
storage = MagicMock()
|
||||
storage.get_image.return_value = None
|
||||
|
||||
with (
|
||||
patch.object(settings, "GOOGLE_API_KEY", None),
|
||||
patch(f"{FACADE}.genai.Client") as client_cls,
|
||||
patch(f"{FACADE}.get_llm", return_value=llm),
|
||||
patch(f"{FACADE}.StorageManager", return_value=storage),
|
||||
):
|
||||
explorer = Explorer(ctx)
|
||||
raw = await explorer.run("find it", "", screenshot, _state(screenshot), version="pro")
|
||||
|
||||
client_cls.assert_not_called()
|
||||
outcome = json.loads(raw)
|
||||
assert outcome["candidates"][0]["coords"] == [100, 200]
|
||||
# The universal engine only sees the pro tier's tools.
|
||||
bound_names = {t["function"]["name"] for t in llm.bind_tools.call_args_list[0].args[0]}
|
||||
assert bound_names == {"ask_perception_tool", "submit_answer"}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Universal dispatch delivers tool images
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_universal_dispatch_appends_image_message(tmp_path):
|
||||
explorer = Explorer(_context(model="claude-sonnet-4"))
|
||||
img_a = tmp_path / "a.jpg"
|
||||
img_b = tmp_path / "b.jpg"
|
||||
img_a.write_bytes(b"aaa")
|
||||
img_b.write_bytes(b"bbb")
|
||||
explorer.exec_ask_perception_tool = AsyncMock(
|
||||
return_value={"text": "perception text", "image_paths": [str(img_a), str(tmp_path / "x")]}
|
||||
)
|
||||
explorer.exec_inspect_region = AsyncMock(
|
||||
return_value={"text": "zoomed", "image_path": str(img_b)}
|
||||
)
|
||||
messages = []
|
||||
tool_calls = [
|
||||
{
|
||||
"name": "ask_perception_tool",
|
||||
"args": {"search_query": "q", "nx": 1, "ny": 2, "detect_queries": ["q"]},
|
||||
"id": "c1",
|
||||
},
|
||||
{
|
||||
"name": "inspect_region",
|
||||
"args": {"x_min": 0, "y_min": 0, "x_max": 10, "y_max": 10},
|
||||
"id": "c2",
|
||||
},
|
||||
]
|
||||
|
||||
await explorer._universal_dispatch_tools(tool_calls, messages, turn=1)
|
||||
|
||||
assert [type(m) for m in messages] == [ToolMessage, ToolMessage, HumanMessage]
|
||||
assert messages[0].content == "perception text"
|
||||
human = messages[2]
|
||||
assert human.content[0] == {
|
||||
"type": "text",
|
||||
"text": "[Annotated image(s) returned by: ask_perception_tool, inspect_region]",
|
||||
}
|
||||
image_blocks = human.content[1:]
|
||||
assert len(image_blocks) == 2 # the missing file is skipped
|
||||
assert all(b["type"] == "image_url" for b in image_blocks)
|
||||
assert image_blocks[0]["image_url"]["url"].startswith("data:image/jpeg;base64,")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_universal_dispatch_without_images_adds_no_human_message():
|
||||
explorer = Explorer(_context(model="claude-sonnet-4"))
|
||||
explorer.exec_get_ocr_list = AsyncMock(return_value={"text": "none", "image_path": None})
|
||||
messages = []
|
||||
await explorer._universal_dispatch_tools(
|
||||
[{"name": "get_ocr_list", "args": {}, "id": "c1"}], messages, turn=1
|
||||
)
|
||||
assert [type(m) for m in messages] == [ToolMessage]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_universal_dispatch_rejects_denylisted_tool():
|
||||
explorer = Explorer(_context(model="claude-sonnet-4"))
|
||||
explorer._apply_tier(get_tier("pro"))
|
||||
explorer.exec_inspect_region = AsyncMock()
|
||||
messages = []
|
||||
await explorer._universal_dispatch_tools(
|
||||
[{"name": "inspect_region", "args": {}, "id": "c1"}], messages, turn=1
|
||||
)
|
||||
explorer.exec_inspect_region.assert_not_called()
|
||||
assert "denylisted" in messages[0].content
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Caching precedence
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_caching_tier_defaults():
|
||||
explorer = Explorer(_context())
|
||||
with patch.object(settings, "EXPLORER_CACHING", None):
|
||||
assert explorer._resolve_caching(get_tier("pro"), None) is False
|
||||
assert explorer._resolve_caching(get_tier("ultra"), None) is True
|
||||
|
||||
|
||||
def test_caching_precedence(monkeypatch):
|
||||
ctx = _context()
|
||||
explorer = Explorer(ctx)
|
||||
pro = get_tier("pro")
|
||||
|
||||
with patch.object(settings, "EXPLORER_CACHING", None):
|
||||
ctx.agent_config.explorer = SimpleNamespace(caching=True)
|
||||
assert explorer._resolve_caching(pro, None) is True # agent config beats tier
|
||||
|
||||
monkeypatch.setenv("ARTEMIS_EXPLORER_CACHING", "false")
|
||||
assert explorer._resolve_caching(pro, None) is False # env beats agent config
|
||||
|
||||
assert explorer._resolve_caching(pro, True) is True # explicit beats env
|
||||
|
||||
with patch.object(settings, "EXPLORER_CACHING", True):
|
||||
monkeypatch.delenv("ARTEMIS_EXPLORER_CACHING")
|
||||
ctx.agent_config.explorer = SimpleNamespace(caching=None)
|
||||
ctx.execution_setup = SimpleNamespace(explorer=SimpleNamespace(caching=False))
|
||||
assert explorer._resolve_caching(pro, None) is False # execution setup beats settings
|
||||
|
||||
ctx.execution_setup = SimpleNamespace(explorer=SimpleNamespace(caching=None))
|
||||
assert explorer._resolve_caching(pro, None) is True # settings beat tier
|
||||
|
||||
|
||||
def test_caching_ignores_mock_config_values():
|
||||
"""A MagicMock context must fall through to the tier default, not force True."""
|
||||
ctx = MagicMock()
|
||||
explorer = Explorer(ctx)
|
||||
with patch.object(settings, "EXPLORER_CACHING", None):
|
||||
assert explorer._resolve_caching(get_tier("pro"), None) is False
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Tier-aware prompt
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_pro_prompt_only_describes_exposed_tools():
|
||||
explorer = Explorer(_context())
|
||||
explorer._apply_tier(get_tier("pro"))
|
||||
prompt, error = explorer._build_prompt_template(get_tier("pro"))
|
||||
assert error is None
|
||||
assert "ask_perception_tool" in prompt
|
||||
assert "inspect_region" not in prompt
|
||||
assert "Image Pool" not in prompt
|
||||
assert "ask_image_processor" not in prompt
|
||||
assert "get_ocr_list" not in prompt
|
||||
assert "# TOOL DENYLIST" not in prompt
|
||||
assert "# EXECUTION CONSTRAINT" in prompt
|
||||
assert "maximum of 3 turns" in prompt
|
||||
|
||||
|
||||
def test_ultra_prompt_describes_all_tools():
|
||||
explorer = Explorer(_context())
|
||||
with patch(f"{FACADE}.is_ocr_configured", return_value=True):
|
||||
explorer._apply_tier(get_tier("ultra"))
|
||||
prompt, error = explorer._build_prompt_template(get_tier("ultra"))
|
||||
assert error is None
|
||||
for tool in PERCEPTION_TOOLS:
|
||||
assert tool in prompt
|
||||
assert "Image Pool" in prompt
|
||||
assert "maximum of 8 turns" in prompt
|
||||
assert "# TOOL DENYLIST" not in prompt
|
||||
|
||||
|
||||
def test_prompt_denylist_section_only_for_user_denied_tier_tools():
|
||||
explorer = Explorer(_context(denylist=["inspect_region", "search_xml_ocr"]))
|
||||
with patch(f"{FACADE}.is_ocr_configured", return_value=True):
|
||||
explorer._apply_tier(get_tier("ultra"))
|
||||
ultra_prompt, _ = explorer._build_prompt_template(get_tier("ultra"))
|
||||
explorer._apply_tier(get_tier("pro"))
|
||||
pro_prompt, _ = explorer._build_prompt_template(get_tier("pro"))
|
||||
|
||||
assert "# TOOL DENYLIST" in ultra_prompt
|
||||
assert "cannot be used: inspect_region" in ultra_prompt
|
||||
assert "search_xml_ocr" not in ultra_prompt
|
||||
# Denied tools lose their guidance bullets too.
|
||||
assert "Targeted Verification" not in ultra_prompt
|
||||
# On pro, inspect_region is tier-hidden, so it is never mentioned.
|
||||
assert "# TOOL DENYLIST" not in pro_prompt
|
||||
assert "inspect_region" not in pro_prompt
|
||||
|
||||
|
||||
def test_prompt_sections_render_shape():
|
||||
explorer = Explorer(_context())
|
||||
explorer._apply_tier(get_tier("pro"))
|
||||
prompt, _ = explorer._build_prompt_template(get_tier("pro"))
|
||||
lines = prompt.splitlines()
|
||||
assert lines[0] == "# IDENTITY"
|
||||
assert any(line.startswith("# OPERATING PRINCIPLES") for line in lines)
|
||||
body = [line for line in lines if line and not line.startswith("#")]
|
||||
assert all(line.startswith("- ") or line.startswith("You are") for line in body)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Flash path
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_flash_run_returns_detected_candidates(screenshot):
|
||||
detection = AsyncMock(
|
||||
return_value={
|
||||
"detected": [
|
||||
{"label": "gear icon", "point": [900, 100]},
|
||||
{"label": "send button", "point": [500, 950]},
|
||||
],
|
||||
"failed": [],
|
||||
}
|
||||
)
|
||||
with (
|
||||
patch(f"{FACADE}._run_object_detection", detection),
|
||||
patch(f"{FACADE}.genai.Client") as client_cls,
|
||||
):
|
||||
explorer = Explorer(_context())
|
||||
raw = await explorer.run(
|
||||
"gear icon | send button", "", screenshot, _state(screenshot), version="flash"
|
||||
)
|
||||
|
||||
client_cls.assert_not_called()
|
||||
outcome = json.loads(raw)
|
||||
assert [c["label"] for c in outcome["candidates"]] == ["D1", "D2"]
|
||||
assert outcome["candidates"][0] == {
|
||||
"label": "D1",
|
||||
"coords": [900, 100],
|
||||
"description": "gear icon",
|
||||
}
|
||||
assert outcome["fallback_message"] == ""
|
||||
assert detection.call_args.args[2] == ["gear icon", "send button"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_flash_run_reports_missing_detection(screenshot):
|
||||
with patch(f"{FACADE}._run_object_detection", AsyncMock(return_value={"detected": []})):
|
||||
raw = await Explorer(_context()).run(
|
||||
"ghost", "", screenshot, _state(screenshot), version="flash"
|
||||
)
|
||||
assert json.loads(raw) == {"candidates": [], "fallback_message": "Failed to detect: ghost"}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Native engine: async SDK usage and resource lifecycle
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_ultra_run_uses_async_cache_api(screenshot):
|
||||
ctx = _context()
|
||||
client = _native_client([_submit_response()])
|
||||
ctx._genai_client = client
|
||||
storage = MagicMock()
|
||||
storage.get_image.return_value = None
|
||||
|
||||
with (
|
||||
patch(f"{FACADE}.genai.Client") as client_cls,
|
||||
patch(f"{FACADE}.StorageManager", return_value=storage),
|
||||
patch.object(settings, "EXPLORER_CACHING", None),
|
||||
):
|
||||
explorer = Explorer(ctx)
|
||||
raw = await explorer.run("target", "", screenshot, _state(screenshot), version="ultra")
|
||||
|
||||
client_cls.assert_not_called()
|
||||
assert json.loads(raw)["candidates"][0]["coords"] == [500, 600]
|
||||
client.aio.caches.create.assert_awaited_once()
|
||||
client.aio.caches.delete.assert_awaited_once_with(name="cachedContents/abc")
|
||||
config = client.aio.models.generate_content.call_args.kwargs["config"]
|
||||
assert config.cached_content == "cachedContents/abc"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_pro_run_skips_cache_by_default(screenshot):
|
||||
ctx = _context()
|
||||
client = _native_client([_submit_response()])
|
||||
ctx._genai_client = client
|
||||
storage = MagicMock()
|
||||
storage.get_image.return_value = None
|
||||
|
||||
with (
|
||||
patch(f"{FACADE}.StorageManager", return_value=storage),
|
||||
patch.object(settings, "EXPLORER_CACHING", None),
|
||||
):
|
||||
await Explorer(ctx).run("target", "", screenshot, _state(screenshot), version="pro")
|
||||
|
||||
client.aio.caches.create.assert_not_awaited()
|
||||
client.aio.caches.delete.assert_not_awaited()
|
||||
config = client.aio.models.generate_content.call_args.kwargs["config"]
|
||||
assert {t.name for t in config.tools[0].function_declarations} == {
|
||||
"ask_perception_tool",
|
||||
"submit_answer",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_file_api_uploads_and_deletes_asynchronously(screenshot, monkeypatch):
|
||||
monkeypatch.setenv("ARTEMIS_USE_FILE_API", "true")
|
||||
ctx = _context()
|
||||
client = _native_client([_submit_response()])
|
||||
ctx._genai_client = client
|
||||
storage = MagicMock()
|
||||
storage.get_image.return_value = None
|
||||
|
||||
with patch(f"{FACADE}.StorageManager", return_value=storage):
|
||||
await Explorer(ctx).run("target", "", screenshot, _state(screenshot), version="pro")
|
||||
|
||||
client.aio.files.upload.assert_awaited_once_with(file=screenshot)
|
||||
client.aio.files.delete.assert_awaited_once_with(name="files/1")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_closes_http_client_on_both_engines(screenshot):
|
||||
storage = MagicMock()
|
||||
storage.get_image.return_value = None
|
||||
|
||||
# Native engine
|
||||
ctx = _context()
|
||||
client = _native_client([_submit_response()])
|
||||
ctx._genai_client = client
|
||||
native = Explorer(ctx)
|
||||
native.http_client = SimpleNamespace(aclose=AsyncMock())
|
||||
with patch(f"{FACADE}.StorageManager", return_value=storage):
|
||||
await native.run("t", "", screenshot, _state(screenshot), version="pro")
|
||||
native_close = native.http_client # already reset
|
||||
assert native_close is None
|
||||
|
||||
# Universal engine (model call fails -> finally still closes)
|
||||
llm = MagicMock()
|
||||
llm.bind_tools.return_value.ainvoke = AsyncMock(side_effect=RuntimeError("boom"))
|
||||
universal = Explorer(_context(model="claude-sonnet-4"))
|
||||
http_client = SimpleNamespace(aclose=AsyncMock())
|
||||
universal.http_client = http_client
|
||||
with (
|
||||
patch(f"{FACADE}.StorageManager", return_value=storage),
|
||||
patch(f"{FACADE}.get_llm", return_value=llm),
|
||||
pytest.raises(RuntimeError),
|
||||
):
|
||||
await universal.run("t", "", screenshot, _state(screenshot), version="pro")
|
||||
http_client.aclose.assert_awaited_once()
|
||||
assert universal.http_client is None
|
||||
|
||||
|
||||
def test_missing_tool_calls_without_candidates_does_not_crash():
|
||||
explorer = Explorer(_context())
|
||||
response = MagicMock()
|
||||
response.candidates = None
|
||||
contents = []
|
||||
turn_record = {"tool_calls": []}
|
||||
explorer._handle_missing_tool_calls(contents, response, turn_record, ["hi"], 3, 1)
|
||||
assert contents[0].role == "model"
|
||||
assert contents[0].parts[0].text == "hi"
|
||||
assert "2 more" in contents[1].parts[0].text
|
||||
assert turn_record["tool_calls"][0]["name"] == "hallucinated_plain_text"
|
||||
|
||||
|
||||
def test_tier_table_is_single_source_for_versions():
|
||||
assert set(EXPLORER_TIERS) == {"flash", "pro", "ultra"}
|
||||
assert get_tier("bogus").name == "flash"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version", [None, "", "turbo", "PRO ", "Ultra"])
|
||||
def test_get_tier_normalizes_or_falls_back(version):
|
||||
"""Unknown or empty names fall back to the default tier; case and spaces are tolerated."""
|
||||
tier = get_tier(version)
|
||||
normalized = str(version or "").strip().lower()
|
||||
expected = normalized if normalized in EXPLORER_TIERS else "flash"
|
||||
assert tier.name == expected
|
||||
@@ -116,9 +116,8 @@ async def test_run_universal_explorer_success(mock_context, tmp_path):
|
||||
screenshot_path=str(fake_screenshot),
|
||||
state=mock_state,
|
||||
minimal_list="",
|
||||
version="pro",
|
||||
prompt_template="You are an explorer.",
|
||||
max_iterations=3,
|
||||
max_turns=3,
|
||||
)
|
||||
|
||||
res = json.loads(res_str)
|
||||
|
||||
@@ -1,213 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for Universal Failure Analyzer."""
|
||||
|
||||
import base64
|
||||
from unittest.mock import AsyncMock, MagicMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import AIMessage
|
||||
|
||||
from artemis.agents.validator.failure_analyzer import (
|
||||
FailureAnalyzer,
|
||||
PixelTargetDisappearedStrategy,
|
||||
TargetDisappearedStrategy,
|
||||
ValidationErrorCategory,
|
||||
VALIDATOR_TOOLS_DECLARATION,
|
||||
)
|
||||
from artemis.context import ArtemisContext
|
||||
|
||||
|
||||
class DummyState:
|
||||
def __init__(self, latest_screenshot=None):
|
||||
self.latest_screenshot = latest_screenshot
|
||||
self.indexed_points = []
|
||||
self.indexed_elements = []
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context(tmp_path):
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
ctx.llm_config = Mock()
|
||||
mock_llm_cfg = Mock()
|
||||
mock_llm_cfg.model = "gemini-2.5-pro"
|
||||
mock_llm_cfg.temperature = 0.1
|
||||
ctx.llm_config.get_agent.return_value = mock_llm_cfg
|
||||
|
||||
ctx.device = Mock()
|
||||
ctx.device.device_width = 1080
|
||||
ctx.device.device_height = 2400
|
||||
|
||||
ctx.data_engine = Mock()
|
||||
ctx.data_engine.base_dir = tmp_path
|
||||
ctx.data_engine.get_agent_friendly_steps.return_value = []
|
||||
ctx.adb_client = None
|
||||
ctx.driver = Mock()
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failure_analyzer_cannot_fix(mock_context, tmp_path):
|
||||
"""Test that FailureAnalyzer natively executes and handles 'cannot_fix' report."""
|
||||
screenshot_file = tmp_path / "shot.jpg"
|
||||
screenshot_file.write_bytes(b"dummy_bytes")
|
||||
|
||||
state = DummyState(latest_screenshot=str(screenshot_file))
|
||||
failed_action = {"action": "click", "coordinates": [100, 200]}
|
||||
error_msg = "Element not found"
|
||||
|
||||
analyzer = FailureAnalyzer(mock_context)
|
||||
|
||||
mock_llm = MagicMock()
|
||||
mock_response = AIMessage(
|
||||
content="I see the error.",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "report_failure_analysis",
|
||||
"args": {
|
||||
"status": "cannot_fix",
|
||||
"analysis": "The element was definitely not there.",
|
||||
},
|
||||
"id": "call_1",
|
||||
}
|
||||
],
|
||||
)
|
||||
mock_bound = MagicMock()
|
||||
mock_bound.ainvoke = AsyncMock(return_value=mock_response)
|
||||
mock_llm.bind_tools.return_value = mock_bound
|
||||
|
||||
with (
|
||||
patch("artemis.agents.validator.failure_analyzer.get_llm", return_value=mock_llm),
|
||||
patch(
|
||||
"artemis.agents.validator.failure_analyzer.UnifiedMobileController"
|
||||
) as mock_controller_cls,
|
||||
):
|
||||
mock_controller = Mock()
|
||||
mock_controller_cls.return_value = mock_controller
|
||||
mock_device_data = Mock()
|
||||
mock_device_data.base64 = base64.b64encode(b"mocked_screenshot").decode()
|
||||
mock_device_data.elements = []
|
||||
mock_device_data.width = 1080
|
||||
mock_device_data.height = 2400
|
||||
mock_controller.get_screen_data = AsyncMock(return_value=mock_device_data)
|
||||
mock_controller.get_ui_elements = AsyncMock(return_value=mock_device_data.elements)
|
||||
|
||||
result = await analyzer.analyze(
|
||||
state,
|
||||
failed_action,
|
||||
error_msg,
|
||||
pre_screenshot=base64.b64encode(b"dummy").decode(),
|
||||
post_screenshot=base64.b64encode(b"dummy").decode(),
|
||||
)
|
||||
|
||||
assert result["status"] == "cannot_fix"
|
||||
assert "not there" in result["analysis"]
|
||||
assert result["new_remaining_actions"] == []
|
||||
|
||||
|
||||
def test_report_failure_analysis_tool_declaration():
|
||||
"""Test that report_failure_analysis tool declaration is correct and matches."""
|
||||
report_tool = next(
|
||||
(t for t in VALIDATOR_TOOLS_DECLARATION if t.name == "report_failure_analysis"),
|
||||
None,
|
||||
)
|
||||
assert report_tool is not None
|
||||
props = report_tool.parameters.get("properties", {})
|
||||
assert "analysis" in props
|
||||
assert "single paragraph" in props["analysis"]["description"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failure_analyzer_strategy_routing(mock_context):
|
||||
"""Test that FailureAnalyzer selects appropriate strategies based on error category."""
|
||||
analyzer = FailureAnalyzer(mock_context)
|
||||
|
||||
strat_disappeared = analyzer._select_strategy(ValidationErrorCategory.TARGET_DISAPPEARED)
|
||||
assert isinstance(strat_disappeared, TargetDisappearedStrategy)
|
||||
assert "target_disappeared" in strat_disappeared.get_prompt_template_name()
|
||||
|
||||
strat_pixel = analyzer._select_strategy(ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED)
|
||||
assert isinstance(strat_pixel, PixelTargetDisappearedStrategy)
|
||||
assert "pixel_target_disappeared" in strat_pixel.get_prompt_template_name()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failure_analyzer_click_sequence(mock_context, tmp_path):
|
||||
"""Test click_sequence execution in FailureAnalyzer."""
|
||||
screenshot_file = tmp_path / "shot.jpg"
|
||||
screenshot_file.write_bytes(b"dummy_bytes")
|
||||
|
||||
state = DummyState(latest_screenshot=str(screenshot_file))
|
||||
analyzer = FailureAnalyzer(mock_context)
|
||||
|
||||
# Turn 1: Call click_sequence tool with direct coordinates
|
||||
call_step_1 = AIMessage(
|
||||
content="Let me click in sequence.",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "click_sequence",
|
||||
"args": {"sequence": [[100, 200], [300, 400]]},
|
||||
"id": "call_seq_1",
|
||||
}
|
||||
],
|
||||
)
|
||||
# Turn 2: Report failure fixed
|
||||
call_step_2 = AIMessage(
|
||||
content="Repaired successfully.",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "report_failure_analysis",
|
||||
"args": {
|
||||
"status": "fixed",
|
||||
"analysis": "Sequence clicked successfully.",
|
||||
},
|
||||
"id": "call_report_2",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
mock_llm = MagicMock()
|
||||
mock_bound = MagicMock()
|
||||
mock_bound.ainvoke = AsyncMock(side_effect=[call_step_1, call_step_2])
|
||||
mock_llm.bind_tools.return_value = mock_bound
|
||||
|
||||
with (
|
||||
patch("artemis.agents.validator.failure_analyzer.get_llm", return_value=mock_llm),
|
||||
patch(
|
||||
"artemis.agents.validator.failure_analyzer.UnifiedMobileController"
|
||||
) as mock_controller_cls,
|
||||
):
|
||||
mock_controller = Mock()
|
||||
mock_controller_cls.return_value = mock_controller
|
||||
mock_device_data = Mock()
|
||||
mock_device_data.base64 = base64.b64encode(b"mocked_screenshot").decode()
|
||||
mock_device_data.elements = []
|
||||
mock_device_data.width = 1080
|
||||
mock_device_data.height = 2400
|
||||
mock_controller.get_screen_data = AsyncMock(return_value=mock_device_data)
|
||||
mock_controller.get_ui_elements = AsyncMock(return_value=mock_device_data.elements)
|
||||
mock_controller.tap_at = AsyncMock(return_value=Mock(error=None))
|
||||
|
||||
result = await analyzer.analyze(
|
||||
state,
|
||||
{"action": "click", "coordinates": [100, 200]},
|
||||
"Element not found",
|
||||
pre_screenshot=base64.b64encode(b"dummy").decode(),
|
||||
post_screenshot=base64.b64encode(b"dummy").decode(),
|
||||
error_category=ValidationErrorCategory.TARGET_DISAPPEARED,
|
||||
)
|
||||
|
||||
assert result["status"] == "fixed"
|
||||
assert mock_controller.tap_at.call_count == 2
|
||||
@@ -1,466 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import base64
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
from artemis.agents.validator.failure_analyzer import (
|
||||
_exec_click,
|
||||
_exec_click_sequence,
|
||||
_exec_input_text,
|
||||
_exec_long_press,
|
||||
_exec_manage_app,
|
||||
_exec_press_key,
|
||||
_exec_swipe,
|
||||
_exec_wait_for_delay,
|
||||
)
|
||||
from artemis.context import ArtemisContext
|
||||
import pytest
|
||||
|
||||
|
||||
class DummyState:
|
||||
def __init__(self, latest_screenshot=None):
|
||||
self.latest_screenshot = latest_screenshot
|
||||
self.indexed_points = []
|
||||
self.indexed_elements = []
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context(tmp_path):
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
ctx.device = Mock()
|
||||
ctx.device.device_width = 1080
|
||||
ctx.device.device_height = 2400
|
||||
ctx.data_engine = Mock()
|
||||
ctx.data_engine.base_dir = tmp_path
|
||||
ctx.data_engine.get_or_create_image.return_value = "dummy_image_name"
|
||||
ctx.data_engine.get_image_path.return_value = tmp_path / "dummy_image_name.jpg"
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_controller():
|
||||
controller = Mock()
|
||||
mock_device_data = Mock()
|
||||
mock_device_data.base64 = base64.b64encode(b"mocked_screenshot").decode()
|
||||
mock_device_data.elements = []
|
||||
mock_device_data.width = 1080
|
||||
mock_device_data.height = 2400
|
||||
controller.get_screen_data = AsyncMock(return_value=mock_device_data)
|
||||
controller.get_ui_elements = AsyncMock(return_value=mock_device_data.elements)
|
||||
controller.tap_at = AsyncMock(return_value=Mock(error=None))
|
||||
controller.type_text = AsyncMock(return_value=True)
|
||||
controller.erase_text = AsyncMock(return_value=True)
|
||||
controller.swipe_coords = AsyncMock(return_value=None)
|
||||
controller.press_enter = AsyncMock(return_value=True)
|
||||
controller.go_back = AsyncMock(return_value=True)
|
||||
controller.go_home = AsyncMock(return_value=True)
|
||||
controller.press_key = AsyncMock(return_value=True)
|
||||
controller.terminate_app = AsyncMock(return_value=True)
|
||||
return controller
|
||||
|
||||
|
||||
# ==========================================
|
||||
# SLIDE 1: click
|
||||
# ==========================================
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_click_happy_path(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
outcome, img_bytes, shot_path, xml_list = await _exec_click(
|
||||
target=[500, 600],
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
times=2,
|
||||
delay_ms=150,
|
||||
)
|
||||
assert "Clicked at [500, 600]" in outcome
|
||||
assert mock_controller.tap_at.call_count == 1
|
||||
mock_controller.tap_at.assert_any_call(540, 1440, times=2, delay_ms=150)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_click_input_validation(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
# Coordinates holding non-integer values
|
||||
outcome, _, _, _ = await _exec_click(
|
||||
target=["invalid", "invalid"],
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Error during click:" in outcome
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_click_edge_cases(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
# Underflow/overflow coordinates should be clamped to valid boundary
|
||||
outcome, _, _, _ = await _exec_click(
|
||||
target=[-10, 1050],
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Clicked at [-10, 1050]" in outcome
|
||||
mock_controller.tap_at.assert_called_with(0, 2399, times=1, delay_ms=100)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_click_graceful_failure(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
mock_controller.tap_at.return_value = Mock(error="Click failed connection lost")
|
||||
outcome, _, _, _ = await _exec_click(
|
||||
target=[500, 600],
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Error executing click: Click failed connection lost" in outcome
|
||||
|
||||
|
||||
# ==========================================
|
||||
# SLIDE 2: click_sequence
|
||||
# ==========================================
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_click_sequence_happy_path(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
state.indexed_points = [[540, 1200]] # Mapped from index 1 (1-indexed)
|
||||
outcome, img_bytes, shot_path, xml_list = await _exec_click_sequence(
|
||||
sequence=[1, [300, 400]],
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Sequence clicked successfully" in outcome
|
||||
assert mock_controller.tap_at.call_count == 2
|
||||
mock_controller.tap_at.assert_any_call(540, 1200)
|
||||
mock_controller.tap_at.assert_any_call(324, 960)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_click_sequence_input_validation(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
state.indexed_points = [[540, 1200], [270, 600]]
|
||||
|
||||
# Subsequent elements can also be integers (mixed array of indexes and coordinates)
|
||||
outcome, _, _, _ = await _exec_click_sequence(
|
||||
sequence=[1, 2, [300, 400]],
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Sequence clicked successfully" in outcome
|
||||
assert mock_controller.tap_at.call_count == 3
|
||||
mock_controller.tap_at.assert_any_call(540, 1200)
|
||||
mock_controller.tap_at.assert_any_call(270, 600)
|
||||
mock_controller.tap_at.assert_any_call(324, 960)
|
||||
|
||||
# Reset indexed_points since _exec_click_sequence overwrites them
|
||||
# via _capture_screenshot_and_parse_ui
|
||||
state.indexed_points = [[540, 1200], [270, 600]]
|
||||
|
||||
# Index out of bounds
|
||||
outcome, _, _, _ = await _exec_click_sequence(
|
||||
sequence=[5, [300, 400]],
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Error during click sequence: Index 5 is out of range" in outcome
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_click_sequence_edge_cases(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
# Coordinates in sequence outside range should be clamped
|
||||
outcome, _, _, _ = await _exec_click_sequence(
|
||||
sequence=[[0, 0], [1200, -50]],
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Sequence clicked successfully" in outcome
|
||||
mock_controller.tap_at.assert_any_call(0, 0)
|
||||
mock_controller.tap_at.assert_any_call(1079, 0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_click_sequence_graceful_failure(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
mock_controller.tap_at.side_effect = [
|
||||
Mock(error=None),
|
||||
Mock(error="Connection error on step 2"),
|
||||
]
|
||||
outcome, _, _, _ = await _exec_click_sequence(
|
||||
sequence=[[100, 200], [300, 400]],
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Error executing click at" in outcome
|
||||
assert "Connection error on step 2" in outcome
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_click_sequence_stringified_formats(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
state.indexed_points = [[540, 1200]]
|
||||
# Test stringified coordinate formats commonly generated by LLMs:
|
||||
# 1. string representation of lists: ["[500, 280]", "[884, 362]"]
|
||||
outcome, _, _, _ = await _exec_click_sequence(
|
||||
sequence=["[500, 280]", "[884, 362]"],
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Sequence clicked successfully" in outcome
|
||||
mock_controller.tap_at.assert_any_call(540, 672)
|
||||
mock_controller.tap_at.assert_any_call(954, 868)
|
||||
|
||||
# 2. comma-separated strings: ["500,280", "884,362"]
|
||||
outcome, _, _, _ = await _exec_click_sequence(
|
||||
sequence=["500,280", "884,362"],
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Sequence clicked successfully" in outcome
|
||||
|
||||
# 3. whole sequence as a string
|
||||
outcome, _, _, _ = await _exec_click_sequence(
|
||||
sequence="[[500, 280], [884, 362]]",
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Sequence clicked successfully" in outcome
|
||||
|
||||
|
||||
# ==========================================
|
||||
# SLIDE 3: long_press
|
||||
# ==========================================
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_long_press_happy_path(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
outcome, img_bytes, shot_path, xml_list = await _exec_long_press(
|
||||
target=[500, 500],
|
||||
duration=1500,
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Long pressed at [500, 500]" in outcome
|
||||
mock_controller.tap_at.assert_called_once_with(
|
||||
540, 1200, long_press=True, long_press_duration=1500
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_long_press_input_validation(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
# Invalid target type
|
||||
outcome, _, _, _ = await _exec_long_press(
|
||||
target="not-a-list",
|
||||
duration=1500,
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Error during long press:" in outcome
|
||||
|
||||
|
||||
# ==========================================
|
||||
# SLIDE 4: input_text
|
||||
# ==========================================
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_input_text_happy_path(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
outcome, img_bytes, shot_path, xml_list = await _exec_input_text(
|
||||
text="hello",
|
||||
target=[500, 500],
|
||||
clear_exist=True,
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Executed typing 'hello'." in outcome
|
||||
# Focus goes through ensure_focus_at_coords (keyword-style tap), and typing
|
||||
# never re-clears: the clear already happened via erase_text.
|
||||
mock_controller.tap_at.assert_called_once_with(x=540, y=1200)
|
||||
mock_controller.erase_text.assert_called_once()
|
||||
mock_controller.type_text.assert_called_once_with("hello", clear_existing=False)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_input_text_input_validation(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
outcome, _, _, _ = await _exec_input_text(
|
||||
text="hello",
|
||||
target="invalid",
|
||||
clear_exist=True,
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Error during input text:" in outcome
|
||||
|
||||
|
||||
# ==========================================
|
||||
# SLIDE 5: swipe
|
||||
# ==========================================
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swipe_happy_path_direction(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
outcome, img_bytes, shot_path, xml_list = await _exec_swipe(
|
||||
action="up",
|
||||
duration=400,
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Swipe completed successfully" in outcome
|
||||
mock_controller.swipe_coords.assert_called_once_with(648, 1680, 648, 720, 400)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swipe_happy_path_coords(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
outcome, img_bytes, shot_path, xml_list = await _exec_swipe(
|
||||
action=[100, 900, 100, 100],
|
||||
duration=400,
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Swipe completed successfully" in outcome
|
||||
mock_controller.swipe_coords.assert_called_once_with(108, 2160, 108, 240, 400)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swipe_input_validation(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
outcome, _, _, _ = await _exec_swipe(
|
||||
action="invalid_direction",
|
||||
duration=400,
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Error during swipe: Invalid direction:" in outcome
|
||||
|
||||
|
||||
# ==========================================
|
||||
# SLIDE 6: press_key
|
||||
# ==========================================
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_press_key_happy_path(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
outcome, img_bytes, shot_path, xml_list = await _exec_press_key(
|
||||
key="BACK",
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Executed key press 'BACK'." in outcome
|
||||
mock_controller.go_back.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_press_key_input_validation(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
outcome, _, _, _ = await _exec_press_key(
|
||||
key="INVALID_KEY",
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
# Unknown keycodes are forwarded verbatim to the driver (historical adb_server
|
||||
# behavior); the mock driver accepts them, so the press reports success.
|
||||
assert "Executed key press 'INVALID_KEY'." in outcome
|
||||
mock_controller.press_key.assert_called_once_with("INVALID_KEY")
|
||||
|
||||
|
||||
# ==========================================
|
||||
# SLIDE 7: manage_app
|
||||
# ==========================================
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manage_app_happy_path(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
with (
|
||||
# The device-call bodies moved into the actuator layer, which imports these
|
||||
# lazily from their source modules -- patch them at the source.
|
||||
patch("artemis.tools.mobile.launch_app.find_package") as mock_find_package,
|
||||
patch("artemis.utils.app_launch_utils.launch_app_with_retries") as mock_launch,
|
||||
):
|
||||
mock_find_package.return_value = "com.google.android.youtube"
|
||||
mock_launch.return_value = (True, "")
|
||||
|
||||
outcome, img_bytes, shot_path, xml_list = await _exec_manage_app(
|
||||
action="launch",
|
||||
app_name="YouTube",
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Launched app 'YouTube' (com.google.android.youtube) successfully." in outcome
|
||||
mock_find_package.assert_called_once_with(mock_context, "YouTube", use_fallback=False)
|
||||
mock_launch.assert_called_once_with(mock_context, "com.google.android.youtube")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manage_app_input_validation(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
with patch("artemis.tools.mobile.launch_app.find_package") as mock_find_package:
|
||||
mock_find_package.return_value = None
|
||||
|
||||
outcome, _, _, _ = await _exec_manage_app(
|
||||
action="launch",
|
||||
app_name="NonExistentApp",
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Error finding package for app: NonExistentApp" in outcome
|
||||
|
||||
|
||||
# ==========================================
|
||||
# SLIDE 8: wait_for_delay
|
||||
# ==========================================
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wait_for_delay_happy_path(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
outcome, img_bytes, shot_path, xml_list = await _exec_wait_for_delay(
|
||||
time_in_ms=100,
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Waited for 100ms successfully." in outcome
|
||||
@@ -18,7 +18,9 @@ from unittest.mock import Mock, patch
|
||||
import pytest
|
||||
from langchain_core.messages import HumanMessage, ToolMessage
|
||||
|
||||
from artemis.agents.explorer.constants import ASK_EXPLORER_DESCRIPTION
|
||||
from artemis.agents.flash.runner import FlashRunner
|
||||
from artemis.agents.validator.tool_declarations import ASK_EXPLORER_TOOL
|
||||
from artemis.context import ArtemisContext
|
||||
|
||||
|
||||
@@ -57,6 +59,31 @@ def test_flash_runner_tools_initialization(mock_context):
|
||||
assert "report_failure_analysis" not in tool_names
|
||||
|
||||
|
||||
def test_flash_runner_executor_follows_the_flash_explorer_tier(mock_context):
|
||||
"""The executor is tagged as the Flash profile so ask_explorer follows flash_mode."""
|
||||
with (
|
||||
patch("artemis.controllers.unified_controller.get_driver"),
|
||||
patch("artemis.agents.flash.runner.McpActionExecutor") as executor_cls,
|
||||
):
|
||||
runner = FlashRunner(mock_context, goal="Test Open Settings")
|
||||
|
||||
executor_cls.assert_called_once()
|
||||
assert executor_cls.call_args.kwargs["agent_name"] == "flash"
|
||||
assert runner.executor is executor_cls.return_value
|
||||
|
||||
|
||||
def test_ask_explorer_declaration_is_tier_agnostic(mock_context):
|
||||
"""Flash binds the shared ask_explorer contract: query + context_feedback, no tier."""
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = FlashRunner(mock_context, goal="Test Open Settings")
|
||||
declaration = next(t for t in runner._get_tools() if t.name == "ask_explorer")
|
||||
|
||||
assert declaration is ASK_EXPLORER_TOOL
|
||||
assert declaration.description == ASK_EXPLORER_DESCRIPTION
|
||||
assert set(declaration.parameters["properties"]) == {"query", "context_feedback"}
|
||||
assert declaration.parameters["required"] == ["query"]
|
||||
|
||||
|
||||
def test_flash_runner_screenshot_pruning(mock_context):
|
||||
"""Verify that earlier screenshot blocks are pruned, keeping only the latest."""
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
|
||||
@@ -0,0 +1,333 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""FlashRunner over the session transcript ledger (Pro-identical history).
|
||||
|
||||
Covers: the ``video_analyzer`` availability gate (same as the Pro graph) and
|
||||
its prompt segment, the unbounded-by-default turn limit, the Pro-shaped
|
||||
observation tail (``# CURRENT OBSERVATION [T+mm:ss]`` header, screenshot,
|
||||
``--- Visible UI Elements ---`` list), and the end-to-end loop: committed
|
||||
turns carry text-only tool messages plus an ``--- Action Execution Result
|
||||
(T+mm:ss) ---`` message, every recorded step of a multi-action turn is
|
||||
registered on the committed turn, the final-turn tool restriction only
|
||||
applies to bounded loops, and a no-tool-call turn is nudged, not terminated.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import re
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
|
||||
|
||||
from artemis.agents.flash.runner import FlashRunner
|
||||
from artemis.agents.validator.tool_declarations import ToolExecutionResult
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.state import State
|
||||
from artemis.memory.transcript import (
|
||||
EXECUTION_RESULT_MARKER,
|
||||
PRO_UI_LIST_MARKER,
|
||||
TranscriptLedger,
|
||||
)
|
||||
|
||||
OBSERVATION_HEADER_RE = re.compile(r"^# CURRENT OBSERVATION \[T\+\d{2,}:\d{2}\]$")
|
||||
RESULT_RE = re.compile(
|
||||
rf"^{re.escape(EXECUTION_RESULT_MARKER)} \(T\+\d{{2,}}:\d{{2}}\) ---\nStatus: success$"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context():
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
ctx.llm_config = Mock()
|
||||
mock_llm_cfg = Mock()
|
||||
mock_llm_cfg.model = "gemini-2.5-flash"
|
||||
mock_llm_cfg.temperature = 0.1
|
||||
ctx.llm_config.get_agent.return_value = mock_llm_cfg
|
||||
ctx.device = Mock()
|
||||
ctx.device.device_width = 1080
|
||||
ctx.device.device_height = 2400
|
||||
ctx.data_engine = None
|
||||
ctx.adb_client = None
|
||||
ctx.driver = Mock()
|
||||
return ctx
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# video_analyzer availability (same gate as the Pro graph)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_video_analyzer_bound_only_when_recording_tools_enabled(mock_context):
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
mock_context.execution_setup = Mock()
|
||||
mock_context.execution_setup.video_recording_tools_enabled = True
|
||||
names = [t.name for t in FlashRunner(mock_context, goal="g")._get_tools()]
|
||||
assert "video_analyzer" in names
|
||||
assert names[-1] == "report_task_status"
|
||||
|
||||
mock_context.execution_setup.video_recording_tools_enabled = False
|
||||
names = [t.name for t in FlashRunner(mock_context, goal="g")._get_tools()]
|
||||
assert "video_analyzer" not in names
|
||||
|
||||
# A Mock attribute (truthy but not True) must not enable the tool.
|
||||
mock_context.execution_setup = Mock()
|
||||
names = [t.name for t in FlashRunner(mock_context, goal="g")._get_tools()]
|
||||
assert "video_analyzer" not in names
|
||||
|
||||
|
||||
def test_recall_history_bound_only_with_a_data_engine_session(mock_context):
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
# No DataEngine: nothing to search, the tool stays out (as in Pro).
|
||||
names = [t.name for t in FlashRunner(mock_context, goal="g")._get_tools()]
|
||||
assert "recall_history" not in names
|
||||
|
||||
mock_context.data_engine = Mock()
|
||||
with patch("artemis.tools.history_recall._recall_config", return_value=None):
|
||||
runner = FlashRunner(mock_context, goal="g")
|
||||
tools = runner._get_tools()
|
||||
names = [t.name for t in tools]
|
||||
assert "recall_history" in names
|
||||
declaration = next(t for t in tools if t.name == "recall_history")
|
||||
assert set(declaration.parameters["properties"]) == {
|
||||
"query",
|
||||
"step_range",
|
||||
"include_details",
|
||||
"include_images",
|
||||
"max_results",
|
||||
}
|
||||
assert declaration.parameters["required"] == ["query"]
|
||||
assert "`recall_history`" in runner._render_system_prompt(tools)
|
||||
|
||||
|
||||
def test_video_analyzer_prompt_segment_follows_availability(mock_context):
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = FlashRunner(mock_context, goal="g")
|
||||
mock_context.execution_setup = Mock()
|
||||
mock_context.execution_setup.video_recording_tools_enabled = True
|
||||
with_video = runner._render_system_prompt(runner._get_tools())
|
||||
assert "`video_analyzer`" in with_video
|
||||
|
||||
mock_context.execution_setup.video_recording_tools_enabled = False
|
||||
without_video = runner._render_system_prompt(runner._get_tools())
|
||||
assert "video_analyzer" not in without_video
|
||||
# Session-clock / history teaching is unconditional.
|
||||
assert "T+mm:ss" in without_video
|
||||
assert "Action Execution Result" in without_video
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Turn limit: unbounded by default, explicit caps honoured
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_turn_limit_semantics(mock_context):
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
assert FlashRunner(mock_context, goal="g", max_turns=0).turn_limit is None
|
||||
assert FlashRunner(mock_context, goal="g", max_turns=7).turn_limit == 7
|
||||
with patch("artemis.agents.flash.runner.load_agent_config", side_effect=RuntimeError):
|
||||
assert FlashRunner(mock_context, goal="g").turn_limit is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Observation tail: Pro shape with the session-relative header
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_observation_tail_has_pro_shape(mock_context):
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = FlashRunner(mock_context, goal="Open Settings")
|
||||
ledger = TranscriptLedger()
|
||||
|
||||
tail = runner._build_tail(ledger, 1, b"IMG", "[1] Settings")
|
||||
texts = [b["text"] for b in tail.content if b["type"] == "text"]
|
||||
assert texts[0] == "Your objective is: Open Settings"
|
||||
assert OBSERVATION_HEADER_RE.match(texts[1])
|
||||
assert texts[2] == "--- Current Screenshot ---"
|
||||
assert texts[3] == f"{PRO_UI_LIST_MARKER}\n[1] Settings"
|
||||
assert sum(1 for b in tail.content if b["type"] == "image_url") == 1
|
||||
|
||||
later = runner._build_tail(
|
||||
ledger, 5, None, None, injected="[INJECTED] stop", notices=["nudge"], is_final=True
|
||||
)
|
||||
texts = [b["text"] for b in later.content]
|
||||
assert OBSERVATION_HEADER_RE.match(texts[0])
|
||||
assert "Your objective is" not in "".join(texts)
|
||||
assert texts[1:] == [
|
||||
"nudge",
|
||||
"[INJECTED] stop",
|
||||
"[WARNING] This is your final turn; only 'report_task_status' is available.",
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# End-to-end loop over the ledger (mocked model / executor)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _exec_result(tc_id, name, post_bytes, ui="[1] next"):
|
||||
return ToolExecutionResult(
|
||||
tool_call_id=tc_id,
|
||||
tool_name=name,
|
||||
status="success",
|
||||
text_summary=f"{name} executed",
|
||||
screenshot_bytes=post_bytes,
|
||||
ui_elements_text=ui,
|
||||
)
|
||||
|
||||
|
||||
def _make_runner(mock_context, responses, max_turns=None):
|
||||
runner = FlashRunner(mock_context, goal="Open Settings", max_turns=max_turns)
|
||||
runner.summarizer = None
|
||||
runner._init_llm = Mock(return_value=Mock())
|
||||
# Snapshot each invocation: the runner appends to the same message list
|
||||
# after the call, so the recorded call args would otherwise drift.
|
||||
runner.calls = []
|
||||
pending = list(responses)
|
||||
|
||||
async def _invoke(llm, tools, messages):
|
||||
runner.calls.append({"tools": list(tools), "messages": list(messages)})
|
||||
return pending.pop(0)
|
||||
|
||||
runner._invoke_model = AsyncMock(side_effect=_invoke)
|
||||
runner.executor.execute = AsyncMock(
|
||||
side_effect=lambda name, args, tc_id, state: _exec_result(
|
||||
tc_id, name, f"IMG-{tc_id}".encode()
|
||||
)
|
||||
)
|
||||
return runner
|
||||
|
||||
|
||||
def _report(status="completed", tc_id="tc-report", **extra):
|
||||
return AIMessage(
|
||||
content="Done.",
|
||||
tool_calls=[
|
||||
{"name": "report_task_status", "args": {"status": status, **extra}, "id": tc_id}
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
_INITIAL_OBSERVATION = ("shot0.png", b"IMG0", "[1] Settings")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_builds_prompt_from_ledger_with_session_offsets(mock_context):
|
||||
"""Turn 2 must see: the static system prefix, the committed turn 1 (its
|
||||
tail, the AI message, text-only tool messages and an execution-result
|
||||
message with a ``T+`` offset) and a fresh observation tail carrying the
|
||||
last action's post screenshot — the Pro transcript shape, with no cap."""
|
||||
responses = [
|
||||
AIMessage(
|
||||
content="I will tap then wait.",
|
||||
tool_calls=[
|
||||
{"name": "click", "args": {"target": [500, 600]}, "id": "tc1"},
|
||||
{"name": "wait_for_delay", "args": {"seconds": 1}, "id": "tc2"},
|
||||
],
|
||||
),
|
||||
_report(explanation="ok"),
|
||||
]
|
||||
with (
|
||||
patch("artemis.controllers.unified_controller.get_driver"),
|
||||
patch(
|
||||
"artemis.agents.flash.runner.capture_screenshot_and_parse_ui",
|
||||
AsyncMock(return_value=_INITIAL_OBSERVATION),
|
||||
),
|
||||
):
|
||||
runner = _make_runner(mock_context, responses, max_turns=0)
|
||||
assert runner.turn_limit is None
|
||||
result = await runner.run(State(initial_goal="Open Settings"))
|
||||
|
||||
assert result == {"status": "completed", "explanation": "ok"}
|
||||
assert runner._invoke_model.await_count == 2
|
||||
|
||||
turn2_messages = runner.calls[1]["messages"]
|
||||
assert isinstance(turn2_messages[0], SystemMessage)
|
||||
assert "T+mm:ss" in turn2_messages[0].content
|
||||
|
||||
tail1 = turn2_messages[1]
|
||||
assert isinstance(tail1, HumanMessage)
|
||||
tail1_texts = [b["text"] for b in tail1.content if b["type"] == "text"]
|
||||
assert tail1_texts[0] == "Your objective is: Open Settings"
|
||||
assert OBSERVATION_HEADER_RE.match(tail1_texts[1])
|
||||
|
||||
assert isinstance(turn2_messages[2], AIMessage)
|
||||
tool_msgs = [m for m in turn2_messages if isinstance(m, ToolMessage)]
|
||||
assert [m.content for m in tool_msgs] == ["click executed", "wait_for_delay executed"]
|
||||
|
||||
result_msg = turn2_messages[5]
|
||||
assert isinstance(result_msg, HumanMessage)
|
||||
assert RESULT_RE.match(result_msg.content[0]["text"])
|
||||
|
||||
tail2 = turn2_messages[6]
|
||||
tail2_texts = [b["text"] for b in tail2.content if b["type"] == "text"]
|
||||
assert OBSERVATION_HEADER_RE.match(tail2_texts[0])
|
||||
assert "Your objective is" not in "".join(tail2_texts)
|
||||
images = [b for b in tail2.content if b["type"] == "image_url"]
|
||||
assert len(images) == 1
|
||||
assert images[0]["image_url"]["url"].endswith(base64.b64encode(b"IMG-tc2").decode())
|
||||
assert tail2_texts[1] == "--- Current Screenshot ---"
|
||||
assert tail2_texts[2] == f"{PRO_UI_LIST_MARKER}\n[1] next"
|
||||
assert len(turn2_messages) == 7
|
||||
|
||||
# Both actions of the multi-action turn were registered on the committed turn.
|
||||
ledger = mock_context.transcript_ledger
|
||||
assert ledger.unchunked_turns()[0]["step_keys"] == ["tc1", "tc2"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_final_turn_restricts_tools_and_fails_without_report(mock_context):
|
||||
responses = [AIMessage(content="I give up.", tool_calls=[])]
|
||||
with (
|
||||
patch("artemis.controllers.unified_controller.get_driver"),
|
||||
patch(
|
||||
"artemis.agents.flash.runner.capture_screenshot_and_parse_ui",
|
||||
AsyncMock(return_value=_INITIAL_OBSERVATION),
|
||||
),
|
||||
):
|
||||
runner = _make_runner(mock_context, responses, max_turns=1)
|
||||
result = await runner.run(State(initial_goal="Open Settings"))
|
||||
|
||||
assert result == {"status": "failed", "explanation": "I give up."}
|
||||
tools = runner.calls[0]["tools"]
|
||||
assert [t.name for t in tools] == ["report_task_status"]
|
||||
tail = runner.calls[0]["messages"][-1]
|
||||
assert any("final turn" in b.get("text", "") for b in tail.content)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_no_tool_call_turn_is_nudged_not_terminated(mock_context):
|
||||
responses = [AIMessage(content="thinking only", tool_calls=[]), _report()]
|
||||
with (
|
||||
patch("artemis.controllers.unified_controller.get_driver"),
|
||||
patch(
|
||||
"artemis.agents.flash.runner.capture_screenshot_and_parse_ui",
|
||||
AsyncMock(return_value=_INITIAL_OBSERVATION),
|
||||
),
|
||||
):
|
||||
runner = _make_runner(mock_context, responses, max_turns=0)
|
||||
result = await runner.run(State(initial_goal="Open Settings"))
|
||||
|
||||
assert result == {"status": "completed"}
|
||||
turn2_messages = runner.calls[1]["messages"]
|
||||
# Unbounded loop: no final-turn restriction was ever applied.
|
||||
tools = runner.calls[1]["tools"]
|
||||
assert "click" in [t.name for t in tools]
|
||||
tail2_texts = [b["text"] for b in turn2_messages[-1].content if b["type"] == "text"]
|
||||
assert any("did not call any tools" in t for t in tail2_texts)
|
||||
# A helper-only / empty turn commits without an execution-result message.
|
||||
assert not any(
|
||||
isinstance(m, HumanMessage)
|
||||
and any(EXECUTION_RESULT_MARKER in b.get("text", "") for b in m.content)
|
||||
for m in turn2_messages
|
||||
)
|
||||
@@ -74,9 +74,7 @@ def _content_fingerprint(msg) -> str:
|
||||
|
||||
|
||||
def _has_image(msg) -> bool:
|
||||
return any(
|
||||
isinstance(b, dict) and b.get("type") in ("image_url", "image") for b in msg.content
|
||||
)
|
||||
return any(isinstance(b, dict) and b.get("type") in ("image_url", "image") for b in msg.content)
|
||||
|
||||
|
||||
def test_success_path_product_behind_the_scrub_edge(mock_context):
|
||||
@@ -204,9 +202,7 @@ def test_race_failed_summary_becomes_unavailable_placeholder(mock_context):
|
||||
_simulate_turns(compressor, messages, [_step(1), _step(2), _step(3)])
|
||||
|
||||
assert not _has_image(messages[0])
|
||||
assert "[visual summary unavailable; evidence at DataEngine step 1]" in str(
|
||||
messages[0].content
|
||||
)
|
||||
assert "[visual summary unavailable; evidence at DataEngine step 1]" in str(messages[0].content)
|
||||
|
||||
|
||||
def test_freeze_invariant_bytes_never_change(mock_context):
|
||||
@@ -232,9 +228,7 @@ def test_freeze_invariant_bytes_never_change(mock_context):
|
||||
for idx in compressor._frozen:
|
||||
fingerprint = _content_fingerprint(messages[idx])
|
||||
if idx in frozen_fingerprints:
|
||||
assert frozen_fingerprints[idx] == fingerprint, (
|
||||
f"frozen message {idx} mutated"
|
||||
)
|
||||
assert frozen_fingerprints[idx] == fingerprint, f"frozen message {idx} mutated"
|
||||
else:
|
||||
frozen_fingerprints[idx] = fingerprint
|
||||
|
||||
|
||||
@@ -551,9 +551,7 @@ async def test_raw_lens_call_meters_llm_usage_without_touching_context_base(mock
|
||||
await summarizer.flush()
|
||||
|
||||
usage_calls = [
|
||||
c
|
||||
for c in engine.record_trace.call_args_list
|
||||
if c.kwargs.get("name") == "llm_usage"
|
||||
c for c in engine.record_trace.call_args_list if c.kwargs.get("name") == "llm_usage"
|
||||
]
|
||||
assert usage_calls, "the raw lens bypass must record an llm_usage trace"
|
||||
payload = usage_calls[0].kwargs["payload"]
|
||||
|
||||
@@ -77,9 +77,7 @@ async def _run(steps, b64=B64_A, cfg=None):
|
||||
builder = PromptBuilder()
|
||||
component = HistoricalStateHintPromptComponent()
|
||||
with patch("artemis.config.load_agent_config", return_value=cfg or _cfg()):
|
||||
await component(
|
||||
builder, MagicMock(), MagicMock(), latest_screenshot_b64=b64, steps=steps
|
||||
)
|
||||
await component(builder, MagicMock(), MagicMock(), latest_screenshot_b64=b64, steps=steps)
|
||||
return [p for p in builder.human_parts if isinstance(p, str)]
|
||||
|
||||
|
||||
|
||||
@@ -354,9 +354,7 @@ async def test_history_analyzer_integration_with_task_tree():
|
||||
"artemis.agents.history_analyzer.history_analyzer.get_note_file_path",
|
||||
return_value=mock_path,
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.history_analyzer.history_analyzer.build_history_for"
|
||||
) as mock_build,
|
||||
patch("artemis.agents.history_analyzer.history_analyzer.build_history_for") as mock_build,
|
||||
):
|
||||
mock_build.return_value = "Mocked operation history"
|
||||
analyzer = HistoryAnalyzer(mock_ctx)
|
||||
|
||||
@@ -414,7 +414,9 @@ async def test_operator_dynamic_prompt():
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
|
||||
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
|
||||
node = OperatorNode(mock_ctx, prompt_components=[MockComponent()], transcript_config=LEGACY_TRANSCRIPT)
|
||||
node = OperatorNode(
|
||||
mock_ctx, prompt_components=[MockComponent()], transcript_config=LEGACY_TRANSCRIPT
|
||||
)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
|
||||
@@ -554,28 +556,6 @@ async def test_operator_history_compression():
|
||||
node_update = await node(mock_state)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_operator_loop_detection():
|
||||
from artemis.agents.operator.operator import OperatorNode
|
||||
from artemis.context import ArtemisContext
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.execution_setup = None
|
||||
mock_ctx.data_engine = None
|
||||
|
||||
mock_state = MagicMock()
|
||||
mock_state.subagent_calls = ["ask_explorer", "ask_explorer", "ask_explorer"]
|
||||
mock_state.initial_goal = "Test goal"
|
||||
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
|
||||
with pytest.raises(RuntimeError) as exc_info:
|
||||
node_update = await node(mock_state)
|
||||
|
||||
assert "Infinite loop detected" in str(exc_info.value)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_operator_tracks_subagent_calls():
|
||||
from artemis.agents.operator.operator import OperatorNode
|
||||
@@ -630,7 +610,9 @@ async def test_operator_tracks_subagent_calls():
|
||||
side_effect=lambda t, ctx: t,
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool], transcript_config=LEGACY_TRANSCRIPT)
|
||||
node = OperatorNode(
|
||||
mock_ctx, tools=[mock_diagnoser_tool], transcript_config=LEGACY_TRANSCRIPT
|
||||
)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
update_dict = node_update
|
||||
@@ -722,7 +704,9 @@ async def test_operator_defer_action_for_gathering():
|
||||
side_effect=lambda t, ctx: t,
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool], transcript_config=LEGACY_TRANSCRIPT)
|
||||
node = OperatorNode(
|
||||
mock_ctx, tools=[mock_diagnoser_tool], transcript_config=LEGACY_TRANSCRIPT
|
||||
)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
# Subagent tool should have been executed
|
||||
@@ -908,7 +892,9 @@ async def test_operator_no_defer_for_note_updating():
|
||||
side_effect=lambda t, ctx: t,
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx, tools=[mock_update_note_tool], transcript_config=LEGACY_TRANSCRIPT)
|
||||
node = OperatorNode(
|
||||
mock_ctx, tools=[mock_update_note_tool], transcript_config=LEGACY_TRANSCRIPT
|
||||
)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
# The update_note tool should have been executed immediately
|
||||
@@ -998,7 +984,9 @@ async def test_operator_defer_for_note_reading():
|
||||
side_effect=lambda t, ctx: t,
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx, tools=[mock_read_note_tool], transcript_config=LEGACY_TRANSCRIPT)
|
||||
node = OperatorNode(
|
||||
mock_ctx, tools=[mock_read_note_tool], transcript_config=LEGACY_TRANSCRIPT
|
||||
)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
# The read_note tool should have been executed
|
||||
@@ -1086,7 +1074,9 @@ async def test_operator_no_defer_for_note_appending():
|
||||
side_effect=lambda t, ctx: t,
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx, tools=[mock_append_note_tool], transcript_config=LEGACY_TRANSCRIPT)
|
||||
node = OperatorNode(
|
||||
mock_ctx, tools=[mock_append_note_tool], transcript_config=LEGACY_TRANSCRIPT
|
||||
)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
# The append_note tool should have been executed immediately
|
||||
@@ -1106,7 +1096,7 @@ async def test_operator_no_defer_for_note_appending():
|
||||
async def test_operator_background_tasks_prompt_injection():
|
||||
from artemis.agents.operator.operator import OperatorNode
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.tools.command_tool import _BACKGROUND_TASKS, BackgroundTask
|
||||
from artemis.tools.command_tool import BackgroundTask, get_adb_task_registry
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
@@ -1140,7 +1130,8 @@ async def test_operator_background_tasks_prompt_injection():
|
||||
cwd="/data/local/tmp",
|
||||
)
|
||||
|
||||
_BACKGROUND_TASKS["task_99999"] = bg_task
|
||||
adb_registry = get_adb_task_registry(mock_ctx)
|
||||
adb_registry.background["task_99999"] = bg_task
|
||||
|
||||
try:
|
||||
captured_messages = []
|
||||
@@ -1194,8 +1185,7 @@ async def test_operator_background_tasks_prompt_injection():
|
||||
assert "term_xyz" in full_content
|
||||
assert "Accumulated Output: 0 lines of logs" in full_content
|
||||
finally:
|
||||
if "task_99999" in _BACKGROUND_TASKS:
|
||||
del _BACKGROUND_TASKS["task_99999"]
|
||||
adb_registry.background.pop("task_99999", None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -1225,11 +1215,16 @@ async def test_operator_task_plan_warning_prompt_injection():
|
||||
mock_llm.ainvoke = AsyncMock(return_value=mock_response)
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
|
||||
# Case 1: task_plan updated in the last two steps (using the "key" argument)
|
||||
# Case 1: task_plan updated in the last step (using the "key" argument)
|
||||
steps_updated = [
|
||||
{
|
||||
"step_id": "step_1",
|
||||
"step_number": 1,
|
||||
"tool_calls": [{"name": "click", "args": {"target": 1}}],
|
||||
},
|
||||
{
|
||||
"step_id": "step_2",
|
||||
"step_number": 2,
|
||||
"tool_calls": [
|
||||
{
|
||||
"name": "update_note",
|
||||
@@ -1237,11 +1232,6 @@ async def test_operator_task_plan_warning_prompt_injection():
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"step_id": "step_2",
|
||||
"step_number": 2,
|
||||
"tool_calls": [{"name": "click", "args": {"target": 1}}],
|
||||
},
|
||||
]
|
||||
|
||||
captured_messages_1 = []
|
||||
@@ -1278,12 +1268,9 @@ async def test_operator_task_plan_warning_prompt_injection():
|
||||
elif isinstance(m.content, str):
|
||||
full_content_1 += m.content + "\n"
|
||||
|
||||
assert (
|
||||
"Reminder: You have not updated the task_plan for two consecutive"
|
||||
" turns." not in full_content_1
|
||||
)
|
||||
assert "Reminder: your last turn executed a Turn-Ending Action" not in full_content_1
|
||||
|
||||
# Case 2: task_plan NOT updated in the last two steps
|
||||
# Case 2: an action turn that did NOT update task_plan
|
||||
steps_not_updated = [
|
||||
{
|
||||
"step_id": "step_1",
|
||||
@@ -1294,6 +1281,7 @@ async def test_operator_task_plan_warning_prompt_injection():
|
||||
"step_id": "step_2",
|
||||
"step_number": 2,
|
||||
"tool_calls": [{"name": "click", "args": {"target": 2}}],
|
||||
"action_taken": {"action": "click", "target": 2},
|
||||
},
|
||||
]
|
||||
|
||||
@@ -1331,10 +1319,7 @@ async def test_operator_task_plan_warning_prompt_injection():
|
||||
elif isinstance(m.content, str):
|
||||
full_content_2 += m.content + "\n"
|
||||
|
||||
assert (
|
||||
"Reminder: You have not updated the task_plan for two consecutive"
|
||||
" turns." in full_content_2
|
||||
)
|
||||
assert "Reminder: your last turn executed a Turn-Ending Action" in full_content_2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -1399,7 +1384,8 @@ async def test_operator_tool_limit_exceeded_warning():
|
||||
full_content += m.content + "\n"
|
||||
|
||||
assert (
|
||||
"You did not execute any screen interaction actions in your last turn" in full_content
|
||||
"your last turn used up its tool-call budget without a Turn-Ending Action"
|
||||
in full_content
|
||||
)
|
||||
|
||||
assert node_update.get("operator_tool_limit_exceeded") is False
|
||||
|
||||
@@ -22,9 +22,19 @@ from artemis.agents.operator.prompts import apply_operator_prompt_contract
|
||||
def test_operator_prompt_contract_matches_note_runtime_semantics():
|
||||
prompt = apply_operator_prompt_contract(load_operator_prompts()["main_template"])
|
||||
|
||||
assert "write-through tools `update_note` and `append_note`" in prompt
|
||||
assert "may accompany at most one Turn-Ending Action" in prompt
|
||||
assert "`read_note`, `list_notes`, `save_note`, and `recall_history`" in prompt
|
||||
assert "**Write-Through Memory Tools** (`save_note`, `update_note`, `append_note`)" in prompt
|
||||
assert "May run alongside the turn's Turn-Ending Action (or Fast-Action Burst)" in prompt
|
||||
# The two execution tiers and the incident workflow are advertised.
|
||||
assert "**Vetted Single Action (default)**" in prompt
|
||||
# The burst ceiling is a context variable filled in by the later template render.
|
||||
assert (
|
||||
"**Fast-Action Burst (time-critical only)**: two to {{ max_burst_actions }}"
|
||||
" Turn-Ending Actions" in prompt
|
||||
)
|
||||
assert "--- Execution Incident (OPEN) ---" in prompt
|
||||
assert "Failure Analyzer" not in prompt
|
||||
assert "(`read_note`, `list_notes`, `recall_history`)" in prompt
|
||||
assert "`save_note`, and `recall_history`" not in prompt
|
||||
assert "Do NOT submit a Turn-Ending Action at the same time" not in prompt
|
||||
assert "memory note tools (`read_note`, `list_notes`, `save_note`, `update_note`" not in prompt
|
||||
|
||||
|
||||
@@ -53,13 +53,17 @@ from artemis.memory.transcript import (
|
||||
|
||||
# SHA-256 of the legacy system message rendered by TemplatePromptComponent
|
||||
# with the fixed inputs below. Originally recorded BEFORE the M2 template
|
||||
# split; re-recorded once in M5 (2026-09-01) after the only intended template
|
||||
# change since — removing the Short-Term Memory instruction block from
|
||||
# operator.json (the <short_term_memory> relay was removed end to end per
|
||||
# pro-context-memory-redesign §3.3). Any other drift here means the flag-off
|
||||
# split; re-recorded in M5 (2026-09-01) after the Short-Term Memory block was
|
||||
# removed, and again on 2026-09-02 when the Failure Analyzer was retired: the
|
||||
# operator.json template gained the Time-Sensitive Tasks principle, the two
|
||||
# execution tiers (vetted single action / fast-action burst), the Execution
|
||||
# Incident workflow section and the Phase 3 burst-discipline rule; and again
|
||||
# on 2026-09-02 after the prompt consolidation (every rule stated once, the
|
||||
# transcript/legacy history-marker split, the recited tool budget, the
|
||||
# recommended diagnosis triggers). Any other drift here means the flag-off
|
||||
# path is no longer byte-identical.
|
||||
GOLDEN_EMPTY_PLAN = "c45aec7d67069dad3a195534dd3c97dadb48454ca186a40eb471d5f9136589fa"
|
||||
GOLDEN_SENTINEL_PLAN = "670580efd48f237c3257d996a814145fbef4ec0bb467aa4dfa6eb2db4a613826"
|
||||
GOLDEN_EMPTY_PLAN = "a20046ff46058fa331fa3b8f2d3ffa4a018be567bf264588b9a29bc64e5b11b8"
|
||||
GOLDEN_SENTINEL_PLAN = "8f4456524c8a964d1614d26d2fafab809bd5bd3f30334fd40498999e4b27638b"
|
||||
|
||||
SCREENSHOT_B64 = base64.b64encode(b"fake-jpeg-bytes").decode("utf-8")
|
||||
|
||||
@@ -253,9 +257,7 @@ async def test_transcript_actionless_turn_commits_without_validator_message():
|
||||
|
||||
ledger = ctx.transcript_ledger
|
||||
assert ledger.turn_count == 1
|
||||
assert not any(
|
||||
EXECUTION_RESULT_MARKER in str(m.content) for m in ledger.active_messages
|
||||
)
|
||||
assert not any(EXECUTION_RESULT_MARKER in str(m.content) for m in ledger.active_messages)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -141,7 +141,6 @@ def _guard_ctx(tmp_path, **setup_kwargs):
|
||||
ctx.planner_task = None
|
||||
ctx.last_validated_plan = None
|
||||
ctx.pending_validated_plan = None
|
||||
ctx.task_plan_content_before = None
|
||||
ctx.pending_checkpoints = []
|
||||
ctx.checkpoint_tasks = {}
|
||||
return ctx
|
||||
|
||||
@@ -26,13 +26,6 @@ from artemis.agents.operator.prompts import (
|
||||
apply_operator_prompt_contract,
|
||||
)
|
||||
from artemis.agents.prompt_assembly import gate_segment, render_tool_enum
|
||||
from artemis.agents.validator.failure_analyzer import (
|
||||
FailureAnalysisStrategy,
|
||||
FailureAnalyzer,
|
||||
PixelTargetDisappearedStrategy,
|
||||
TargetDisappearedStrategy,
|
||||
)
|
||||
from artemis.agents.validator.validator import ValidationErrorCategory
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
@@ -115,12 +108,15 @@ def test_removed_swipe_takes_its_teaching_along():
|
||||
|
||||
|
||||
def test_removed_wait_for_delay_keeps_interval_line_coherent():
|
||||
"""Only the wait_for_delay parenthetical goes; the SOP grammar line stays."""
|
||||
"""Only the wait_for_delay parenthetical goes; the SOP pacing step stays."""
|
||||
template = load_operator_prompts()["main_template"]
|
||||
out = apply_operator_prompt_contract(
|
||||
template, available_tools=OPERATOR_PROMPT_TOOLSET - {"wait_for_delay"}
|
||||
)
|
||||
assert "`Interval: <Interval>`: Declared polling cadence." in out
|
||||
assert (
|
||||
"Adhere strictly to the exact duration declared in `<Interval>`, then execute"
|
||||
" a refresh action" in out
|
||||
)
|
||||
assert "Loading & Transitions" not in out
|
||||
|
||||
|
||||
@@ -131,12 +127,10 @@ def test_reduced_enums_stay_well_formed():
|
||||
available_tools=OPERATOR_PROMPT_TOOLSET - {"manage_app", "wait_for_delay"},
|
||||
)
|
||||
assert (
|
||||
"Physical device actions (`click`, `input_text`, `swipe`, `press_key`,"
|
||||
" `long_press`)" in out
|
||||
"Physical device actions (`click`, `input_text`, `swipe`, `press_key`, `long_press`)" in out
|
||||
)
|
||||
assert (
|
||||
"Turn-Ending Action (`click`, `swipe`, `input_text`, `long_press`, or"
|
||||
" `press_key`)" in out
|
||||
"Turn-Ending Action (`click`, `swipe`, `input_text`, `long_press`, or `press_key`)" in out
|
||||
)
|
||||
|
||||
|
||||
@@ -152,20 +146,20 @@ def test_full_set_enum_slots_render_verbatim():
|
||||
"Turn-Ending Action (`click`, `swipe`, `input_text`, `long_press`,"
|
||||
" `press_key`, `manage_app`, or `wait_for_delay`)" in out
|
||||
)
|
||||
assert (
|
||||
"(`ask_explorer`, `ask_diagnoser`, `video_analyzer`, `run_adb_command`,"
|
||||
" `manage_task`, `analyze_task_output`, `read_note`, `list_notes`,"
|
||||
" `save_note`, and `recall_history`)" in out
|
||||
)
|
||||
assert "Helper/Subagent tools (`ask_explorer`, `ask_diagnoser`, `video_analyzer`)" in out
|
||||
assert "ADB/task tools (`run_adb_command`, `manage_task`)" in out
|
||||
assert "(`read_note`, `list_notes`, `recall_history`)" in out
|
||||
# save_note is a write-through tool, never a result-dependent one; the on-demand
|
||||
# analyze_task_output is not advertised.
|
||||
assert "`save_note`, and" not in out
|
||||
assert "analyze_task_output" not in out
|
||||
|
||||
|
||||
# --- Flash prompt assembly -----------------------------------------------------------
|
||||
|
||||
|
||||
def _render_flash(available_tools: frozenset[str]) -> str:
|
||||
template_text = (REPO_ROOT / "artemis/agents/flash/flash_runner.md").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
template_text = (REPO_ROOT / "artemis/agents/flash/flash_runner.md").read_text(encoding="utf-8")
|
||||
return Template(template_text).render(goal="test goal", available_tools=available_tools)
|
||||
|
||||
|
||||
@@ -184,7 +178,7 @@ def test_flash_without_manage_app_drops_app_launching():
|
||||
assert "**App Launching**" not in out
|
||||
assert not _tool_refs(out, "manage_app")
|
||||
# The neighbouring numbered rule survives.
|
||||
assert "**Screenshot Pruning Awareness**" in out
|
||||
assert "**Do not determine element coordinates based on guesswork.**" in out
|
||||
|
||||
|
||||
def test_flash_without_wait_for_delay_drops_timed_waiting():
|
||||
@@ -192,42 +186,3 @@ def test_flash_without_wait_for_delay_drops_timed_waiting():
|
||||
assert "**Timed Waiting & Transitions**" not in out
|
||||
assert not _tool_refs(out, "wait_for_delay")
|
||||
assert "**State Drift & Progression Tolerance**" in out
|
||||
|
||||
|
||||
# --- FailureAnalyzer strategy gating -------------------------------------------------
|
||||
|
||||
|
||||
def _analyzer() -> FailureAnalyzer:
|
||||
return FailureAnalyzer.__new__(FailureAnalyzer) # _select_strategy needs no ctx
|
||||
|
||||
|
||||
def test_strategy_selection_unchanged_without_gating():
|
||||
analyzer = _analyzer()
|
||||
assert isinstance(
|
||||
analyzer._select_strategy(ValidationErrorCategory.TARGET_DISAPPEARED),
|
||||
TargetDisappearedStrategy,
|
||||
)
|
||||
assert isinstance(
|
||||
analyzer._select_strategy(ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED),
|
||||
PixelTargetDisappearedStrategy,
|
||||
)
|
||||
|
||||
|
||||
def test_strategy_falls_back_when_required_tool_missing():
|
||||
"""Strategy + prompt are one unit: no click_sequence, no disappeared-prompts."""
|
||||
analyzer = _analyzer()
|
||||
strategy = analyzer._select_strategy(
|
||||
ValidationErrorCategory.TARGET_DISAPPEARED,
|
||||
available_tools=frozenset({"click", "swipe"}),
|
||||
)
|
||||
assert type(strategy) is FailureAnalysisStrategy
|
||||
assert strategy.get_prompt_template_name() == "failure_analyzer.md"
|
||||
|
||||
|
||||
def test_strategy_kept_when_required_tool_present():
|
||||
analyzer = _analyzer()
|
||||
strategy = analyzer._select_strategy(
|
||||
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
|
||||
available_tools=frozenset({"click_sequence"}),
|
||||
)
|
||||
assert isinstance(strategy, PixelTargetDisappearedStrategy)
|
||||
@@ -136,9 +136,7 @@ async def test_summarizer_falls_back_to_structured_decisions(mock_context):
|
||||
kwargs = mock_context.step_memory.dispatch.call_args.kwargs
|
||||
assert kwargs["action_name"] == "swipe"
|
||||
assert kwargs["action_args"]["coordinates"] == [1, 2, 3, 4]
|
||||
assert kwargs["action_args"]["additional_actions"] == [
|
||||
{"action": "tap", "coordinates": [9, 9]}
|
||||
]
|
||||
assert kwargs["action_args"]["additional_actions"] == [{"action": "tap", "coordinates": [9, 9]}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
+178
-154
@@ -27,7 +27,7 @@ from unittest.mock import AsyncMock, MagicMock, Mock, patch
|
||||
from langchain_core.messages import AIMessage
|
||||
import pytest
|
||||
|
||||
from artemis.agents.validator.failure_analyzer import ValidationErrorCategory
|
||||
from artemis.agents.validator.categories import ValidationErrorCategory
|
||||
from artemis.agents.validator.validator import ValidatorNode
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.mcp.action_types import ActionCode, ActionResult
|
||||
@@ -39,10 +39,12 @@ class DummyState:
|
||||
structured_decisions,
|
||||
current_step_id=None,
|
||||
latest_screenshot=None,
|
||||
open_incident=None,
|
||||
):
|
||||
self.structured_decisions = structured_decisions
|
||||
self.current_step_id = current_step_id
|
||||
self.latest_screenshot = latest_screenshot
|
||||
self.open_incident = open_incident
|
||||
|
||||
|
||||
class FakeActionSession:
|
||||
@@ -143,10 +145,9 @@ async def test_validator_success(mock_mcp, mock_context, temp_screenshot):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_failure_analysis(mock_mcp, mock_context, temp_screenshot):
|
||||
"""Test that ValidatorNode triggers failure analysis on error and handles
|
||||
cannot_fix via tool.
|
||||
"""
|
||||
async def test_validator_exec_error_opens_incident(mock_mcp, mock_context, temp_screenshot):
|
||||
"""A vetted single action that the device rejects is retried once, then an
|
||||
execution incident is opened for the Operator (no repair agent)."""
|
||||
mock_mcp.action_handler = lambda name, args: (
|
||||
(False, "Error: Element not found") if name == "click" else (True, "")
|
||||
)
|
||||
@@ -155,80 +156,168 @@ async def test_validator_failure_analysis(mock_mcp, mock_context, temp_screensho
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
|
||||
node = ValidatorNode(mock_context)
|
||||
|
||||
with (
|
||||
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
|
||||
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
|
||||
):
|
||||
mock_analyze.return_value = {
|
||||
"status": "cannot_fix",
|
||||
"analysis": "Analysis: Coordinates are likely wrong.",
|
||||
}
|
||||
|
||||
with patch("artemis.utils.image_diff.check_ui_change", return_value=False):
|
||||
result = await node(state)
|
||||
|
||||
assert "last_execution_result" in result
|
||||
report = result["last_execution_result"]
|
||||
assert "execution" in report
|
||||
assert report["status"] == "failed"
|
||||
assert report["burst"] is False
|
||||
assert len(report["execution"]) == 1
|
||||
assert report["execution"][0]["action"] == "tap"
|
||||
assert report["execution"][0]["attempts"] == [
|
||||
"Error: Element not found",
|
||||
"Error: Element not found",
|
||||
]
|
||||
assert report["execution"][0]["repair"] == "Analysis: Coordinates are likely wrong."
|
||||
assert "repair" not in report["execution"][0]
|
||||
|
||||
incident = report["incident"]
|
||||
assert incident["kind"] == "exec_error"
|
||||
assert incident["category"] == "general"
|
||||
assert incident["reason"] == "Error: Element not found"
|
||||
assert incident["consecutive_failures"] == 1
|
||||
assert incident["burst_size"] == 1
|
||||
assert incident["action_description"] == "Tapped element at [105, 205]"
|
||||
# The same record rides in graph state for the Operator prompt.
|
||||
assert result["open_incident"] == incident
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_repair_success(mock_mcp, mock_context, temp_screenshot):
|
||||
"""Test that ValidatorNode terminates successfully and clears remaining actions
|
||||
after successful repair.
|
||||
"""
|
||||
tap_count = 0
|
||||
async def test_validator_burst_skips_safety_net_and_aborts_on_first_failure(
|
||||
mock_mcp, mock_context, temp_screenshot
|
||||
):
|
||||
"""Two or more actions form a fast-action burst: no precondition gate, a
|
||||
single dispatch per member, and the first failure aborts the rest."""
|
||||
calls = []
|
||||
|
||||
def handler(name, args):
|
||||
nonlocal tap_count
|
||||
if name == "click":
|
||||
tap_count += 1
|
||||
if tap_count <= 2: # First action fails on both validator attempts
|
||||
return False, "Error: Element not found"
|
||||
calls.append(name)
|
||||
if name == "click" and len([c for c in calls if c == "click"]) == 2:
|
||||
return False, "Error: tap rejected"
|
||||
return True, ""
|
||||
|
||||
mock_mcp.action_handler = handler
|
||||
|
||||
# Two actions planned initially
|
||||
decisions = json.dumps(
|
||||
[
|
||||
{"action": "tap", "coordinates": [105, 205]},
|
||||
{"action": "tap", "coordinates": [205, 305]},
|
||||
{"action": "press_key", "keycode": "BACK"},
|
||||
]
|
||||
)
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
|
||||
node = ValidatorNode(mock_context)
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.validator.execution_loop._run_precondition_gate",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_gate,
|
||||
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
|
||||
):
|
||||
result = await node(state)
|
||||
|
||||
mock_gate.assert_not_called()
|
||||
|
||||
report = result["last_execution_result"]
|
||||
assert report["burst"] is True
|
||||
assert report["status"] == "failed"
|
||||
assert len(report["execution"]) == 3
|
||||
# First member executed cleanly.
|
||||
assert "attempts" not in report["execution"][0]
|
||||
# Second member failed after exactly one dispatch (bursts never retry).
|
||||
assert report["execution"][1]["attempts"] == ["Error: tap rejected"]
|
||||
# Third member never fired.
|
||||
assert report["execution"][2]["attempts"] == ["Skipped (burst aborted)"]
|
||||
assert len(mock_mcp.calls_for("click")) == 2
|
||||
assert mock_mcp.calls_for("press_key") == []
|
||||
|
||||
incident = report["incident"]
|
||||
assert incident["kind"] == "exec_error"
|
||||
assert incident["action_index"] == 1
|
||||
assert incident["burst_size"] == 3
|
||||
assert result["open_incident"] == incident
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_burst_success_executes_every_member(
|
||||
mock_mcp, mock_context, temp_screenshot
|
||||
):
|
||||
decisions = json.dumps(
|
||||
[
|
||||
{"action": "tap", "coordinates": [105, 205]},
|
||||
{"action": "wait_for_delay", "time_in_ms": 1},
|
||||
{"action": "tap", "coordinates": [205, 305]},
|
||||
]
|
||||
)
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
|
||||
node = ValidatorNode(mock_context)
|
||||
|
||||
with (
|
||||
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
|
||||
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
|
||||
):
|
||||
mock_analyze.return_value = {
|
||||
"status": "fixed",
|
||||
"analysis": "Fixed by updating actions",
|
||||
}
|
||||
|
||||
with patch("artemis.utils.image_diff.check_ui_change", return_value=True):
|
||||
result = await node(state)
|
||||
|
||||
assert "last_execution_result" in result
|
||||
report = result["last_execution_result"]
|
||||
assert "execution" in report
|
||||
assert len(report["execution"]) == 1 # Only the first failed action is in report
|
||||
assert report["status"] == "success"
|
||||
assert report["burst"] is True
|
||||
assert report["incident"] is None
|
||||
assert result["open_incident"] is None
|
||||
assert result["last_closed_incident"] is None
|
||||
assert len(mock_mcp.calls_for("click")) == 2
|
||||
|
||||
assert report["execution"][0]["coordinates"] == [105, 205]
|
||||
assert report["execution"][0]["attempts"] == [
|
||||
"Error: Element not found",
|
||||
"Error: Element not found",
|
||||
]
|
||||
assert report["execution"][0]["repair"] == "Fixed by updating actions"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_success_closes_open_incident(mock_mcp, mock_context, temp_screenshot):
|
||||
"""A successful turn closes whatever incident was still open."""
|
||||
previous = {
|
||||
"kind": "safety_net",
|
||||
"category": "target_disappeared",
|
||||
"reason": "gone",
|
||||
"action": {"action": "tap"},
|
||||
"action_description": "Tapped element",
|
||||
"consecutive_failures": 2,
|
||||
}
|
||||
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
|
||||
state = DummyState(
|
||||
structured_decisions=decisions,
|
||||
latest_screenshot=temp_screenshot,
|
||||
open_incident=previous,
|
||||
)
|
||||
|
||||
node = ValidatorNode(mock_context)
|
||||
with patch("artemis.utils.image_diff.check_ui_change", return_value=True):
|
||||
result = await node(state)
|
||||
|
||||
assert result["last_execution_result"]["status"] == "success"
|
||||
assert result["open_incident"] is None
|
||||
# The closed record is handed over once so the Operator settles the intent.
|
||||
assert result["last_closed_incident"]["kind"] == "safety_net"
|
||||
assert "closed_at_step" in result["last_closed_incident"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_consecutive_failures_escalate_incident(
|
||||
mock_mcp, mock_context, temp_screenshot
|
||||
):
|
||||
"""Failing again while an incident is open continues its failure count."""
|
||||
previous = {
|
||||
"kind": "exec_error",
|
||||
"category": "general",
|
||||
"reason": "Error: Element not found",
|
||||
"action": {"action": "tap"},
|
||||
"action_description": "Tapped element",
|
||||
"consecutive_failures": 2,
|
||||
}
|
||||
mock_mcp.action_handler = lambda name, args: (False, "Error: Element not found")
|
||||
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
|
||||
state = DummyState(
|
||||
structured_decisions=decisions,
|
||||
latest_screenshot=temp_screenshot,
|
||||
open_incident=previous,
|
||||
)
|
||||
|
||||
node = ValidatorNode(mock_context)
|
||||
with patch("artemis.utils.image_diff.check_ui_change", return_value=False):
|
||||
result = await node(state)
|
||||
|
||||
assert result["open_incident"]["consecutive_failures"] == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -271,7 +360,7 @@ async def test_validator_focus_and_clear_text_no_ui_change(mock_mcp, mock_contex
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_silent_failure_treated_as_success(mock_mcp, mock_context, temp_screenshot):
|
||||
"""Test that silent failure (exec succeeds but no UI change) is treated as
|
||||
success and does not trigger FailureAnalyzer.
|
||||
success and does not open an execution incident.
|
||||
"""
|
||||
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
@@ -281,13 +370,12 @@ async def test_validator_silent_failure_treated_as_success(mock_mcp, mock_contex
|
||||
with (
|
||||
patch("artemis.agents.validator.validator.VALIDATOR_POLL_TIMEOUT", 0.1),
|
||||
patch("artemis.agents.validator.validator.VALIDATOR_POLL_INTERVAL", 0.01),
|
||||
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
|
||||
patch("artemis.utils.image_diff.check_ui_change", return_value=False), # No UI change
|
||||
):
|
||||
result = await node(state)
|
||||
|
||||
# Assert FailureAnalyzer was NOT called
|
||||
mock_analyze.assert_not_called()
|
||||
# No incident is opened for a silent (no-UI-change) success.
|
||||
assert result["open_incident"] is None
|
||||
|
||||
# Assert the action is considered executed successfully
|
||||
assert "last_execution_result" in result
|
||||
@@ -299,13 +387,11 @@ async def test_validator_silent_failure_treated_as_success(mock_mcp, mock_contex
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_records_skipped_actions_on_cannot_fix(
|
||||
async def test_validator_burst_first_member_failure_marks_rest_skipped(
|
||||
mock_mcp, mock_context, temp_screenshot
|
||||
):
|
||||
"""Test that ValidatorNode marks unexecuted actions as Skipped when repair fails."""
|
||||
mock_mcp.action_handler = lambda name, args: (False, "Error: Connection lost")
|
||||
|
||||
# Two actions planned
|
||||
decisions = json.dumps(
|
||||
[
|
||||
{"action": "tap", "coordinates": [100, 200]},
|
||||
@@ -315,42 +401,27 @@ async def test_validator_records_skipped_actions_on_cannot_fix(
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
|
||||
node = ValidatorNode(mock_context)
|
||||
|
||||
with (
|
||||
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
|
||||
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
|
||||
):
|
||||
mock_analyze.return_value = {
|
||||
"status": "cannot_fix",
|
||||
"analysis": "Cannot fix: System crashed.",
|
||||
}
|
||||
|
||||
with patch("artemis.utils.image_diff.check_ui_change", return_value=False):
|
||||
result = await node(state)
|
||||
|
||||
assert "last_execution_result" in result
|
||||
report = result["last_execution_result"]
|
||||
assert "execution" in report
|
||||
assert len(report["execution"]) == 2
|
||||
|
||||
# First action failed
|
||||
assert report["execution"][0]["action"] == "tap"
|
||||
assert report["execution"][0]["attempts"] == [
|
||||
"Error: Connection lost",
|
||||
"Error: Connection lost",
|
||||
]
|
||||
assert report["execution"][0]["repair"] == "Cannot fix: System crashed."
|
||||
|
||||
# Second action was skipped
|
||||
# A burst member is dispatched exactly once.
|
||||
assert report["execution"][0]["attempts"] == ["Error: Connection lost"]
|
||||
assert "repair" not in report["execution"][0]
|
||||
assert report["execution"][1]["action"] == "press_key"
|
||||
assert report["execution"][1]["attempts"] == ["Skipped"]
|
||||
assert report["execution"][1]["attempts"] == ["Skipped (burst aborted)"]
|
||||
assert report["incident"]["action_index"] == 0
|
||||
assert report["incident"]["burst_size"] == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_pre_execution_validation_and_repair(
|
||||
async def test_validator_pre_execution_validation_opens_incident(
|
||||
mock_mcp, mock_context, temp_screenshot
|
||||
):
|
||||
"""Test that pre-execution validation fails when element changes, and is
|
||||
repaired by FailureAnalyzer.
|
||||
"""Test that pre-execution validation fails when the element changes, and the
|
||||
interception is handed to the Operator as an execution incident.
|
||||
"""
|
||||
mock_mcp.hierarchy = [
|
||||
{
|
||||
@@ -376,7 +447,6 @@ async def test_validator_pre_execution_validation_and_repair(
|
||||
node = ValidatorNode(mock_context)
|
||||
|
||||
with (
|
||||
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
|
||||
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
|
||||
patch.object(
|
||||
ValidatorNode,
|
||||
@@ -389,29 +459,20 @@ async def test_validator_pre_execution_validation_and_repair(
|
||||
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
|
||||
"Pixel check failed",
|
||||
)
|
||||
mock_analyze.return_value = {
|
||||
"status": "fixed",
|
||||
"analysis": ("Pre-exec failure matched and resolved by switching to Sign Up button."),
|
||||
}
|
||||
|
||||
result = await node(state)
|
||||
|
||||
mock_analyze.assert_called_once()
|
||||
args, kwargs = mock_analyze.call_args
|
||||
assert "Pre-execution validation failed" in args[2]
|
||||
assert "Login" in args[2]
|
||||
assert kwargs.get("error_category") == ValidationErrorCategory.TARGET_OCCUPIED
|
||||
incident = result["last_execution_result"]["incident"]
|
||||
assert "Pre-execution validation failed" in incident["reason"]
|
||||
assert "Login" in incident["reason"]
|
||||
assert incident["category"] == ValidationErrorCategory.TARGET_OCCUPIED.value
|
||||
|
||||
assert "last_execution_result" in result
|
||||
report = result["last_execution_result"]
|
||||
assert len(report["execution"]) == 1
|
||||
assert report["execution"][0]["action"] == "tap"
|
||||
assert "Pre-execution validation failed" in report["execution"][0]["attempts"][0]
|
||||
assert (
|
||||
report["execution"][0]["repair"]
|
||||
== "Pre-exec failure matched and resolved by switching to Sign Up"
|
||||
" button."
|
||||
)
|
||||
assert report["incident"]["kind"] == "safety_net"
|
||||
assert report["status"] == "failed"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -443,7 +504,7 @@ async def test_validator_pre_execution_validation_self_healing(
|
||||
node = ValidatorNode(mock_context)
|
||||
|
||||
with patch("artemis.utils.image_diff.check_ui_change", return_value=True):
|
||||
await node(state)
|
||||
result = await node(state)
|
||||
|
||||
# Center of [100, 220][300, 320] is pixel [200, 270]; the healed coordinates
|
||||
# travel to the canonical click tool normalized to 0-1000 on a 1080x2400 screen:
|
||||
@@ -487,7 +548,6 @@ async def test_validator_pre_execution_validation_anonymous_occupant(
|
||||
node = ValidatorNode(mock_context)
|
||||
|
||||
with (
|
||||
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
|
||||
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
|
||||
patch.object(
|
||||
ValidatorNode,
|
||||
@@ -500,19 +560,16 @@ async def test_validator_pre_execution_validation_anonymous_occupant(
|
||||
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
|
||||
"Pixel check failed",
|
||||
)
|
||||
mock_analyze.return_value = {
|
||||
"status": "fixed",
|
||||
"analysis": "Pre-exec failure bypassed.",
|
||||
}
|
||||
|
||||
await node(state)
|
||||
result = await node(state)
|
||||
|
||||
# Assert that the validator correctly classified this as TARGET_OCCUPIED
|
||||
# (due to the clickable anonymous view blocking the click)
|
||||
mock_analyze.assert_called_once()
|
||||
args, kwargs = mock_analyze.call_args
|
||||
assert kwargs.get("error_category") == ValidationErrorCategory.TARGET_OCCUPIED
|
||||
assert "occupied/intercepted by a different element: interactive anonymous element" in args[2]
|
||||
incident = result["last_execution_result"]["incident"]
|
||||
assert incident["category"] == ValidationErrorCategory.TARGET_OCCUPIED.value
|
||||
assert (
|
||||
"occupied/intercepted by a different element: interactive anonymous element"
|
||||
in incident["reason"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -558,7 +615,7 @@ async def test_validator_pixel_validation_success(mock_mcp, mock_context, temp_s
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_pixel_validation_failure(mock_mcp, mock_context, temp_screenshot):
|
||||
"""Test that the pixel safety net fails and triggers FailureAnalyzer
|
||||
"""Test that the pixel safety net fails and opens an execution incident
|
||||
when Gemini reports target is missing.
|
||||
"""
|
||||
decisions = json.dumps([{"action": "tap", "coordinates": [100, 200]}])
|
||||
@@ -588,20 +645,13 @@ async def test_validator_pixel_validation_failure(mock_mcp, mock_context, temp_s
|
||||
),
|
||||
patch("artemis.agents.validator.validator.get_llm", return_value=mock_llm),
|
||||
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
|
||||
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
|
||||
):
|
||||
mock_analyze.return_value = {
|
||||
"status": "cannot_fix",
|
||||
"analysis": "Pixel safety net flagged missing target.",
|
||||
}
|
||||
|
||||
node = ValidatorNode(mock_context)
|
||||
await node(state)
|
||||
result = await node(state)
|
||||
|
||||
mock_analyze.assert_called_once()
|
||||
args, kwargs = mock_analyze.call_args
|
||||
assert "Pixel-level validation failed" in args[2]
|
||||
assert kwargs.get("error_category") == ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED
|
||||
incident = result["last_execution_result"]["incident"]
|
||||
assert "Pixel-level validation failed" in incident["reason"]
|
||||
assert incident["category"] == ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED.value
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -639,14 +689,8 @@ async def test_validator_launch_app_failure_no_retry(mock_mcp, mock_context, tem
|
||||
with (
|
||||
patch("artemis.agents.validator.validator.VALIDATOR_POLL_TIMEOUT", 0.05),
|
||||
patch("artemis.agents.validator.validator.VALIDATOR_POLL_INTERVAL", 0.01),
|
||||
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
|
||||
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
|
||||
):
|
||||
mock_analyze.return_value = {
|
||||
"status": "cannot_fix",
|
||||
"analysis": "Cannot launch app.",
|
||||
}
|
||||
|
||||
node = ValidatorNode(mock_context)
|
||||
result = await node(state)
|
||||
|
||||
@@ -693,7 +737,6 @@ async def test_validator_pre_execution_validation_disappeared_not_shifted(
|
||||
node = ValidatorNode(mock_context)
|
||||
|
||||
with (
|
||||
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
|
||||
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
|
||||
patch.object(
|
||||
ValidatorNode,
|
||||
@@ -706,18 +749,12 @@ async def test_validator_pre_execution_validation_disappeared_not_shifted(
|
||||
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
|
||||
"Pixel check failed",
|
||||
)
|
||||
mock_analyze.return_value = {
|
||||
"status": "cannot_fix",
|
||||
"analysis": "Sign Up button disappeared.",
|
||||
}
|
||||
|
||||
await node(state)
|
||||
result = await node(state)
|
||||
|
||||
# Assert that the validator correctly classified this as TARGET_DISAPPEARED
|
||||
# (since size mismatch prevents shift matching)
|
||||
mock_analyze.assert_called_once()
|
||||
args, kwargs = mock_analyze.call_args
|
||||
assert kwargs.get("error_category") == ValidationErrorCategory.TARGET_DISAPPEARED
|
||||
incident = result["last_execution_result"]["incident"]
|
||||
assert incident["category"] == ValidationErrorCategory.TARGET_DISAPPEARED.value
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -800,7 +837,6 @@ async def test_validator_pre_execution_xml_failure_not_overridden_when_pixel_byp
|
||||
node = ValidatorNode(mock_context)
|
||||
|
||||
with (
|
||||
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
|
||||
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
|
||||
patch.object(
|
||||
ValidatorNode,
|
||||
@@ -814,18 +850,12 @@ async def test_validator_pre_execution_xml_failure_not_overridden_when_pixel_byp
|
||||
ValidationErrorCategory.PIXEL_BYPASSED,
|
||||
"",
|
||||
)
|
||||
mock_analyze.return_value = {
|
||||
"status": "cannot_fix",
|
||||
"analysis": ("XML validation failed and pixel bypass did not override."),
|
||||
}
|
||||
result = await node(state)
|
||||
|
||||
await node(state)
|
||||
|
||||
# FailureAnalyzer SHOULD be called because XML validation failed
|
||||
# An incident SHOULD be opened because XML validation failed
|
||||
# and PIXEL_BYPASSED did not override it
|
||||
mock_analyze.assert_called_once()
|
||||
args, kwargs = mock_analyze.call_args
|
||||
assert "Pre-execution validation failed" in args[2]
|
||||
incident = result["last_execution_result"]["incident"]
|
||||
assert "Pre-execution validation failed" in incident["reason"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -845,14 +875,8 @@ async def test_validator_failure_screenshot_mcp_error_handled_safely(
|
||||
node = ValidatorNode(mock_context)
|
||||
|
||||
with (
|
||||
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
|
||||
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
|
||||
):
|
||||
mock_analyze.return_value = {
|
||||
"status": "cannot_fix",
|
||||
"analysis": "Action failed and cannot fix.",
|
||||
}
|
||||
|
||||
result = await node(state)
|
||||
|
||||
assert "last_execution_result" in result
|
||||
|
||||
@@ -209,9 +209,7 @@ async def test_video_coordinator_retries_transient_failures():
|
||||
async def test_audio_analysis_reuses_exact_completed_coverage(analyzer_context):
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
|
||||
analyzer = VideoAnalyzer(analyzer_context)
|
||||
claim = analyzer.blackboard.claim_segment(
|
||||
2.0, 6.0, "hear notification", modality="audio"
|
||||
)
|
||||
claim = analyzer.blackboard.claim_segment(2.0, 6.0, "hear notification", modality="audio")
|
||||
analyzer.blackboard.complete_segment(
|
||||
2.0,
|
||||
6.0,
|
||||
@@ -222,12 +220,8 @@ async def test_audio_analysis_reuses_exact_completed_coverage(analyzer_context):
|
||||
modality="audio",
|
||||
)
|
||||
|
||||
with patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.get_controller"
|
||||
) as controller:
|
||||
result = await analyzer.exec_analyze_audio_only(
|
||||
2.0, 6.0, "hear notification"
|
||||
)
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.get_controller") as controller:
|
||||
result = await analyzer.exec_analyze_audio_only(2.0, 6.0, "hear notification")
|
||||
|
||||
controller.assert_not_called()
|
||||
assert result.startswith("CACHED AUDIO ANALYSIS:")
|
||||
@@ -272,9 +266,7 @@ def test_session_deletion_removes_video_blackboard_rows(tmp_path):
|
||||
engine.record_video_start(video_id, "device", recording)
|
||||
|
||||
board = VideoBlackboard(f"video:{video_id}", db_path=engine.storage.db_path)
|
||||
board.add_observation(
|
||||
{"start": 0.0, "end": 1.0, "target": "cleanup", "summary": "evidence"}
|
||||
)
|
||||
board.add_observation({"start": 0.0, "end": 1.0, "target": "cleanup", "summary": "evidence"})
|
||||
assert board.list_observations()
|
||||
|
||||
engine.storage.delete_session(session_id)
|
||||
|
||||
@@ -138,9 +138,7 @@ async def test_native_chunk_exhaustion_commits_universal_fallback(tmp_path):
|
||||
duration_seconds=5.0,
|
||||
warning=None,
|
||||
)
|
||||
controller = SimpleNamespace(
|
||||
extract_segment_metadata=AsyncMock(return_value=extracted)
|
||||
)
|
||||
controller = SimpleNamespace(extract_segment_metadata=AsyncMock(return_value=extracted))
|
||||
fallback_text = "[from 0.0s to 5.0s] Summary: fallback recovered Analysis: grounded"
|
||||
|
||||
with (
|
||||
|
||||
Reference in new issue
Block a user