feat: improve task execution, verification and packaging

Add execution incidents, action bursts and shared Flash history.

Expose verification and Explorer settings with streamed check results.

Support graceful cancellation, recording recovery and bundled UI resources.

Add CI quality checks and clean up comments and documentation.
This commit is contained in:
somew1nd committed 2026-09-02 21:45:51 -07:00
1 parent 5ccf06d677
commit 67248c887e
289 files changed
+15883 -6919

No files matched your search

@@ -67,7 +67,9 @@ def test_planner_blocks_with_checks_mount_generation_and_audit():
assert "Never silently promise" in generation
audit = data["blocks"]["check_audit"]
assert "REJECT" in audit
# Advisory review: the audit flags weakened check standards, never blocks.
assert "FLAG" in audit
assert "REJECT" not in audit
def _mock_ctx(tmp_path=None, plan: str | None = None, **setup_kwargs):
@@ -108,7 +110,7 @@ async def test_explainer_renders_iff_plan_has_check_lines(tmp_path):
await component(builder2, _state(), ctx2)
joined = "\n".join(p for p in builder2.human_parts if isinstance(p, str))
assert "check lines" in joined
assert "independent Checker" in joined
assert "never declare a check passed yourself" in joined
assert "automatically restored" in joined
@@ -148,13 +150,13 @@ async def test_main_template_diagnosis_trigger_gated_on_verification():
with_checks = await _render_main_template(
disable_checker=False, disable_planner_validation=True
)
assert "Ambiguous Validation Rejection or Verification Finding" in with_checks
assert "Ambiguous Verification Finding" in with_checks
# Planner validation alone can also produce rejections.
# Planner validation alone can also produce (advisory) findings.
with_validation = await _render_main_template(
disable_checker=True, disable_planner_validation=False
)
assert "Ambiguous Validation Rejection or Verification Finding" in with_validation
assert "Ambiguous Verification Finding" in with_validation
@pytest.mark.asyncio
@@ -0,0 +1,55 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""The structured verdict request must never end on a model turn (Gemini
rejects such requests with 400 "Requests ending with a model turn are not
supported"), which is the natural state after the loop's tool-free reply."""
from unittest.mock import AsyncMock, MagicMock
import pytest
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage
from artemis.agents.checker.checker import CheckReport, _structured_report
def _llm(report: CheckReport):
structured = MagicMock()
structured.ainvoke = AsyncMock(return_value=report)
llm = MagicMock()
llm.with_structured_output.return_value = structured
return llm, structured
@pytest.mark.asyncio
async def test_structured_report_appends_user_turn_after_model_reply():
llm, structured = _llm(CheckReport(verdicts=[]))
messages = [SystemMessage(content="s"), HumanMessage(content="h"), AIMessage(content="done")]
await _structured_report(llm, messages)
sent = structured.ainvoke.call_args.args[0]
assert isinstance(sent[-1], HumanMessage)
assert sent[-2] is messages[-1]
assert len(messages) == 3 # caller's list untouched
@pytest.mark.asyncio
async def test_structured_report_keeps_user_ending_conversation_as_is():
llm, structured = _llm(CheckReport(verdicts=[]))
messages = [SystemMessage(content="s"), HumanMessage(content="final iteration")]
await _structured_report(llm, messages)
assert structured.ainvoke.call_args.args[0] is messages
@@ -0,0 +1,197 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""The Operator-facing execution incident block and the incident record."""
from types import SimpleNamespace
import pytest
from artemis.agents.operator.prompts import (
EXECUTION_INCIDENT_MARKER,
ExecutionIncidentPromptComponent,
PromptBuilder,
render_closed_incident,
render_execution_incident,
)
from artemis.agents.validator.categories import ValidationErrorCategory
from artemis.agents.validator.incidents import (
KIND_EXEC_ERROR,
KIND_SAFETY_NET,
ExecutionIncident,
open_incident,
)
STEPS = [
{
"step_number": 11,
"action_taken": [{"action": "click", "coordinates": [540, 1800], "target_text": "player"}],
"last_execution_result": {"status": "success"},
},
{
"step_number": 12,
"action_taken": [{"action": "click", "target_text": "Skip"}],
"last_execution_result": {"status": "failed"},
},
]
def _disappeared(consecutive=1, **overrides):
incident = open_incident(
previous=None,
kind=KIND_SAFETY_NET,
category=ValidationErrorCategory.TARGET_DISAPPEARED,
reason="Target element 'Skip' was not found on the screen.",
action_item={
"action": "tap",
"coordinates": [950, 288],
"normalized_coordinates": [880, 120],
"target_text": "Skip",
"attempts": ["Pre-execution validation failed: ..."],
},
action_description="Tapped 'Skip' at [950, 288]",
step_number=12,
)
incident["consecutive_failures"] = consecutive
incident.update(overrides)
return incident
# --- record -------------------------------------------------------------------------
def test_open_incident_strips_internal_keys_and_starts_at_one():
incident = _disappeared()
assert incident["consecutive_failures"] == 1
assert "attempts" not in incident["action"]
assert incident["category"] == "target_disappeared"
assert incident["burst_size"] == 1 and incident["action_index"] == 0
def test_open_incident_continues_the_previous_count():
previous = _disappeared(consecutive=2)
incident = open_incident(
previous=previous,
kind=KIND_EXEC_ERROR,
category=ValidationErrorCategory.GENERAL,
reason="Error: tap rejected",
action_item={"action": "tap"},
action_description="Tapped element",
)
assert incident["consecutive_failures"] == 3
assert ExecutionIncident.from_dict(incident).is_burst is False
assert ExecutionIncident.from_dict(None) is None
assert ExecutionIncident.from_dict({"kind": "safety_net", "unknown_key": 1}) is not None
# --- rendering ----------------------------------------------------------------------
def test_render_disappeared_names_target_trigger_and_burst_exception():
text = render_execution_incident(_disappeared(), STEPS)
assert text.startswith(EXECUTION_INCIDENT_MARKER)
assert "Opened at Step 12; consecutive failed turns: 1." in text
assert "was NOT executed. The pre-execution safety net refused it" in text
# Recorded target with normalized coordinates and text.
assert 'normalized [880, 120] "Skip"' in text
# The last successful action before the incident is the trigger candidate.
assert "Your last successfully executed action was `Tapped 'player' at [540, 1800]`" in text
assert "(Step 11)" in text
# Facts and evidence only: the response protocol is stated once, in the
# static system prompt, so the per-turn block never restates it.
assert "How to respond" not in text
assert "settle the original intent" not in text
assert "e.g." not in text
# No escalation ladder: the Operator keeps its freedom.
assert "Escalation" not in text
assert "ask_diagnoser" not in text
def test_render_shifted_and_occupied_evidence():
shifted = _disappeared(
category="target_shifted",
evidence={"new_location": [500, 640], "new_bounds": [400, 600, 600, 680]},
)
text = render_execution_incident(shifted, STEPS)
assert "has moved at normalized [500, 640] (bounds [400, 600, 600, 680])" in text
occupied = _disappeared(category="target_occupied", evidence={"occupant": "'Sign Up' button"})
text = render_execution_incident(occupied, STEPS)
assert "now covered by 'Sign Up' button" in text
def test_render_burst_abort_and_exec_error():
burst = _disappeared(kind=KIND_EXEC_ERROR, category="general", burst_size=3, action_index=1)
burst["reason"] = "Error: tap rejected"
text = render_execution_incident(burst, STEPS)
assert "action 2 of your 3-action fast burst" in text
assert "The 1 action(s) after it were NOT executed" in text
plain = _disappeared(kind=KIND_EXEC_ERROR, category="general")
text = render_execution_incident(plain, STEPS)
assert "was dispatched, but the device/executor reported" in text
def test_render_without_history_omits_the_trigger_hint():
text = render_execution_incident(_disappeared(), [])
assert "last successfully executed action" not in text
assert 'Its recorded target was normalized [880, 120] "Skip".' in text
# --- component ----------------------------------------------------------------------
@pytest.mark.asyncio
async def test_component_renders_only_while_an_incident_is_open():
builder = PromptBuilder()
ctx = SimpleNamespace(data_engine=None)
await ExecutionIncidentPromptComponent()(
builder, SimpleNamespace(open_incident=None), ctx, steps=STEPS
)
assert builder.human_parts == []
await ExecutionIncidentPromptComponent()(
builder, SimpleNamespace(open_incident=_disappeared()), ctx, steps=STEPS
)
assert len(builder.human_parts) == 1
# Its own text part: the transcript scrub keeps it verbatim across turns.
assert builder.human_parts[0].startswith(EXECUTION_INCIDENT_MARKER)
def test_render_closed_notice_settles_the_intent():
closed = dict(_disappeared(), closed_at_step=15)
text = render_closed_incident(closed)
assert text.startswith("--- Execution Incident (CLOSED at Step 15) ---")
assert "opened at Step 12 on `Tapped 'Skip' at [950, 288]`" in text
assert "already served, still pending, or no longer needed" in text
@pytest.mark.asyncio
async def test_component_renders_closed_notice_once_no_incident_is_open():
builder = PromptBuilder()
ctx = SimpleNamespace(data_engine=None)
state = SimpleNamespace(
open_incident=None, last_closed_incident=dict(_disappeared(), closed_at_step=15)
)
await ExecutionIncidentPromptComponent()(builder, state, ctx, steps=STEPS)
assert len(builder.human_parts) == 1
assert builder.human_parts[0].startswith("--- Execution Incident (CLOSED at Step 15) ---")
# An open incident takes precedence over a stale closed notice.
builder = PromptBuilder()
state = SimpleNamespace(
open_incident=_disappeared(), last_closed_incident=dict(_disappeared(), closed_at_step=15)
)
await ExecutionIncidentPromptComponent()(builder, state, ctx, steps=STEPS)
assert builder.human_parts[0].startswith(EXECUTION_INCIDENT_MARKER)
+67 -74
View File
@@ -1041,25 +1041,26 @@ async def test_explorer_inspect_region():
@pytest.mark.asyncio
async def test_explorer_final_turn_tool_stripping():
"""Test that on the final iteration (turn 10), Explorer strips other tools and injects warning."""
"""On the final turn the Explorer strips other tools and injects the warning."""
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.device = MagicMock()
mock_ctx.device.device_width = 1080
mock_ctx.device.device_height = 2400
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = "/tmp/test_session"
explorer = Explorer(mock_ctx)
mock_ctx.llm_config = MagicMock()
mock_ctx.llm_config.explorer = MagicMock()
mock_ctx.llm_config.explorer.model = "gemini-3.7-flash"
mock_state = MagicMock(spec=State)
mock_state.latest_screenshot = "/tmp/test_screenshot.jpg"
# Set up generate_content mock to capture call arguments
# A client already shared on the context selects the native engine and is
# reused lazily by ``run`` (no ``genai.Client`` construction).
mock_client = MagicMock()
mock_ctx._genai_client = mock_client
explorer = Explorer(mock_ctx)
assert explorer.use_native_gemini is True
mock_state = MagicMock(spec=State)
mock_state.latest_screenshot = "/tmp/test_screenshot.jpg"
# Turn 1: model returns search_xml_ocr
fc_search = MagicMock()
@@ -1094,14 +1095,14 @@ async def test_explorer_final_turn_tool_stripping():
minimal_list="test list",
)
# Total number of iterations is max_iterations = 3 under "pro" mode default
# The default "pro" tier budgets 3 turns (tiers.EXPLORER_TIERS)
assert mock_client.aio.models.generate_content.call_count == 3
# Check the final turn call (index 2) keyword args
call_args_3 = mock_client.aio.models.generate_content.call_args_list[2]
config = call_args_3.kwargs["config"]
# Tools in the 10th call should only contain "submit_answer"
# Tools in the final call should only contain "submit_answer"
tools = config.tools[0].function_declarations
assert len(tools) == 1
assert tools[0].name == "submit_answer"
@@ -1120,9 +1121,16 @@ async def test_explorer_final_turn_tool_stripping():
@pytest.mark.asyncio
async def test_explorer_ask_perception_tool():
"""The perception bundle answers from the in-memory screen index.
No Data Engine lookup is involved: the text search, the coordinate
audit and the OCR inventory all read ``explorer.screen_index``.
"""
import numpy as np
from unittest.mock import mock_open
from artemis.agents.explorer.screen_index import ScreenIndex
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = "/tmp/test_session"
@@ -1144,37 +1152,24 @@ async def test_explorer_ask_perception_tool():
"description": "Original complete screenshot",
}
}
# Mock search_ui_func
mock_search_ui_res = {
"matches": [
explorer.screen_index = ScreenIndex.from_hierarchy(
[
{
"matched_text": "Settings",
"bounds": [540, 480, 540, 480],
"type": "xml",
"bounds": "[500,400][600,560]",
"text": "Settings",
"resource-id": "com.android.settings:id/btn_settings",
"class": "android.widget.Button",
"clickable": "true",
},
{
"matched_text": "Sign In",
"bounds": [100, 100, 200, 200],
"type": "ocr",
"bounds": "[100,100][200,200]",
"class": "android.widget.FrameLayout [OCR]",
"ocr_elements": [{"text": "Settings page", "bounds": "[100,100][200,200]"}],
},
]
}
# Mock storage record
mock_record = MagicMock()
mock_record.ui_tree = [
{
"bounds": "[500,400][600,560]",
"text": "Settings Button",
"resource-id": "com.android.settings:id/btn_settings",
"class": "android.widget.Button",
}
]
mock_record.ocr_result = []
mock_storage = MagicMock()
mock_storage.get_image.return_value = mock_record
],
1080,
2400,
)
# Mock _run_object_detection
mock_detector_res = {
@@ -1186,18 +1181,7 @@ async def test_explorer_ask_perception_tool():
mock_draw_dots = MagicMock()
with (
patch(
"artemis.agents.explorer.explorer.search_ui_func",
return_value=mock_search_ui_res,
),
patch(
"artemis.agents.explorer.explorer.StorageManager",
return_value=mock_storage,
),
patch(
"artemis.mcp.xml_search_server.StorageManager",
return_value=mock_storage,
),
patch("artemis.agents.explorer.explorer.StorageManager") as storage_cls,
patch(
"artemis.agents.explorer.explorer._run_object_detection",
return_value=mock_detector_res,
@@ -1216,37 +1200,46 @@ async def test_explorer_ask_perception_tool():
detect_queries=["profile icon"],
)
assert "Text Search Results are:" in result["text"]
assert "Coordinate Search Results are:" in result["text"]
assert "Object Detection Results are:" in result["text"]
storage_cls.assert_not_called()
assert "Text Search Results are:" in result["text"]
assert "Coordinate Search Results are:" in result["text"]
assert "Object Detection Results are:" in result["text"]
# XML Text search should return label X1
assert "[X1]" in result["text"]
# OCR Text search should return label O2
assert "[O2]" in result["text"]
# Coordinate search should return label X3
assert "[X3]" in result["text"]
# Object detection should return label D4 (since global label idx is shared/contiguous)
assert "[D4]" in result["text"]
# UI-tree text match -> X1, OCR text match -> O2 (both from the index)
assert "[X1] 'Settings' at [509,200]" in result["text"]
assert "[O2] 'Settings page' at [138,62]" in result["text"]
# Coordinate audit -> X3 with the innermost element's description
assert (
"Matched element at [550,200]: [X3] | 'Settings' class=android.widget.Button"
" id=com.android.settings:id/btn_settings source=ui-tree interactive"
) in result["text"]
# Object detection -> D4 (the label counter is shared and contiguous)
assert "[D4]" in result["text"]
assert len(result["image_paths"]) == 3
# Element-backed labels are registered; detection points are not.
assert explorer.label_registry["X1"].text == "Settings"
assert explorer.label_registry["O2"].source == "ocr"
assert explorer.label_registry["X3"] is explorer.label_registry["X1"]
assert "D4" not in explorer.label_registry
# Verify call arguments of draw_dots
assert mock_draw_dots.call_count == 3
# First call (XML/OCR Text search): green color
first_call = mock_draw_dots.call_args_list[0]
assert first_call.kwargs.get("color") == "green"
assert first_call.args[2] == ["X1", "O2"]
assert len(result["image_paths"]) == 3
# Second call (XML Coordinate audit): blue color
second_call = mock_draw_dots.call_args_list[1]
assert second_call.kwargs.get("color") == "blue"
assert second_call.args[2] == ["X3"]
# Verify call arguments of draw_dots
assert mock_draw_dots.call_count == 3
# First call (XML/OCR Text search): green color
first_call = mock_draw_dots.call_args_list[0]
assert first_call.kwargs.get("color") == "green"
assert first_call.args[2] == ["X1", "O2"]
# Third call (Object detection): red color
third_call = mock_draw_dots.call_args_list[2]
assert third_call.kwargs.get("color") == "red"
assert third_call.args[2] == ["D4"]
# Second call (XML Coordinate audit): blue color
second_call = mock_draw_dots.call_args_list[1]
assert second_call.kwargs.get("color") == "blue"
assert second_call.args[2] == ["X3"]
# Third call (Object detection): red color
third_call = mock_draw_dots.call_args_list[2]
assert third_call.kwargs.get("color") == "red"
assert third_call.args[2] == ["D4"]
def test_explorer_prune_historical_images():
+150
View File
@@ -0,0 +1,150 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Screen geometry shared by the Explorer and the ``ask_explorer`` tool."""
from types import SimpleNamespace
import pytest
from artemis.agents.explorer.geometry import (
FALLBACK_SCREEN_SIZE,
NORMALIZED_MAX,
is_valid_norm_point,
norm_to_pixel,
pixel_to_norm,
resolve_screen_size,
)
def _ctx(width=None, height=None, device=True):
if not device:
return SimpleNamespace(device=None)
return SimpleNamespace(device=SimpleNamespace(device_width=width, device_height=height))
def _state(raw):
return SimpleNamespace(operator_raw_data=raw)
# ---------------------------------------------------------------------------
# resolve_screen_size
# ---------------------------------------------------------------------------
def test_operator_observation_wins_over_the_device_context():
ctx = _ctx(1080, 2400)
state = _state({"width": 720, "height": 1600})
assert resolve_screen_size(ctx, state) == (720, 1600)
def test_device_context_is_used_when_the_observation_has_no_size():
ctx = _ctx(1080, 2400)
assert resolve_screen_size(ctx, _state(None)) == (1080, 2400)
assert resolve_screen_size(ctx, _state({})) == (1080, 2400)
assert resolve_screen_size(ctx, _state("not a dict")) == (1080, 2400)
@pytest.mark.parametrize(
"raw",
[
{"width": 0, "height": 1600},
{"width": 720, "height": -1},
{"width": "720", "height": 1600},
{"width": 720.0, "height": 1600},
{"width": None, "height": None},
{"width": 720},
],
)
def test_non_int_or_non_positive_observation_sizes_are_ignored(raw):
ctx = _ctx(1080, 2400)
assert resolve_screen_size(ctx, _state(raw)) == (1080, 2400)
def test_fallback_when_neither_source_reports_a_size():
assert resolve_screen_size(_ctx(device=False), _state(None)) == FALLBACK_SCREEN_SIZE
assert resolve_screen_size(_ctx(None, None), _state(None)) == FALLBACK_SCREEN_SIZE
assert resolve_screen_size(SimpleNamespace(), SimpleNamespace()) == FALLBACK_SCREEN_SIZE
def test_device_dimensions_fall_back_independently():
fb_w, fb_h = FALLBACK_SCREEN_SIZE
assert resolve_screen_size(_ctx(1440, 0), _state(None)) == (1440, fb_h)
assert resolve_screen_size(_ctx("1440", 3200), _state(None)) == (fb_w, 3200)
assert resolve_screen_size(_ctx(None, 3200), _state(None)) == (fb_w, 3200)
# ---------------------------------------------------------------------------
# norm_to_pixel / pixel_to_norm
# ---------------------------------------------------------------------------
def test_norm_to_pixel_scales_on_the_0_1000_grid():
assert norm_to_pixel(500, 500, 1080, 2400) == (540, 1200)
assert norm_to_pixel(0, 0, 1080, 2400) == (0, 0)
assert norm_to_pixel(NORMALIZED_MAX, NORMALIZED_MAX, 1080, 2400) == (1080, 2400)
assert norm_to_pixel(250, 750, 1080, 2400) == (270, 1800)
def test_norm_to_pixel_clamps_out_of_range_inputs():
assert norm_to_pixel(-50, -1, 1080, 2400) == (0, 0)
assert norm_to_pixel(1500, 2000, 1080, 2400) == (1080, 2400)
def test_pixel_to_norm_scales_back_to_the_0_1000_grid():
assert pixel_to_norm(540, 1200, 1080, 2400) == (500, 500)
assert pixel_to_norm(0, 0, 1080, 2400) == (0, 0)
assert pixel_to_norm(1080, 2400, 1080, 2400) == (NORMALIZED_MAX, NORMALIZED_MAX)
def test_pixel_to_norm_clamps_out_of_range_inputs():
assert pixel_to_norm(-10, -10, 1080, 2400) == (0, 0)
assert pixel_to_norm(5000, 9000, 1080, 2400) == (NORMALIZED_MAX, NORMALIZED_MAX)
def test_conversions_round_trip_on_grid_points():
for nx, ny in [(0, 0), (500, 500), (1000, 1000), (250, 750)]:
px, py = norm_to_pixel(nx, ny, 1000, 2000)
assert pixel_to_norm(px, py, 1000, 2000) == (nx, ny)
# ---------------------------------------------------------------------------
# is_valid_norm_point
# ---------------------------------------------------------------------------
@pytest.mark.parametrize(
"coords",
[[0, 0], [500, 500], (1000, 1000), ["500", "250"], [12.7, 999.2]],
)
def test_valid_norm_points(coords):
assert is_valid_norm_point(coords)
@pytest.mark.parametrize(
"coords",
[
None,
"500,500",
[500],
[500, 500, 500],
[-1, 0],
[0, 1001],
["x", 0],
[None, 5],
{"x": 1, "y": 2},
],
)
def test_invalid_norm_points(coords):
assert not is_valid_norm_point(coords)
@@ -0,0 +1,65 @@
"""Regression tests for Explorer perception-tool failure paths."""
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from artemis.agents.explorer.perception_tools import PerceptionToolsMixin
from artemis.agents.explorer.screen_index import ScreenIndex
def _perception_host() -> PerceptionToolsMixin:
host = PerceptionToolsMixin()
host.ctx = MagicMock()
host.global_label_idx = 1
host.width = 1080
host.height = 2400
host.image_name = "screen-1"
host.screenshot_path = "screen-1.jpg"
host.image_pool = {
"img_0": {
"path": "screen-1.jpg",
"transform": {
"offset_x": 0.0,
"offset_y": 0.0,
"scale_x": 1.0,
"scale_y": 1.0,
},
}
}
host.next_img_id = 1
host.screen_index = ScreenIndex.empty(1080, 2400)
host.label_registry = {}
return host
@pytest.mark.asyncio
async def test_get_ocr_list_reads_the_screen_index_without_storage() -> None:
host = _perception_host()
with patch("artemis.agents.explorer.explorer.StorageManager") as storage_cls:
result = await host.exec_get_ocr_list()
assert result == {
"text": "No text elements detected on the screen.",
"image_path": None,
}
storage_cls.assert_not_called()
@pytest.mark.asyncio
async def test_image_processor_empty_outputs_returns_stable_error() -> None:
host = _perception_host()
processor = MagicMock()
processor.run = AsyncMock(return_value={"outputs": [], "summary": "nothing found"})
with patch(
"artemis.agents.explorer.perception_tools.ImageProcessor",
return_value=processor,
):
result = await host.exec_ask_image_processor("crop", target_image_id="img_0")
assert result == {
"text": "ImageProcessor error: no output images were produced.",
"image_paths": [],
}
+463
View File
@@ -0,0 +1,463 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Explorer in-memory screen search, label registry, bounds enrichment and pre-pass."""
import json
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from artemis.agents.explorer.explorer import Explorer
from artemis.agents.explorer.screen_index import ScreenElement, ScreenIndex
from artemis.context import ArtemisContext
from artemis.graph.state import State
FACADE = "artemis.agents.explorer.explorer"
W, H = 1080, 2400
#: One "Settings" button, two "Send" buttons and two OCR-only texts.
HIERARCHY = [
{
"text": "Settings",
"bounds": "[500,400][600,560]",
"class": "android.widget.Button",
"resource-id": "app:id/settings",
"clickable": "true",
},
{"text": "Send", "bounds": "[100,2000][300,2100]", "class": "android.widget.Button"},
{"text": "Send", "bounds": "[700,2000][900,2100]", "class": "android.widget.Button"},
{
"bounds": "[0,0][1080,2400]",
"class": "android.widget.FrameLayout",
"ocr_elements": [
{"text": "Settings page", "bounds": "[100,100][300,160]"},
{"text": "Welcome", "bounds": "[100,200][400,260]"},
],
},
]
#: "Settings" [500,400][600,560] on a 1080x2400 screen, in normalized units.
SETTINGS_COORDS = [509, 200]
SETTINGS_BOUNDS = [462, 166, 555, 233]
def _context() -> MagicMock:
ctx = MagicMock(spec=ArtemisContext)
ctx.device = MagicMock()
ctx.device.device_width = W
ctx.device.device_height = H
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = None
ctx.llm_config = MagicMock()
ctx.llm_config.explorer = MagicMock()
ctx.llm_config.explorer.model = "gemini-3.8-flash"
ctx.agent_config = MagicMock()
ctx.agent_config.denylisted_tools = {}
return ctx
def _state(screenshot: str, hierarchy=None) -> MagicMock:
state = MagicMock(spec=State)
state.latest_screenshot = screenshot
state.latest_ui_hierarchy = HIERARCHY if hierarchy is None else hierarchy
state.operator_raw_data = {}
return state
def _indexed_explorer(hierarchy=None) -> Explorer:
"""An Explorer whose screen is already indexed (no ``run`` involved)."""
explorer = Explorer(_context())
explorer.width, explorer.height = W, H
explorer.screenshot_path = "/tmp/screen.jpg"
explorer.screen_index = ScreenIndex.from_hierarchy(
HIERARCHY if hierarchy is None else hierarchy, W, H
)
return explorer
def _storage(record=None) -> MagicMock:
storage = MagicMock()
storage.get_image.return_value = record
return storage
def _outcome(candidates=None, fallback: str = "") -> str:
return json.dumps({"candidates": candidates or [], "fallback_message": fallback})
@pytest.fixture
def screenshot(tmp_path):
path = tmp_path / "screen.jpg"
path.write_bytes(b"fake-jpeg-bytes")
return str(path)
@pytest.fixture
def no_drawing():
"""Keeps the annotation helpers away from the file system."""
with (
patch(f"{FACADE}.draw_dots") as draw_dots,
patch("pathlib.Path.mkdir"),
patch("glob.glob", return_value=[]),
):
yield draw_dots
# --------------------------------------------------------------------------- #
# Screen index construction inside run()
# --------------------------------------------------------------------------- #
@pytest.mark.asyncio
async def test_index_is_built_from_state_hierarchy_when_record_is_missing(screenshot, no_drawing):
explorer = Explorer(_context())
explorer._run_flash = AsyncMock(return_value=_outcome(fallback="nothing"))
with (
patch(f"{FACADE}.StorageManager", return_value=_storage(None)),
patch(f"{FACADE}.perform_ocr", new_callable=AsyncMock) as perform_ocr,
):
await explorer.run("gear icon", "", screenshot, _state(screenshot), version="flash")
text = (await explorer._search_ui_helper("Settings", prefix="X"))["text"]
perform_ocr.assert_not_awaited()
assert len(explorer.screen_index) == 5
assert "not found in Data Engine" not in text
assert "[X1] 'Settings' at [509,200]" in text
@pytest.mark.asyncio
async def test_index_ignores_state_hierarchy_for_another_screenshot(screenshot):
explorer = Explorer(_context())
explorer._run_flash = AsyncMock(return_value=_outcome())
state = _state("/tmp/other.jpg")
with patch(f"{FACADE}.StorageManager", return_value=_storage(None)):
await explorer.run("Settings", "", screenshot, state, version="flash")
assert len(explorer.screen_index) == 0
explorer._run_flash.assert_awaited_once_with("Settings", screenshot)
assert (await explorer._search_ui_helper("Settings"))["text"] == (
"No UI-tree data is available for this screen; rely on visual detection."
)
# --------------------------------------------------------------------------- #
# Perception helpers answer from the index
# --------------------------------------------------------------------------- #
@pytest.mark.asyncio
async def test_search_ui_helper_merges_lenient_matches_without_duplicates(no_drawing):
hierarchy = [*HIERARCHY, {"text": "Settled", "bounds": "[100,300][300,360]"}]
explorer = _indexed_explorer(hierarchy)
spy = MagicMock(wraps=explorer.screen_index.search_text)
explorer.screen_index.search_text = spy
result = await explorer._search_ui_helper("Settings", prefix="X", color="green")
assert [call.args[1] for call in spy.call_args_list] == [0.7, 0.4]
parts = result["text"].split(" | ")
assert parts[0] == "[X1] 'Settings' at [509,200]"
assert parts[1] == "[O2] 'Settings page' at [185,54]"
# "Settled" (similarity ~0.53) only arrives through the lenient pass...
assert any(part.startswith("[X") and "'Settled'" in part for part in parts)
# ...and the strict hits are not listed a second time.
assert sum("'Settings'" in part for part in parts) == 1
assert explorer.label_registry["X1"].text == "Settings"
assert explorer.label_registry["O2"].source == "ocr"
assert no_drawing.call_args.kwargs["color"] == "green"
assert result["image_path"] is not None
@pytest.mark.asyncio
async def test_search_ui_helper_skips_lenient_pass_with_enough_strict_hits(no_drawing):
explorer = _indexed_explorer()
strict = explorer.screen_index.search_text("Settings", 0.4) # 5 hits, all "strict"
explorer.screen_index.search_text = MagicMock(return_value=strict)
await explorer._search_ui_helper("Settings", prefix="X")
explorer.screen_index.search_text.assert_called_once_with("Settings", 0.7)
@pytest.mark.asyncio
async def test_search_by_coords_helper_reports_innermost_first(no_drawing):
explorer = _indexed_explorer(
[
{"text": "Card", "bounds": "[0,0][1080,1200]", "class": "android.widget.FrameLayout"},
{"text": "Title", "bounds": "[500,400][600,560]", "class": "android.widget.TextView"},
]
)
hit = await explorer._search_by_coords_helper(550, 200, prefix="X", color="blue")
miss = await explorer._search_by_coords_helper(10, 990, prefix="X", color="blue")
assert hit["text"] == (
"Matched element at [550,200]: [X1]"
" | 'Title' class=android.widget.TextView source=ui-tree"
" | 'Card' class=android.widget.FrameLayout source=ui-tree"
)
assert explorer.label_registry["X1"].text == "Title"
assert miss["text"] == "No elements found at [10,990]."
assert "X2" not in explorer.label_registry
# The blue dot is drawn for both probes at the pixel position.
assert no_drawing.call_count == 2
assert no_drawing.call_args_list[0].args[1] == [[594, 480]]
assert no_drawing.call_args_list[0].args[2] == ["X1"]
@pytest.mark.asyncio
async def test_get_ocr_list_reads_the_index(no_drawing):
explorer = _indexed_explorer()
with patch(f"{FACADE}.StorageManager") as storage_cls:
result = await explorer.exec_get_ocr_list()
storage_cls.assert_not_called()
assert result["text"].splitlines() == [
"[O1] 'Settings page' coords: [185,54]",
"[O2] 'Welcome' coords: [231,95]",
]
assert result["image_path"] is not None
assert {k: v.source for k, v in explorer.label_registry.items()} == {"O1": "ocr", "O2": "ocr"}
@pytest.mark.asyncio
async def test_get_ocr_list_without_ocr_elements():
explorer = _indexed_explorer([{"text": "Only XML", "bounds": "[0,0][100,100]"}])
result = await explorer.exec_get_ocr_list()
assert result == {"text": "No text elements detected on the screen.", "image_path": None}
# --------------------------------------------------------------------------- #
# Label registry from the initial numbered list
# --------------------------------------------------------------------------- #
def test_initial_list_registers_numbered_labels(no_drawing):
explorer = _indexed_explorer()
minimal_list, _marked = explorer._annotate_initial_screenshot(
HIERARCHY, "/tmp/screen.jpg", None, ""
)
assert minimal_list.splitlines()[0].startswith("[1] Text: 'Settings'")
assert sorted(explorer.label_registry) == ["1", "2", "3", "4", "5"]
settings = explorer.label_registry["1"]
assert settings.bounds == (500, 400, 600, 560)
assert settings.source == "xml" and settings.interactive # taken from the index
assert explorer.label_registry["4"].source == "ocr"
assert explorer.label_registry["4"].text == "Settings page"
assert explorer.global_label_idx == 6
assert no_drawing.call_args.args[1][0] == [550, 480]
# --------------------------------------------------------------------------- #
# Submit enrichment
# --------------------------------------------------------------------------- #
def test_enrich_candidates_adds_normalized_bounds_only_when_label_and_coords_agree():
explorer = _indexed_explorer()
settings = explorer.screen_index.exact_matches("Settings")[0]
explorer._register_label("X1", settings)
inside = {"label": "X1", "coords": SETTINGS_COORDS, "description": "gear"}
# 614px is 14px right of the element, inside the 2% (21.6px) tolerance.
near = {"label": "X1", "coords": [569, 200], "description": "gear"}
far = {"label": "X1", "coords": [100, 900], "description": "gear"}
unknown = {"label": "D7", "coords": SETTINGS_COORDS, "description": "gear"}
invalid = {"label": "X1", "coords": [5000, 5], "description": "gear"}
enriched = explorer._enrich_candidates([inside, near, far, unknown, invalid, "junk"])
assert enriched[0] == {**inside, "bounds": SETTINGS_BOUNDS, "source": "xml"}
assert enriched[1] == {**near, "bounds": SETTINGS_BOUNDS, "source": "xml"}
assert enriched[2:] == [far, unknown, invalid, "junk"]
assert "bounds" not in far and "bounds" not in inside # inputs are never mutated
def test_enrich_candidates_withholds_degenerate_bounds():
explorer = _indexed_explorer()
explorer._register_label("X9", ScreenElement("dot", (10, 10, 11, 11), "xml"))
cand = {"label": "X9", "coords": [9, 4]}
assert explorer._enrich_candidates([cand]) == [cand]
def test_native_submit_enriches_registered_candidates():
explorer = _indexed_explorer()
explorer._register_label("O2", explorer.screen_index.ocr_elements[0])
call = MagicMock()
call.name = "submit_answer"
call.args = {
"candidates": [
{"label": "O2", "coords": [185, 54], "description": "header"},
{"label": "D3", "coords": [900, 100], "description": "icon"},
],
"fallback_message": "",
}
turn_record = {"tool_calls": []}
raw = explorer._process_submit_answer([call], turn_record, [])
data = json.loads(raw)
assert list(data) == ["candidates", "fallback_message"]
assert data["candidates"][0] == {
"label": "O2",
"coords": [185, 54],
"description": "header",
"bounds": [92, 41, 277, 66],
"source": "ocr",
}
assert data["candidates"][1] == call.args["candidates"][1]
def test_universal_submit_enriches_registered_candidates():
explorer = _indexed_explorer()
explorer._register_label("1", explorer.screen_index.exact_matches("Settings")[0])
tool_calls = [
{
"name": "submit_answer",
"args": {
"candidates": [
{"label": "1", "coords": SETTINGS_COORDS, "description": "gear"},
{"label": "1", "coords": [5, 5000], "description": "bad"},
],
"fallback_message": "note",
},
"id": "c1",
}
]
data = json.loads(explorer._universal_submit_outcome(tool_calls))
assert data["fallback_message"] == "note"
assert data["candidates"] == [
{
"label": "1",
"coords": SETTINGS_COORDS,
"description": "gear",
"bounds": SETTINGS_BOUNDS,
"source": "xml",
}
]
# --------------------------------------------------------------------------- #
# Structural pre-pass
# --------------------------------------------------------------------------- #
@pytest.mark.parametrize("version", ["flash", "pro"])
@pytest.mark.asyncio
async def test_prepass_answers_unique_exact_match_without_the_engine(screenshot, version):
explorer = Explorer(_context())
explorer._run_flash = AsyncMock()
explorer._run_loop = AsyncMock()
with patch(f"{FACADE}.StorageManager", return_value=_storage(None)):
raw = await explorer.run(" settings ", "", screenshot, _state(screenshot), version=version)
explorer._run_flash.assert_not_called()
explorer._run_loop.assert_not_called()
assert json.loads(raw) == {
"candidates": [
{
"label": "T1",
"coords": SETTINGS_COORDS,
"bounds": SETTINGS_BOUNDS,
"source": "xml",
"description": "Settings (exact text match)",
}
],
"fallback_message": "",
}
assert explorer.label_registry["T1"].text == "Settings"
@pytest.mark.asyncio
async def test_prepass_leaves_ambiguous_matches_to_the_engine(screenshot):
explorer = Explorer(_context())
engine = _outcome([{"label": "D1", "coords": [185, 854], "description": "Send"}])
explorer._run_flash = AsyncMock(return_value=engine)
with patch(f"{FACADE}.StorageManager", return_value=_storage(None)):
raw = await explorer.run("Send", "", screenshot, _state(screenshot), version="flash")
explorer._run_flash.assert_awaited_once_with("Send", screenshot)
assert raw == engine
@pytest.mark.asyncio
async def test_prepass_without_pipe_or_match_passes_the_query_through(screenshot):
explorer = Explorer(_context())
engine = _outcome(fallback="Failed to detect: gear icon")
explorer._run_flash = AsyncMock(return_value=engine)
with patch(f"{FACADE}.StorageManager", return_value=_storage(None)):
raw = await explorer.run("gear icon", "", screenshot, _state(screenshot), version="flash")
explorer._run_flash.assert_awaited_once_with("gear icon", screenshot)
assert raw == engine
@pytest.mark.asyncio
async def test_prepass_merges_partial_results_with_the_engine_outcome(screenshot):
explorer = Explorer(_context())
detected = {"label": "D1", "coords": [185, 854], "description": "Send"}
explorer._run_loop = AsyncMock(return_value=_outcome([detected], fallback="two Send buttons"))
with patch(f"{FACADE}.StorageManager", return_value=_storage(None)):
raw = await explorer.run(
"Settings | Send | Welcome", "", screenshot, _state(screenshot), version="pro"
)
# Only the unresolved part reaches the engine; "Welcome" is an OCR exact match.
assert explorer._run_loop.await_args.args[1] == "Send"
data = json.loads(raw)
assert [c["label"] for c in data["candidates"]] == ["T1", "T2", "D1"]
assert data["candidates"][1]["source"] == "ocr"
assert data["candidates"][1]["description"] == "Welcome (exact text match)"
assert data["candidates"][2] == detected
assert data["fallback_message"] == "two Send buttons"
def test_merge_prepass_keeps_structural_hits_over_an_unreadable_engine_answer():
prepass = [{"label": "T1", "coords": [1, 2]}]
merged = json.loads(Explorer._merge_prepass(prepass, "Error: max iterations"))
assert merged == {"candidates": prepass, "fallback_message": "Error: max iterations"}
assert Explorer._merge_prepass([], "raw") == "raw"
@pytest.mark.asyncio
async def test_prepass_never_runs_on_the_fly_ocr_for_flash(screenshot):
record = SimpleNamespace(ui_tree=HIERARCHY[:3], ocr_result=None)
explorer = Explorer(_context())
explorer._run_flash = AsyncMock(return_value=_outcome())
explorer._run_loop = AsyncMock(return_value=_outcome())
with (
patch(f"{FACADE}.StorageManager", return_value=_storage(record)),
patch(f"{FACADE}.is_ocr_configured", return_value=True),
patch(f"{FACADE}.perform_ocr", new_callable=AsyncMock, return_value=[]) as perform_ocr,
):
await explorer.run("gear icon", "", screenshot, _state(screenshot, []), version="flash")
perform_ocr.assert_not_awaited()
assert len(explorer.screen_index) == 3 # the record still feeds the index
await explorer.run("gear icon", "", screenshot, _state(screenshot, []), version="pro")
perform_ocr.assert_awaited_once()
@@ -0,0 +1,120 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for the Explorer's in-memory screen index."""
import pytest
from artemis.agents.explorer.screen_index import (
ScreenElement,
ScreenIndex,
normalize_text,
text_similarity,
)
W, H = 1080, 2400
def _node(text=None, bounds="[0,0][100,100]", desc=None, **attrs):
node = {"bounds": bounds, "class": "android.widget.TextView"}
if text is not None:
node["text"] = text
if desc is not None:
node["content-desc"] = desc
node.update(attrs)
return node
def _hierarchy():
return [
_node("Gmail", "[300,1600][500,1800]", clickable="true", **{"resource-id": "app:gmail"}),
_node(None, "[0,0][1080,2400]"), # unlabeled root wrapper is skipped
_node(desc="Settings", bounds="[900,50][1000,150]", clickable="true"),
{
"bounds": "[100,2100][900,2300]",
"class": "android.widget.FrameLayout [OCR]",
"ocr_elements": [{"text": "Search", "bounds": "[150,2150][350,2250]"}],
},
_node("Search", "[120,2120][380,2280]"), # overlaps the OCR "Search"
_node("Offscreen", "[-500,-500][-400,-400]"),
_node("Zero", "[10,10][10,10]"),
]
def test_normalize_and_similarity():
assert normalize_text(" Send\n Message ") == "send message"
assert text_similarity("gmail", "Gmail") == 1.0
assert text_similarity("send", "Send message") >= 0.85
assert text_similarity("ok", "Look here") < 0.85
assert text_similarity("", "x") == 0.0
def test_from_hierarchy_skips_invisible_and_unlabeled():
index = ScreenIndex.from_hierarchy(_hierarchy(), W, H)
texts = sorted(e.text for e in index.elements)
assert texts == ["Gmail", "Search", "Search", "Settings"]
assert len(index.ocr_elements) == 1
gmail = next(e for e in index.elements if e.text == "Gmail")
assert gmail.interactive and gmail.resource_id == "app:gmail" and gmail.source == "xml"
assert gmail.center == (400, 1700)
def test_from_hierarchy_tolerates_garbage():
index = ScreenIndex.from_hierarchy([None, "x", {"bounds": "bad"}, {}], W, H)
assert len(index) == 0
assert ScreenIndex.empty(W, H).search_text("x") == []
def test_search_text_orders_by_score_and_respects_threshold():
index = ScreenIndex.from_hierarchy(_hierarchy(), W, H)
matches = index.search_text("Gmai", threshold=0.6)
assert matches and matches[0].element.text == "Gmail"
assert all(m.score > 0.6 for m in matches)
assert index.search_text("zzzz", threshold=0.6) == []
assert len(index.search_text("Search", threshold=0.4, limit=1)) == 1
def test_exact_matches_collapse_overlapping_duplicates_preferring_xml():
index = ScreenIndex.from_hierarchy(_hierarchy(), W, H)
hits = index.exact_matches(" search ")
assert len(hits) == 1
assert hits[0].source == "xml"
assert index.exact_matches("settings")[0].text == "Settings"
assert index.exact_matches("") == []
assert index.exact_matches("nope") == []
def test_elements_at_reports_innermost_then_ocr():
elements = [
ScreenElement("outer", (0, 0, 1000, 1000), "xml"),
ScreenElement("middle", (100, 100, 900, 900), "xml", interactive=False),
ScreenElement("button", (400, 400, 600, 600), "xml", interactive=True),
ScreenElement("label", (450, 450, 550, 550), "ocr"),
]
index = ScreenIndex(elements, W, H)
found = index.elements_at(500, 500)
# innermost interactive node stops the UI-tree walk; OCR appended after
assert [e.text for e in found] == ["button", "label"]
found = index.elements_at(150, 150)
assert [e.text for e in found] == ["middle", "outer"]
assert index.elements_at(1050, 1050) == []
def test_describe_and_overlap():
a = ScreenElement("A", (0, 0, 100, 100), "xml", class_name="Btn", resource_id="id/a")
b = ScreenElement("B", (50, 50, 150, 150), "ocr")
assert "class=Btn" in a.describe() and "id=id/a" in a.describe()
assert "source=ocr" in b.describe()
assert a.overlap_ratio(b) == pytest.approx(0.25)
assert a.overlap_ratio(ScreenElement("C", (200, 200, 300, 300), "xml")) == 0.0
+555
View File
@@ -0,0 +1,555 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Explorer tier integration: engine selection, tool exposure, caching, prompts."""
import json
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
from google.genai import types
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
import pytest
from artemis.agents.explorer.explorer import Explorer
from artemis.agents.explorer.tiers import EXPLORER_TIERS, PERCEPTION_TOOLS, get_tier
from artemis.config import settings
from artemis.context import ArtemisContext
from artemis.graph.state import State
FACADE = "artemis.agents.explorer.explorer"
def _context(model: str = "gemini-3.8-flash", denylist: list[str] | None = None) -> MagicMock:
ctx = MagicMock(spec=ArtemisContext)
ctx.device = MagicMock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = None
ctx.llm_config = MagicMock()
ctx.llm_config.explorer = MagicMock()
ctx.llm_config.explorer.model = model
ctx.llm_config.explorer.temperature = 0.1
ctx.llm_config.explorer.thinking_level = None
ctx.llm_config.explorer.fallback = None
ctx.agent_config = MagicMock()
ctx.agent_config.denylisted_tools = {"explorer": denylist or []}
ctx.agent_config.explorer = SimpleNamespace(caching=None)
ctx.execution_setup = SimpleNamespace(explorer=SimpleNamespace(caching=None))
return ctx
def _state(screenshot: str) -> MagicMock:
state = MagicMock(spec=State)
state.latest_screenshot = screenshot
state.latest_ui_hierarchy = []
state.operator_raw_data = {}
return state
def _submit_response(coords=(500, 600)) -> MagicMock:
call = MagicMock()
call.name = "submit_answer"
call.args = {
"candidates": [{"label": "D1", "coords": list(coords), "description": "target"}],
"fallback_message": "",
}
response = MagicMock()
response.function_calls = [call]
response.candidates = [MagicMock()]
response.candidates[0].content = types.Content(
role="model",
parts=[types.Part.from_function_call(name="submit_answer", args=call.args)],
)
return response
def _native_client(responses) -> MagicMock:
client = MagicMock()
client.aio.models.generate_content = AsyncMock(side_effect=list(responses))
cache = SimpleNamespace(name="cachedContents/abc")
client.aio.caches.create = AsyncMock(return_value=cache)
client.aio.caches.delete = AsyncMock()
client.aio.files.upload = AsyncMock(
return_value=SimpleNamespace(uri="files/1", mime_type="image/jpeg", name="files/1")
)
client.aio.files.delete = AsyncMock()
return client
@pytest.fixture
def screenshot(tmp_path):
path = tmp_path / "screen.jpg"
path.write_bytes(b"fake-jpeg-bytes")
return str(path)
@pytest.fixture(autouse=True)
def _isolated_env(monkeypatch):
monkeypatch.delenv("ARTEMIS_EXPLORER_CACHING", raising=False)
monkeypatch.setenv("ARTEMIS_USE_FILE_API", "false")
# --------------------------------------------------------------------------- #
# Tier -> turn budget / tool exposure
# --------------------------------------------------------------------------- #
def test_construction_creates_no_genai_client():
with patch(f"{FACADE}.genai.Client") as client_cls:
explorer = Explorer(_context())
client_cls.assert_not_called()
assert explorer.client is None
assert explorer.tier is None
@pytest.mark.parametrize(
("version", "max_turns", "expected_tools"),
[
("pro", 3, {"ask_perception_tool", "submit_answer"}),
("ultra", 8, PERCEPTION_TOOLS | {"submit_answer"}),
],
)
def test_tier_sets_turn_budget_and_exposed_tools(version, max_turns, expected_tools):
explorer = Explorer(_context())
with patch(f"{FACADE}.is_ocr_configured", return_value=True):
explorer._apply_tier(get_tier(version))
native_names = {t.name for t in explorer.get_exposed_tools()}
universal_names = {t["function"]["name"] for t in explorer._universal_exposed_tools()}
assert explorer.max_turns == max_turns
assert native_names == expected_tools
assert universal_names == expected_tools
assert explorer.denylisted_tools >= get_tier(version).hidden_tools
def test_unconfigured_ocr_and_user_denylist_hide_tools_on_ultra():
explorer = Explorer(_context(denylist=["inspect_region"]))
with patch(f"{FACADE}.is_ocr_configured", return_value=False):
explorer._apply_tier(get_tier("ultra"))
names = {t.name for t in explorer.get_exposed_tools()}
assert "get_ocr_list" not in names
assert "inspect_region" not in names
assert {"detect_objects", "ask_image_processor", "ask_perception_tool"} <= names
# --------------------------------------------------------------------------- #
# Engine selection
# --------------------------------------------------------------------------- #
def test_gemini_model_without_google_key_uses_universal_engine():
with patch.object(settings, "GOOGLE_API_KEY", None):
explorer = Explorer(_context(model="gemini-3.8-flash"))
assert explorer.use_native_gemini is False
def test_gemini_model_with_shared_client_uses_native_engine():
ctx = _context(model="google/gemini-3.8-pro")
ctx._genai_client = MagicMock()
with patch.object(settings, "GOOGLE_API_KEY", None):
explorer = Explorer(ctx)
assert explorer.use_native_gemini is True
assert explorer.model_name == "gemini-3.8-pro"
@pytest.mark.asyncio
async def test_universal_run_without_google_key_never_touches_genai(screenshot):
ctx = _context(model="claude-sonnet-4")
response = AIMessage(
content="",
tool_calls=[
{
"name": "submit_answer",
"args": {
"candidates": [{"label": "1", "coords": [100, 200], "description": "ok"}],
"fallback_message": "",
},
"id": "call_1",
}
],
)
llm = MagicMock()
bound = MagicMock()
bound.ainvoke = AsyncMock(return_value=response)
llm.bind_tools.return_value = bound
storage = MagicMock()
storage.get_image.return_value = None
with (
patch.object(settings, "GOOGLE_API_KEY", None),
patch(f"{FACADE}.genai.Client") as client_cls,
patch(f"{FACADE}.get_llm", return_value=llm),
patch(f"{FACADE}.StorageManager", return_value=storage),
):
explorer = Explorer(ctx)
raw = await explorer.run("find it", "", screenshot, _state(screenshot), version="pro")
client_cls.assert_not_called()
outcome = json.loads(raw)
assert outcome["candidates"][0]["coords"] == [100, 200]
# The universal engine only sees the pro tier's tools.
bound_names = {t["function"]["name"] for t in llm.bind_tools.call_args_list[0].args[0]}
assert bound_names == {"ask_perception_tool", "submit_answer"}
# --------------------------------------------------------------------------- #
# Universal dispatch delivers tool images
# --------------------------------------------------------------------------- #
@pytest.mark.asyncio
async def test_universal_dispatch_appends_image_message(tmp_path):
explorer = Explorer(_context(model="claude-sonnet-4"))
img_a = tmp_path / "a.jpg"
img_b = tmp_path / "b.jpg"
img_a.write_bytes(b"aaa")
img_b.write_bytes(b"bbb")
explorer.exec_ask_perception_tool = AsyncMock(
return_value={"text": "perception text", "image_paths": [str(img_a), str(tmp_path / "x")]}
)
explorer.exec_inspect_region = AsyncMock(
return_value={"text": "zoomed", "image_path": str(img_b)}
)
messages = []
tool_calls = [
{
"name": "ask_perception_tool",
"args": {"search_query": "q", "nx": 1, "ny": 2, "detect_queries": ["q"]},
"id": "c1",
},
{
"name": "inspect_region",
"args": {"x_min": 0, "y_min": 0, "x_max": 10, "y_max": 10},
"id": "c2",
},
]
await explorer._universal_dispatch_tools(tool_calls, messages, turn=1)
assert [type(m) for m in messages] == [ToolMessage, ToolMessage, HumanMessage]
assert messages[0].content == "perception text"
human = messages[2]
assert human.content[0] == {
"type": "text",
"text": "[Annotated image(s) returned by: ask_perception_tool, inspect_region]",
}
image_blocks = human.content[1:]
assert len(image_blocks) == 2 # the missing file is skipped
assert all(b["type"] == "image_url" for b in image_blocks)
assert image_blocks[0]["image_url"]["url"].startswith("data:image/jpeg;base64,")
@pytest.mark.asyncio
async def test_universal_dispatch_without_images_adds_no_human_message():
explorer = Explorer(_context(model="claude-sonnet-4"))
explorer.exec_get_ocr_list = AsyncMock(return_value={"text": "none", "image_path": None})
messages = []
await explorer._universal_dispatch_tools(
[{"name": "get_ocr_list", "args": {}, "id": "c1"}], messages, turn=1
)
assert [type(m) for m in messages] == [ToolMessage]
@pytest.mark.asyncio
async def test_universal_dispatch_rejects_denylisted_tool():
explorer = Explorer(_context(model="claude-sonnet-4"))
explorer._apply_tier(get_tier("pro"))
explorer.exec_inspect_region = AsyncMock()
messages = []
await explorer._universal_dispatch_tools(
[{"name": "inspect_region", "args": {}, "id": "c1"}], messages, turn=1
)
explorer.exec_inspect_region.assert_not_called()
assert "denylisted" in messages[0].content
# --------------------------------------------------------------------------- #
# Caching precedence
# --------------------------------------------------------------------------- #
def test_caching_tier_defaults():
explorer = Explorer(_context())
with patch.object(settings, "EXPLORER_CACHING", None):
assert explorer._resolve_caching(get_tier("pro"), None) is False
assert explorer._resolve_caching(get_tier("ultra"), None) is True
def test_caching_precedence(monkeypatch):
ctx = _context()
explorer = Explorer(ctx)
pro = get_tier("pro")
with patch.object(settings, "EXPLORER_CACHING", None):
ctx.agent_config.explorer = SimpleNamespace(caching=True)
assert explorer._resolve_caching(pro, None) is True # agent config beats tier
monkeypatch.setenv("ARTEMIS_EXPLORER_CACHING", "false")
assert explorer._resolve_caching(pro, None) is False # env beats agent config
assert explorer._resolve_caching(pro, True) is True # explicit beats env
with patch.object(settings, "EXPLORER_CACHING", True):
monkeypatch.delenv("ARTEMIS_EXPLORER_CACHING")
ctx.agent_config.explorer = SimpleNamespace(caching=None)
ctx.execution_setup = SimpleNamespace(explorer=SimpleNamespace(caching=False))
assert explorer._resolve_caching(pro, None) is False # execution setup beats settings
ctx.execution_setup = SimpleNamespace(explorer=SimpleNamespace(caching=None))
assert explorer._resolve_caching(pro, None) is True # settings beat tier
def test_caching_ignores_mock_config_values():
"""A MagicMock context must fall through to the tier default, not force True."""
ctx = MagicMock()
explorer = Explorer(ctx)
with patch.object(settings, "EXPLORER_CACHING", None):
assert explorer._resolve_caching(get_tier("pro"), None) is False
# --------------------------------------------------------------------------- #
# Tier-aware prompt
# --------------------------------------------------------------------------- #
def test_pro_prompt_only_describes_exposed_tools():
explorer = Explorer(_context())
explorer._apply_tier(get_tier("pro"))
prompt, error = explorer._build_prompt_template(get_tier("pro"))
assert error is None
assert "ask_perception_tool" in prompt
assert "inspect_region" not in prompt
assert "Image Pool" not in prompt
assert "ask_image_processor" not in prompt
assert "get_ocr_list" not in prompt
assert "# TOOL DENYLIST" not in prompt
assert "# EXECUTION CONSTRAINT" in prompt
assert "maximum of 3 turns" in prompt
def test_ultra_prompt_describes_all_tools():
explorer = Explorer(_context())
with patch(f"{FACADE}.is_ocr_configured", return_value=True):
explorer._apply_tier(get_tier("ultra"))
prompt, error = explorer._build_prompt_template(get_tier("ultra"))
assert error is None
for tool in PERCEPTION_TOOLS:
assert tool in prompt
assert "Image Pool" in prompt
assert "maximum of 8 turns" in prompt
assert "# TOOL DENYLIST" not in prompt
def test_prompt_denylist_section_only_for_user_denied_tier_tools():
explorer = Explorer(_context(denylist=["inspect_region", "search_xml_ocr"]))
with patch(f"{FACADE}.is_ocr_configured", return_value=True):
explorer._apply_tier(get_tier("ultra"))
ultra_prompt, _ = explorer._build_prompt_template(get_tier("ultra"))
explorer._apply_tier(get_tier("pro"))
pro_prompt, _ = explorer._build_prompt_template(get_tier("pro"))
assert "# TOOL DENYLIST" in ultra_prompt
assert "cannot be used: inspect_region" in ultra_prompt
assert "search_xml_ocr" not in ultra_prompt
# Denied tools lose their guidance bullets too.
assert "Targeted Verification" not in ultra_prompt
# On pro, inspect_region is tier-hidden, so it is never mentioned.
assert "# TOOL DENYLIST" not in pro_prompt
assert "inspect_region" not in pro_prompt
def test_prompt_sections_render_shape():
explorer = Explorer(_context())
explorer._apply_tier(get_tier("pro"))
prompt, _ = explorer._build_prompt_template(get_tier("pro"))
lines = prompt.splitlines()
assert lines[0] == "# IDENTITY"
assert any(line.startswith("# OPERATING PRINCIPLES") for line in lines)
body = [line for line in lines if line and not line.startswith("#")]
assert all(line.startswith("- ") or line.startswith("You are") for line in body)
# --------------------------------------------------------------------------- #
# Flash path
# --------------------------------------------------------------------------- #
@pytest.mark.asyncio
async def test_flash_run_returns_detected_candidates(screenshot):
detection = AsyncMock(
return_value={
"detected": [
{"label": "gear icon", "point": [900, 100]},
{"label": "send button", "point": [500, 950]},
],
"failed": [],
}
)
with (
patch(f"{FACADE}._run_object_detection", detection),
patch(f"{FACADE}.genai.Client") as client_cls,
):
explorer = Explorer(_context())
raw = await explorer.run(
"gear icon | send button", "", screenshot, _state(screenshot), version="flash"
)
client_cls.assert_not_called()
outcome = json.loads(raw)
assert [c["label"] for c in outcome["candidates"]] == ["D1", "D2"]
assert outcome["candidates"][0] == {
"label": "D1",
"coords": [900, 100],
"description": "gear icon",
}
assert outcome["fallback_message"] == ""
assert detection.call_args.args[2] == ["gear icon", "send button"]
@pytest.mark.asyncio
async def test_flash_run_reports_missing_detection(screenshot):
with patch(f"{FACADE}._run_object_detection", AsyncMock(return_value={"detected": []})):
raw = await Explorer(_context()).run(
"ghost", "", screenshot, _state(screenshot), version="flash"
)
assert json.loads(raw) == {"candidates": [], "fallback_message": "Failed to detect: ghost"}
# --------------------------------------------------------------------------- #
# Native engine: async SDK usage and resource lifecycle
# --------------------------------------------------------------------------- #
@pytest.mark.asyncio
async def test_native_ultra_run_uses_async_cache_api(screenshot):
ctx = _context()
client = _native_client([_submit_response()])
ctx._genai_client = client
storage = MagicMock()
storage.get_image.return_value = None
with (
patch(f"{FACADE}.genai.Client") as client_cls,
patch(f"{FACADE}.StorageManager", return_value=storage),
patch.object(settings, "EXPLORER_CACHING", None),
):
explorer = Explorer(ctx)
raw = await explorer.run("target", "", screenshot, _state(screenshot), version="ultra")
client_cls.assert_not_called()
assert json.loads(raw)["candidates"][0]["coords"] == [500, 600]
client.aio.caches.create.assert_awaited_once()
client.aio.caches.delete.assert_awaited_once_with(name="cachedContents/abc")
config = client.aio.models.generate_content.call_args.kwargs["config"]
assert config.cached_content == "cachedContents/abc"
@pytest.mark.asyncio
async def test_native_pro_run_skips_cache_by_default(screenshot):
ctx = _context()
client = _native_client([_submit_response()])
ctx._genai_client = client
storage = MagicMock()
storage.get_image.return_value = None
with (
patch(f"{FACADE}.StorageManager", return_value=storage),
patch.object(settings, "EXPLORER_CACHING", None),
):
await Explorer(ctx).run("target", "", screenshot, _state(screenshot), version="pro")
client.aio.caches.create.assert_not_awaited()
client.aio.caches.delete.assert_not_awaited()
config = client.aio.models.generate_content.call_args.kwargs["config"]
assert {t.name for t in config.tools[0].function_declarations} == {
"ask_perception_tool",
"submit_answer",
}
@pytest.mark.asyncio
async def test_native_file_api_uploads_and_deletes_asynchronously(screenshot, monkeypatch):
monkeypatch.setenv("ARTEMIS_USE_FILE_API", "true")
ctx = _context()
client = _native_client([_submit_response()])
ctx._genai_client = client
storage = MagicMock()
storage.get_image.return_value = None
with patch(f"{FACADE}.StorageManager", return_value=storage):
await Explorer(ctx).run("target", "", screenshot, _state(screenshot), version="pro")
client.aio.files.upload.assert_awaited_once_with(file=screenshot)
client.aio.files.delete.assert_awaited_once_with(name="files/1")
@pytest.mark.asyncio
async def test_run_closes_http_client_on_both_engines(screenshot):
storage = MagicMock()
storage.get_image.return_value = None
# Native engine
ctx = _context()
client = _native_client([_submit_response()])
ctx._genai_client = client
native = Explorer(ctx)
native.http_client = SimpleNamespace(aclose=AsyncMock())
with patch(f"{FACADE}.StorageManager", return_value=storage):
await native.run("t", "", screenshot, _state(screenshot), version="pro")
native_close = native.http_client # already reset
assert native_close is None
# Universal engine (model call fails -> finally still closes)
llm = MagicMock()
llm.bind_tools.return_value.ainvoke = AsyncMock(side_effect=RuntimeError("boom"))
universal = Explorer(_context(model="claude-sonnet-4"))
http_client = SimpleNamespace(aclose=AsyncMock())
universal.http_client = http_client
with (
patch(f"{FACADE}.StorageManager", return_value=storage),
patch(f"{FACADE}.get_llm", return_value=llm),
pytest.raises(RuntimeError),
):
await universal.run("t", "", screenshot, _state(screenshot), version="pro")
http_client.aclose.assert_awaited_once()
assert universal.http_client is None
def test_missing_tool_calls_without_candidates_does_not_crash():
explorer = Explorer(_context())
response = MagicMock()
response.candidates = None
contents = []
turn_record = {"tool_calls": []}
explorer._handle_missing_tool_calls(contents, response, turn_record, ["hi"], 3, 1)
assert contents[0].role == "model"
assert contents[0].parts[0].text == "hi"
assert "2 more" in contents[1].parts[0].text
assert turn_record["tool_calls"][0]["name"] == "hallucinated_plain_text"
def test_tier_table_is_single_source_for_versions():
assert set(EXPLORER_TIERS) == {"flash", "pro", "ultra"}
assert get_tier("bogus").name == "flash"
@pytest.mark.parametrize("version", [None, "", "turbo", "PRO ", "Ultra"])
def test_get_tier_normalizes_or_falls_back(version):
"""Unknown or empty names fall back to the default tier; case and spaces are tolerated."""
tier = get_tier(version)
normalized = str(version or "").strip().lower()
expected = normalized if normalized in EXPLORER_TIERS else "flash"
assert tier.name == expected
+1 -2
View File
@@ -116,9 +116,8 @@ async def test_run_universal_explorer_success(mock_context, tmp_path):
screenshot_path=str(fake_screenshot),
state=mock_state,
minimal_list="",
version="pro",
prompt_template="You are an explorer.",
max_iterations=3,
max_turns=3,
)
res = json.loads(res_str)
-213
View File
@@ -1,213 +0,0 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for Universal Failure Analyzer."""
import base64
from unittest.mock import AsyncMock, MagicMock, Mock, patch
import pytest
from langchain_core.messages import AIMessage
from artemis.agents.validator.failure_analyzer import (
FailureAnalyzer,
PixelTargetDisappearedStrategy,
TargetDisappearedStrategy,
ValidationErrorCategory,
VALIDATOR_TOOLS_DECLARATION,
)
from artemis.context import ArtemisContext
class DummyState:
def __init__(self, latest_screenshot=None):
self.latest_screenshot = latest_screenshot
self.indexed_points = []
self.indexed_elements = []
@pytest.fixture
def mock_context(tmp_path):
ctx = Mock(spec=ArtemisContext)
ctx.llm_config = Mock()
mock_llm_cfg = Mock()
mock_llm_cfg.model = "gemini-2.5-pro"
mock_llm_cfg.temperature = 0.1
ctx.llm_config.get_agent.return_value = mock_llm_cfg
ctx.device = Mock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
ctx.data_engine = Mock()
ctx.data_engine.base_dir = tmp_path
ctx.data_engine.get_agent_friendly_steps.return_value = []
ctx.adb_client = None
ctx.driver = Mock()
return ctx
@pytest.mark.asyncio
async def test_failure_analyzer_cannot_fix(mock_context, tmp_path):
"""Test that FailureAnalyzer natively executes and handles 'cannot_fix' report."""
screenshot_file = tmp_path / "shot.jpg"
screenshot_file.write_bytes(b"dummy_bytes")
state = DummyState(latest_screenshot=str(screenshot_file))
failed_action = {"action": "click", "coordinates": [100, 200]}
error_msg = "Element not found"
analyzer = FailureAnalyzer(mock_context)
mock_llm = MagicMock()
mock_response = AIMessage(
content="I see the error.",
tool_calls=[
{
"name": "report_failure_analysis",
"args": {
"status": "cannot_fix",
"analysis": "The element was definitely not there.",
},
"id": "call_1",
}
],
)
mock_bound = MagicMock()
mock_bound.ainvoke = AsyncMock(return_value=mock_response)
mock_llm.bind_tools.return_value = mock_bound
with (
patch("artemis.agents.validator.failure_analyzer.get_llm", return_value=mock_llm),
patch(
"artemis.agents.validator.failure_analyzer.UnifiedMobileController"
) as mock_controller_cls,
):
mock_controller = Mock()
mock_controller_cls.return_value = mock_controller
mock_device_data = Mock()
mock_device_data.base64 = base64.b64encode(b"mocked_screenshot").decode()
mock_device_data.elements = []
mock_device_data.width = 1080
mock_device_data.height = 2400
mock_controller.get_screen_data = AsyncMock(return_value=mock_device_data)
mock_controller.get_ui_elements = AsyncMock(return_value=mock_device_data.elements)
result = await analyzer.analyze(
state,
failed_action,
error_msg,
pre_screenshot=base64.b64encode(b"dummy").decode(),
post_screenshot=base64.b64encode(b"dummy").decode(),
)
assert result["status"] == "cannot_fix"
assert "not there" in result["analysis"]
assert result["new_remaining_actions"] == []
def test_report_failure_analysis_tool_declaration():
"""Test that report_failure_analysis tool declaration is correct and matches."""
report_tool = next(
(t for t in VALIDATOR_TOOLS_DECLARATION if t.name == "report_failure_analysis"),
None,
)
assert report_tool is not None
props = report_tool.parameters.get("properties", {})
assert "analysis" in props
assert "single paragraph" in props["analysis"]["description"]
@pytest.mark.asyncio
async def test_failure_analyzer_strategy_routing(mock_context):
"""Test that FailureAnalyzer selects appropriate strategies based on error category."""
analyzer = FailureAnalyzer(mock_context)
strat_disappeared = analyzer._select_strategy(ValidationErrorCategory.TARGET_DISAPPEARED)
assert isinstance(strat_disappeared, TargetDisappearedStrategy)
assert "target_disappeared" in strat_disappeared.get_prompt_template_name()
strat_pixel = analyzer._select_strategy(ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED)
assert isinstance(strat_pixel, PixelTargetDisappearedStrategy)
assert "pixel_target_disappeared" in strat_pixel.get_prompt_template_name()
@pytest.mark.asyncio
async def test_failure_analyzer_click_sequence(mock_context, tmp_path):
"""Test click_sequence execution in FailureAnalyzer."""
screenshot_file = tmp_path / "shot.jpg"
screenshot_file.write_bytes(b"dummy_bytes")
state = DummyState(latest_screenshot=str(screenshot_file))
analyzer = FailureAnalyzer(mock_context)
# Turn 1: Call click_sequence tool with direct coordinates
call_step_1 = AIMessage(
content="Let me click in sequence.",
tool_calls=[
{
"name": "click_sequence",
"args": {"sequence": [[100, 200], [300, 400]]},
"id": "call_seq_1",
}
],
)
# Turn 2: Report failure fixed
call_step_2 = AIMessage(
content="Repaired successfully.",
tool_calls=[
{
"name": "report_failure_analysis",
"args": {
"status": "fixed",
"analysis": "Sequence clicked successfully.",
},
"id": "call_report_2",
}
],
)
mock_llm = MagicMock()
mock_bound = MagicMock()
mock_bound.ainvoke = AsyncMock(side_effect=[call_step_1, call_step_2])
mock_llm.bind_tools.return_value = mock_bound
with (
patch("artemis.agents.validator.failure_analyzer.get_llm", return_value=mock_llm),
patch(
"artemis.agents.validator.failure_analyzer.UnifiedMobileController"
) as mock_controller_cls,
):
mock_controller = Mock()
mock_controller_cls.return_value = mock_controller
mock_device_data = Mock()
mock_device_data.base64 = base64.b64encode(b"mocked_screenshot").decode()
mock_device_data.elements = []
mock_device_data.width = 1080
mock_device_data.height = 2400
mock_controller.get_screen_data = AsyncMock(return_value=mock_device_data)
mock_controller.get_ui_elements = AsyncMock(return_value=mock_device_data.elements)
mock_controller.tap_at = AsyncMock(return_value=Mock(error=None))
result = await analyzer.analyze(
state,
{"action": "click", "coordinates": [100, 200]},
"Element not found",
pre_screenshot=base64.b64encode(b"dummy").decode(),
post_screenshot=base64.b64encode(b"dummy").decode(),
error_category=ValidationErrorCategory.TARGET_DISAPPEARED,
)
assert result["status"] == "fixed"
assert mock_controller.tap_at.call_count == 2
@@ -1,466 +0,0 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import base64
from unittest.mock import AsyncMock, Mock, patch
from artemis.agents.validator.failure_analyzer import (
_exec_click,
_exec_click_sequence,
_exec_input_text,
_exec_long_press,
_exec_manage_app,
_exec_press_key,
_exec_swipe,
_exec_wait_for_delay,
)
from artemis.context import ArtemisContext
import pytest
class DummyState:
def __init__(self, latest_screenshot=None):
self.latest_screenshot = latest_screenshot
self.indexed_points = []
self.indexed_elements = []
@pytest.fixture
def mock_context(tmp_path):
ctx = Mock(spec=ArtemisContext)
ctx.device = Mock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
ctx.data_engine = Mock()
ctx.data_engine.base_dir = tmp_path
ctx.data_engine.get_or_create_image.return_value = "dummy_image_name"
ctx.data_engine.get_image_path.return_value = tmp_path / "dummy_image_name.jpg"
return ctx
@pytest.fixture
def mock_controller():
controller = Mock()
mock_device_data = Mock()
mock_device_data.base64 = base64.b64encode(b"mocked_screenshot").decode()
mock_device_data.elements = []
mock_device_data.width = 1080
mock_device_data.height = 2400
controller.get_screen_data = AsyncMock(return_value=mock_device_data)
controller.get_ui_elements = AsyncMock(return_value=mock_device_data.elements)
controller.tap_at = AsyncMock(return_value=Mock(error=None))
controller.type_text = AsyncMock(return_value=True)
controller.erase_text = AsyncMock(return_value=True)
controller.swipe_coords = AsyncMock(return_value=None)
controller.press_enter = AsyncMock(return_value=True)
controller.go_back = AsyncMock(return_value=True)
controller.go_home = AsyncMock(return_value=True)
controller.press_key = AsyncMock(return_value=True)
controller.terminate_app = AsyncMock(return_value=True)
return controller
# ==========================================
# SLIDE 1: click
# ==========================================
@pytest.mark.asyncio
async def test_click_happy_path(mock_context, mock_controller):
state = DummyState()
outcome, img_bytes, shot_path, xml_list = await _exec_click(
target=[500, 600],
state=state,
controller=mock_controller,
ctx=mock_context,
times=2,
delay_ms=150,
)
assert "Clicked at [500, 600]" in outcome
assert mock_controller.tap_at.call_count == 1
mock_controller.tap_at.assert_any_call(540, 1440, times=2, delay_ms=150)
@pytest.mark.asyncio
async def test_click_input_validation(mock_context, mock_controller):
state = DummyState()
# Coordinates holding non-integer values
outcome, _, _, _ = await _exec_click(
target=["invalid", "invalid"],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Error during click:" in outcome
@pytest.mark.asyncio
async def test_click_edge_cases(mock_context, mock_controller):
state = DummyState()
# Underflow/overflow coordinates should be clamped to valid boundary
outcome, _, _, _ = await _exec_click(
target=[-10, 1050],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Clicked at [-10, 1050]" in outcome
mock_controller.tap_at.assert_called_with(0, 2399, times=1, delay_ms=100)
@pytest.mark.asyncio
async def test_click_graceful_failure(mock_context, mock_controller):
state = DummyState()
mock_controller.tap_at.return_value = Mock(error="Click failed connection lost")
outcome, _, _, _ = await _exec_click(
target=[500, 600],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Error executing click: Click failed connection lost" in outcome
# ==========================================
# SLIDE 2: click_sequence
# ==========================================
@pytest.mark.asyncio
async def test_click_sequence_happy_path(mock_context, mock_controller):
state = DummyState()
state.indexed_points = [[540, 1200]] # Mapped from index 1 (1-indexed)
outcome, img_bytes, shot_path, xml_list = await _exec_click_sequence(
sequence=[1, [300, 400]],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Sequence clicked successfully" in outcome
assert mock_controller.tap_at.call_count == 2
mock_controller.tap_at.assert_any_call(540, 1200)
mock_controller.tap_at.assert_any_call(324, 960)
@pytest.mark.asyncio
async def test_click_sequence_input_validation(mock_context, mock_controller):
state = DummyState()
state.indexed_points = [[540, 1200], [270, 600]]
# Subsequent elements can also be integers (mixed array of indexes and coordinates)
outcome, _, _, _ = await _exec_click_sequence(
sequence=[1, 2, [300, 400]],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Sequence clicked successfully" in outcome
assert mock_controller.tap_at.call_count == 3
mock_controller.tap_at.assert_any_call(540, 1200)
mock_controller.tap_at.assert_any_call(270, 600)
mock_controller.tap_at.assert_any_call(324, 960)
# Reset indexed_points since _exec_click_sequence overwrites them
# via _capture_screenshot_and_parse_ui
state.indexed_points = [[540, 1200], [270, 600]]
# Index out of bounds
outcome, _, _, _ = await _exec_click_sequence(
sequence=[5, [300, 400]],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Error during click sequence: Index 5 is out of range" in outcome
@pytest.mark.asyncio
async def test_click_sequence_edge_cases(mock_context, mock_controller):
state = DummyState()
# Coordinates in sequence outside range should be clamped
outcome, _, _, _ = await _exec_click_sequence(
sequence=[[0, 0], [1200, -50]],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Sequence clicked successfully" in outcome
mock_controller.tap_at.assert_any_call(0, 0)
mock_controller.tap_at.assert_any_call(1079, 0)
@pytest.mark.asyncio
async def test_click_sequence_graceful_failure(mock_context, mock_controller):
state = DummyState()
mock_controller.tap_at.side_effect = [
Mock(error=None),
Mock(error="Connection error on step 2"),
]
outcome, _, _, _ = await _exec_click_sequence(
sequence=[[100, 200], [300, 400]],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Error executing click at" in outcome
assert "Connection error on step 2" in outcome
@pytest.mark.asyncio
async def test_click_sequence_stringified_formats(mock_context, mock_controller):
state = DummyState()
state.indexed_points = [[540, 1200]]
# Test stringified coordinate formats commonly generated by LLMs:
# 1. string representation of lists: ["[500, 280]", "[884, 362]"]
outcome, _, _, _ = await _exec_click_sequence(
sequence=["[500, 280]", "[884, 362]"],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Sequence clicked successfully" in outcome
mock_controller.tap_at.assert_any_call(540, 672)
mock_controller.tap_at.assert_any_call(954, 868)
# 2. comma-separated strings: ["500,280", "884,362"]
outcome, _, _, _ = await _exec_click_sequence(
sequence=["500,280", "884,362"],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Sequence clicked successfully" in outcome
# 3. whole sequence as a string
outcome, _, _, _ = await _exec_click_sequence(
sequence="[[500, 280], [884, 362]]",
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Sequence clicked successfully" in outcome
# ==========================================
# SLIDE 3: long_press
# ==========================================
@pytest.mark.asyncio
async def test_long_press_happy_path(mock_context, mock_controller):
state = DummyState()
outcome, img_bytes, shot_path, xml_list = await _exec_long_press(
target=[500, 500],
duration=1500,
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Long pressed at [500, 500]" in outcome
mock_controller.tap_at.assert_called_once_with(
540, 1200, long_press=True, long_press_duration=1500
)
@pytest.mark.asyncio
async def test_long_press_input_validation(mock_context, mock_controller):
state = DummyState()
# Invalid target type
outcome, _, _, _ = await _exec_long_press(
target="not-a-list",
duration=1500,
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Error during long press:" in outcome
# ==========================================
# SLIDE 4: input_text
# ==========================================
@pytest.mark.asyncio
async def test_input_text_happy_path(mock_context, mock_controller):
state = DummyState()
outcome, img_bytes, shot_path, xml_list = await _exec_input_text(
text="hello",
target=[500, 500],
clear_exist=True,
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Executed typing 'hello'." in outcome
# Focus goes through ensure_focus_at_coords (keyword-style tap), and typing
# never re-clears: the clear already happened via erase_text.
mock_controller.tap_at.assert_called_once_with(x=540, y=1200)
mock_controller.erase_text.assert_called_once()
mock_controller.type_text.assert_called_once_with("hello", clear_existing=False)
@pytest.mark.asyncio
async def test_input_text_input_validation(mock_context, mock_controller):
state = DummyState()
outcome, _, _, _ = await _exec_input_text(
text="hello",
target="invalid",
clear_exist=True,
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Error during input text:" in outcome
# ==========================================
# SLIDE 5: swipe
# ==========================================
@pytest.mark.asyncio
async def test_swipe_happy_path_direction(mock_context, mock_controller):
state = DummyState()
outcome, img_bytes, shot_path, xml_list = await _exec_swipe(
action="up",
duration=400,
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Swipe completed successfully" in outcome
mock_controller.swipe_coords.assert_called_once_with(648, 1680, 648, 720, 400)
@pytest.mark.asyncio
async def test_swipe_happy_path_coords(mock_context, mock_controller):
state = DummyState()
outcome, img_bytes, shot_path, xml_list = await _exec_swipe(
action=[100, 900, 100, 100],
duration=400,
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Swipe completed successfully" in outcome
mock_controller.swipe_coords.assert_called_once_with(108, 2160, 108, 240, 400)
@pytest.mark.asyncio
async def test_swipe_input_validation(mock_context, mock_controller):
state = DummyState()
outcome, _, _, _ = await _exec_swipe(
action="invalid_direction",
duration=400,
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Error during swipe: Invalid direction:" in outcome
# ==========================================
# SLIDE 6: press_key
# ==========================================
@pytest.mark.asyncio
async def test_press_key_happy_path(mock_context, mock_controller):
state = DummyState()
outcome, img_bytes, shot_path, xml_list = await _exec_press_key(
key="BACK",
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Executed key press 'BACK'." in outcome
mock_controller.go_back.assert_called_once()
@pytest.mark.asyncio
async def test_press_key_input_validation(mock_context, mock_controller):
state = DummyState()
outcome, _, _, _ = await _exec_press_key(
key="INVALID_KEY",
state=state,
controller=mock_controller,
ctx=mock_context,
)
# Unknown keycodes are forwarded verbatim to the driver (historical adb_server
# behavior); the mock driver accepts them, so the press reports success.
assert "Executed key press 'INVALID_KEY'." in outcome
mock_controller.press_key.assert_called_once_with("INVALID_KEY")
# ==========================================
# SLIDE 7: manage_app
# ==========================================
@pytest.mark.asyncio
async def test_manage_app_happy_path(mock_context, mock_controller):
state = DummyState()
with (
# The device-call bodies moved into the actuator layer, which imports these
# lazily from their source modules -- patch them at the source.
patch("artemis.tools.mobile.launch_app.find_package") as mock_find_package,
patch("artemis.utils.app_launch_utils.launch_app_with_retries") as mock_launch,
):
mock_find_package.return_value = "com.google.android.youtube"
mock_launch.return_value = (True, "")
outcome, img_bytes, shot_path, xml_list = await _exec_manage_app(
action="launch",
app_name="YouTube",
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Launched app 'YouTube' (com.google.android.youtube) successfully." in outcome
mock_find_package.assert_called_once_with(mock_context, "YouTube", use_fallback=False)
mock_launch.assert_called_once_with(mock_context, "com.google.android.youtube")
@pytest.mark.asyncio
async def test_manage_app_input_validation(mock_context, mock_controller):
state = DummyState()
with patch("artemis.tools.mobile.launch_app.find_package") as mock_find_package:
mock_find_package.return_value = None
outcome, _, _, _ = await _exec_manage_app(
action="launch",
app_name="NonExistentApp",
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Error finding package for app: NonExistentApp" in outcome
# ==========================================
# SLIDE 8: wait_for_delay
# ==========================================
@pytest.mark.asyncio
async def test_wait_for_delay_happy_path(mock_context, mock_controller):
state = DummyState()
outcome, img_bytes, shot_path, xml_list = await _exec_wait_for_delay(
time_in_ms=100,
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Waited for 100ms successfully." in outcome
+27
View File
@@ -18,7 +18,9 @@ from unittest.mock import Mock, patch
import pytest
from langchain_core.messages import HumanMessage, ToolMessage
from artemis.agents.explorer.constants import ASK_EXPLORER_DESCRIPTION
from artemis.agents.flash.runner import FlashRunner
from artemis.agents.validator.tool_declarations import ASK_EXPLORER_TOOL
from artemis.context import ArtemisContext
@@ -57,6 +59,31 @@ def test_flash_runner_tools_initialization(mock_context):
assert "report_failure_analysis" not in tool_names
def test_flash_runner_executor_follows_the_flash_explorer_tier(mock_context):
"""The executor is tagged as the Flash profile so ask_explorer follows flash_mode."""
with (
patch("artemis.controllers.unified_controller.get_driver"),
patch("artemis.agents.flash.runner.McpActionExecutor") as executor_cls,
):
runner = FlashRunner(mock_context, goal="Test Open Settings")
executor_cls.assert_called_once()
assert executor_cls.call_args.kwargs["agent_name"] == "flash"
assert runner.executor is executor_cls.return_value
def test_ask_explorer_declaration_is_tier_agnostic(mock_context):
"""Flash binds the shared ask_explorer contract: query + context_feedback, no tier."""
with patch("artemis.controllers.unified_controller.get_driver"):
runner = FlashRunner(mock_context, goal="Test Open Settings")
declaration = next(t for t in runner._get_tools() if t.name == "ask_explorer")
assert declaration is ASK_EXPLORER_TOOL
assert declaration.description == ASK_EXPLORER_DESCRIPTION
assert set(declaration.parameters["properties"]) == {"query", "context_feedback"}
assert declaration.parameters["required"] == ["query"]
def test_flash_runner_screenshot_pruning(mock_context):
"""Verify that earlier screenshot blocks are pruned, keeping only the latest."""
with patch("artemis.controllers.unified_controller.get_driver"):
@@ -0,0 +1,333 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""FlashRunner over the session transcript ledger (Pro-identical history).
Covers: the ``video_analyzer`` availability gate (same as the Pro graph) and
its prompt segment, the unbounded-by-default turn limit, the Pro-shaped
observation tail (``# CURRENT OBSERVATION [T+mm:ss]`` header, screenshot,
``--- Visible UI Elements ---`` list), and the end-to-end loop: committed
turns carry text-only tool messages plus an ``--- Action Execution Result
(T+mm:ss) ---`` message, every recorded step of a multi-action turn is
registered on the committed turn, the final-turn tool restriction only
applies to bounded loops, and a no-tool-call turn is nudged, not terminated.
"""
import base64
import re
from unittest.mock import AsyncMock, Mock, patch
import pytest
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
from artemis.agents.flash.runner import FlashRunner
from artemis.agents.validator.tool_declarations import ToolExecutionResult
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.memory.transcript import (
EXECUTION_RESULT_MARKER,
PRO_UI_LIST_MARKER,
TranscriptLedger,
)
OBSERVATION_HEADER_RE = re.compile(r"^# CURRENT OBSERVATION \[T\+\d{2,}:\d{2}\]$")
RESULT_RE = re.compile(
rf"^{re.escape(EXECUTION_RESULT_MARKER)} \(T\+\d{{2,}}:\d{{2}}\) ---\nStatus: success$"
)
@pytest.fixture
def mock_context():
ctx = Mock(spec=ArtemisContext)
ctx.llm_config = Mock()
mock_llm_cfg = Mock()
mock_llm_cfg.model = "gemini-2.5-flash"
mock_llm_cfg.temperature = 0.1
ctx.llm_config.get_agent.return_value = mock_llm_cfg
ctx.device = Mock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
ctx.data_engine = None
ctx.adb_client = None
ctx.driver = Mock()
return ctx
# ---------------------------------------------------------------------------
# video_analyzer availability (same gate as the Pro graph)
# ---------------------------------------------------------------------------
def test_video_analyzer_bound_only_when_recording_tools_enabled(mock_context):
with patch("artemis.controllers.unified_controller.get_driver"):
mock_context.execution_setup = Mock()
mock_context.execution_setup.video_recording_tools_enabled = True
names = [t.name for t in FlashRunner(mock_context, goal="g")._get_tools()]
assert "video_analyzer" in names
assert names[-1] == "report_task_status"
mock_context.execution_setup.video_recording_tools_enabled = False
names = [t.name for t in FlashRunner(mock_context, goal="g")._get_tools()]
assert "video_analyzer" not in names
# A Mock attribute (truthy but not True) must not enable the tool.
mock_context.execution_setup = Mock()
names = [t.name for t in FlashRunner(mock_context, goal="g")._get_tools()]
assert "video_analyzer" not in names
def test_recall_history_bound_only_with_a_data_engine_session(mock_context):
with patch("artemis.controllers.unified_controller.get_driver"):
# No DataEngine: nothing to search, the tool stays out (as in Pro).
names = [t.name for t in FlashRunner(mock_context, goal="g")._get_tools()]
assert "recall_history" not in names
mock_context.data_engine = Mock()
with patch("artemis.tools.history_recall._recall_config", return_value=None):
runner = FlashRunner(mock_context, goal="g")
tools = runner._get_tools()
names = [t.name for t in tools]
assert "recall_history" in names
declaration = next(t for t in tools if t.name == "recall_history")
assert set(declaration.parameters["properties"]) == {
"query",
"step_range",
"include_details",
"include_images",
"max_results",
}
assert declaration.parameters["required"] == ["query"]
assert "`recall_history`" in runner._render_system_prompt(tools)
def test_video_analyzer_prompt_segment_follows_availability(mock_context):
with patch("artemis.controllers.unified_controller.get_driver"):
runner = FlashRunner(mock_context, goal="g")
mock_context.execution_setup = Mock()
mock_context.execution_setup.video_recording_tools_enabled = True
with_video = runner._render_system_prompt(runner._get_tools())
assert "`video_analyzer`" in with_video
mock_context.execution_setup.video_recording_tools_enabled = False
without_video = runner._render_system_prompt(runner._get_tools())
assert "video_analyzer" not in without_video
# Session-clock / history teaching is unconditional.
assert "T+mm:ss" in without_video
assert "Action Execution Result" in without_video
# ---------------------------------------------------------------------------
# Turn limit: unbounded by default, explicit caps honoured
# ---------------------------------------------------------------------------
def test_turn_limit_semantics(mock_context):
with patch("artemis.controllers.unified_controller.get_driver"):
assert FlashRunner(mock_context, goal="g", max_turns=0).turn_limit is None
assert FlashRunner(mock_context, goal="g", max_turns=7).turn_limit == 7
with patch("artemis.agents.flash.runner.load_agent_config", side_effect=RuntimeError):
assert FlashRunner(mock_context, goal="g").turn_limit is None
# ---------------------------------------------------------------------------
# Observation tail: Pro shape with the session-relative header
# ---------------------------------------------------------------------------
def test_observation_tail_has_pro_shape(mock_context):
with patch("artemis.controllers.unified_controller.get_driver"):
runner = FlashRunner(mock_context, goal="Open Settings")
ledger = TranscriptLedger()
tail = runner._build_tail(ledger, 1, b"IMG", "[1] Settings")
texts = [b["text"] for b in tail.content if b["type"] == "text"]
assert texts[0] == "Your objective is: Open Settings"
assert OBSERVATION_HEADER_RE.match(texts[1])
assert texts[2] == "--- Current Screenshot ---"
assert texts[3] == f"{PRO_UI_LIST_MARKER}\n[1] Settings"
assert sum(1 for b in tail.content if b["type"] == "image_url") == 1
later = runner._build_tail(
ledger, 5, None, None, injected="[INJECTED] stop", notices=["nudge"], is_final=True
)
texts = [b["text"] for b in later.content]
assert OBSERVATION_HEADER_RE.match(texts[0])
assert "Your objective is" not in "".join(texts)
assert texts[1:] == [
"nudge",
"[INJECTED] stop",
"[WARNING] This is your final turn; only 'report_task_status' is available.",
]
# ---------------------------------------------------------------------------
# End-to-end loop over the ledger (mocked model / executor)
# ---------------------------------------------------------------------------
def _exec_result(tc_id, name, post_bytes, ui="[1] next"):
return ToolExecutionResult(
tool_call_id=tc_id,
tool_name=name,
status="success",
text_summary=f"{name} executed",
screenshot_bytes=post_bytes,
ui_elements_text=ui,
)
def _make_runner(mock_context, responses, max_turns=None):
runner = FlashRunner(mock_context, goal="Open Settings", max_turns=max_turns)
runner.summarizer = None
runner._init_llm = Mock(return_value=Mock())
# Snapshot each invocation: the runner appends to the same message list
# after the call, so the recorded call args would otherwise drift.
runner.calls = []
pending = list(responses)
async def _invoke(llm, tools, messages):
runner.calls.append({"tools": list(tools), "messages": list(messages)})
return pending.pop(0)
runner._invoke_model = AsyncMock(side_effect=_invoke)
runner.executor.execute = AsyncMock(
side_effect=lambda name, args, tc_id, state: _exec_result(
tc_id, name, f"IMG-{tc_id}".encode()
)
)
return runner
def _report(status="completed", tc_id="tc-report", **extra):
return AIMessage(
content="Done.",
tool_calls=[
{"name": "report_task_status", "args": {"status": status, **extra}, "id": tc_id}
],
)
_INITIAL_OBSERVATION = ("shot0.png", b"IMG0", "[1] Settings")
@pytest.mark.asyncio
async def test_run_builds_prompt_from_ledger_with_session_offsets(mock_context):
"""Turn 2 must see: the static system prefix, the committed turn 1 (its
tail, the AI message, text-only tool messages and an execution-result
message with a ``T+`` offset) and a fresh observation tail carrying the
last action's post screenshot — the Pro transcript shape, with no cap."""
responses = [
AIMessage(
content="I will tap then wait.",
tool_calls=[
{"name": "click", "args": {"target": [500, 600]}, "id": "tc1"},
{"name": "wait_for_delay", "args": {"seconds": 1}, "id": "tc2"},
],
),
_report(explanation="ok"),
]
with (
patch("artemis.controllers.unified_controller.get_driver"),
patch(
"artemis.agents.flash.runner.capture_screenshot_and_parse_ui",
AsyncMock(return_value=_INITIAL_OBSERVATION),
),
):
runner = _make_runner(mock_context, responses, max_turns=0)
assert runner.turn_limit is None
result = await runner.run(State(initial_goal="Open Settings"))
assert result == {"status": "completed", "explanation": "ok"}
assert runner._invoke_model.await_count == 2
turn2_messages = runner.calls[1]["messages"]
assert isinstance(turn2_messages[0], SystemMessage)
assert "T+mm:ss" in turn2_messages[0].content
tail1 = turn2_messages[1]
assert isinstance(tail1, HumanMessage)
tail1_texts = [b["text"] for b in tail1.content if b["type"] == "text"]
assert tail1_texts[0] == "Your objective is: Open Settings"
assert OBSERVATION_HEADER_RE.match(tail1_texts[1])
assert isinstance(turn2_messages[2], AIMessage)
tool_msgs = [m for m in turn2_messages if isinstance(m, ToolMessage)]
assert [m.content for m in tool_msgs] == ["click executed", "wait_for_delay executed"]
result_msg = turn2_messages[5]
assert isinstance(result_msg, HumanMessage)
assert RESULT_RE.match(result_msg.content[0]["text"])
tail2 = turn2_messages[6]
tail2_texts = [b["text"] for b in tail2.content if b["type"] == "text"]
assert OBSERVATION_HEADER_RE.match(tail2_texts[0])
assert "Your objective is" not in "".join(tail2_texts)
images = [b for b in tail2.content if b["type"] == "image_url"]
assert len(images) == 1
assert images[0]["image_url"]["url"].endswith(base64.b64encode(b"IMG-tc2").decode())
assert tail2_texts[1] == "--- Current Screenshot ---"
assert tail2_texts[2] == f"{PRO_UI_LIST_MARKER}\n[1] next"
assert len(turn2_messages) == 7
# Both actions of the multi-action turn were registered on the committed turn.
ledger = mock_context.transcript_ledger
assert ledger.unchunked_turns()[0]["step_keys"] == ["tc1", "tc2"]
@pytest.mark.asyncio
async def test_run_final_turn_restricts_tools_and_fails_without_report(mock_context):
responses = [AIMessage(content="I give up.", tool_calls=[])]
with (
patch("artemis.controllers.unified_controller.get_driver"),
patch(
"artemis.agents.flash.runner.capture_screenshot_and_parse_ui",
AsyncMock(return_value=_INITIAL_OBSERVATION),
),
):
runner = _make_runner(mock_context, responses, max_turns=1)
result = await runner.run(State(initial_goal="Open Settings"))
assert result == {"status": "failed", "explanation": "I give up."}
tools = runner.calls[0]["tools"]
assert [t.name for t in tools] == ["report_task_status"]
tail = runner.calls[0]["messages"][-1]
assert any("final turn" in b.get("text", "") for b in tail.content)
@pytest.mark.asyncio
async def test_run_no_tool_call_turn_is_nudged_not_terminated(mock_context):
responses = [AIMessage(content="thinking only", tool_calls=[]), _report()]
with (
patch("artemis.controllers.unified_controller.get_driver"),
patch(
"artemis.agents.flash.runner.capture_screenshot_and_parse_ui",
AsyncMock(return_value=_INITIAL_OBSERVATION),
),
):
runner = _make_runner(mock_context, responses, max_turns=0)
result = await runner.run(State(initial_goal="Open Settings"))
assert result == {"status": "completed"}
turn2_messages = runner.calls[1]["messages"]
# Unbounded loop: no final-turn restriction was ever applied.
tools = runner.calls[1]["tools"]
assert "click" in [t.name for t in tools]
tail2_texts = [b["text"] for b in turn2_messages[-1].content if b["type"] == "text"]
assert any("did not call any tools" in t for t in tail2_texts)
# A helper-only / empty turn commits without an execution-result message.
assert not any(
isinstance(m, HumanMessage)
and any(EXECUTION_RESULT_MARKER in b.get("text", "") for b in m.content)
for m in turn2_messages
)
+3 -9
View File
@@ -74,9 +74,7 @@ def _content_fingerprint(msg) -> str:
def _has_image(msg) -> bool:
return any(
isinstance(b, dict) and b.get("type") in ("image_url", "image") for b in msg.content
)
return any(isinstance(b, dict) and b.get("type") in ("image_url", "image") for b in msg.content)
def test_success_path_product_behind_the_scrub_edge(mock_context):
@@ -204,9 +202,7 @@ def test_race_failed_summary_becomes_unavailable_placeholder(mock_context):
_simulate_turns(compressor, messages, [_step(1), _step(2), _step(3)])
assert not _has_image(messages[0])
assert "[visual summary unavailable; evidence at DataEngine step 1]" in str(
messages[0].content
)
assert "[visual summary unavailable; evidence at DataEngine step 1]" in str(messages[0].content)
def test_freeze_invariant_bytes_never_change(mock_context):
@@ -232,9 +228,7 @@ def test_freeze_invariant_bytes_never_change(mock_context):
for idx in compressor._frozen:
fingerprint = _content_fingerprint(messages[idx])
if idx in frozen_fingerprints:
assert frozen_fingerprints[idx] == fingerprint, (
f"frozen message {idx} mutated"
)
assert frozen_fingerprints[idx] == fingerprint, f"frozen message {idx} mutated"
else:
frozen_fingerprints[idx] = fingerprint
@@ -551,9 +551,7 @@ async def test_raw_lens_call_meters_llm_usage_without_touching_context_base(mock
await summarizer.flush()
usage_calls = [
c
for c in engine.record_trace.call_args_list
if c.kwargs.get("name") == "llm_usage"
c for c in engine.record_trace.call_args_list if c.kwargs.get("name") == "llm_usage"
]
assert usage_calls, "the raw lens bypass must record an llm_usage trace"
payload = usage_calls[0].kwargs["payload"]
@@ -77,9 +77,7 @@ async def _run(steps, b64=B64_A, cfg=None):
builder = PromptBuilder()
component = HistoricalStateHintPromptComponent()
with patch("artemis.config.load_agent_config", return_value=cfg or _cfg()):
await component(
builder, MagicMock(), MagicMock(), latest_screenshot_b64=b64, steps=steps
)
await component(builder, MagicMock(), MagicMock(), latest_screenshot_b64=b64, steps=steps)
return [p for p in builder.human_parts if isinstance(p, str)]
+1 -3
View File
@@ -354,9 +354,7 @@ async def test_history_analyzer_integration_with_task_tree():
"artemis.agents.history_analyzer.history_analyzer.get_note_file_path",
return_value=mock_path,
),
patch(
"artemis.agents.history_analyzer.history_analyzer.build_history_for"
) as mock_build,
patch("artemis.agents.history_analyzer.history_analyzer.build_history_for") as mock_build,
):
mock_build.return_value = "Mocked operation history"
analyzer = HistoryAnalyzer(mock_ctx)
+34 -48
View File
@@ -414,7 +414,9 @@ async def test_operator_dynamic_prompt():
mock_llm.bind_tools.return_value = mock_llm
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
node = OperatorNode(mock_ctx, prompt_components=[MockComponent()], transcript_config=LEGACY_TRANSCRIPT)
node = OperatorNode(
mock_ctx, prompt_components=[MockComponent()], transcript_config=LEGACY_TRANSCRIPT
)
node_update = await node(mock_state)
@@ -554,28 +556,6 @@ async def test_operator_history_compression():
node_update = await node(mock_state)
@pytest.mark.asyncio
async def test_operator_loop_detection():
from artemis.agents.operator.operator import OperatorNode
from artemis.context import ArtemisContext
from unittest.mock import MagicMock
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.execution_setup = None
mock_ctx.data_engine = None
mock_state = MagicMock()
mock_state.subagent_calls = ["ask_explorer", "ask_explorer", "ask_explorer"]
mock_state.initial_goal = "Test goal"
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
with pytest.raises(RuntimeError) as exc_info:
node_update = await node(mock_state)
assert "Infinite loop detected" in str(exc_info.value)
@pytest.mark.asyncio
async def test_operator_tracks_subagent_calls():
from artemis.agents.operator.operator import OperatorNode
@@ -630,7 +610,9 @@ async def test_operator_tracks_subagent_calls():
side_effect=lambda t, ctx: t,
),
):
node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool], transcript_config=LEGACY_TRANSCRIPT)
node = OperatorNode(
mock_ctx, tools=[mock_diagnoser_tool], transcript_config=LEGACY_TRANSCRIPT
)
node_update = await node(mock_state)
update_dict = node_update
@@ -722,7 +704,9 @@ async def test_operator_defer_action_for_gathering():
side_effect=lambda t, ctx: t,
),
):
node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool], transcript_config=LEGACY_TRANSCRIPT)
node = OperatorNode(
mock_ctx, tools=[mock_diagnoser_tool], transcript_config=LEGACY_TRANSCRIPT
)
node_update = await node(mock_state)
# Subagent tool should have been executed
@@ -908,7 +892,9 @@ async def test_operator_no_defer_for_note_updating():
side_effect=lambda t, ctx: t,
),
):
node = OperatorNode(mock_ctx, tools=[mock_update_note_tool], transcript_config=LEGACY_TRANSCRIPT)
node = OperatorNode(
mock_ctx, tools=[mock_update_note_tool], transcript_config=LEGACY_TRANSCRIPT
)
node_update = await node(mock_state)
# The update_note tool should have been executed immediately
@@ -998,7 +984,9 @@ async def test_operator_defer_for_note_reading():
side_effect=lambda t, ctx: t,
),
):
node = OperatorNode(mock_ctx, tools=[mock_read_note_tool], transcript_config=LEGACY_TRANSCRIPT)
node = OperatorNode(
mock_ctx, tools=[mock_read_note_tool], transcript_config=LEGACY_TRANSCRIPT
)
node_update = await node(mock_state)
# The read_note tool should have been executed
@@ -1086,7 +1074,9 @@ async def test_operator_no_defer_for_note_appending():
side_effect=lambda t, ctx: t,
),
):
node = OperatorNode(mock_ctx, tools=[mock_append_note_tool], transcript_config=LEGACY_TRANSCRIPT)
node = OperatorNode(
mock_ctx, tools=[mock_append_note_tool], transcript_config=LEGACY_TRANSCRIPT
)
node_update = await node(mock_state)
# The append_note tool should have been executed immediately
@@ -1106,7 +1096,7 @@ async def test_operator_no_defer_for_note_appending():
async def test_operator_background_tasks_prompt_injection():
from artemis.agents.operator.operator import OperatorNode
from artemis.context import ArtemisContext
from artemis.tools.command_tool import _BACKGROUND_TASKS, BackgroundTask
from artemis.tools.command_tool import BackgroundTask, get_adb_task_registry
from unittest.mock import AsyncMock, MagicMock, patch
mock_ctx = MagicMock(spec=ArtemisContext)
@@ -1140,7 +1130,8 @@ async def test_operator_background_tasks_prompt_injection():
cwd="/data/local/tmp",
)
_BACKGROUND_TASKS["task_99999"] = bg_task
adb_registry = get_adb_task_registry(mock_ctx)
adb_registry.background["task_99999"] = bg_task
try:
captured_messages = []
@@ -1194,8 +1185,7 @@ async def test_operator_background_tasks_prompt_injection():
assert "term_xyz" in full_content
assert "Accumulated Output: 0 lines of logs" in full_content
finally:
if "task_99999" in _BACKGROUND_TASKS:
del _BACKGROUND_TASKS["task_99999"]
adb_registry.background.pop("task_99999", None)
@pytest.mark.asyncio
@@ -1225,11 +1215,16 @@ async def test_operator_task_plan_warning_prompt_injection():
mock_llm.ainvoke = AsyncMock(return_value=mock_response)
mock_llm.bind_tools.return_value = mock_llm
# Case 1: task_plan updated in the last two steps (using the "key" argument)
# Case 1: task_plan updated in the last step (using the "key" argument)
steps_updated = [
{
"step_id": "step_1",
"step_number": 1,
"tool_calls": [{"name": "click", "args": {"target": 1}}],
},
{
"step_id": "step_2",
"step_number": 2,
"tool_calls": [
{
"name": "update_note",
@@ -1237,11 +1232,6 @@ async def test_operator_task_plan_warning_prompt_injection():
}
],
},
{
"step_id": "step_2",
"step_number": 2,
"tool_calls": [{"name": "click", "args": {"target": 1}}],
},
]
captured_messages_1 = []
@@ -1278,12 +1268,9 @@ async def test_operator_task_plan_warning_prompt_injection():
elif isinstance(m.content, str):
full_content_1 += m.content + "\n"
assert (
"Reminder: You have not updated the task_plan for two consecutive"
" turns." not in full_content_1
)
assert "Reminder: your last turn executed a Turn-Ending Action" not in full_content_1
# Case 2: task_plan NOT updated in the last two steps
# Case 2: an action turn that did NOT update task_plan
steps_not_updated = [
{
"step_id": "step_1",
@@ -1294,6 +1281,7 @@ async def test_operator_task_plan_warning_prompt_injection():
"step_id": "step_2",
"step_number": 2,
"tool_calls": [{"name": "click", "args": {"target": 2}}],
"action_taken": {"action": "click", "target": 2},
},
]
@@ -1331,10 +1319,7 @@ async def test_operator_task_plan_warning_prompt_injection():
elif isinstance(m.content, str):
full_content_2 += m.content + "\n"
assert (
"Reminder: You have not updated the task_plan for two consecutive"
" turns." in full_content_2
)
assert "Reminder: your last turn executed a Turn-Ending Action" in full_content_2
@pytest.mark.asyncio
@@ -1399,7 +1384,8 @@ async def test_operator_tool_limit_exceeded_warning():
full_content += m.content + "\n"
assert (
"You did not execute any screen interaction actions in your last turn" in full_content
"your last turn used up its tool-call budget without a Turn-Ending Action"
in full_content
)
assert node_update.get("operator_tool_limit_exceeded") is False
+13 -3
View File
@@ -22,9 +22,19 @@ from artemis.agents.operator.prompts import apply_operator_prompt_contract
def test_operator_prompt_contract_matches_note_runtime_semantics():
prompt = apply_operator_prompt_contract(load_operator_prompts()["main_template"])
assert "write-through tools `update_note` and `append_note`" in prompt
assert "may accompany at most one Turn-Ending Action" in prompt
assert "`read_note`, `list_notes`, `save_note`, and `recall_history`" in prompt
assert "**Write-Through Memory Tools** (`save_note`, `update_note`, `append_note`)" in prompt
assert "May run alongside the turn's Turn-Ending Action (or Fast-Action Burst)" in prompt
# The two execution tiers and the incident workflow are advertised.
assert "**Vetted Single Action (default)**" in prompt
# The burst ceiling is a context variable filled in by the later template render.
assert (
"**Fast-Action Burst (time-critical only)**: two to {{ max_burst_actions }}"
" Turn-Ending Actions" in prompt
)
assert "--- Execution Incident (OPEN) ---" in prompt
assert "Failure Analyzer" not in prompt
assert "(`read_note`, `list_notes`, `recall_history`)" in prompt
assert "`save_note`, and `recall_history`" not in prompt
assert "Do NOT submit a Turn-Ending Action at the same time" not in prompt
assert "memory note tools (`read_note`, `list_notes`, `save_note`, `update_note`" not in prompt
+11 -9
View File
@@ -53,13 +53,17 @@ from artemis.memory.transcript import (
# SHA-256 of the legacy system message rendered by TemplatePromptComponent
# with the fixed inputs below. Originally recorded BEFORE the M2 template
# split; re-recorded once in M5 (2026-09-01) after the only intended template
# change since — removing the Short-Term Memory instruction block from
# operator.json (the <short_term_memory> relay was removed end to end per
# pro-context-memory-redesign §3.3). Any other drift here means the flag-off
# split; re-recorded in M5 (2026-09-01) after the Short-Term Memory block was
# removed, and again on 2026-09-02 when the Failure Analyzer was retired: the
# operator.json template gained the Time-Sensitive Tasks principle, the two
# execution tiers (vetted single action / fast-action burst), the Execution
# Incident workflow section and the Phase 3 burst-discipline rule; and again
# on 2026-09-02 after the prompt consolidation (every rule stated once, the
# transcript/legacy history-marker split, the recited tool budget, the
# recommended diagnosis triggers). Any other drift here means the flag-off
# path is no longer byte-identical.
GOLDEN_EMPTY_PLAN = "c45aec7d67069dad3a195534dd3c97dadb48454ca186a40eb471d5f9136589fa"
GOLDEN_SENTINEL_PLAN = "670580efd48f237c3257d996a814145fbef4ec0bb467aa4dfa6eb2db4a613826"
GOLDEN_EMPTY_PLAN = "a20046ff46058fa331fa3b8f2d3ffa4a018be567bf264588b9a29bc64e5b11b8"
GOLDEN_SENTINEL_PLAN = "8f4456524c8a964d1614d26d2fafab809bd5bd3f30334fd40498999e4b27638b"
SCREENSHOT_B64 = base64.b64encode(b"fake-jpeg-bytes").decode("utf-8")
@@ -253,9 +257,7 @@ async def test_transcript_actionless_turn_commits_without_validator_message():
ledger = ctx.transcript_ledger
assert ledger.turn_count == 1
assert not any(
EXECUTION_RESULT_MARKER in str(m.content) for m in ledger.active_messages
)
assert not any(EXECUTION_RESULT_MARKER in str(m.content) for m in ledger.active_messages)
@pytest.mark.asyncio
@@ -141,7 +141,6 @@ def _guard_ctx(tmp_path, **setup_kwargs):
ctx.planner_task = None
ctx.last_validated_plan = None
ctx.pending_validated_plan = None
ctx.task_plan_content_before = None
ctx.pending_checkpoints = []
ctx.checkpoint_tasks = {}
return ctx
+16 -61
View File
@@ -26,13 +26,6 @@ from artemis.agents.operator.prompts import (
apply_operator_prompt_contract,
)
from artemis.agents.prompt_assembly import gate_segment, render_tool_enum
from artemis.agents.validator.failure_analyzer import (
FailureAnalysisStrategy,
FailureAnalyzer,
PixelTargetDisappearedStrategy,
TargetDisappearedStrategy,
)
from artemis.agents.validator.validator import ValidationErrorCategory
REPO_ROOT = Path(__file__).resolve().parents[3]
@@ -115,12 +108,15 @@ def test_removed_swipe_takes_its_teaching_along():
def test_removed_wait_for_delay_keeps_interval_line_coherent():
"""Only the wait_for_delay parenthetical goes; the SOP grammar line stays."""
"""Only the wait_for_delay parenthetical goes; the SOP pacing step stays."""
template = load_operator_prompts()["main_template"]
out = apply_operator_prompt_contract(
template, available_tools=OPERATOR_PROMPT_TOOLSET - {"wait_for_delay"}
)
assert "`Interval: <Interval>`: Declared polling cadence." in out
assert (
"Adhere strictly to the exact duration declared in `<Interval>`, then execute"
" a refresh action" in out
)
assert "Loading & Transitions" not in out
@@ -131,12 +127,10 @@ def test_reduced_enums_stay_well_formed():
available_tools=OPERATOR_PROMPT_TOOLSET - {"manage_app", "wait_for_delay"},
)
assert (
"Physical device actions (`click`, `input_text`, `swipe`, `press_key`,"
" `long_press`)" in out
"Physical device actions (`click`, `input_text`, `swipe`, `press_key`, `long_press`)" in out
)
assert (
"Turn-Ending Action (`click`, `swipe`, `input_text`, `long_press`, or"
" `press_key`)" in out
"Turn-Ending Action (`click`, `swipe`, `input_text`, `long_press`, or `press_key`)" in out
)
@@ -152,20 +146,20 @@ def test_full_set_enum_slots_render_verbatim():
"Turn-Ending Action (`click`, `swipe`, `input_text`, `long_press`,"
" `press_key`, `manage_app`, or `wait_for_delay`)" in out
)
assert (
"(`ask_explorer`, `ask_diagnoser`, `video_analyzer`, `run_adb_command`,"
" `manage_task`, `analyze_task_output`, `read_note`, `list_notes`,"
" `save_note`, and `recall_history`)" in out
)
assert "Helper/Subagent tools (`ask_explorer`, `ask_diagnoser`, `video_analyzer`)" in out
assert "ADB/task tools (`run_adb_command`, `manage_task`)" in out
assert "(`read_note`, `list_notes`, `recall_history`)" in out
# save_note is a write-through tool, never a result-dependent one; the on-demand
# analyze_task_output is not advertised.
assert "`save_note`, and" not in out
assert "analyze_task_output" not in out
# --- Flash prompt assembly -----------------------------------------------------------
def _render_flash(available_tools: frozenset[str]) -> str:
template_text = (REPO_ROOT / "artemis/agents/flash/flash_runner.md").read_text(
encoding="utf-8"
)
template_text = (REPO_ROOT / "artemis/agents/flash/flash_runner.md").read_text(encoding="utf-8")
return Template(template_text).render(goal="test goal", available_tools=available_tools)
@@ -184,7 +178,7 @@ def test_flash_without_manage_app_drops_app_launching():
assert "**App Launching**" not in out
assert not _tool_refs(out, "manage_app")
# The neighbouring numbered rule survives.
assert "**Screenshot Pruning Awareness**" in out
assert "**Do not determine element coordinates based on guesswork.**" in out
def test_flash_without_wait_for_delay_drops_timed_waiting():
@@ -192,42 +186,3 @@ def test_flash_without_wait_for_delay_drops_timed_waiting():
assert "**Timed Waiting & Transitions**" not in out
assert not _tool_refs(out, "wait_for_delay")
assert "**State Drift & Progression Tolerance**" in out
# --- FailureAnalyzer strategy gating -------------------------------------------------
def _analyzer() -> FailureAnalyzer:
return FailureAnalyzer.__new__(FailureAnalyzer) # _select_strategy needs no ctx
def test_strategy_selection_unchanged_without_gating():
analyzer = _analyzer()
assert isinstance(
analyzer._select_strategy(ValidationErrorCategory.TARGET_DISAPPEARED),
TargetDisappearedStrategy,
)
assert isinstance(
analyzer._select_strategy(ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED),
PixelTargetDisappearedStrategy,
)
def test_strategy_falls_back_when_required_tool_missing():
"""Strategy + prompt are one unit: no click_sequence, no disappeared-prompts."""
analyzer = _analyzer()
strategy = analyzer._select_strategy(
ValidationErrorCategory.TARGET_DISAPPEARED,
available_tools=frozenset({"click", "swipe"}),
)
assert type(strategy) is FailureAnalysisStrategy
assert strategy.get_prompt_template_name() == "failure_analyzer.md"
def test_strategy_kept_when_required_tool_present():
analyzer = _analyzer()
strategy = analyzer._select_strategy(
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
available_tools=frozenset({"click_sequence"}),
)
assert isinstance(strategy, PixelTargetDisappearedStrategy)
+1 -3
View File
@@ -136,9 +136,7 @@ async def test_summarizer_falls_back_to_structured_decisions(mock_context):
kwargs = mock_context.step_memory.dispatch.call_args.kwargs
assert kwargs["action_name"] == "swipe"
assert kwargs["action_args"]["coordinates"] == [1, 2, 3, 4]
assert kwargs["action_args"]["additional_actions"] == [
{"action": "tap", "coordinates": [9, 9]}
]
assert kwargs["action_args"]["additional_actions"] == [{"action": "tap", "coordinates": [9, 9]}]
@pytest.mark.asyncio
+178 -154
View File
@@ -27,7 +27,7 @@ from unittest.mock import AsyncMock, MagicMock, Mock, patch
from langchain_core.messages import AIMessage
import pytest
from artemis.agents.validator.failure_analyzer import ValidationErrorCategory
from artemis.agents.validator.categories import ValidationErrorCategory
from artemis.agents.validator.validator import ValidatorNode
from artemis.context import ArtemisContext
from artemis.mcp.action_types import ActionCode, ActionResult
@@ -39,10 +39,12 @@ class DummyState:
structured_decisions,
current_step_id=None,
latest_screenshot=None,
open_incident=None,
):
self.structured_decisions = structured_decisions
self.current_step_id = current_step_id
self.latest_screenshot = latest_screenshot
self.open_incident = open_incident
class FakeActionSession:
@@ -143,10 +145,9 @@ async def test_validator_success(mock_mcp, mock_context, temp_screenshot):
@pytest.mark.asyncio
async def test_validator_failure_analysis(mock_mcp, mock_context, temp_screenshot):
"""Test that ValidatorNode triggers failure analysis on error and handles
cannot_fix via tool.
"""
async def test_validator_exec_error_opens_incident(mock_mcp, mock_context, temp_screenshot):
"""A vetted single action that the device rejects is retried once, then an
execution incident is opened for the Operator (no repair agent)."""
mock_mcp.action_handler = lambda name, args: (
(False, "Error: Element not found") if name == "click" else (True, "")
)
@@ -155,80 +156,168 @@ async def test_validator_failure_analysis(mock_mcp, mock_context, temp_screensho
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
node = ValidatorNode(mock_context)
with (
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
):
mock_analyze.return_value = {
"status": "cannot_fix",
"analysis": "Analysis: Coordinates are likely wrong.",
}
with patch("artemis.utils.image_diff.check_ui_change", return_value=False):
result = await node(state)
assert "last_execution_result" in result
report = result["last_execution_result"]
assert "execution" in report
assert report["status"] == "failed"
assert report["burst"] is False
assert len(report["execution"]) == 1
assert report["execution"][0]["action"] == "tap"
assert report["execution"][0]["attempts"] == [
"Error: Element not found",
"Error: Element not found",
]
assert report["execution"][0]["repair"] == "Analysis: Coordinates are likely wrong."
assert "repair" not in report["execution"][0]
incident = report["incident"]
assert incident["kind"] == "exec_error"
assert incident["category"] == "general"
assert incident["reason"] == "Error: Element not found"
assert incident["consecutive_failures"] == 1
assert incident["burst_size"] == 1
assert incident["action_description"] == "Tapped element at [105, 205]"
# The same record rides in graph state for the Operator prompt.
assert result["open_incident"] == incident
@pytest.mark.asyncio
async def test_validator_repair_success(mock_mcp, mock_context, temp_screenshot):
"""Test that ValidatorNode terminates successfully and clears remaining actions
after successful repair.
"""
tap_count = 0
async def test_validator_burst_skips_safety_net_and_aborts_on_first_failure(
mock_mcp, mock_context, temp_screenshot
):
"""Two or more actions form a fast-action burst: no precondition gate, a
single dispatch per member, and the first failure aborts the rest."""
calls = []
def handler(name, args):
nonlocal tap_count
if name == "click":
tap_count += 1
if tap_count <= 2: # First action fails on both validator attempts
return False, "Error: Element not found"
calls.append(name)
if name == "click" and len([c for c in calls if c == "click"]) == 2:
return False, "Error: tap rejected"
return True, ""
mock_mcp.action_handler = handler
# Two actions planned initially
decisions = json.dumps(
[
{"action": "tap", "coordinates": [105, 205]},
{"action": "tap", "coordinates": [205, 305]},
{"action": "press_key", "keycode": "BACK"},
]
)
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
node = ValidatorNode(mock_context)
with (
patch(
"artemis.agents.validator.execution_loop._run_precondition_gate",
new_callable=AsyncMock,
) as mock_gate,
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
):
result = await node(state)
mock_gate.assert_not_called()
report = result["last_execution_result"]
assert report["burst"] is True
assert report["status"] == "failed"
assert len(report["execution"]) == 3
# First member executed cleanly.
assert "attempts" not in report["execution"][0]
# Second member failed after exactly one dispatch (bursts never retry).
assert report["execution"][1]["attempts"] == ["Error: tap rejected"]
# Third member never fired.
assert report["execution"][2]["attempts"] == ["Skipped (burst aborted)"]
assert len(mock_mcp.calls_for("click")) == 2
assert mock_mcp.calls_for("press_key") == []
incident = report["incident"]
assert incident["kind"] == "exec_error"
assert incident["action_index"] == 1
assert incident["burst_size"] == 3
assert result["open_incident"] == incident
@pytest.mark.asyncio
async def test_validator_burst_success_executes_every_member(
mock_mcp, mock_context, temp_screenshot
):
decisions = json.dumps(
[
{"action": "tap", "coordinates": [105, 205]},
{"action": "wait_for_delay", "time_in_ms": 1},
{"action": "tap", "coordinates": [205, 305]},
]
)
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
node = ValidatorNode(mock_context)
with (
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
):
mock_analyze.return_value = {
"status": "fixed",
"analysis": "Fixed by updating actions",
}
with patch("artemis.utils.image_diff.check_ui_change", return_value=True):
result = await node(state)
assert "last_execution_result" in result
report = result["last_execution_result"]
assert "execution" in report
assert len(report["execution"]) == 1 # Only the first failed action is in report
assert report["status"] == "success"
assert report["burst"] is True
assert report["incident"] is None
assert result["open_incident"] is None
assert result["last_closed_incident"] is None
assert len(mock_mcp.calls_for("click")) == 2
assert report["execution"][0]["coordinates"] == [105, 205]
assert report["execution"][0]["attempts"] == [
"Error: Element not found",
"Error: Element not found",
]
assert report["execution"][0]["repair"] == "Fixed by updating actions"
@pytest.mark.asyncio
async def test_validator_success_closes_open_incident(mock_mcp, mock_context, temp_screenshot):
"""A successful turn closes whatever incident was still open."""
previous = {
"kind": "safety_net",
"category": "target_disappeared",
"reason": "gone",
"action": {"action": "tap"},
"action_description": "Tapped element",
"consecutive_failures": 2,
}
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
state = DummyState(
structured_decisions=decisions,
latest_screenshot=temp_screenshot,
open_incident=previous,
)
node = ValidatorNode(mock_context)
with patch("artemis.utils.image_diff.check_ui_change", return_value=True):
result = await node(state)
assert result["last_execution_result"]["status"] == "success"
assert result["open_incident"] is None
# The closed record is handed over once so the Operator settles the intent.
assert result["last_closed_incident"]["kind"] == "safety_net"
assert "closed_at_step" in result["last_closed_incident"]
@pytest.mark.asyncio
async def test_validator_consecutive_failures_escalate_incident(
mock_mcp, mock_context, temp_screenshot
):
"""Failing again while an incident is open continues its failure count."""
previous = {
"kind": "exec_error",
"category": "general",
"reason": "Error: Element not found",
"action": {"action": "tap"},
"action_description": "Tapped element",
"consecutive_failures": 2,
}
mock_mcp.action_handler = lambda name, args: (False, "Error: Element not found")
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
state = DummyState(
structured_decisions=decisions,
latest_screenshot=temp_screenshot,
open_incident=previous,
)
node = ValidatorNode(mock_context)
with patch("artemis.utils.image_diff.check_ui_change", return_value=False):
result = await node(state)
assert result["open_incident"]["consecutive_failures"] == 3
@pytest.mark.asyncio
@@ -271,7 +360,7 @@ async def test_validator_focus_and_clear_text_no_ui_change(mock_mcp, mock_contex
@pytest.mark.asyncio
async def test_validator_silent_failure_treated_as_success(mock_mcp, mock_context, temp_screenshot):
"""Test that silent failure (exec succeeds but no UI change) is treated as
success and does not trigger FailureAnalyzer.
success and does not open an execution incident.
"""
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
@@ -281,13 +370,12 @@ async def test_validator_silent_failure_treated_as_success(mock_mcp, mock_contex
with (
patch("artemis.agents.validator.validator.VALIDATOR_POLL_TIMEOUT", 0.1),
patch("artemis.agents.validator.validator.VALIDATOR_POLL_INTERVAL", 0.01),
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=False), # No UI change
):
result = await node(state)
# Assert FailureAnalyzer was NOT called
mock_analyze.assert_not_called()
# No incident is opened for a silent (no-UI-change) success.
assert result["open_incident"] is None
# Assert the action is considered executed successfully
assert "last_execution_result" in result
@@ -299,13 +387,11 @@ async def test_validator_silent_failure_treated_as_success(mock_mcp, mock_contex
@pytest.mark.asyncio
async def test_validator_records_skipped_actions_on_cannot_fix(
async def test_validator_burst_first_member_failure_marks_rest_skipped(
mock_mcp, mock_context, temp_screenshot
):
"""Test that ValidatorNode marks unexecuted actions as Skipped when repair fails."""
mock_mcp.action_handler = lambda name, args: (False, "Error: Connection lost")
# Two actions planned
decisions = json.dumps(
[
{"action": "tap", "coordinates": [100, 200]},
@@ -315,42 +401,27 @@ async def test_validator_records_skipped_actions_on_cannot_fix(
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
node = ValidatorNode(mock_context)
with (
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
):
mock_analyze.return_value = {
"status": "cannot_fix",
"analysis": "Cannot fix: System crashed.",
}
with patch("artemis.utils.image_diff.check_ui_change", return_value=False):
result = await node(state)
assert "last_execution_result" in result
report = result["last_execution_result"]
assert "execution" in report
assert len(report["execution"]) == 2
# First action failed
assert report["execution"][0]["action"] == "tap"
assert report["execution"][0]["attempts"] == [
"Error: Connection lost",
"Error: Connection lost",
]
assert report["execution"][0]["repair"] == "Cannot fix: System crashed."
# Second action was skipped
# A burst member is dispatched exactly once.
assert report["execution"][0]["attempts"] == ["Error: Connection lost"]
assert "repair" not in report["execution"][0]
assert report["execution"][1]["action"] == "press_key"
assert report["execution"][1]["attempts"] == ["Skipped"]
assert report["execution"][1]["attempts"] == ["Skipped (burst aborted)"]
assert report["incident"]["action_index"] == 0
assert report["incident"]["burst_size"] == 2
@pytest.mark.asyncio
async def test_validator_pre_execution_validation_and_repair(
async def test_validator_pre_execution_validation_opens_incident(
mock_mcp, mock_context, temp_screenshot
):
"""Test that pre-execution validation fails when element changes, and is
repaired by FailureAnalyzer.
"""Test that pre-execution validation fails when the element changes, and the
interception is handed to the Operator as an execution incident.
"""
mock_mcp.hierarchy = [
{
@@ -376,7 +447,6 @@ async def test_validator_pre_execution_validation_and_repair(
node = ValidatorNode(mock_context)
with (
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
patch.object(
ValidatorNode,
@@ -389,29 +459,20 @@ async def test_validator_pre_execution_validation_and_repair(
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
"Pixel check failed",
)
mock_analyze.return_value = {
"status": "fixed",
"analysis": ("Pre-exec failure matched and resolved by switching to Sign Up button."),
}
result = await node(state)
mock_analyze.assert_called_once()
args, kwargs = mock_analyze.call_args
assert "Pre-execution validation failed" in args[2]
assert "Login" in args[2]
assert kwargs.get("error_category") == ValidationErrorCategory.TARGET_OCCUPIED
incident = result["last_execution_result"]["incident"]
assert "Pre-execution validation failed" in incident["reason"]
assert "Login" in incident["reason"]
assert incident["category"] == ValidationErrorCategory.TARGET_OCCUPIED.value
assert "last_execution_result" in result
report = result["last_execution_result"]
assert len(report["execution"]) == 1
assert report["execution"][0]["action"] == "tap"
assert "Pre-execution validation failed" in report["execution"][0]["attempts"][0]
assert (
report["execution"][0]["repair"]
== "Pre-exec failure matched and resolved by switching to Sign Up"
" button."
)
assert report["incident"]["kind"] == "safety_net"
assert report["status"] == "failed"
@pytest.mark.asyncio
@@ -443,7 +504,7 @@ async def test_validator_pre_execution_validation_self_healing(
node = ValidatorNode(mock_context)
with patch("artemis.utils.image_diff.check_ui_change", return_value=True):
await node(state)
result = await node(state)
# Center of [100, 220][300, 320] is pixel [200, 270]; the healed coordinates
# travel to the canonical click tool normalized to 0-1000 on a 1080x2400 screen:
@@ -487,7 +548,6 @@ async def test_validator_pre_execution_validation_anonymous_occupant(
node = ValidatorNode(mock_context)
with (
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
patch.object(
ValidatorNode,
@@ -500,19 +560,16 @@ async def test_validator_pre_execution_validation_anonymous_occupant(
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
"Pixel check failed",
)
mock_analyze.return_value = {
"status": "fixed",
"analysis": "Pre-exec failure bypassed.",
}
await node(state)
result = await node(state)
# Assert that the validator correctly classified this as TARGET_OCCUPIED
# (due to the clickable anonymous view blocking the click)
mock_analyze.assert_called_once()
args, kwargs = mock_analyze.call_args
assert kwargs.get("error_category") == ValidationErrorCategory.TARGET_OCCUPIED
assert "occupied/intercepted by a different element: interactive anonymous element" in args[2]
incident = result["last_execution_result"]["incident"]
assert incident["category"] == ValidationErrorCategory.TARGET_OCCUPIED.value
assert (
"occupied/intercepted by a different element: interactive anonymous element"
in incident["reason"]
)
@pytest.mark.asyncio
@@ -558,7 +615,7 @@ async def test_validator_pixel_validation_success(mock_mcp, mock_context, temp_s
@pytest.mark.asyncio
async def test_validator_pixel_validation_failure(mock_mcp, mock_context, temp_screenshot):
"""Test that the pixel safety net fails and triggers FailureAnalyzer
"""Test that the pixel safety net fails and opens an execution incident
when Gemini reports target is missing.
"""
decisions = json.dumps([{"action": "tap", "coordinates": [100, 200]}])
@@ -588,20 +645,13 @@ async def test_validator_pixel_validation_failure(mock_mcp, mock_context, temp_s
),
patch("artemis.agents.validator.validator.get_llm", return_value=mock_llm),
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
):
mock_analyze.return_value = {
"status": "cannot_fix",
"analysis": "Pixel safety net flagged missing target.",
}
node = ValidatorNode(mock_context)
await node(state)
result = await node(state)
mock_analyze.assert_called_once()
args, kwargs = mock_analyze.call_args
assert "Pixel-level validation failed" in args[2]
assert kwargs.get("error_category") == ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED
incident = result["last_execution_result"]["incident"]
assert "Pixel-level validation failed" in incident["reason"]
assert incident["category"] == ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED.value
@pytest.mark.asyncio
@@ -639,14 +689,8 @@ async def test_validator_launch_app_failure_no_retry(mock_mcp, mock_context, tem
with (
patch("artemis.agents.validator.validator.VALIDATOR_POLL_TIMEOUT", 0.05),
patch("artemis.agents.validator.validator.VALIDATOR_POLL_INTERVAL", 0.01),
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
):
mock_analyze.return_value = {
"status": "cannot_fix",
"analysis": "Cannot launch app.",
}
node = ValidatorNode(mock_context)
result = await node(state)
@@ -693,7 +737,6 @@ async def test_validator_pre_execution_validation_disappeared_not_shifted(
node = ValidatorNode(mock_context)
with (
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
patch.object(
ValidatorNode,
@@ -706,18 +749,12 @@ async def test_validator_pre_execution_validation_disappeared_not_shifted(
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
"Pixel check failed",
)
mock_analyze.return_value = {
"status": "cannot_fix",
"analysis": "Sign Up button disappeared.",
}
await node(state)
result = await node(state)
# Assert that the validator correctly classified this as TARGET_DISAPPEARED
# (since size mismatch prevents shift matching)
mock_analyze.assert_called_once()
args, kwargs = mock_analyze.call_args
assert kwargs.get("error_category") == ValidationErrorCategory.TARGET_DISAPPEARED
incident = result["last_execution_result"]["incident"]
assert incident["category"] == ValidationErrorCategory.TARGET_DISAPPEARED.value
@pytest.mark.asyncio
@@ -800,7 +837,6 @@ async def test_validator_pre_execution_xml_failure_not_overridden_when_pixel_byp
node = ValidatorNode(mock_context)
with (
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
patch.object(
ValidatorNode,
@@ -814,18 +850,12 @@ async def test_validator_pre_execution_xml_failure_not_overridden_when_pixel_byp
ValidationErrorCategory.PIXEL_BYPASSED,
"",
)
mock_analyze.return_value = {
"status": "cannot_fix",
"analysis": ("XML validation failed and pixel bypass did not override."),
}
result = await node(state)
await node(state)
# FailureAnalyzer SHOULD be called because XML validation failed
# An incident SHOULD be opened because XML validation failed
# and PIXEL_BYPASSED did not override it
mock_analyze.assert_called_once()
args, kwargs = mock_analyze.call_args
assert "Pre-execution validation failed" in args[2]
incident = result["last_execution_result"]["incident"]
assert "Pre-execution validation failed" in incident["reason"]
@pytest.mark.asyncio
@@ -845,14 +875,8 @@ async def test_validator_failure_screenshot_mcp_error_handled_safely(
node = ValidatorNode(mock_context)
with (
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
):
mock_analyze.return_value = {
"status": "cannot_fix",
"analysis": "Action failed and cannot fix.",
}
result = await node(state)
assert "last_execution_result" in result
+4 -12
View File
@@ -209,9 +209,7 @@ async def test_video_coordinator_retries_transient_failures():
async def test_audio_analysis_reuses_exact_completed_coverage(analyzer_context):
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
analyzer = VideoAnalyzer(analyzer_context)
claim = analyzer.blackboard.claim_segment(
2.0, 6.0, "hear notification", modality="audio"
)
claim = analyzer.blackboard.claim_segment(2.0, 6.0, "hear notification", modality="audio")
analyzer.blackboard.complete_segment(
2.0,
6.0,
@@ -222,12 +220,8 @@ async def test_audio_analysis_reuses_exact_completed_coverage(analyzer_context):
modality="audio",
)
with patch(
"artemis.agents.video_analyzer.video_analyzer.get_controller"
) as controller:
result = await analyzer.exec_analyze_audio_only(
2.0, 6.0, "hear notification"
)
with patch("artemis.agents.video_analyzer.video_analyzer.get_controller") as controller:
result = await analyzer.exec_analyze_audio_only(2.0, 6.0, "hear notification")
controller.assert_not_called()
assert result.startswith("CACHED AUDIO ANALYSIS:")
@@ -272,9 +266,7 @@ def test_session_deletion_removes_video_blackboard_rows(tmp_path):
engine.record_video_start(video_id, "device", recording)
board = VideoBlackboard(f"video:{video_id}", db_path=engine.storage.db_path)
board.add_observation(
{"start": 0.0, "end": 1.0, "target": "cleanup", "summary": "evidence"}
)
board.add_observation({"start": 0.0, "end": 1.0, "target": "cleanup", "summary": "evidence"})
assert board.list_observations()
engine.storage.delete_session(session_id)
+1 -3
View File
@@ -138,9 +138,7 @@ async def test_native_chunk_exhaustion_commits_universal_fallback(tmp_path):
duration_seconds=5.0,
warning=None,
)
controller = SimpleNamespace(
extract_segment_metadata=AsyncMock(return_value=extracted)
)
controller = SimpleNamespace(extract_segment_metadata=AsyncMock(return_value=extracted))
fallback_text = "[from 0.0s to 5.0s] Summary: fallback recovered Analysis: grounded"
with (