somew1nd
67248c887e
Add execution incidents, action bursts and shared Flash history. Expose verification and Explorer settings with streamed check results. Support graceful cancellation, recording recovery and bundled UI resources. Add CI quality checks and clean up comments and documentation.
464 lines
18 KiB
Python
464 lines
18 KiB
Python
# Copyright 2026 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""Explorer in-memory screen search, label registry, bounds enrichment and pre-pass."""
|
|
|
|
import json
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from artemis.agents.explorer.explorer import Explorer
|
|
from artemis.agents.explorer.screen_index import ScreenElement, ScreenIndex
|
|
from artemis.context import ArtemisContext
|
|
from artemis.graph.state import State
|
|
|
|
FACADE = "artemis.agents.explorer.explorer"
|
|
W, H = 1080, 2400
|
|
|
|
#: One "Settings" button, two "Send" buttons and two OCR-only texts.
|
|
HIERARCHY = [
|
|
{
|
|
"text": "Settings",
|
|
"bounds": "[500,400][600,560]",
|
|
"class": "android.widget.Button",
|
|
"resource-id": "app:id/settings",
|
|
"clickable": "true",
|
|
},
|
|
{"text": "Send", "bounds": "[100,2000][300,2100]", "class": "android.widget.Button"},
|
|
{"text": "Send", "bounds": "[700,2000][900,2100]", "class": "android.widget.Button"},
|
|
{
|
|
"bounds": "[0,0][1080,2400]",
|
|
"class": "android.widget.FrameLayout",
|
|
"ocr_elements": [
|
|
{"text": "Settings page", "bounds": "[100,100][300,160]"},
|
|
{"text": "Welcome", "bounds": "[100,200][400,260]"},
|
|
],
|
|
},
|
|
]
|
|
|
|
#: "Settings" [500,400][600,560] on a 1080x2400 screen, in normalized units.
|
|
SETTINGS_COORDS = [509, 200]
|
|
SETTINGS_BOUNDS = [462, 166, 555, 233]
|
|
|
|
|
|
def _context() -> MagicMock:
|
|
ctx = MagicMock(spec=ArtemisContext)
|
|
ctx.device = MagicMock()
|
|
ctx.device.device_width = W
|
|
ctx.device.device_height = H
|
|
ctx.data_engine = MagicMock()
|
|
ctx.data_engine.base_dir = None
|
|
ctx.llm_config = MagicMock()
|
|
ctx.llm_config.explorer = MagicMock()
|
|
ctx.llm_config.explorer.model = "gemini-3.8-flash"
|
|
ctx.agent_config = MagicMock()
|
|
ctx.agent_config.denylisted_tools = {}
|
|
return ctx
|
|
|
|
|
|
def _state(screenshot: str, hierarchy=None) -> MagicMock:
|
|
state = MagicMock(spec=State)
|
|
state.latest_screenshot = screenshot
|
|
state.latest_ui_hierarchy = HIERARCHY if hierarchy is None else hierarchy
|
|
state.operator_raw_data = {}
|
|
return state
|
|
|
|
|
|
def _indexed_explorer(hierarchy=None) -> Explorer:
|
|
"""An Explorer whose screen is already indexed (no ``run`` involved)."""
|
|
explorer = Explorer(_context())
|
|
explorer.width, explorer.height = W, H
|
|
explorer.screenshot_path = "/tmp/screen.jpg"
|
|
explorer.screen_index = ScreenIndex.from_hierarchy(
|
|
HIERARCHY if hierarchy is None else hierarchy, W, H
|
|
)
|
|
return explorer
|
|
|
|
|
|
def _storage(record=None) -> MagicMock:
|
|
storage = MagicMock()
|
|
storage.get_image.return_value = record
|
|
return storage
|
|
|
|
|
|
def _outcome(candidates=None, fallback: str = "") -> str:
|
|
return json.dumps({"candidates": candidates or [], "fallback_message": fallback})
|
|
|
|
|
|
@pytest.fixture
|
|
def screenshot(tmp_path):
|
|
path = tmp_path / "screen.jpg"
|
|
path.write_bytes(b"fake-jpeg-bytes")
|
|
return str(path)
|
|
|
|
|
|
@pytest.fixture
|
|
def no_drawing():
|
|
"""Keeps the annotation helpers away from the file system."""
|
|
with (
|
|
patch(f"{FACADE}.draw_dots") as draw_dots,
|
|
patch("pathlib.Path.mkdir"),
|
|
patch("glob.glob", return_value=[]),
|
|
):
|
|
yield draw_dots
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Screen index construction inside run()
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_index_is_built_from_state_hierarchy_when_record_is_missing(screenshot, no_drawing):
|
|
explorer = Explorer(_context())
|
|
explorer._run_flash = AsyncMock(return_value=_outcome(fallback="nothing"))
|
|
|
|
with (
|
|
patch(f"{FACADE}.StorageManager", return_value=_storage(None)),
|
|
patch(f"{FACADE}.perform_ocr", new_callable=AsyncMock) as perform_ocr,
|
|
):
|
|
await explorer.run("gear icon", "", screenshot, _state(screenshot), version="flash")
|
|
text = (await explorer._search_ui_helper("Settings", prefix="X"))["text"]
|
|
|
|
perform_ocr.assert_not_awaited()
|
|
assert len(explorer.screen_index) == 5
|
|
assert "not found in Data Engine" not in text
|
|
assert "[X1] 'Settings' at [509,200]" in text
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_index_ignores_state_hierarchy_for_another_screenshot(screenshot):
|
|
explorer = Explorer(_context())
|
|
explorer._run_flash = AsyncMock(return_value=_outcome())
|
|
state = _state("/tmp/other.jpg")
|
|
|
|
with patch(f"{FACADE}.StorageManager", return_value=_storage(None)):
|
|
await explorer.run("Settings", "", screenshot, state, version="flash")
|
|
|
|
assert len(explorer.screen_index) == 0
|
|
explorer._run_flash.assert_awaited_once_with("Settings", screenshot)
|
|
assert (await explorer._search_ui_helper("Settings"))["text"] == (
|
|
"No UI-tree data is available for this screen; rely on visual detection."
|
|
)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Perception helpers answer from the index
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_search_ui_helper_merges_lenient_matches_without_duplicates(no_drawing):
|
|
hierarchy = [*HIERARCHY, {"text": "Settled", "bounds": "[100,300][300,360]"}]
|
|
explorer = _indexed_explorer(hierarchy)
|
|
spy = MagicMock(wraps=explorer.screen_index.search_text)
|
|
explorer.screen_index.search_text = spy
|
|
|
|
result = await explorer._search_ui_helper("Settings", prefix="X", color="green")
|
|
|
|
assert [call.args[1] for call in spy.call_args_list] == [0.7, 0.4]
|
|
parts = result["text"].split(" | ")
|
|
assert parts[0] == "[X1] 'Settings' at [509,200]"
|
|
assert parts[1] == "[O2] 'Settings page' at [185,54]"
|
|
# "Settled" (similarity ~0.53) only arrives through the lenient pass...
|
|
assert any(part.startswith("[X") and "'Settled'" in part for part in parts)
|
|
# ...and the strict hits are not listed a second time.
|
|
assert sum("'Settings'" in part for part in parts) == 1
|
|
assert explorer.label_registry["X1"].text == "Settings"
|
|
assert explorer.label_registry["O2"].source == "ocr"
|
|
assert no_drawing.call_args.kwargs["color"] == "green"
|
|
assert result["image_path"] is not None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_search_ui_helper_skips_lenient_pass_with_enough_strict_hits(no_drawing):
|
|
explorer = _indexed_explorer()
|
|
strict = explorer.screen_index.search_text("Settings", 0.4) # 5 hits, all "strict"
|
|
explorer.screen_index.search_text = MagicMock(return_value=strict)
|
|
|
|
await explorer._search_ui_helper("Settings", prefix="X")
|
|
|
|
explorer.screen_index.search_text.assert_called_once_with("Settings", 0.7)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_search_by_coords_helper_reports_innermost_first(no_drawing):
|
|
explorer = _indexed_explorer(
|
|
[
|
|
{"text": "Card", "bounds": "[0,0][1080,1200]", "class": "android.widget.FrameLayout"},
|
|
{"text": "Title", "bounds": "[500,400][600,560]", "class": "android.widget.TextView"},
|
|
]
|
|
)
|
|
|
|
hit = await explorer._search_by_coords_helper(550, 200, prefix="X", color="blue")
|
|
miss = await explorer._search_by_coords_helper(10, 990, prefix="X", color="blue")
|
|
|
|
assert hit["text"] == (
|
|
"Matched element at [550,200]: [X1]"
|
|
" | 'Title' class=android.widget.TextView source=ui-tree"
|
|
" | 'Card' class=android.widget.FrameLayout source=ui-tree"
|
|
)
|
|
assert explorer.label_registry["X1"].text == "Title"
|
|
assert miss["text"] == "No elements found at [10,990]."
|
|
assert "X2" not in explorer.label_registry
|
|
# The blue dot is drawn for both probes at the pixel position.
|
|
assert no_drawing.call_count == 2
|
|
assert no_drawing.call_args_list[0].args[1] == [[594, 480]]
|
|
assert no_drawing.call_args_list[0].args[2] == ["X1"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_ocr_list_reads_the_index(no_drawing):
|
|
explorer = _indexed_explorer()
|
|
|
|
with patch(f"{FACADE}.StorageManager") as storage_cls:
|
|
result = await explorer.exec_get_ocr_list()
|
|
|
|
storage_cls.assert_not_called()
|
|
assert result["text"].splitlines() == [
|
|
"[O1] 'Settings page' coords: [185,54]",
|
|
"[O2] 'Welcome' coords: [231,95]",
|
|
]
|
|
assert result["image_path"] is not None
|
|
assert {k: v.source for k, v in explorer.label_registry.items()} == {"O1": "ocr", "O2": "ocr"}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_ocr_list_without_ocr_elements():
|
|
explorer = _indexed_explorer([{"text": "Only XML", "bounds": "[0,0][100,100]"}])
|
|
result = await explorer.exec_get_ocr_list()
|
|
assert result == {"text": "No text elements detected on the screen.", "image_path": None}
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Label registry from the initial numbered list
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_initial_list_registers_numbered_labels(no_drawing):
|
|
explorer = _indexed_explorer()
|
|
|
|
minimal_list, _marked = explorer._annotate_initial_screenshot(
|
|
HIERARCHY, "/tmp/screen.jpg", None, ""
|
|
)
|
|
|
|
assert minimal_list.splitlines()[0].startswith("[1] Text: 'Settings'")
|
|
assert sorted(explorer.label_registry) == ["1", "2", "3", "4", "5"]
|
|
settings = explorer.label_registry["1"]
|
|
assert settings.bounds == (500, 400, 600, 560)
|
|
assert settings.source == "xml" and settings.interactive # taken from the index
|
|
assert explorer.label_registry["4"].source == "ocr"
|
|
assert explorer.label_registry["4"].text == "Settings page"
|
|
assert explorer.global_label_idx == 6
|
|
assert no_drawing.call_args.args[1][0] == [550, 480]
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Submit enrichment
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_enrich_candidates_adds_normalized_bounds_only_when_label_and_coords_agree():
|
|
explorer = _indexed_explorer()
|
|
settings = explorer.screen_index.exact_matches("Settings")[0]
|
|
explorer._register_label("X1", settings)
|
|
|
|
inside = {"label": "X1", "coords": SETTINGS_COORDS, "description": "gear"}
|
|
# 614px is 14px right of the element, inside the 2% (21.6px) tolerance.
|
|
near = {"label": "X1", "coords": [569, 200], "description": "gear"}
|
|
far = {"label": "X1", "coords": [100, 900], "description": "gear"}
|
|
unknown = {"label": "D7", "coords": SETTINGS_COORDS, "description": "gear"}
|
|
invalid = {"label": "X1", "coords": [5000, 5], "description": "gear"}
|
|
|
|
enriched = explorer._enrich_candidates([inside, near, far, unknown, invalid, "junk"])
|
|
|
|
assert enriched[0] == {**inside, "bounds": SETTINGS_BOUNDS, "source": "xml"}
|
|
assert enriched[1] == {**near, "bounds": SETTINGS_BOUNDS, "source": "xml"}
|
|
assert enriched[2:] == [far, unknown, invalid, "junk"]
|
|
assert "bounds" not in far and "bounds" not in inside # inputs are never mutated
|
|
|
|
|
|
def test_enrich_candidates_withholds_degenerate_bounds():
|
|
explorer = _indexed_explorer()
|
|
explorer._register_label("X9", ScreenElement("dot", (10, 10, 11, 11), "xml"))
|
|
cand = {"label": "X9", "coords": [9, 4]}
|
|
assert explorer._enrich_candidates([cand]) == [cand]
|
|
|
|
|
|
def test_native_submit_enriches_registered_candidates():
|
|
explorer = _indexed_explorer()
|
|
explorer._register_label("O2", explorer.screen_index.ocr_elements[0])
|
|
call = MagicMock()
|
|
call.name = "submit_answer"
|
|
call.args = {
|
|
"candidates": [
|
|
{"label": "O2", "coords": [185, 54], "description": "header"},
|
|
{"label": "D3", "coords": [900, 100], "description": "icon"},
|
|
],
|
|
"fallback_message": "",
|
|
}
|
|
turn_record = {"tool_calls": []}
|
|
|
|
raw = explorer._process_submit_answer([call], turn_record, [])
|
|
|
|
data = json.loads(raw)
|
|
assert list(data) == ["candidates", "fallback_message"]
|
|
assert data["candidates"][0] == {
|
|
"label": "O2",
|
|
"coords": [185, 54],
|
|
"description": "header",
|
|
"bounds": [92, 41, 277, 66],
|
|
"source": "ocr",
|
|
}
|
|
assert data["candidates"][1] == call.args["candidates"][1]
|
|
|
|
|
|
def test_universal_submit_enriches_registered_candidates():
|
|
explorer = _indexed_explorer()
|
|
explorer._register_label("1", explorer.screen_index.exact_matches("Settings")[0])
|
|
tool_calls = [
|
|
{
|
|
"name": "submit_answer",
|
|
"args": {
|
|
"candidates": [
|
|
{"label": "1", "coords": SETTINGS_COORDS, "description": "gear"},
|
|
{"label": "1", "coords": [5, 5000], "description": "bad"},
|
|
],
|
|
"fallback_message": "note",
|
|
},
|
|
"id": "c1",
|
|
}
|
|
]
|
|
|
|
data = json.loads(explorer._universal_submit_outcome(tool_calls))
|
|
|
|
assert data["fallback_message"] == "note"
|
|
assert data["candidates"] == [
|
|
{
|
|
"label": "1",
|
|
"coords": SETTINGS_COORDS,
|
|
"description": "gear",
|
|
"bounds": SETTINGS_BOUNDS,
|
|
"source": "xml",
|
|
}
|
|
]
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Structural pre-pass
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
@pytest.mark.parametrize("version", ["flash", "pro"])
|
|
@pytest.mark.asyncio
|
|
async def test_prepass_answers_unique_exact_match_without_the_engine(screenshot, version):
|
|
explorer = Explorer(_context())
|
|
explorer._run_flash = AsyncMock()
|
|
explorer._run_loop = AsyncMock()
|
|
|
|
with patch(f"{FACADE}.StorageManager", return_value=_storage(None)):
|
|
raw = await explorer.run(" settings ", "", screenshot, _state(screenshot), version=version)
|
|
|
|
explorer._run_flash.assert_not_called()
|
|
explorer._run_loop.assert_not_called()
|
|
assert json.loads(raw) == {
|
|
"candidates": [
|
|
{
|
|
"label": "T1",
|
|
"coords": SETTINGS_COORDS,
|
|
"bounds": SETTINGS_BOUNDS,
|
|
"source": "xml",
|
|
"description": "Settings (exact text match)",
|
|
}
|
|
],
|
|
"fallback_message": "",
|
|
}
|
|
assert explorer.label_registry["T1"].text == "Settings"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prepass_leaves_ambiguous_matches_to_the_engine(screenshot):
|
|
explorer = Explorer(_context())
|
|
engine = _outcome([{"label": "D1", "coords": [185, 854], "description": "Send"}])
|
|
explorer._run_flash = AsyncMock(return_value=engine)
|
|
|
|
with patch(f"{FACADE}.StorageManager", return_value=_storage(None)):
|
|
raw = await explorer.run("Send", "", screenshot, _state(screenshot), version="flash")
|
|
|
|
explorer._run_flash.assert_awaited_once_with("Send", screenshot)
|
|
assert raw == engine
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prepass_without_pipe_or_match_passes_the_query_through(screenshot):
|
|
explorer = Explorer(_context())
|
|
engine = _outcome(fallback="Failed to detect: gear icon")
|
|
explorer._run_flash = AsyncMock(return_value=engine)
|
|
|
|
with patch(f"{FACADE}.StorageManager", return_value=_storage(None)):
|
|
raw = await explorer.run("gear icon", "", screenshot, _state(screenshot), version="flash")
|
|
|
|
explorer._run_flash.assert_awaited_once_with("gear icon", screenshot)
|
|
assert raw == engine
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prepass_merges_partial_results_with_the_engine_outcome(screenshot):
|
|
explorer = Explorer(_context())
|
|
detected = {"label": "D1", "coords": [185, 854], "description": "Send"}
|
|
explorer._run_loop = AsyncMock(return_value=_outcome([detected], fallback="two Send buttons"))
|
|
|
|
with patch(f"{FACADE}.StorageManager", return_value=_storage(None)):
|
|
raw = await explorer.run(
|
|
"Settings | Send | Welcome", "", screenshot, _state(screenshot), version="pro"
|
|
)
|
|
|
|
# Only the unresolved part reaches the engine; "Welcome" is an OCR exact match.
|
|
assert explorer._run_loop.await_args.args[1] == "Send"
|
|
data = json.loads(raw)
|
|
assert [c["label"] for c in data["candidates"]] == ["T1", "T2", "D1"]
|
|
assert data["candidates"][1]["source"] == "ocr"
|
|
assert data["candidates"][1]["description"] == "Welcome (exact text match)"
|
|
assert data["candidates"][2] == detected
|
|
assert data["fallback_message"] == "two Send buttons"
|
|
|
|
|
|
def test_merge_prepass_keeps_structural_hits_over_an_unreadable_engine_answer():
|
|
prepass = [{"label": "T1", "coords": [1, 2]}]
|
|
merged = json.loads(Explorer._merge_prepass(prepass, "Error: max iterations"))
|
|
assert merged == {"candidates": prepass, "fallback_message": "Error: max iterations"}
|
|
assert Explorer._merge_prepass([], "raw") == "raw"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prepass_never_runs_on_the_fly_ocr_for_flash(screenshot):
|
|
record = SimpleNamespace(ui_tree=HIERARCHY[:3], ocr_result=None)
|
|
explorer = Explorer(_context())
|
|
explorer._run_flash = AsyncMock(return_value=_outcome())
|
|
explorer._run_loop = AsyncMock(return_value=_outcome())
|
|
|
|
with (
|
|
patch(f"{FACADE}.StorageManager", return_value=_storage(record)),
|
|
patch(f"{FACADE}.is_ocr_configured", return_value=True),
|
|
patch(f"{FACADE}.perform_ocr", new_callable=AsyncMock, return_value=[]) as perform_ocr,
|
|
):
|
|
await explorer.run("gear icon", "", screenshot, _state(screenshot, []), version="flash")
|
|
perform_ocr.assert_not_awaited()
|
|
assert len(explorer.screen_index) == 3 # the record still feeds the index
|
|
|
|
await explorer.run("gear icon", "", screenshot, _state(screenshot, []), version="pro")
|
|
perform_ocr.assert_awaited_once()
|