somew1nd
67248c887e
Add execution incidents, action bursts and shared Flash history. Expose verification and Explorer settings with streamed check results. Support graceful cancellation, recording recovery and bundled UI resources. Add CI quality checks and clean up comments and documentation.
645 lines
25 KiB
Python
645 lines
25 KiB
Python
# Copyright 2026 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""``ask_explorer``: tier-agnostic contract and the locate / register / render pipeline."""
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from artemis.agents.explorer.constants import (
|
|
ASK_EXPLORER_CONTEXT_FEEDBACK_DESCRIPTION,
|
|
ASK_EXPLORER_DESCRIPTION,
|
|
ASK_EXPLORER_QUERY_DESCRIPTION,
|
|
)
|
|
from artemis.context import ArtemisContext
|
|
from artemis.graph.state import State
|
|
from artemis.tools.base import ArtemisTool, ToolRegistry
|
|
from artemis.tools.explorer_tool import (
|
|
AskExplorerArgs,
|
|
AskExplorerTool,
|
|
ExplorerCandidate,
|
|
ExplorerOutcome,
|
|
RegisteredCandidate,
|
|
_run_explorer_logic,
|
|
ask_explorer,
|
|
ask_explorer_text,
|
|
ask_explorer_wrapper,
|
|
get_ask_explorer_tool,
|
|
locate,
|
|
register_candidates,
|
|
render_operator_blocks,
|
|
render_text,
|
|
)
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Fixtures
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _no_env_tier(monkeypatch):
|
|
"""The environment override must not leak into tier-resolution assertions."""
|
|
monkeypatch.delenv("ARTEMIS_EXPLORER_VERSION", raising=False)
|
|
|
|
|
|
def _ctx(width: int | None = 1080, height: int | None = 2400, base_dir: Path | None = None):
|
|
ctx = MagicMock(spec=ArtemisContext)
|
|
ctx.agent_config = None
|
|
ctx.llm_config = None
|
|
ctx.execution_setup = None
|
|
if base_dir is None:
|
|
ctx.data_engine = None
|
|
else:
|
|
ctx.data_engine = MagicMock()
|
|
ctx.data_engine.base_dir = str(base_dir)
|
|
if width is None:
|
|
ctx.device = None
|
|
else:
|
|
ctx.device = MagicMock()
|
|
ctx.device.device_width = width
|
|
ctx.device.device_height = height
|
|
return ctx
|
|
|
|
|
|
def _state(screenshot: str | None = "/tmp/test.jpg", raw: dict | None = None):
|
|
state = MagicMock(spec=State)
|
|
state.latest_screenshot = screenshot
|
|
state.indexed_points = []
|
|
state.indexed_elements = []
|
|
state.operator_raw_data = raw
|
|
return state
|
|
|
|
|
|
def _explorer_returning(raw):
|
|
instance = MagicMock()
|
|
instance.run = AsyncMock(return_value=raw)
|
|
return patch("artemis.tools.explorer_tool.Explorer", return_value=instance), instance
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Contract
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_tool_contract_is_tier_agnostic():
|
|
assert issubclass(AskExplorerTool, ArtemisTool)
|
|
assert isinstance(ask_explorer, AskExplorerTool)
|
|
assert ask_explorer.name == "ask_explorer"
|
|
assert ask_explorer.category == "explorer"
|
|
assert ask_explorer.args_schema is AskExplorerArgs
|
|
|
|
fields = AskExplorerArgs.model_fields
|
|
assert set(fields) == {"query", "context_feedback"}
|
|
assert fields["query"].is_required()
|
|
assert fields["query"].description == ASK_EXPLORER_QUERY_DESCRIPTION
|
|
assert fields["context_feedback"].default == ""
|
|
assert fields["context_feedback"].description == ASK_EXPLORER_CONTEXT_FEEDBACK_DESCRIPTION
|
|
|
|
declaration = ask_explorer.to_genai_declaration()
|
|
assert declaration.name == "ask_explorer"
|
|
assert set(declaration.parameters.properties) == {"query", "context_feedback"}
|
|
assert declaration.parameters.required == ["query"]
|
|
|
|
|
|
@pytest.mark.parametrize("version", [None, "flash", "pro", "ultra"])
|
|
def test_description_does_not_depend_on_the_tier(version):
|
|
tool = AskExplorerTool(version=version)
|
|
assert tool.description == ASK_EXPLORER_DESCRIPTION
|
|
assert tool.version == version
|
|
|
|
|
|
def test_registry_and_wrapper_export():
|
|
reg_tool = ToolRegistry.get("ask_explorer")
|
|
assert isinstance(reg_tool, AskExplorerTool)
|
|
assert ask_explorer_wrapper.tool_fn_getter is get_ask_explorer_tool
|
|
assert ask_explorer_wrapper.on_success_fn("x") == "x"
|
|
assert "boom" in ask_explorer_wrapper.on_failure_fn("boom")
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# ExplorerOutcome.from_raw
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_from_raw_parses_candidates_and_message():
|
|
raw = json.dumps(
|
|
{
|
|
"candidates": [
|
|
{"label": "S1", "coords": [500, 500], "description": "First Button"},
|
|
{"label": "S2", "coords": [250, 750], "description": "Second Button"},
|
|
],
|
|
"fallback_message": "Please choose carefully.",
|
|
}
|
|
)
|
|
outcome = ExplorerOutcome.from_raw(raw)
|
|
assert outcome.found and not outcome.error
|
|
assert outcome.candidates == [
|
|
ExplorerCandidate(label="S1", coords=(500, 500), description="First Button"),
|
|
ExplorerCandidate(label="S2", coords=(250, 750), description="Second Button"),
|
|
]
|
|
assert outcome.message == "Please choose carefully."
|
|
assert outcome.raw == raw
|
|
|
|
|
|
def test_from_raw_accepts_a_dict_and_fills_defaults():
|
|
outcome = ExplorerOutcome.from_raw(
|
|
{"candidates": [{"coords": ["10", "20"]}]}, default_label="gear icon"
|
|
)
|
|
assert len(outcome.candidates) == 1
|
|
cand = outcome.candidates[0]
|
|
assert cand.label == "gear icon"
|
|
assert cand.description == "gear icon"
|
|
assert cand.coords == (10, 20)
|
|
assert outcome.message == ""
|
|
|
|
|
|
def test_from_raw_drops_candidates_with_invalid_coords():
|
|
outcome = ExplorerOutcome.from_raw(
|
|
{
|
|
"candidates": [
|
|
{"label": "bad1", "coords": [1500, 10]},
|
|
{"label": "bad2", "coords": [10]},
|
|
{"label": "bad3", "coords": "10,10"},
|
|
{"label": "bad4"},
|
|
"not a dict",
|
|
{"label": "ok", "coords": [10, 10]},
|
|
]
|
|
}
|
|
)
|
|
assert [c.label for c in outcome.candidates] == ["ok"]
|
|
assert not outcome.error
|
|
|
|
|
|
def test_from_raw_unreadable_text_is_an_error_outcome():
|
|
outcome = ExplorerOutcome.from_raw("I could not parse the screen {")
|
|
assert outcome.error and not outcome.found
|
|
assert "unreadable answer" in outcome.message
|
|
assert outcome.raw == "I could not parse the screen {"
|
|
|
|
|
|
def test_from_raw_non_object_json_is_an_error_outcome():
|
|
outcome = ExplorerOutcome.from_raw("[1, 2, 3]")
|
|
assert outcome.error
|
|
assert "unexpected answer" in outcome.message
|
|
|
|
|
|
def test_from_raw_empty_candidates_is_a_clean_not_found():
|
|
outcome = ExplorerOutcome.from_raw({"candidates": [], "fallback_message": "Covered."})
|
|
assert not outcome.found and not outcome.error
|
|
assert outcome.message == "Covered."
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# register_candidates
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_register_candidates_appends_after_existing_elements_using_observation_size():
|
|
ctx = _ctx()
|
|
state = _state(raw={"width": 1000, "height": 2000})
|
|
state.indexed_points = [[100, 200]]
|
|
state.indexed_elements = [{"index": 1, "center": [100, 200], "text": "Pre-existing"}]
|
|
outcome = ExplorerOutcome(
|
|
candidates=[
|
|
ExplorerCandidate(label="S1", coords=(500, 500), description="First Button"),
|
|
ExplorerCandidate(label="S2", coords=(250, 750), description="Second Button"),
|
|
]
|
|
)
|
|
|
|
registered = register_candidates(ctx, state, outcome)
|
|
|
|
assert registered == [
|
|
RegisteredCandidate(
|
|
index=2, pixel=(500, 1000), coords=(500, 500), description="First Button"
|
|
),
|
|
RegisteredCandidate(
|
|
index=3, pixel=(250, 1500), coords=(250, 750), description="Second Button"
|
|
),
|
|
]
|
|
assert state.indexed_points == [[100, 200], [500, 1000], [250, 1500]]
|
|
assert len(state.indexed_elements) == 3
|
|
assert state.indexed_elements[1] == {
|
|
"index": 2,
|
|
"center": [500, 1000],
|
|
"text": "First Button",
|
|
"bounds": None,
|
|
"class": "ExplorerCandidate",
|
|
"resource_id": None,
|
|
"is_ocr": False,
|
|
}
|
|
assert state.indexed_elements[2]["index"] == 3
|
|
assert state.indexed_elements[2]["center"] == [250, 1500]
|
|
|
|
|
|
def test_register_candidates_falls_back_to_the_device_size():
|
|
ctx = _ctx(1080, 2400)
|
|
state = _state(raw=None)
|
|
outcome = ExplorerOutcome(candidates=[ExplorerCandidate("S1", (500, 500), "Btn")])
|
|
registered = register_candidates(ctx, state, outcome)
|
|
assert registered[0].pixel == (540, 1200)
|
|
assert registered[0].index == 1
|
|
|
|
|
|
def test_register_candidates_creates_the_lists_when_the_state_has_none():
|
|
ctx = _ctx()
|
|
state = MagicMock(spec=State)
|
|
state.indexed_points = None
|
|
state.indexed_elements = None
|
|
state.operator_raw_data = None
|
|
outcome = ExplorerOutcome(candidates=[ExplorerCandidate("S1", (0, 0), "Origin")])
|
|
registered = register_candidates(ctx, state, outcome)
|
|
assert registered[0].index == 1
|
|
assert state.indexed_points == [[0, 0]]
|
|
assert state.indexed_elements[0]["text"] == "Origin"
|
|
|
|
|
|
def test_register_candidates_is_a_no_op_without_candidates():
|
|
ctx = _ctx()
|
|
state = _state()
|
|
assert register_candidates(ctx, state, ExplorerOutcome(message="nope")) == []
|
|
assert register_candidates(ctx, state, ExplorerOutcome.failure("boom")) == []
|
|
assert state.indexed_points == []
|
|
assert state.indexed_elements == []
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# render_text / render_operator_blocks
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_render_text_found_lists_indices_and_coordinates():
|
|
outcome = ExplorerOutcome(
|
|
candidates=[ExplorerCandidate("S1", (500, 500), "First Button")],
|
|
message="Please choose carefully.",
|
|
)
|
|
registered = [RegisteredCandidate(2, (540, 1200), (500, 500), "First Button")]
|
|
text = render_text("Find buttons", outcome, registered)
|
|
assert "Explorer located 1 candidate(s) for 'Find buttons'" in text
|
|
assert "- [2] 'First Button' at normalized [500, 500]" in text
|
|
assert "target=[index]" in text
|
|
assert "Explorer notes: Please choose carefully." in text
|
|
|
|
|
|
def test_render_text_not_found_explains_and_suggests_rephrasing():
|
|
text = render_text("hidden button", ExplorerOutcome(message="Covered by keyboard."), [])
|
|
assert text.startswith("Explorer could not locate 'hidden button'. Covered by keyboard.")
|
|
assert "describing the target differently" in text
|
|
# Without an explanation the text still reads as a complete sentence.
|
|
text = render_text("x", ExplorerOutcome(), [])
|
|
assert "It gave no further detail." in text
|
|
|
|
|
|
def test_render_text_error_reports_the_failure():
|
|
text = render_text("gear icon", ExplorerOutcome.failure("Explorer failed: quota"), [])
|
|
assert text == "Explorer could not run for 'gear icon'. Explorer failed: quota"
|
|
|
|
|
|
def test_render_operator_blocks_adds_the_annotated_image(tmp_path):
|
|
ctx = _ctx(base_dir=tmp_path)
|
|
state = _state(screenshot=str(tmp_path / "shot.jpg"))
|
|
outcome = ExplorerOutcome(candidates=[ExplorerCandidate("S1", (500, 500), "First Button")])
|
|
registered = [RegisteredCandidate(2, (540, 1200), (500, 500), "First Button")]
|
|
|
|
def fake_draw_dots(screenshot_path, points, labels, output_path, **kwargs):
|
|
Path(output_path).write_bytes(b"fake_annotated_image_bytes")
|
|
|
|
with patch("artemis.tools.explorer_tool.draw_dots", side_effect=fake_draw_dots) as draw:
|
|
result = render_operator_blocks(ctx, state, "Find buttons", outcome, registered)
|
|
|
|
draw.assert_called_once()
|
|
args, kwargs = draw.call_args
|
|
assert args[0] == str(tmp_path / "shot.jpg")
|
|
assert args[1] == [[540, 1200]]
|
|
assert args[2] == ["2"]
|
|
assert Path(args[3]).parent == tmp_path / "images" / "explorer_tool"
|
|
assert kwargs["color"] == "magenta"
|
|
|
|
assert isinstance(result, list) and len(result) == 2
|
|
assert result[0]["type"] == "text"
|
|
assert "Explorer located 1 candidate(s)" in result[0]["text"]
|
|
assert result[1]["type"] == "image_url"
|
|
assert result[1]["image_url"]["url"].startswith("data:image/jpeg;base64,")
|
|
|
|
|
|
def test_render_operator_blocks_numbers_annotations_sequentially(tmp_path):
|
|
ctx = _ctx(base_dir=tmp_path)
|
|
state = _state(screenshot=str(tmp_path / "shot.jpg"))
|
|
outcome = ExplorerOutcome(candidates=[ExplorerCandidate("S1", (500, 500), "B")])
|
|
registered = [RegisteredCandidate(1, (540, 1200), (500, 500), "B")]
|
|
seen: list[str] = []
|
|
|
|
def fake_draw_dots(screenshot_path, points, labels, output_path, **kwargs):
|
|
seen.append(Path(output_path).name)
|
|
Path(output_path).write_bytes(b"img")
|
|
|
|
with patch("artemis.tools.explorer_tool.draw_dots", side_effect=fake_draw_dots):
|
|
render_operator_blocks(ctx, state, "B", outcome, registered)
|
|
render_operator_blocks(ctx, state, "B", outcome, registered)
|
|
assert seen == ["explorer_output_1.jpg", "explorer_output_2.jpg"]
|
|
|
|
|
|
def test_render_operator_blocks_degrades_to_text_when_drawing_fails(tmp_path):
|
|
ctx = _ctx(base_dir=tmp_path)
|
|
state = _state(screenshot=str(tmp_path / "shot.jpg"))
|
|
outcome = ExplorerOutcome(candidates=[ExplorerCandidate("S1", (500, 500), "First Button")])
|
|
registered = [RegisteredCandidate(2, (540, 1200), (500, 500), "First Button")]
|
|
|
|
with patch("artemis.tools.explorer_tool.draw_dots", side_effect=OSError("no PIL")):
|
|
result = render_operator_blocks(ctx, state, "Find buttons", outcome, registered)
|
|
|
|
assert isinstance(result, str)
|
|
assert "Explorer located 1 candidate(s)" in result
|
|
|
|
|
|
def test_render_operator_blocks_is_plain_text_without_candidates(tmp_path):
|
|
ctx = _ctx(base_dir=tmp_path)
|
|
state = _state(screenshot=str(tmp_path / "shot.jpg"))
|
|
with patch("artemis.tools.explorer_tool.draw_dots") as draw:
|
|
result = render_operator_blocks(ctx, state, "x", ExplorerOutcome(message="Nope."), [])
|
|
draw.assert_not_called()
|
|
assert isinstance(result, str)
|
|
assert "Explorer could not locate 'x'. Nope." in result
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# locate
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_locate_runs_the_resolved_tier_and_parses_the_answer():
|
|
ctx = _ctx()
|
|
state = _state()
|
|
raw = json.dumps({"candidates": [{"label": "S1", "coords": [500, 500]}]})
|
|
explorer_patch, instance = _explorer_returning(raw)
|
|
with (
|
|
explorer_patch,
|
|
patch(
|
|
"artemis.tools.explorer_tool.resolve_explorer_version", return_value="ultra"
|
|
) as resolve,
|
|
):
|
|
outcome = await locate(ctx, state, "gear icon", "was wrong", agent_name="validator")
|
|
|
|
resolve.assert_called_once_with(ctx, explicit_version=None, agent_or_profile_name="validator")
|
|
instance.run.assert_awaited_once_with(
|
|
"gear icon", "was wrong", "/tmp/test.jpg", state, version="ultra"
|
|
)
|
|
assert outcome.found and not outcome.error
|
|
assert outcome.candidates[0].coords == (500, 500)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_locate_forwards_a_programmatic_version_pin():
|
|
ctx = _ctx()
|
|
state = _state()
|
|
explorer_patch, instance = _explorer_returning(json.dumps({"candidates": []}))
|
|
with (
|
|
explorer_patch,
|
|
patch(
|
|
"artemis.tools.explorer_tool.resolve_explorer_version", return_value="pro"
|
|
) as resolve,
|
|
):
|
|
await locate(ctx, state, "q", version="pro", agent_name="flash")
|
|
resolve.assert_called_once_with(ctx, explicit_version="pro", agent_or_profile_name="flash")
|
|
assert instance.run.await_args.kwargs["version"] == "pro"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_locate_unknown_resolved_tier_falls_back_to_the_default_tier():
|
|
ctx = _ctx()
|
|
state = _state()
|
|
explorer_patch, instance = _explorer_returning(json.dumps({"candidates": []}))
|
|
with (
|
|
explorer_patch,
|
|
patch("artemis.tools.explorer_tool.resolve_explorer_version", return_value="turbo"),
|
|
):
|
|
await locate(ctx, state, "q")
|
|
assert instance.run.await_args.kwargs["version"] == "flash"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_locate_without_a_screenshot_is_an_error_outcome():
|
|
ctx = _ctx()
|
|
state = _state(screenshot=None)
|
|
with patch("artemis.tools.explorer_tool.Explorer") as explorer_cls:
|
|
outcome = await locate(ctx, state, "gear icon")
|
|
explorer_cls.assert_not_called()
|
|
assert outcome.error
|
|
assert "No screenshot" in outcome.message
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_locate_contains_explorer_exceptions_as_error_outcomes():
|
|
ctx = _ctx()
|
|
state = _state()
|
|
instance = MagicMock()
|
|
instance.run = AsyncMock(side_effect=RuntimeError("quota exhausted"))
|
|
with patch("artemis.tools.explorer_tool.Explorer", return_value=instance):
|
|
outcome = await locate(ctx, state, "gear icon")
|
|
assert outcome.error and not outcome.found
|
|
assert outcome.message == "Explorer failed: quota exhausted"
|
|
assert "could not run for 'gear icon'" in render_text("gear icon", outcome, [])
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Entry points
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_ask_explorer_text_end_to_end():
|
|
ctx = _ctx()
|
|
state = _state()
|
|
state.indexed_points = [[100, 200]]
|
|
state.indexed_elements = [{"index": 1, "center": [100, 200], "text": "Pre-existing"}]
|
|
raw = json.dumps(
|
|
{
|
|
"candidates": [
|
|
{"label": "S1", "coords": [500, 500], "description": "First Button"},
|
|
{"label": "S2", "coords": [250, 750], "description": "Second Button"},
|
|
],
|
|
"fallback_message": "Please choose carefully.",
|
|
}
|
|
)
|
|
explorer_patch, instance = _explorer_returning(raw)
|
|
with explorer_patch:
|
|
text = await ask_explorer_text(ctx, state, "Find buttons", "Attempt 1", agent_name="flash")
|
|
|
|
instance.run.assert_awaited_once()
|
|
assert instance.run.await_args.args[:2] == ("Find buttons", "Attempt 1")
|
|
assert state.indexed_points == [[100, 200], [540, 1200], [270, 1800]]
|
|
assert state.indexed_elements[1]["text"] == "First Button"
|
|
assert state.indexed_elements[2]["text"] == "Second Button"
|
|
assert "Explorer located 2 candidate(s) for 'Find buttons'" in text
|
|
assert "- [2] 'First Button' at normalized [500, 500]" in text
|
|
assert "- [3] 'Second Button' at normalized [250, 750]" in text
|
|
assert "Explorer notes: Please choose carefully." in text
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_ask_explorer_text_not_found_leaves_the_state_untouched():
|
|
ctx = _ctx()
|
|
state = _state()
|
|
explorer_patch, _ = _explorer_returning(
|
|
json.dumps({"candidates": [], "fallback_message": "Covered by a keyboard."})
|
|
)
|
|
with explorer_patch:
|
|
text = await ask_explorer_text(ctx, state, "hidden button")
|
|
assert state.indexed_points == []
|
|
assert state.indexed_elements == []
|
|
assert "Explorer could not locate 'hidden button'. Covered by a keyboard." in text
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_langchain_tool_uses_the_operator_presentation(tmp_path):
|
|
ctx = _ctx(base_dir=tmp_path)
|
|
state = _state(screenshot=str(tmp_path / "shot.jpg"))
|
|
raw = json.dumps({"candidates": [{"label": "S1", "coords": [500, 500], "description": "B"}]})
|
|
explorer_patch, _ = _explorer_returning(raw)
|
|
|
|
def fake_draw_dots(screenshot_path, points, labels, output_path, **kwargs):
|
|
Path(output_path).write_bytes(b"img")
|
|
|
|
with explorer_patch, patch("artemis.tools.explorer_tool.draw_dots", side_effect=fake_draw_dots):
|
|
tool = get_ask_explorer_tool(ctx)
|
|
result = await tool.ainvoke(
|
|
{"query": "Find buttons", "context_feedback": "", "state": state}
|
|
)
|
|
|
|
assert tool.name == "ask_explorer"
|
|
assert isinstance(result, list) and result[1]["type"] == "image_url"
|
|
assert "- [1] 'B' at normalized [500, 500]" in result[0]["text"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_direct_execute_accepts_a_version_pin_and_legacy_arg_names():
|
|
ctx = _ctx(width=None)
|
|
state = _state()
|
|
explorer_patch, instance = _explorer_returning(
|
|
json.dumps({"candidates": [], "fallback_message": "Direct execution not found."})
|
|
)
|
|
with explorer_patch:
|
|
result = await ask_explorer.execute(
|
|
ctx=ctx, state=state, Query="Find icon", ContextFeedback="cf", version="pro"
|
|
)
|
|
assert instance.run.await_args.args[:2] == ("Find icon", "cf")
|
|
assert instance.run.await_args.kwargs["version"] == "pro"
|
|
assert "Direct execution not found." in result
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_run_explorer_logic_replay_shim_returns_operator_blocks(tmp_path):
|
|
ctx = _ctx(base_dir=tmp_path)
|
|
state = _state(screenshot=str(tmp_path / "shot.jpg"))
|
|
raw = json.dumps({"candidates": [{"label": "S1", "coords": [500, 500], "description": "B"}]})
|
|
explorer_patch, instance = _explorer_returning(raw)
|
|
|
|
def fake_draw_dots(screenshot_path, points, labels, output_path, **kwargs):
|
|
Path(output_path).write_bytes(b"img")
|
|
|
|
with explorer_patch, patch("artemis.tools.explorer_tool.draw_dots", side_effect=fake_draw_dots):
|
|
result = await _run_explorer_logic(ctx, state, "B", "", version="ultra")
|
|
|
|
assert instance.run.await_args.kwargs["version"] == "ultra"
|
|
assert isinstance(result, list) and len(result) == 2
|
|
assert result[0]["type"] == "text" and result[1]["type"] == "image_url"
|
|
assert state.indexed_points == [[540, 1200]]
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Bounds and de-duplication against the existing index
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_from_raw_parses_bounds_and_source_and_drops_invalid_bounds():
|
|
raw = json.dumps(
|
|
{
|
|
"candidates": [
|
|
{
|
|
"label": "X1",
|
|
"coords": [500, 500],
|
|
"bounds": [400, 450, 600, 550],
|
|
"source": "xml",
|
|
},
|
|
{"label": "D1", "coords": [100, 100], "bounds": [600, 550, 400, 450]},
|
|
{"label": "D2", "coords": [100, 200], "bounds": [0, 0, 1200, 10]},
|
|
]
|
|
}
|
|
)
|
|
outcome = ExplorerOutcome.from_raw(raw)
|
|
assert outcome.candidates[0].bounds == (400, 450, 600, 550)
|
|
assert outcome.candidates[0].source == "xml"
|
|
assert outcome.candidates[1].bounds is None
|
|
assert outcome.candidates[2].bounds is None
|
|
assert outcome.candidates[1].source == ""
|
|
|
|
|
|
def test_register_candidates_reuses_an_indexed_element_that_contains_the_point():
|
|
ctx = _ctx()
|
|
state = _state(raw={"width": 1000, "height": 2000})
|
|
state.indexed_points = [[300, 400], [800, 1600]]
|
|
state.indexed_elements = [
|
|
{"index": 1, "center": [300, 400], "text": "Gmail", "bounds": [200, 300, 400, 500]},
|
|
{"index": 2, "center": [800, 1600], "text": "Send", "bounds": [700, 1500, 900, 1700]},
|
|
]
|
|
outcome = ExplorerOutcome(candidates=[ExplorerCandidate("D1", (310, 210), "Gmail app icon")])
|
|
registered = register_candidates(ctx, state, outcome)
|
|
assert registered == [
|
|
RegisteredCandidate(
|
|
index=1,
|
|
pixel=(310, 420),
|
|
coords=(310, 210),
|
|
description="Gmail app icon",
|
|
bounds=(200, 300, 400, 500),
|
|
reused=True,
|
|
)
|
|
]
|
|
assert len(state.indexed_points) == 2 # nothing appended
|
|
assert "already in your indexed list" in render_text("Gmail app icon", outcome, registered)
|
|
|
|
|
|
def test_register_candidates_reuses_a_bare_center_within_the_dedup_radius_only():
|
|
ctx = _ctx()
|
|
state = _state(raw={"width": 1000, "height": 2000})
|
|
state.indexed_points = [[500, 1000]]
|
|
state.indexed_elements = [
|
|
{"index": 1, "center": [500, 1000], "text": "Earlier candidate", "bounds": None}
|
|
]
|
|
near = ExplorerCandidate("D1", (510, 505), "Same spot") # 10 x 10 px away
|
|
far = ExplorerCandidate("D2", (560, 500), "Sixty px right")
|
|
registered = register_candidates(ctx, state, ExplorerOutcome(candidates=[near, far]))
|
|
assert [r.reused for r in registered] == [True, False]
|
|
assert registered[0].index == 1
|
|
assert registered[1].index == 2
|
|
assert state.indexed_points == [[500, 1000], [560, 1000]]
|
|
|
|
|
|
def test_register_candidates_stores_pixel_bounds_and_ocr_flag():
|
|
ctx = _ctx()
|
|
state = _state(raw={"width": 1000, "height": 2000})
|
|
outcome = ExplorerOutcome(
|
|
candidates=[
|
|
ExplorerCandidate("O3", (500, 500), "Search", bounds=(400, 450, 600, 550), source="ocr")
|
|
]
|
|
)
|
|
registered = register_candidates(ctx, state, outcome)
|
|
assert registered[0].bounds == (400, 900, 600, 1100)
|
|
assert registered[0].reused is False
|
|
element = state.indexed_elements[0]
|
|
assert element["bounds"] == [400, 900, 600, 1100]
|
|
assert element["is_ocr"] is True
|