feat: improve task execution, verification and packaging
Add execution incidents, action bursts and shared Flash history. Expose verification and Explorer settings with streamed check results. Support graceful cancellation, recording recovery and bundled UI resources. Add CI quality checks and clean up comments and documentation.
This commit is contained in:
somew1nd committed
1 parent
5ccf06d677
commit
67248c887e
289 files changed
+15883
-6919
No files matched your search
@@ -12,364 +12,633 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import json
|
||||
from unittest.mock import AsyncMock, MagicMock, mock_open, patch
|
||||
"""``ask_explorer``: tier-agnostic contract and the locate / register / render pipeline."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.agents.explorer.constants import (
|
||||
ASK_EXPLORER_CONTEXT_FEEDBACK_DESCRIPTION,
|
||||
ASK_EXPLORER_DESCRIPTION,
|
||||
ASK_EXPLORER_QUERY_DESCRIPTION,
|
||||
)
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.explorer_tool import (
|
||||
AskExplorer,
|
||||
AskExplorerArgs,
|
||||
AskExplorerTool,
|
||||
AskExplorerToolAlias,
|
||||
ExplorerCandidate,
|
||||
ExplorerOutcome,
|
||||
RegisteredCandidate,
|
||||
_run_explorer_logic,
|
||||
ask_explorer,
|
||||
ask_explorer_text,
|
||||
ask_explorer_wrapper,
|
||||
get_ask_explorer_tool,
|
||||
locate,
|
||||
register_candidates,
|
||||
render_operator_blocks,
|
||||
render_text,
|
||||
)
|
||||
import pytest
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Fixtures
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_ask_explorer_tool_subclass_and_registry():
|
||||
"""Verify AskExplorerTool is a subclass of ArtemisTool and properly registered."""
|
||||
@pytest.fixture(autouse=True)
|
||||
def _no_env_tier(monkeypatch):
|
||||
"""The environment override must not leak into tier-resolution assertions."""
|
||||
monkeypatch.delenv("ARTEMIS_EXPLORER_VERSION", raising=False)
|
||||
|
||||
|
||||
def _ctx(width: int | None = 1080, height: int | None = 2400, base_dir: Path | None = None):
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.agent_config = None
|
||||
ctx.llm_config = None
|
||||
ctx.execution_setup = None
|
||||
if base_dir is None:
|
||||
ctx.data_engine = None
|
||||
else:
|
||||
ctx.data_engine = MagicMock()
|
||||
ctx.data_engine.base_dir = str(base_dir)
|
||||
if width is None:
|
||||
ctx.device = None
|
||||
else:
|
||||
ctx.device = MagicMock()
|
||||
ctx.device.device_width = width
|
||||
ctx.device.device_height = height
|
||||
return ctx
|
||||
|
||||
|
||||
def _state(screenshot: str | None = "/tmp/test.jpg", raw: dict | None = None):
|
||||
state = MagicMock(spec=State)
|
||||
state.latest_screenshot = screenshot
|
||||
state.indexed_points = []
|
||||
state.indexed_elements = []
|
||||
state.operator_raw_data = raw
|
||||
return state
|
||||
|
||||
|
||||
def _explorer_returning(raw):
|
||||
instance = MagicMock()
|
||||
instance.run = AsyncMock(return_value=raw)
|
||||
return patch("artemis.tools.explorer_tool.Explorer", return_value=instance), instance
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Contract
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_tool_contract_is_tier_agnostic():
|
||||
assert issubclass(AskExplorerTool, ArtemisTool)
|
||||
assert issubclass(AskExplorer, ArtemisTool)
|
||||
assert issubclass(AskExplorerToolAlias, ArtemisTool)
|
||||
assert isinstance(ask_explorer, ArtemisTool)
|
||||
assert isinstance(ask_explorer, AskExplorerTool)
|
||||
|
||||
assert ask_explorer.name == "ask_explorer"
|
||||
assert ask_explorer.category == "explorer"
|
||||
assert ask_explorer.args_schema == AskExplorerArgs
|
||||
assert ask_explorer.args_schema is AskExplorerArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("ask_explorer")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, AskExplorerTool)
|
||||
fields = AskExplorerArgs.model_fields
|
||||
assert set(fields) == {"query", "context_feedback"}
|
||||
assert fields["query"].is_required()
|
||||
assert fields["query"].description == ASK_EXPLORER_QUERY_DESCRIPTION
|
||||
assert fields["context_feedback"].default == ""
|
||||
assert fields["context_feedback"].description == ASK_EXPLORER_CONTEXT_FEEDBACK_DESCRIPTION
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = ask_explorer.to_genai_declaration()
|
||||
assert declaration.name == "ask_explorer"
|
||||
assert "query" in declaration.parameters.properties
|
||||
|
||||
# Wrapper check
|
||||
assert ask_explorer_wrapper is not None
|
||||
assert ask_explorer_wrapper.tool_fn_getter == get_ask_explorer_tool
|
||||
assert set(declaration.parameters.properties) == {"query", "context_feedback"}
|
||||
assert declaration.parameters.required == ["query"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ask_explorer_multiple_candidates():
|
||||
# Mock Context and State
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = None
|
||||
mock_ctx.agent_config = None
|
||||
mock_ctx.llm_config = None
|
||||
mock_ctx.execution_setup = None
|
||||
mock_ctx.device = MagicMock()
|
||||
mock_ctx.device.device_width = 1080
|
||||
mock_ctx.device.device_height = 2400
|
||||
@pytest.mark.parametrize("version", [None, "flash", "pro", "ultra"])
|
||||
def test_description_does_not_depend_on_the_tier(version):
|
||||
tool = AskExplorerTool(version=version)
|
||||
assert tool.description == ASK_EXPLORER_DESCRIPTION
|
||||
assert tool.version == version
|
||||
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.latest_screenshot = "/tmp/test.jpg"
|
||||
mock_state.indexed_points = [[100, 200]] # Pre-existing point
|
||||
mock_state.indexed_elements = [
|
||||
|
||||
def test_registry_and_wrapper_export():
|
||||
reg_tool = ToolRegistry.get("ask_explorer")
|
||||
assert isinstance(reg_tool, AskExplorerTool)
|
||||
assert ask_explorer_wrapper.tool_fn_getter is get_ask_explorer_tool
|
||||
assert ask_explorer_wrapper.on_success_fn("x") == "x"
|
||||
assert "boom" in ask_explorer_wrapper.on_failure_fn("boom")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# ExplorerOutcome.from_raw
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_from_raw_parses_candidates_and_message():
|
||||
raw = json.dumps(
|
||||
{
|
||||
"index": 1,
|
||||
"center": [100, 200],
|
||||
"text": "Pre-existing",
|
||||
"bounds": None,
|
||||
"class": "Test",
|
||||
"resource_id": None,
|
||||
"is_ocr": False,
|
||||
"candidates": [
|
||||
{"label": "S1", "coords": [500, 500], "description": "First Button"},
|
||||
{"label": "S2", "coords": [250, 750], "description": "Second Button"},
|
||||
],
|
||||
"fallback_message": "Please choose carefully.",
|
||||
}
|
||||
)
|
||||
outcome = ExplorerOutcome.from_raw(raw)
|
||||
assert outcome.found and not outcome.error
|
||||
assert outcome.candidates == [
|
||||
ExplorerCandidate(label="S1", coords=(500, 500), description="First Button"),
|
||||
ExplorerCandidate(label="S2", coords=(250, 750), description="Second Button"),
|
||||
]
|
||||
assert outcome.message == "Please choose carefully."
|
||||
assert outcome.raw == raw
|
||||
|
||||
# Mock Explorer Agent Outcome with multiple candidates and fallback message
|
||||
explorer_outcome = {
|
||||
"candidates": [
|
||||
{
|
||||
"label": "S1",
|
||||
"coords": [500, 500],
|
||||
"description": "First Button",
|
||||
},
|
||||
{
|
||||
"label": "S2",
|
||||
"coords": [250, 750],
|
||||
"description": "Second Button",
|
||||
},
|
||||
],
|
||||
"fallback_message": "Please choose carefully.",
|
||||
}
|
||||
explorer_output_str = json.dumps(explorer_outcome)
|
||||
|
||||
# We need to patch the Explorer class inside the tool
|
||||
mock_explorer_instance = MagicMock()
|
||||
mock_explorer_instance.run = AsyncMock(return_value=explorer_output_str)
|
||||
def test_from_raw_accepts_a_dict_and_fills_defaults():
|
||||
outcome = ExplorerOutcome.from_raw(
|
||||
{"candidates": [{"coords": ["10", "20"]}]}, default_label="gear icon"
|
||||
)
|
||||
assert len(outcome.candidates) == 1
|
||||
cand = outcome.candidates[0]
|
||||
assert cand.label == "gear icon"
|
||||
assert cand.description == "gear icon"
|
||||
assert cand.coords == (10, 20)
|
||||
assert outcome.message == ""
|
||||
|
||||
with patch(
|
||||
"artemis.tools.explorer_tool.Explorer",
|
||||
return_value=mock_explorer_instance,
|
||||
):
|
||||
# Retrieve the tool
|
||||
ask_explorer_tool = get_ask_explorer_tool(mock_ctx, version="pro")
|
||||
|
||||
result = await ask_explorer_tool.ainvoke(
|
||||
{
|
||||
"query": "Find buttons",
|
||||
"context_feedback": "Attempt 1",
|
||||
"state": mock_state,
|
||||
}
|
||||
)
|
||||
def test_from_raw_drops_candidates_with_invalid_coords():
|
||||
outcome = ExplorerOutcome.from_raw(
|
||||
{
|
||||
"candidates": [
|
||||
{"label": "bad1", "coords": [1500, 10]},
|
||||
{"label": "bad2", "coords": [10]},
|
||||
{"label": "bad3", "coords": "10,10"},
|
||||
{"label": "bad4"},
|
||||
"not a dict",
|
||||
{"label": "ok", "coords": [10, 10]},
|
||||
]
|
||||
}
|
||||
)
|
||||
assert [c.label for c in outcome.candidates] == ["ok"]
|
||||
assert not outcome.error
|
||||
|
||||
assert mock_state.indexed_points == [
|
||||
[100, 200],
|
||||
[540, 1200],
|
||||
[270, 1800],
|
||||
|
||||
def test_from_raw_unreadable_text_is_an_error_outcome():
|
||||
outcome = ExplorerOutcome.from_raw("I could not parse the screen {")
|
||||
assert outcome.error and not outcome.found
|
||||
assert "unreadable answer" in outcome.message
|
||||
assert outcome.raw == "I could not parse the screen {"
|
||||
|
||||
|
||||
def test_from_raw_non_object_json_is_an_error_outcome():
|
||||
outcome = ExplorerOutcome.from_raw("[1, 2, 3]")
|
||||
assert outcome.error
|
||||
assert "unexpected answer" in outcome.message
|
||||
|
||||
|
||||
def test_from_raw_empty_candidates_is_a_clean_not_found():
|
||||
outcome = ExplorerOutcome.from_raw({"candidates": [], "fallback_message": "Covered."})
|
||||
assert not outcome.found and not outcome.error
|
||||
assert outcome.message == "Covered."
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# register_candidates
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_register_candidates_appends_after_existing_elements_using_observation_size():
|
||||
ctx = _ctx()
|
||||
state = _state(raw={"width": 1000, "height": 2000})
|
||||
state.indexed_points = [[100, 200]]
|
||||
state.indexed_elements = [{"index": 1, "center": [100, 200], "text": "Pre-existing"}]
|
||||
outcome = ExplorerOutcome(
|
||||
candidates=[
|
||||
ExplorerCandidate(label="S1", coords=(500, 500), description="First Button"),
|
||||
ExplorerCandidate(label="S2", coords=(250, 750), description="Second Button"),
|
||||
]
|
||||
)
|
||||
|
||||
assert len(mock_state.indexed_elements) == 3
|
||||
assert mock_state.indexed_elements[1]["index"] == 2
|
||||
assert mock_state.indexed_elements[1]["center"] == [540, 1200]
|
||||
assert mock_state.indexed_elements[1]["text"] == "First Button"
|
||||
assert mock_state.indexed_elements[2]["index"] == 3
|
||||
assert mock_state.indexed_elements[2]["center"] == [270, 1800]
|
||||
assert mock_state.indexed_elements[2]["text"] == "Second Button"
|
||||
registered = register_candidates(ctx, state, outcome)
|
||||
|
||||
assert isinstance(result, str)
|
||||
assert "Explorer successfully located the following candidate(s):" in result
|
||||
assert "- [2] 'First Button' at coordinate [500, 500]" in result
|
||||
assert "- [3] 'Second Button' at coordinate [250, 750]" in result
|
||||
assert (
|
||||
"You can click/act on them directly by calling perform_action with"
|
||||
" their respective index." in result
|
||||
)
|
||||
assert "Additional Notes from Explorer: Please choose carefully." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ask_explorer_only_fallback_message():
|
||||
# Mock Context and State
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = None
|
||||
mock_ctx.agent_config = None
|
||||
mock_ctx.llm_config = None
|
||||
mock_ctx.execution_setup = None
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.latest_screenshot = "/tmp/test.jpg"
|
||||
mock_state.indexed_points = []
|
||||
mock_state.indexed_elements = []
|
||||
|
||||
explorer_outcome = {
|
||||
"candidates": [],
|
||||
"fallback_message": ("Element not found because it is covered by a keyboard."),
|
||||
}
|
||||
explorer_output_str = json.dumps(explorer_outcome)
|
||||
|
||||
mock_explorer_instance = MagicMock()
|
||||
mock_explorer_instance.run = AsyncMock(return_value=explorer_output_str)
|
||||
|
||||
with patch(
|
||||
"artemis.tools.explorer_tool.Explorer",
|
||||
return_value=mock_explorer_instance,
|
||||
):
|
||||
ask_explorer_tool = get_ask_explorer_tool(mock_ctx, version="pro")
|
||||
|
||||
result = await ask_explorer_tool.ainvoke(
|
||||
{
|
||||
"query": "Find hidden button",
|
||||
"context_feedback": "",
|
||||
"state": mock_state,
|
||||
}
|
||||
)
|
||||
|
||||
assert mock_state.indexed_points == []
|
||||
assert mock_state.indexed_elements == []
|
||||
|
||||
assert (
|
||||
"Explorer could not locate the element. Message: Element not found"
|
||||
" because it is covered by a keyboard." in result
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ask_explorer_multimodal_success():
|
||||
# Mock Context and State
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = None
|
||||
mock_ctx.agent_config = None
|
||||
mock_ctx.llm_config = None
|
||||
mock_ctx.execution_setup = None
|
||||
mock_ctx.device = MagicMock()
|
||||
mock_ctx.device.device_width = 1080
|
||||
mock_ctx.device.device_height = 2400
|
||||
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.latest_screenshot = "/tmp/test.jpg"
|
||||
mock_state.indexed_points = [[100, 200]]
|
||||
mock_state.indexed_elements = [
|
||||
{
|
||||
"index": 1,
|
||||
"center": [100, 200],
|
||||
"text": "Pre-existing",
|
||||
"bounds": None,
|
||||
"class": "Test",
|
||||
"resource_id": None,
|
||||
"is_ocr": False,
|
||||
}
|
||||
assert registered == [
|
||||
RegisteredCandidate(
|
||||
index=2, pixel=(500, 1000), coords=(500, 500), description="First Button"
|
||||
),
|
||||
RegisteredCandidate(
|
||||
index=3, pixel=(250, 1500), coords=(250, 750), description="Second Button"
|
||||
),
|
||||
]
|
||||
|
||||
explorer_outcome = {
|
||||
"candidates": [
|
||||
{
|
||||
"label": "S1",
|
||||
"coords": [500, 500],
|
||||
"description": "First Button",
|
||||
},
|
||||
],
|
||||
"fallback_message": "",
|
||||
assert state.indexed_points == [[100, 200], [500, 1000], [250, 1500]]
|
||||
assert len(state.indexed_elements) == 3
|
||||
assert state.indexed_elements[1] == {
|
||||
"index": 2,
|
||||
"center": [500, 1000],
|
||||
"text": "First Button",
|
||||
"bounds": None,
|
||||
"class": "ExplorerCandidate",
|
||||
"resource_id": None,
|
||||
"is_ocr": False,
|
||||
}
|
||||
explorer_output_str = json.dumps(explorer_outcome)
|
||||
|
||||
mock_explorer_instance = MagicMock()
|
||||
mock_explorer_instance.run = AsyncMock(return_value=explorer_output_str)
|
||||
|
||||
m_open = mock_open(read_data=b"fake_annotated_image_bytes")
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.tools.explorer_tool.Explorer",
|
||||
return_value=mock_explorer_instance,
|
||||
),
|
||||
patch("artemis.tools.explorer_tool.draw_dots") as mock_draw_dots,
|
||||
patch("builtins.open", m_open),
|
||||
patch("pathlib.Path.mkdir"),
|
||||
):
|
||||
ask_explorer_tool = get_ask_explorer_tool(mock_ctx, version="pro")
|
||||
|
||||
result = await ask_explorer_tool.ainvoke(
|
||||
{
|
||||
"query": "Find buttons",
|
||||
"context_feedback": "",
|
||||
"state": mock_state,
|
||||
}
|
||||
)
|
||||
|
||||
assert isinstance(result, list)
|
||||
assert len(result) == 2
|
||||
assert result[0]["type"] == "text"
|
||||
assert "Explorer successfully located" in result[0]["text"]
|
||||
assert result[1]["type"] == "image_url"
|
||||
assert result[1]["image_url"]["url"].startswith("data:image/jpeg;base64,")
|
||||
|
||||
mock_draw_dots.assert_called_once()
|
||||
points_arg = mock_draw_dots.call_args[0][1]
|
||||
assert points_arg == [[100, 200], [540, 1200]]
|
||||
labels_arg = mock_draw_dots.call_args[0][2]
|
||||
assert labels_arg == ["1", "2"]
|
||||
assert state.indexed_elements[2]["index"] == 3
|
||||
assert state.indexed_elements[2]["center"] == [250, 1500]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ask_explorer_direct_execution():
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = None
|
||||
mock_ctx.agent_config = None
|
||||
mock_ctx.llm_config = None
|
||||
mock_ctx.execution_setup = None
|
||||
mock_ctx.device = None
|
||||
def test_register_candidates_falls_back_to_the_device_size():
|
||||
ctx = _ctx(1080, 2400)
|
||||
state = _state(raw=None)
|
||||
outcome = ExplorerOutcome(candidates=[ExplorerCandidate("S1", (500, 500), "Btn")])
|
||||
registered = register_candidates(ctx, state, outcome)
|
||||
assert registered[0].pixel == (540, 1200)
|
||||
assert registered[0].index == 1
|
||||
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.latest_screenshot = "/tmp/test.jpg"
|
||||
mock_state.indexed_points = []
|
||||
mock_state.indexed_elements = []
|
||||
|
||||
mock_explorer_instance = MagicMock()
|
||||
mock_explorer_instance.run = AsyncMock(
|
||||
return_value=json.dumps(
|
||||
{
|
||||
"candidates": [],
|
||||
"fallback_message": "Direct execution not found.",
|
||||
}
|
||||
)
|
||||
def test_register_candidates_creates_the_lists_when_the_state_has_none():
|
||||
ctx = _ctx()
|
||||
state = MagicMock(spec=State)
|
||||
state.indexed_points = None
|
||||
state.indexed_elements = None
|
||||
state.operator_raw_data = None
|
||||
outcome = ExplorerOutcome(candidates=[ExplorerCandidate("S1", (0, 0), "Origin")])
|
||||
registered = register_candidates(ctx, state, outcome)
|
||||
assert registered[0].index == 1
|
||||
assert state.indexed_points == [[0, 0]]
|
||||
assert state.indexed_elements[0]["text"] == "Origin"
|
||||
|
||||
|
||||
def test_register_candidates_is_a_no_op_without_candidates():
|
||||
ctx = _ctx()
|
||||
state = _state()
|
||||
assert register_candidates(ctx, state, ExplorerOutcome(message="nope")) == []
|
||||
assert register_candidates(ctx, state, ExplorerOutcome.failure("boom")) == []
|
||||
assert state.indexed_points == []
|
||||
assert state.indexed_elements == []
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# render_text / render_operator_blocks
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_render_text_found_lists_indices_and_coordinates():
|
||||
outcome = ExplorerOutcome(
|
||||
candidates=[ExplorerCandidate("S1", (500, 500), "First Button")],
|
||||
message="Please choose carefully.",
|
||||
)
|
||||
registered = [RegisteredCandidate(2, (540, 1200), (500, 500), "First Button")]
|
||||
text = render_text("Find buttons", outcome, registered)
|
||||
assert "Explorer located 1 candidate(s) for 'Find buttons'" in text
|
||||
assert "- [2] 'First Button' at normalized [500, 500]" in text
|
||||
assert "target=[index]" in text
|
||||
assert "Explorer notes: Please choose carefully." in text
|
||||
|
||||
with patch(
|
||||
"artemis.tools.explorer_tool.Explorer",
|
||||
return_value=mock_explorer_instance,
|
||||
):
|
||||
result = await ask_explorer.execute(
|
||||
ctx=mock_ctx, state=mock_state, query="Find icon", version="pro"
|
||||
)
|
||||
assert "Direct execution not found." in result
|
||||
|
||||
def test_render_text_not_found_explains_and_suggests_rephrasing():
|
||||
text = render_text("hidden button", ExplorerOutcome(message="Covered by keyboard."), [])
|
||||
assert text.startswith("Explorer could not locate 'hidden button'. Covered by keyboard.")
|
||||
assert "describing the target differently" in text
|
||||
# Without an explanation the text still reads as a complete sentence.
|
||||
text = render_text("x", ExplorerOutcome(), [])
|
||||
assert "It gave no further detail." in text
|
||||
|
||||
|
||||
def test_render_text_error_reports_the_failure():
|
||||
text = render_text("gear icon", ExplorerOutcome.failure("Explorer failed: quota"), [])
|
||||
assert text == "Explorer could not run for 'gear icon'. Explorer failed: quota"
|
||||
|
||||
|
||||
def test_render_operator_blocks_adds_the_annotated_image(tmp_path):
|
||||
ctx = _ctx(base_dir=tmp_path)
|
||||
state = _state(screenshot=str(tmp_path / "shot.jpg"))
|
||||
outcome = ExplorerOutcome(candidates=[ExplorerCandidate("S1", (500, 500), "First Button")])
|
||||
registered = [RegisteredCandidate(2, (540, 1200), (500, 500), "First Button")]
|
||||
|
||||
def fake_draw_dots(screenshot_path, points, labels, output_path, **kwargs):
|
||||
Path(output_path).write_bytes(b"fake_annotated_image_bytes")
|
||||
|
||||
with patch("artemis.tools.explorer_tool.draw_dots", side_effect=fake_draw_dots) as draw:
|
||||
result = render_operator_blocks(ctx, state, "Find buttons", outcome, registered)
|
||||
|
||||
draw.assert_called_once()
|
||||
args, kwargs = draw.call_args
|
||||
assert args[0] == str(tmp_path / "shot.jpg")
|
||||
assert args[1] == [[540, 1200]]
|
||||
assert args[2] == ["2"]
|
||||
assert Path(args[3]).parent == tmp_path / "images" / "explorer_tool"
|
||||
assert kwargs["color"] == "magenta"
|
||||
|
||||
assert isinstance(result, list) and len(result) == 2
|
||||
assert result[0]["type"] == "text"
|
||||
assert "Explorer located 1 candidate(s)" in result[0]["text"]
|
||||
assert result[1]["type"] == "image_url"
|
||||
assert result[1]["image_url"]["url"].startswith("data:image/jpeg;base64,")
|
||||
|
||||
|
||||
def test_render_operator_blocks_numbers_annotations_sequentially(tmp_path):
|
||||
ctx = _ctx(base_dir=tmp_path)
|
||||
state = _state(screenshot=str(tmp_path / "shot.jpg"))
|
||||
outcome = ExplorerOutcome(candidates=[ExplorerCandidate("S1", (500, 500), "B")])
|
||||
registered = [RegisteredCandidate(1, (540, 1200), (500, 500), "B")]
|
||||
seen: list[str] = []
|
||||
|
||||
def fake_draw_dots(screenshot_path, points, labels, output_path, **kwargs):
|
||||
seen.append(Path(output_path).name)
|
||||
Path(output_path).write_bytes(b"img")
|
||||
|
||||
with patch("artemis.tools.explorer_tool.draw_dots", side_effect=fake_draw_dots):
|
||||
render_operator_blocks(ctx, state, "B", outcome, registered)
|
||||
render_operator_blocks(ctx, state, "B", outcome, registered)
|
||||
assert seen == ["explorer_output_1.jpg", "explorer_output_2.jpg"]
|
||||
|
||||
|
||||
def test_render_operator_blocks_degrades_to_text_when_drawing_fails(tmp_path):
|
||||
ctx = _ctx(base_dir=tmp_path)
|
||||
state = _state(screenshot=str(tmp_path / "shot.jpg"))
|
||||
outcome = ExplorerOutcome(candidates=[ExplorerCandidate("S1", (500, 500), "First Button")])
|
||||
registered = [RegisteredCandidate(2, (540, 1200), (500, 500), "First Button")]
|
||||
|
||||
with patch("artemis.tools.explorer_tool.draw_dots", side_effect=OSError("no PIL")):
|
||||
result = render_operator_blocks(ctx, state, "Find buttons", outcome, registered)
|
||||
|
||||
assert isinstance(result, str)
|
||||
assert "Explorer located 1 candidate(s)" in result
|
||||
|
||||
|
||||
def test_render_operator_blocks_is_plain_text_without_candidates(tmp_path):
|
||||
ctx = _ctx(base_dir=tmp_path)
|
||||
state = _state(screenshot=str(tmp_path / "shot.jpg"))
|
||||
with patch("artemis.tools.explorer_tool.draw_dots") as draw:
|
||||
result = render_operator_blocks(ctx, state, "x", ExplorerOutcome(message="Nope."), [])
|
||||
draw.assert_not_called()
|
||||
assert isinstance(result, str)
|
||||
assert "Explorer could not locate 'x'. Nope." in result
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# locate
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ask_explorer_callable_execution():
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = None
|
||||
mock_ctx.agent_config = None
|
||||
mock_ctx.llm_config = None
|
||||
mock_ctx.execution_setup = None
|
||||
mock_ctx.device = None
|
||||
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.latest_screenshot = "/tmp/test.jpg"
|
||||
mock_state.indexed_points = []
|
||||
mock_state.indexed_elements = []
|
||||
|
||||
mock_explorer_instance = MagicMock()
|
||||
mock_explorer_instance.run = AsyncMock(
|
||||
return_value=json.dumps(
|
||||
{
|
||||
"candidates": [],
|
||||
"fallback_message": "Callable test message.",
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
with patch(
|
||||
"artemis.tools.explorer_tool.Explorer",
|
||||
return_value=mock_explorer_instance,
|
||||
):
|
||||
result = await ask_explorer(
|
||||
ctx=mock_ctx, state=mock_state, query="Search bar", version="pro"
|
||||
)
|
||||
assert "Callable test message." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ask_explorer_flash_mode():
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = None
|
||||
mock_ctx.agent_config = None
|
||||
mock_ctx.llm_config = None
|
||||
mock_ctx.execution_setup = None
|
||||
mock_ctx.device = MagicMock()
|
||||
mock_ctx.device.device_width = 1080
|
||||
mock_ctx.device.device_height = 2400
|
||||
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.latest_screenshot = "/tmp/test.jpg"
|
||||
mock_state.indexed_points = []
|
||||
mock_state.indexed_elements = []
|
||||
|
||||
async def test_locate_runs_the_resolved_tier_and_parses_the_answer():
|
||||
ctx = _ctx()
|
||||
state = _state()
|
||||
raw = json.dumps({"candidates": [{"label": "S1", "coords": [500, 500]}]})
|
||||
explorer_patch, instance = _explorer_returning(raw)
|
||||
with (
|
||||
explorer_patch,
|
||||
patch(
|
||||
"artemis.tools.explorer_tool._run_object_detection",
|
||||
new_callable=AsyncMock,
|
||||
return_value={
|
||||
"detected": [
|
||||
{"label": "Icon", "point": [500, 500]},
|
||||
],
|
||||
"failed": [],
|
||||
},
|
||||
),
|
||||
patch(
|
||||
"builtins.open",
|
||||
mock_open(read_data=json.dumps({"templates": [], "instructions": ""})),
|
||||
),
|
||||
"artemis.tools.explorer_tool.resolve_explorer_version", return_value="ultra"
|
||||
) as resolve,
|
||||
):
|
||||
result = await ask_explorer.execute(
|
||||
ctx=mock_ctx, state=mock_state, query="Icon", version="flash"
|
||||
outcome = await locate(ctx, state, "gear icon", "was wrong", agent_name="validator")
|
||||
|
||||
resolve.assert_called_once_with(ctx, explicit_version=None, agent_or_profile_name="validator")
|
||||
instance.run.assert_awaited_once_with(
|
||||
"gear icon", "was wrong", "/tmp/test.jpg", state, version="ultra"
|
||||
)
|
||||
assert outcome.found and not outcome.error
|
||||
assert outcome.candidates[0].coords == (500, 500)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_locate_forwards_a_programmatic_version_pin():
|
||||
ctx = _ctx()
|
||||
state = _state()
|
||||
explorer_patch, instance = _explorer_returning(json.dumps({"candidates": []}))
|
||||
with (
|
||||
explorer_patch,
|
||||
patch(
|
||||
"artemis.tools.explorer_tool.resolve_explorer_version", return_value="pro"
|
||||
) as resolve,
|
||||
):
|
||||
await locate(ctx, state, "q", version="pro", agent_name="flash")
|
||||
resolve.assert_called_once_with(ctx, explicit_version="pro", agent_or_profile_name="flash")
|
||||
assert instance.run.await_args.kwargs["version"] == "pro"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_locate_unknown_resolved_tier_falls_back_to_the_default_tier():
|
||||
ctx = _ctx()
|
||||
state = _state()
|
||||
explorer_patch, instance = _explorer_returning(json.dumps({"candidates": []}))
|
||||
with (
|
||||
explorer_patch,
|
||||
patch("artemis.tools.explorer_tool.resolve_explorer_version", return_value="turbo"),
|
||||
):
|
||||
await locate(ctx, state, "q")
|
||||
assert instance.run.await_args.kwargs["version"] == "flash"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_locate_without_a_screenshot_is_an_error_outcome():
|
||||
ctx = _ctx()
|
||||
state = _state(screenshot=None)
|
||||
with patch("artemis.tools.explorer_tool.Explorer") as explorer_cls:
|
||||
outcome = await locate(ctx, state, "gear icon")
|
||||
explorer_cls.assert_not_called()
|
||||
assert outcome.error
|
||||
assert "No screenshot" in outcome.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_locate_contains_explorer_exceptions_as_error_outcomes():
|
||||
ctx = _ctx()
|
||||
state = _state()
|
||||
instance = MagicMock()
|
||||
instance.run = AsyncMock(side_effect=RuntimeError("quota exhausted"))
|
||||
with patch("artemis.tools.explorer_tool.Explorer", return_value=instance):
|
||||
outcome = await locate(ctx, state, "gear icon")
|
||||
assert outcome.error and not outcome.found
|
||||
assert outcome.message == "Explorer failed: quota exhausted"
|
||||
assert "could not run for 'gear icon'" in render_text("gear icon", outcome, [])
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Entry points
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ask_explorer_text_end_to_end():
|
||||
ctx = _ctx()
|
||||
state = _state()
|
||||
state.indexed_points = [[100, 200]]
|
||||
state.indexed_elements = [{"index": 1, "center": [100, 200], "text": "Pre-existing"}]
|
||||
raw = json.dumps(
|
||||
{
|
||||
"candidates": [
|
||||
{"label": "S1", "coords": [500, 500], "description": "First Button"},
|
||||
{"label": "S2", "coords": [250, 750], "description": "Second Button"},
|
||||
],
|
||||
"fallback_message": "Please choose carefully.",
|
||||
}
|
||||
)
|
||||
explorer_patch, instance = _explorer_returning(raw)
|
||||
with explorer_patch:
|
||||
text = await ask_explorer_text(ctx, state, "Find buttons", "Attempt 1", agent_name="flash")
|
||||
|
||||
instance.run.assert_awaited_once()
|
||||
assert instance.run.await_args.args[:2] == ("Find buttons", "Attempt 1")
|
||||
assert state.indexed_points == [[100, 200], [540, 1200], [270, 1800]]
|
||||
assert state.indexed_elements[1]["text"] == "First Button"
|
||||
assert state.indexed_elements[2]["text"] == "Second Button"
|
||||
assert "Explorer located 2 candidate(s) for 'Find buttons'" in text
|
||||
assert "- [2] 'First Button' at normalized [500, 500]" in text
|
||||
assert "- [3] 'Second Button' at normalized [250, 750]" in text
|
||||
assert "Explorer notes: Please choose carefully." in text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ask_explorer_text_not_found_leaves_the_state_untouched():
|
||||
ctx = _ctx()
|
||||
state = _state()
|
||||
explorer_patch, _ = _explorer_returning(
|
||||
json.dumps({"candidates": [], "fallback_message": "Covered by a keyboard."})
|
||||
)
|
||||
with explorer_patch:
|
||||
text = await ask_explorer_text(ctx, state, "hidden button")
|
||||
assert state.indexed_points == []
|
||||
assert state.indexed_elements == []
|
||||
assert "Explorer could not locate 'hidden button'. Covered by a keyboard." in text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_langchain_tool_uses_the_operator_presentation(tmp_path):
|
||||
ctx = _ctx(base_dir=tmp_path)
|
||||
state = _state(screenshot=str(tmp_path / "shot.jpg"))
|
||||
raw = json.dumps({"candidates": [{"label": "S1", "coords": [500, 500], "description": "B"}]})
|
||||
explorer_patch, _ = _explorer_returning(raw)
|
||||
|
||||
def fake_draw_dots(screenshot_path, points, labels, output_path, **kwargs):
|
||||
Path(output_path).write_bytes(b"img")
|
||||
|
||||
with explorer_patch, patch("artemis.tools.explorer_tool.draw_dots", side_effect=fake_draw_dots):
|
||||
tool = get_ask_explorer_tool(ctx)
|
||||
result = await tool.ainvoke(
|
||||
{"query": "Find buttons", "context_feedback": "", "state": state}
|
||||
)
|
||||
assert "Explorer successfully located" in result
|
||||
assert "Icon" in result
|
||||
|
||||
assert tool.name == "ask_explorer"
|
||||
assert isinstance(result, list) and result[1]["type"] == "image_url"
|
||||
assert "- [1] 'B' at normalized [500, 500]" in result[0]["text"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_direct_execute_accepts_a_version_pin_and_legacy_arg_names():
|
||||
ctx = _ctx(width=None)
|
||||
state = _state()
|
||||
explorer_patch, instance = _explorer_returning(
|
||||
json.dumps({"candidates": [], "fallback_message": "Direct execution not found."})
|
||||
)
|
||||
with explorer_patch:
|
||||
result = await ask_explorer.execute(
|
||||
ctx=ctx, state=state, Query="Find icon", ContextFeedback="cf", version="pro"
|
||||
)
|
||||
assert instance.run.await_args.args[:2] == ("Find icon", "cf")
|
||||
assert instance.run.await_args.kwargs["version"] == "pro"
|
||||
assert "Direct execution not found." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_explorer_logic_replay_shim_returns_operator_blocks(tmp_path):
|
||||
ctx = _ctx(base_dir=tmp_path)
|
||||
state = _state(screenshot=str(tmp_path / "shot.jpg"))
|
||||
raw = json.dumps({"candidates": [{"label": "S1", "coords": [500, 500], "description": "B"}]})
|
||||
explorer_patch, instance = _explorer_returning(raw)
|
||||
|
||||
def fake_draw_dots(screenshot_path, points, labels, output_path, **kwargs):
|
||||
Path(output_path).write_bytes(b"img")
|
||||
|
||||
with explorer_patch, patch("artemis.tools.explorer_tool.draw_dots", side_effect=fake_draw_dots):
|
||||
result = await _run_explorer_logic(ctx, state, "B", "", version="ultra")
|
||||
|
||||
assert instance.run.await_args.kwargs["version"] == "ultra"
|
||||
assert isinstance(result, list) and len(result) == 2
|
||||
assert result[0]["type"] == "text" and result[1]["type"] == "image_url"
|
||||
assert state.indexed_points == [[540, 1200]]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Bounds and de-duplication against the existing index
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_from_raw_parses_bounds_and_source_and_drops_invalid_bounds():
|
||||
raw = json.dumps(
|
||||
{
|
||||
"candidates": [
|
||||
{
|
||||
"label": "X1",
|
||||
"coords": [500, 500],
|
||||
"bounds": [400, 450, 600, 550],
|
||||
"source": "xml",
|
||||
},
|
||||
{"label": "D1", "coords": [100, 100], "bounds": [600, 550, 400, 450]},
|
||||
{"label": "D2", "coords": [100, 200], "bounds": [0, 0, 1200, 10]},
|
||||
]
|
||||
}
|
||||
)
|
||||
outcome = ExplorerOutcome.from_raw(raw)
|
||||
assert outcome.candidates[0].bounds == (400, 450, 600, 550)
|
||||
assert outcome.candidates[0].source == "xml"
|
||||
assert outcome.candidates[1].bounds is None
|
||||
assert outcome.candidates[2].bounds is None
|
||||
assert outcome.candidates[1].source == ""
|
||||
|
||||
|
||||
def test_register_candidates_reuses_an_indexed_element_that_contains_the_point():
|
||||
ctx = _ctx()
|
||||
state = _state(raw={"width": 1000, "height": 2000})
|
||||
state.indexed_points = [[300, 400], [800, 1600]]
|
||||
state.indexed_elements = [
|
||||
{"index": 1, "center": [300, 400], "text": "Gmail", "bounds": [200, 300, 400, 500]},
|
||||
{"index": 2, "center": [800, 1600], "text": "Send", "bounds": [700, 1500, 900, 1700]},
|
||||
]
|
||||
outcome = ExplorerOutcome(candidates=[ExplorerCandidate("D1", (310, 210), "Gmail app icon")])
|
||||
registered = register_candidates(ctx, state, outcome)
|
||||
assert registered == [
|
||||
RegisteredCandidate(
|
||||
index=1,
|
||||
pixel=(310, 420),
|
||||
coords=(310, 210),
|
||||
description="Gmail app icon",
|
||||
bounds=(200, 300, 400, 500),
|
||||
reused=True,
|
||||
)
|
||||
]
|
||||
assert len(state.indexed_points) == 2 # nothing appended
|
||||
assert "already in your indexed list" in render_text("Gmail app icon", outcome, registered)
|
||||
|
||||
|
||||
def test_register_candidates_reuses_a_bare_center_within_the_dedup_radius_only():
|
||||
ctx = _ctx()
|
||||
state = _state(raw={"width": 1000, "height": 2000})
|
||||
state.indexed_points = [[500, 1000]]
|
||||
state.indexed_elements = [
|
||||
{"index": 1, "center": [500, 1000], "text": "Earlier candidate", "bounds": None}
|
||||
]
|
||||
near = ExplorerCandidate("D1", (510, 505), "Same spot") # 10 x 10 px away
|
||||
far = ExplorerCandidate("D2", (560, 500), "Sixty px right")
|
||||
registered = register_candidates(ctx, state, ExplorerOutcome(candidates=[near, far]))
|
||||
assert [r.reused for r in registered] == [True, False]
|
||||
assert registered[0].index == 1
|
||||
assert registered[1].index == 2
|
||||
assert state.indexed_points == [[500, 1000], [560, 1000]]
|
||||
|
||||
|
||||
def test_register_candidates_stores_pixel_bounds_and_ocr_flag():
|
||||
ctx = _ctx()
|
||||
state = _state(raw={"width": 1000, "height": 2000})
|
||||
outcome = ExplorerOutcome(
|
||||
candidates=[
|
||||
ExplorerCandidate("O3", (500, 500), "Search", bounds=(400, 450, 600, 550), source="ocr")
|
||||
]
|
||||
)
|
||||
registered = register_candidates(ctx, state, outcome)
|
||||
assert registered[0].bounds == (400, 900, 600, 1100)
|
||||
assert registered[0].reused is False
|
||||
element = state.indexed_elements[0]
|
||||
assert element["bounds"] == [400, 900, 600, 1100]
|
||||
assert element["is_ocr"] is True
|
||||
Reference in new issue
Block a user