# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os os.environ["ARTEMIS_USE_FILE_API"] = "false" import pytest from unittest.mock import AsyncMock, MagicMock, patch import json from artemis.agents.explorer.explorer import Explorer from artemis.context import ArtemisContext from artemis.graph.state import State MOCK_PROMPT_JSON = """{ "IDENTITY": "You are the agentic UI Explorer designed to identify objects, scan text, and extract UI attributes on a phone screen.", "OPERATING PRINCIPLES": [ "Confidence-Driven Cost-Benefit Strategy: Maximize localization accuracy...", "Tool Capabilities: You have three classes of tools...", "Parallel vs Sequential Invocations: You are fully supported..." ] }""" @pytest.mark.asyncio async def test_explorer_inspect_region(): import pathlib import numpy as np # Mock context mock_ctx = MagicMock(spec=ArtemisContext) mock_ctx.device = MagicMock() mock_ctx.device.device_width = 1080 mock_ctx.device.device_height = 2400 mock_ctx.data_engine = MagicMock() explorer = Explorer(mock_ctx) explorer.width = 1080 explorer.height = 2400 explorer.screenshot_path = "/tmp/test_screenshot.jpg" explorer.image_name = "test_image" dummy_img = np.zeros((2400, 1080, 3), dtype=np.uint8) with ( patch("cv2.imread", return_value=dummy_img) as mock_imread, patch("cv2.resize", return_value=dummy_img) as mock_resize, patch("cv2.imwrite", return_value=True) as mock_imwrite, patch("pathlib.Path.mkdir") as mock_mkdir, patch("glob.glob", return_value=[]), patch("artemis.agents.explorer.perception_tools.settings") as mock_settings, ): mock_settings.TRACES_PATH = pathlib.Path("/tmp/traces") res = await explorer.exec_inspect_region( x_min=100, y_min=200, x_max=300, y_max=400, zoom_factor=2.0 ) assert res["image_path"] is not None assert "Inspected region coordinates" in res["text"] mock_imread.assert_called_once_with("/tmp/test_screenshot.jpg") # Verify resize calculations: # px_start = 100 * 1080 / 1000 = 108 # py_start = 200 * 2400 / 1000 = 480 # px_end = 300 * 1080 / 1000 = 324 # py_end = 400 * 2400 / 1000 = 960 # width = 324 - 108 = 216 # height = 960 - 480 = 480 # new_width = 216 * 2.0 = 432 # new_height = 480 * 2.0 = 960 mock_resize.assert_called_once() resize_args = mock_resize.call_args[0] assert resize_args[1] == (432, 960) mock_imwrite.assert_called_once() @pytest.mark.asyncio async def test_explorer_ask_perception_tool(): """The perception bundle answers from the in-memory screen index. No Data Engine lookup is involved: the text search, the coordinate audit and the OCR inventory all read ``explorer.screen_index``. """ import numpy as np from unittest.mock import mock_open from artemis.agents.explorer.screen_index import ScreenIndex mock_ctx = MagicMock(spec=ArtemisContext) mock_ctx.data_engine = MagicMock() mock_ctx.data_engine.base_dir = "/tmp/test_session" explorer = Explorer(mock_ctx) explorer.image_name = "dummy_hash" explorer.screenshot_path = "/tmp/test_screenshot.jpg" explorer.width = 1080 explorer.height = 2400 explorer.global_label_idx = 1 explorer.image_pool = { "img_0": { "path": "/tmp/test_screenshot.jpg", "transform": { "offset_x": 0.0, "offset_y": 0.0, "scale_x": 1.0, "scale_y": 1.0, }, "description": "Original complete screenshot", } } explorer.screen_index = ScreenIndex.from_hierarchy( [ { "bounds": "[500,400][600,560]", "text": "Settings", "resource-id": "com.android.settings:id/btn_settings", "class": "android.widget.Button", "clickable": "true", }, { "bounds": "[100,100][200,200]", "class": "android.widget.FrameLayout [OCR]", "ocr_elements": [{"text": "Settings page", "bounds": "[100,100][200,200]"}], }, ], 1080, 2400, ) # Mock _run_object_detection mock_detector_res = { "detected": [{"label": "profile icon", "point": [900, 100]}], "failed": [], } # Mock draw_dots mock_draw_dots = MagicMock() with ( patch("artemis.agents.explorer.run_setup.StorageManager") as storage_cls, patch( "artemis.agents.explorer.perception_tools._run_object_detection", return_value=mock_detector_res, ), patch("artemis.agents.explorer.perception_tools.draw_dots", mock_draw_dots), patch("artemis.utils.visualization.draw_dots", mock_draw_dots), patch("pathlib.Path.mkdir"), patch("glob.glob", return_value=[]), patch("builtins.open", mock_open(read_data=b'{"templates": []}')), patch("cv2.imread", return_value=np.zeros((2400, 1080, 3), dtype=np.uint8)), ): result = await explorer.exec_ask_perception_tool( search_query="Settings", nx=550, ny=200, detect_queries=["profile icon"], ) storage_cls.assert_not_called() assert "Text Search Results are:" in result["text"] assert "Coordinate Search Results are:" in result["text"] assert "Object Detection Results are:" in result["text"] # UI-tree text match -> X1, OCR text match -> O2 (both from the index) assert "[X1] 'Settings' at [509,200]" in result["text"] assert "[O2] 'Settings page' at [138,62]" in result["text"] # Coordinate audit -> X3 with the innermost element's description assert ( "Matched element at [550,200]: [X3] | 'Settings' class=android.widget.Button" " id=com.android.settings:id/btn_settings source=ui-tree interactive" ) in result["text"] # Object detection -> D4 (the label counter is shared and contiguous) assert "[D4]" in result["text"] # Element-backed labels are registered; detection points are not. assert explorer.label_registry["X1"].text == "Settings" assert explorer.label_registry["O2"].source == "ocr" assert explorer.label_registry["X3"] is explorer.label_registry["X1"] assert "D4" not in explorer.label_registry assert len(result["image_paths"]) == 3 # Verify call arguments of draw_dots assert mock_draw_dots.call_count == 3 # First call (XML/OCR Text search): green color first_call = mock_draw_dots.call_args_list[0] assert first_call.kwargs.get("color") == "green" assert first_call.args[2] == ["X1", "O2"] # Second call (XML Coordinate audit): blue color second_call = mock_draw_dots.call_args_list[1] assert second_call.kwargs.get("color") == "blue" assert second_call.args[2] == ["X3"] # Third call (Object detection): red color third_call = mock_draw_dots.call_args_list[2] assert third_call.kwargs.get("color") == "red" assert third_call.args[2] == ["D4"]