chenmingxuan
2ddec53954
Localize the web console and persist custom model and capability settings. Use OpenAI Responses exclusively and move runtime limits into environment configuration. Bootstrap uv and the virtual environment on Windows, support optional China mirrors, and package source distributions without environments, build output, or credentials. Update regression tests and repository quality checks for the new behavior.
208 lines
7.6 KiB
Python
208 lines
7.6 KiB
Python
# Copyright 2026 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import os
|
|
|
|
os.environ["ARTEMIS_USE_FILE_API"] = "false"
|
|
import pytest
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
import json
|
|
|
|
from artemis.agents.explorer.explorer import Explorer
|
|
from artemis.context import ArtemisContext
|
|
from artemis.graph.state import State
|
|
|
|
MOCK_PROMPT_JSON = """{
|
|
"IDENTITY": "You are the agentic UI Explorer designed to identify objects, scan text, and extract UI attributes on a phone screen.",
|
|
"OPERATING PRINCIPLES": [
|
|
"Confidence-Driven Cost-Benefit Strategy: Maximize localization accuracy...",
|
|
"Tool Capabilities: You have three classes of tools...",
|
|
"Parallel vs Sequential Invocations: You are fully supported..."
|
|
]
|
|
}"""
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_explorer_inspect_region():
|
|
import pathlib
|
|
import numpy as np
|
|
|
|
# Mock context
|
|
mock_ctx = MagicMock(spec=ArtemisContext)
|
|
mock_ctx.device = MagicMock()
|
|
mock_ctx.device.device_width = 1080
|
|
mock_ctx.device.device_height = 2400
|
|
mock_ctx.data_engine = MagicMock()
|
|
|
|
explorer = Explorer(mock_ctx)
|
|
explorer.width = 1080
|
|
explorer.height = 2400
|
|
explorer.screenshot_path = "/tmp/test_screenshot.jpg"
|
|
explorer.image_name = "test_image"
|
|
|
|
dummy_img = np.zeros((2400, 1080, 3), dtype=np.uint8)
|
|
|
|
with (
|
|
patch("cv2.imread", return_value=dummy_img) as mock_imread,
|
|
patch("cv2.resize", return_value=dummy_img) as mock_resize,
|
|
patch("cv2.imwrite", return_value=True) as mock_imwrite,
|
|
patch("pathlib.Path.mkdir") as mock_mkdir,
|
|
patch("glob.glob", return_value=[]),
|
|
patch("artemis.agents.explorer.perception_tools.settings") as mock_settings,
|
|
):
|
|
mock_settings.TRACES_PATH = pathlib.Path("/tmp/traces")
|
|
res = await explorer.exec_inspect_region(
|
|
x_min=100, y_min=200, x_max=300, y_max=400, zoom_factor=2.0
|
|
)
|
|
assert res["image_path"] is not None
|
|
assert "Inspected region coordinates" in res["text"]
|
|
mock_imread.assert_called_once_with("/tmp/test_screenshot.jpg")
|
|
|
|
# Verify resize calculations:
|
|
# px_start = 100 * 1080 / 1000 = 108
|
|
# py_start = 200 * 2400 / 1000 = 480
|
|
# px_end = 300 * 1080 / 1000 = 324
|
|
# py_end = 400 * 2400 / 1000 = 960
|
|
# width = 324 - 108 = 216
|
|
# height = 960 - 480 = 480
|
|
# new_width = 216 * 2.0 = 432
|
|
# new_height = 480 * 2.0 = 960
|
|
mock_resize.assert_called_once()
|
|
resize_args = mock_resize.call_args[0]
|
|
assert resize_args[1] == (432, 960)
|
|
mock_imwrite.assert_called_once()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_explorer_ask_perception_tool():
|
|
"""The perception bundle answers from the in-memory screen index.
|
|
|
|
No Data Engine lookup is involved: the text search, the coordinate
|
|
audit and the OCR inventory all read ``explorer.screen_index``.
|
|
"""
|
|
import numpy as np
|
|
from unittest.mock import mock_open
|
|
|
|
from artemis.agents.explorer.screen_index import ScreenIndex
|
|
|
|
mock_ctx = MagicMock(spec=ArtemisContext)
|
|
mock_ctx.data_engine = MagicMock()
|
|
mock_ctx.data_engine.base_dir = "/tmp/test_session"
|
|
explorer = Explorer(mock_ctx)
|
|
explorer.image_name = "dummy_hash"
|
|
explorer.screenshot_path = "/tmp/test_screenshot.jpg"
|
|
explorer.width = 1080
|
|
explorer.height = 2400
|
|
explorer.global_label_idx = 1
|
|
explorer.image_pool = {
|
|
"img_0": {
|
|
"path": "/tmp/test_screenshot.jpg",
|
|
"transform": {
|
|
"offset_x": 0.0,
|
|
"offset_y": 0.0,
|
|
"scale_x": 1.0,
|
|
"scale_y": 1.0,
|
|
},
|
|
"description": "Original complete screenshot",
|
|
}
|
|
}
|
|
explorer.screen_index = ScreenIndex.from_hierarchy(
|
|
[
|
|
{
|
|
"bounds": "[500,400][600,560]",
|
|
"text": "Settings",
|
|
"resource-id": "com.android.settings:id/btn_settings",
|
|
"class": "android.widget.Button",
|
|
"clickable": "true",
|
|
},
|
|
{
|
|
"bounds": "[100,100][200,200]",
|
|
"class": "android.widget.FrameLayout [OCR]",
|
|
"ocr_elements": [{"text": "Settings page", "bounds": "[100,100][200,200]"}],
|
|
},
|
|
],
|
|
1080,
|
|
2400,
|
|
)
|
|
|
|
# Mock _run_object_detection
|
|
mock_detector_res = {
|
|
"detected": [{"label": "profile icon", "point": [900, 100]}],
|
|
"failed": [],
|
|
}
|
|
|
|
# Mock draw_dots
|
|
mock_draw_dots = MagicMock()
|
|
|
|
with (
|
|
patch("artemis.agents.explorer.run_setup.StorageManager") as storage_cls,
|
|
patch(
|
|
"artemis.agents.explorer.perception_tools._run_object_detection",
|
|
return_value=mock_detector_res,
|
|
),
|
|
patch("artemis.agents.explorer.perception_tools.draw_dots", mock_draw_dots),
|
|
patch("artemis.utils.visualization.draw_dots", mock_draw_dots),
|
|
patch("pathlib.Path.mkdir"),
|
|
patch("glob.glob", return_value=[]),
|
|
patch("builtins.open", mock_open(read_data=b'{"templates": []}')),
|
|
patch("cv2.imread", return_value=np.zeros((2400, 1080, 3), dtype=np.uint8)),
|
|
):
|
|
result = await explorer.exec_ask_perception_tool(
|
|
search_query="Settings",
|
|
nx=550,
|
|
ny=200,
|
|
detect_queries=["profile icon"],
|
|
)
|
|
|
|
storage_cls.assert_not_called()
|
|
assert "Text Search Results are:" in result["text"]
|
|
assert "Coordinate Search Results are:" in result["text"]
|
|
assert "Object Detection Results are:" in result["text"]
|
|
|
|
# UI-tree text match -> X1, OCR text match -> O2 (both from the index)
|
|
assert "[X1] 'Settings' at [509,200]" in result["text"]
|
|
assert "[O2] 'Settings page' at [138,62]" in result["text"]
|
|
# Coordinate audit -> X3 with the innermost element's description
|
|
assert (
|
|
"Matched element at [550,200]: [X3] | 'Settings' class=android.widget.Button"
|
|
" id=com.android.settings:id/btn_settings source=ui-tree interactive"
|
|
) in result["text"]
|
|
# Object detection -> D4 (the label counter is shared and contiguous)
|
|
assert "[D4]" in result["text"]
|
|
|
|
# Element-backed labels are registered; detection points are not.
|
|
assert explorer.label_registry["X1"].text == "Settings"
|
|
assert explorer.label_registry["O2"].source == "ocr"
|
|
assert explorer.label_registry["X3"] is explorer.label_registry["X1"]
|
|
assert "D4" not in explorer.label_registry
|
|
|
|
assert len(result["image_paths"]) == 3
|
|
|
|
# Verify call arguments of draw_dots
|
|
assert mock_draw_dots.call_count == 3
|
|
# First call (XML/OCR Text search): green color
|
|
first_call = mock_draw_dots.call_args_list[0]
|
|
assert first_call.kwargs.get("color") == "green"
|
|
assert first_call.args[2] == ["X1", "O2"]
|
|
|
|
# Second call (XML Coordinate audit): blue color
|
|
second_call = mock_draw_dots.call_args_list[1]
|
|
assert second_call.kwargs.get("color") == "blue"
|
|
assert second_call.args[2] == ["X3"]
|
|
|
|
# Third call (Object detection): red color
|
|
third_call = mock_draw_dots.call_args_list[2]
|
|
assert third_call.kwargs.get("color") == "red"
|
|
assert third_call.args[2] == ["D4"]
|