chenmingxuan
2ddec53954
Localize the web console and persist custom model and capability settings. Use OpenAI Responses exclusively and move runtime limits into environment configuration. Bootstrap uv and the virtual environment on Windows, support optional China mirrors, and package source distributions without environments, build output, or credentials. Update regression tests and repository quality checks for the new behavior.
361 lines
13 KiB
Python
361 lines
13 KiB
Python
# Copyright 2026 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""Explorer tier integration: engine selection, tool exposure, caching, prompts."""
|
|
|
|
import json
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
|
|
import pytest
|
|
|
|
from artemis.agents.explorer.explorer import Explorer
|
|
from artemis.agents.explorer.tiers import EXPLORER_TIERS, PERCEPTION_TOOLS, get_tier
|
|
from artemis.config import settings
|
|
from artemis.context import ArtemisContext
|
|
from artemis.graph.state import State
|
|
|
|
EXPLORER = "artemis.agents.explorer.explorer"
|
|
RUN_SETUP = "artemis.agents.explorer.run_setup"
|
|
UNIVERSAL_RUNNER = "artemis.agents.explorer.universal_runner"
|
|
|
|
|
|
def _context(model: str = "gemini-3.8-flash", denylist: list[str] | None = None) -> MagicMock:
|
|
ctx = MagicMock(spec=ArtemisContext)
|
|
ctx.device = MagicMock()
|
|
ctx.device.device_width = 1080
|
|
ctx.device.device_height = 2400
|
|
ctx.data_engine = MagicMock()
|
|
ctx.data_engine.base_dir = None
|
|
ctx.llm_config = MagicMock()
|
|
ctx.llm_config.explorer = MagicMock()
|
|
ctx.llm_config.explorer.model = model
|
|
ctx.llm_config.explorer.temperature = 0.1
|
|
ctx.llm_config.explorer.thinking_level = None
|
|
ctx.llm_config.explorer.fallback = None
|
|
ctx.agent_config = MagicMock()
|
|
ctx.agent_config.denylisted_tools = {"explorer": denylist or []}
|
|
ctx.agent_config.explorer = SimpleNamespace(caching=None)
|
|
ctx.execution_setup = SimpleNamespace(explorer=SimpleNamespace(caching=None))
|
|
return ctx
|
|
|
|
|
|
def _state(screenshot: str) -> MagicMock:
|
|
state = MagicMock(spec=State)
|
|
state.latest_screenshot = screenshot
|
|
state.latest_ui_hierarchy = []
|
|
state.operator_raw_data = {}
|
|
return state
|
|
|
|
|
|
@pytest.fixture
|
|
def screenshot(tmp_path):
|
|
path = tmp_path / "screen.jpg"
|
|
path.write_bytes(b"fake-jpeg-bytes")
|
|
return str(path)
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _isolated_env(monkeypatch):
|
|
monkeypatch.delenv("ARTEMIS_EXPLORER_CACHING", raising=False)
|
|
monkeypatch.setenv("ARTEMIS_USE_FILE_API", "false")
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Tier -> turn budget / tool exposure
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_construction_creates_no_genai_client():
|
|
explorer = Explorer(_context())
|
|
assert explorer.client is None
|
|
assert explorer.tier is None
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("version", "max_turns", "expected_tools"),
|
|
[
|
|
("pro", 3, {"ask_perception_tool", "submit_answer"}),
|
|
("ultra", 8, PERCEPTION_TOOLS | {"submit_answer"}),
|
|
],
|
|
)
|
|
def test_tier_sets_turn_budget_and_exposed_tools(version, max_turns, expected_tools):
|
|
explorer = Explorer(_context())
|
|
with patch(f"{EXPLORER}.is_ocr_configured", return_value=True):
|
|
explorer._apply_tier(get_tier(version))
|
|
native_names = {t["function"]["name"] for t in explorer.get_exposed_tools()}
|
|
universal_names = {t["function"]["name"] for t in explorer._universal_exposed_tools()}
|
|
|
|
assert explorer.max_turns == max_turns
|
|
assert native_names == expected_tools
|
|
assert universal_names == expected_tools
|
|
assert explorer.denylisted_tools >= get_tier(version).hidden_tools
|
|
|
|
|
|
def test_unconfigured_ocr_and_user_denylist_hide_tools_on_ultra():
|
|
explorer = Explorer(_context(denylist=["inspect_region"]))
|
|
with patch(f"{EXPLORER}.is_ocr_configured", return_value=False):
|
|
explorer._apply_tier(get_tier("ultra"))
|
|
names = {t["function"]["name"] for t in explorer.get_exposed_tools()}
|
|
assert "get_ocr_list" not in names
|
|
assert "inspect_region" not in names
|
|
assert {"detect_objects", "ask_image_processor", "ask_perception_tool"} <= names
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Engine selection
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_universal_run_without_google_key_never_touches_genai(screenshot):
|
|
ctx = _context(model="claude-sonnet-4")
|
|
response = AIMessage(
|
|
content="",
|
|
tool_calls=[
|
|
{
|
|
"name": "submit_answer",
|
|
"args": {
|
|
"candidates": [{"label": "1", "coords": [100, 200], "description": "ok"}],
|
|
"fallback_message": "",
|
|
},
|
|
"id": "call_1",
|
|
}
|
|
],
|
|
)
|
|
llm = MagicMock()
|
|
bound = MagicMock()
|
|
bound.ainvoke = AsyncMock(return_value=response)
|
|
llm.bind_tools.return_value = bound
|
|
storage = MagicMock()
|
|
storage.get_image.return_value = None
|
|
|
|
with (
|
|
patch.object(settings, "OPENAI_API_KEY", None),
|
|
patch(f"{UNIVERSAL_RUNNER}.get_llm", return_value=llm),
|
|
patch(f"{RUN_SETUP}.StorageManager", return_value=storage),
|
|
):
|
|
explorer = Explorer(ctx)
|
|
raw = await explorer.run("find it", "", screenshot, _state(screenshot), version="pro")
|
|
|
|
outcome = json.loads(raw)
|
|
assert outcome["candidates"][0]["coords"] == [100, 200]
|
|
# The universal engine only sees the pro tier's tools.
|
|
bound_names = {t["function"]["name"] for t in llm.bind_tools.call_args_list[0].args[0]}
|
|
assert bound_names == {"ask_perception_tool", "submit_answer"}
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Universal dispatch delivers tool images
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_universal_dispatch_appends_image_message(tmp_path):
|
|
explorer = Explorer(_context(model="claude-sonnet-4"))
|
|
img_a = tmp_path / "a.jpg"
|
|
img_b = tmp_path / "b.jpg"
|
|
img_a.write_bytes(b"aaa")
|
|
img_b.write_bytes(b"bbb")
|
|
explorer.exec_ask_perception_tool = AsyncMock(
|
|
return_value={"text": "perception text", "image_paths": [str(img_a), str(tmp_path / "x")]}
|
|
)
|
|
explorer.exec_inspect_region = AsyncMock(
|
|
return_value={"text": "zoomed", "image_path": str(img_b)}
|
|
)
|
|
messages = []
|
|
tool_calls = [
|
|
{
|
|
"name": "ask_perception_tool",
|
|
"args": {"search_query": "q", "nx": 1, "ny": 2, "detect_queries": ["q"]},
|
|
"id": "c1",
|
|
},
|
|
{
|
|
"name": "inspect_region",
|
|
"args": {"x_min": 0, "y_min": 0, "x_max": 10, "y_max": 10},
|
|
"id": "c2",
|
|
},
|
|
]
|
|
|
|
await explorer._universal_dispatch_tools(tool_calls, messages, turn=1)
|
|
|
|
assert [type(m) for m in messages] == [ToolMessage, ToolMessage, HumanMessage]
|
|
assert messages[0].content == "perception text"
|
|
human = messages[2]
|
|
assert human.content[0] == {
|
|
"type": "text",
|
|
"text": "[Annotated image(s) returned by: ask_perception_tool, inspect_region]",
|
|
}
|
|
image_blocks = human.content[1:]
|
|
assert len(image_blocks) == 2 # the missing file is skipped
|
|
assert all(b["type"] == "image_url" for b in image_blocks)
|
|
assert image_blocks[0]["image_url"]["url"].startswith("data:image/jpeg;base64,")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_universal_dispatch_without_images_adds_no_human_message():
|
|
explorer = Explorer(_context(model="claude-sonnet-4"))
|
|
explorer.exec_get_ocr_list = AsyncMock(return_value={"text": "none", "image_path": None})
|
|
messages = []
|
|
await explorer._universal_dispatch_tools(
|
|
[{"name": "get_ocr_list", "args": {}, "id": "c1"}], messages, turn=1
|
|
)
|
|
assert [type(m) for m in messages] == [ToolMessage]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_universal_dispatch_rejects_denylisted_tool():
|
|
explorer = Explorer(_context(model="claude-sonnet-4"))
|
|
explorer._apply_tier(get_tier("pro"))
|
|
explorer.exec_inspect_region = AsyncMock()
|
|
messages = []
|
|
await explorer._universal_dispatch_tools(
|
|
[{"name": "inspect_region", "args": {}, "id": "c1"}], messages, turn=1
|
|
)
|
|
explorer.exec_inspect_region.assert_not_called()
|
|
assert "denylisted" in messages[0].content
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Caching precedence
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Tier-aware prompt
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_pro_prompt_only_describes_exposed_tools():
|
|
explorer = Explorer(_context())
|
|
explorer._apply_tier(get_tier("pro"))
|
|
prompt, error = explorer._build_prompt_template(get_tier("pro"))
|
|
assert error is None
|
|
assert "ask_perception_tool" in prompt
|
|
assert "inspect_region" not in prompt
|
|
assert "Image Pool" not in prompt
|
|
assert "ask_image_processor" not in prompt
|
|
assert "get_ocr_list" not in prompt
|
|
assert "# TOOL DENYLIST" not in prompt
|
|
assert "# EXECUTION CONSTRAINT" in prompt
|
|
assert "maximum of 3 turns" in prompt
|
|
|
|
|
|
def test_ultra_prompt_describes_all_tools():
|
|
explorer = Explorer(_context())
|
|
with patch(f"{EXPLORER}.is_ocr_configured", return_value=True):
|
|
explorer._apply_tier(get_tier("ultra"))
|
|
prompt, error = explorer._build_prompt_template(get_tier("ultra"))
|
|
assert error is None
|
|
for tool in PERCEPTION_TOOLS:
|
|
assert tool in prompt
|
|
assert "Image Pool" in prompt
|
|
assert "maximum of 8 turns" in prompt
|
|
assert "# TOOL DENYLIST" not in prompt
|
|
|
|
|
|
def test_prompt_denylist_section_only_for_user_denied_tier_tools():
|
|
explorer = Explorer(_context(denylist=["inspect_region", "search_xml_ocr"]))
|
|
with patch(f"{EXPLORER}.is_ocr_configured", return_value=True):
|
|
explorer._apply_tier(get_tier("ultra"))
|
|
ultra_prompt, _ = explorer._build_prompt_template(get_tier("ultra"))
|
|
explorer._apply_tier(get_tier("pro"))
|
|
pro_prompt, _ = explorer._build_prompt_template(get_tier("pro"))
|
|
|
|
assert "# TOOL DENYLIST" in ultra_prompt
|
|
assert "cannot be used: inspect_region" in ultra_prompt
|
|
assert "search_xml_ocr" not in ultra_prompt
|
|
# Denied tools lose their guidance bullets too.
|
|
assert "Targeted Verification" not in ultra_prompt
|
|
# On pro, inspect_region is tier-hidden, so it is never mentioned.
|
|
assert "# TOOL DENYLIST" not in pro_prompt
|
|
assert "inspect_region" not in pro_prompt
|
|
|
|
|
|
def test_prompt_sections_render_shape():
|
|
explorer = Explorer(_context())
|
|
explorer._apply_tier(get_tier("pro"))
|
|
prompt, _ = explorer._build_prompt_template(get_tier("pro"))
|
|
lines = prompt.splitlines()
|
|
assert lines[0] == "# IDENTITY"
|
|
assert any(line.startswith("# OPERATING PRINCIPLES") for line in lines)
|
|
body = [line for line in lines if line and not line.startswith("#")]
|
|
assert "输出语言要求" in prompt
|
|
assert "简体中文" in prompt
|
|
assert any(line.startswith("You are") for line in body)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Flash path
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_flash_run_returns_detected_candidates(screenshot):
|
|
detection = AsyncMock(
|
|
return_value={
|
|
"detected": [
|
|
{"label": "gear icon", "point": [900, 100]},
|
|
{"label": "send button", "point": [500, 950]},
|
|
],
|
|
"failed": [],
|
|
}
|
|
)
|
|
with (
|
|
patch(f"{RUN_SETUP}._run_object_detection", detection),
|
|
):
|
|
explorer = Explorer(_context())
|
|
raw = await explorer.run(
|
|
"gear icon | send button", "", screenshot, _state(screenshot), version="flash"
|
|
)
|
|
|
|
outcome = json.loads(raw)
|
|
assert [c["label"] for c in outcome["candidates"]] == ["D1", "D2"]
|
|
assert outcome["candidates"][0] == {
|
|
"label": "D1",
|
|
"coords": [900, 100],
|
|
"description": "gear icon",
|
|
}
|
|
assert outcome["fallback_message"] == ""
|
|
assert detection.call_args.args[2] == ["gear icon", "send button"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_flash_run_reports_missing_detection(screenshot):
|
|
with patch(f"{RUN_SETUP}._run_object_detection", AsyncMock(return_value={"detected": []})):
|
|
raw = await Explorer(_context()).run(
|
|
"ghost", "", screenshot, _state(screenshot), version="flash"
|
|
)
|
|
assert json.loads(raw) == {"candidates": [], "fallback_message": "Failed to detect: ghost"}
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Native engine: async SDK usage and resource lifecycle
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_tier_table_is_single_source_for_versions():
|
|
assert set(EXPLORER_TIERS) == {"flash", "pro", "ultra"}
|
|
assert get_tier("bogus").name == "flash"
|
|
|
|
|
|
@pytest.mark.parametrize("version", [None, "", "turbo", "PRO ", "Ultra"])
|
|
def test_get_tier_normalizes_or_falls_back(version):
|
|
"""Unknown or empty names fall back to the default tier; case and spaces are tolerated."""
|
|
tier = get_tier(version)
|
|
normalized = str(version or "").strip().lower()
|
|
expected = normalized if normalized in EXPLORER_TIERS else "flash"
|
|
assert tier.name == expected
|