test: add comprehensive unit, integration, and e2e test suites

Introduce complete test coverage for the ARTEMIS platform across multiple layers:
Unit tests: Add tests for agent nodes (Planner, Operator, Validator, Diagnoser, FlashRunner, Explorer), tools, utils (OCR/XML fusion, UI hierarchy), and data engine.
Integration tests: Add live tool verification, reactive runner, IME, and standalone workflow tests.
E2E tests: Add end-to-end scenario tests for media playback and failure recovery flows.
Fixtures & mocks: Add mock inputs (audio, video, screenshot) and test utilities.
This commit is contained in:
chiwanko11 committed 2026-08-13 12:18:32 -07:00
1 parent c1803fcb86
commit 5d2283deaa
115 files changed
+22813

No files matched your search

+98
View File
@@ -0,0 +1,98 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.agents.checker.checker import CheckerResult, run_async_check
from artemis.context import ArtemisContext
import pytest
@pytest.mark.asyncio
async def test_run_async_check(tmp_path):
notes_dir = tmp_path / "notes"
notes_dir.mkdir(parents=True, exist_ok=True)
# Put a dummy task plan file
(notes_dir / "task_plan.md").write_text("Plan")
# Put a dummy verification_chat file
import hashlib
subgoal_text = "Open WhatsApp"
s_hash = hashlib.md5(subgoal_text.encode("utf-8")).hexdigest()
(notes_dir / f"verification_chat_{s_hash}.json").write_text("[]")
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = str(tmp_path)
mock_ctx.data_engine.get_agent_friendly_steps.return_value = []
mock_ctx.execution_setup = None
# Mock controller for screen data
mock_controller = MagicMock()
mock_controller.take_screenshot = AsyncMock(return_value="dummy_screenshot_base64")
from artemis.controllers.device_controller import ScreenDataResponse
mock_screen_data = ScreenDataResponse(
base64="dummy_screenshot_base64",
elements=[],
width=1080,
height=2400,
platform="android",
)
mock_controller.get_screen_data = AsyncMock(return_value=mock_screen_data)
# Mock LLM response with with_structured_output
mock_llm = MagicMock()
mock_response = MagicMock()
mock_response.content = "Task completed successfully."
mock_response.tool_calls = []
async def mock_astream(*args, **kwargs):
yield mock_response
mock_llm.astream.side_effect = mock_astream
mock_llm.bind_tools.return_value = mock_llm
mock_structured_llm = MagicMock()
mock_structured_llm.ainvoke = AsyncMock(
return_value=CheckerResult(success=True, reason="Task completed successfully.")
)
mock_llm.with_structured_output.return_value = mock_structured_llm
with (
patch(
"artemis.agents.checker.checker.UnifiedMobileController",
return_value=mock_controller,
),
patch("artemis.agents.checker.checker.get_llm", return_value=mock_llm),
patch(
"artemis.utils.task_tree.build_plan_and_history",
return_value="Task tree",
),
):
result = await run_async_check(
mock_ctx,
subgoal_text=subgoal_text,
subgoal_hash=s_hash,
raw_perception_data={"screenshot_b64": "dummy", "width": 1080, "height": 2400},
latest_ui_hierarchy=[],
)
assert result["status"] == "success"
assert result["reason"] == "Task completed successfully."
assert mock_llm.astream.called
assert mock_llm.with_structured_output.called
+308
View File
@@ -0,0 +1,308 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, Mock, patch
from langchain_core.messages import AIMessage, ToolMessage
from langgraph.types import Command
from artemis.agents.diagnoser.diagnoser import Diagnoser
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.graph.state import State
import pytest
@pytest.fixture
def mock_context():
ctx = Mock(spec=ArtemisContext)
ctx.llm_config = Mock()
ctx.data_engine = Mock()
ctx.data_engine.base_dir = "/tmp"
ctx.data_engine.get_agent_friendly_steps.return_value = []
# Mock execution_setup
mock_setup = Mock()
mock_setup.video_recording_tools_enabled = True
ctx.execution_setup = mock_setup
return ctx
@pytest.fixture
def mock_state():
state = Mock(spec=State)
state.initial_goal = "Test goal"
state.latest_screenshot = None
state.latest_ui_hierarchy = []
return state
@patch("artemis.agents.diagnoser.diagnoser.get_llm")
@patch("artemis.agents.diagnoser.diagnoser.get_video_analyzer_tool")
@patch("artemis.agents.diagnoser.diagnoser.get_analyze_logs_tool")
@pytest.mark.asyncio
async def test_diagnostic_agent_simple_call(
mock_get_log_tool,
mock_get_video_tool,
mock_get_llm,
mock_context,
mock_state,
):
"""Test that DiagnosticAgent calls LLM and returns result."""
mock_llm = Mock()
mock_llm.bind_tools.return_value = mock_llm
mock_response = AIMessage(content="I diagnosed the issue: it is a network error.")
async def mock_astream(*args, **kwargs):
yield mock_response
mock_llm.astream.side_effect = mock_astream
mock_get_llm.return_value = mock_llm
# Mock tools
mock_video_tool = Mock()
mock_video_tool.name = "video_analyzer"
mock_get_video_tool.return_value = mock_video_tool
mock_log_tool = Mock()
mock_log_tool.name = "analyze_logs"
mock_get_log_tool.return_value = mock_log_tool
agent = Diagnoser(mock_context)
result = await agent.run("Diagnose why the app crashed", mock_state)
assert result == "I diagnosed the issue: it is a network error."
@patch("artemis.agents.diagnoser.diagnoser.get_llm")
@patch("artemis.agents.diagnoser.diagnoser.get_video_analyzer_tool")
@patch("artemis.agents.diagnoser.diagnoser.get_analyze_logs_tool")
@pytest.mark.asyncio
async def test_diagnostic_agent_tool_calls(
mock_get_log_tool,
mock_get_video_tool,
mock_get_llm,
mock_context,
mock_state,
):
"""Test that DiagnosticAgent handles tool calls."""
mock_llm = Mock()
mock_llm.bind_tools.return_value = mock_llm
# Simulate response with tool calls
mock_response = AIMessage(
content="I need to check logs and video.",
tool_calls=[
{
"name": "analyze_logs",
"args": {"specific_query": "Check for crashes"},
"id": "call_1",
},
{
"name": "video_analyzer",
"args": {
"time_description": "around 10s",
"purpose": "Check UI",
},
"id": "call_2",
},
],
)
# Second call returns final answer
mock_final_response = AIMessage(content="Diagnosis complete.")
responses = [mock_response, mock_final_response]
call_count = 0
async def mock_astream(*args, **kwargs):
nonlocal call_count
if call_count < len(responses):
yield responses[call_count]
call_count += 1
mock_llm.astream.side_effect = mock_astream
mock_get_llm.return_value = mock_llm
# Mock tools
mock_log_tool = AsyncMock()
mock_log_tool.name = "analyze_logs"
mock_log_tool.args = {"specific_query": None, "state": None}
mock_log_tool.ainvoke.return_value = "Logs show no crash."
mock_get_log_tool.return_value = mock_log_tool
mock_video_tool = AsyncMock()
mock_video_tool.name = "video_analyzer"
mock_video_tool.args = {
"time_description": None,
"purpose": None,
"state": None,
}
mock_command = Command(
update={
VALIDATOR_MESSAGES_KEY: [
ToolMessage(content="Video shows UI loading.", tool_call_id="call_2")
]
}
)
mock_video_tool.ainvoke.return_value = mock_command
mock_get_video_tool.return_value = mock_video_tool
agent = Diagnoser(mock_context)
result = await agent.run("Diagnose issue", mock_state)
assert result == "Diagnosis complete."
mock_log_tool.ainvoke.assert_called_once_with(
{"specific_query": "Check for crashes", "state": mock_state}
)
mock_video_tool.ainvoke.assert_called_once()
@patch("artemis.agents.diagnoser.diagnoser.get_llm")
@patch("artemis.agents.diagnoser.diagnoser.get_video_analyzer_tool")
@patch("artemis.agents.diagnoser.diagnoser.get_analyze_logs_tool")
@pytest.mark.asyncio
async def test_diagnostic_agent_no_video_tool_when_disabled(
mock_get_log_tool,
mock_get_video_tool,
mock_get_llm,
mock_context,
mock_state,
):
"""Test that Diagnoser does not bind video tool when disabled."""
# Set video recording to False
mock_context.execution_setup.video_recording_tools_enabled = False
mock_llm = Mock()
mock_llm.bind_tools.return_value = mock_llm
mock_response = AIMessage(content="Diagnosis complete.")
async def mock_astream(*args, **kwargs):
yield mock_response
mock_llm.astream.side_effect = mock_astream
mock_get_llm.return_value = mock_llm
# Mock tools
mock_video_tool = Mock()
mock_video_tool.name = "video_analyzer"
mock_get_video_tool.return_value = mock_video_tool
mock_log_tool = Mock()
mock_log_tool.name = "analyze_logs"
mock_get_log_tool.return_value = mock_log_tool
agent = Diagnoser(mock_context)
await agent.run("Diagnose issue", mock_state)
# Verify bind_tools was called
mock_llm.bind_tools.assert_called_once()
bound_tools = mock_llm.bind_tools.call_args[1]["tools"]
# Verify video_analyzer is NOT in bound tools
tool_names = [t.name for t in bound_tools if hasattr(t, "name")]
assert "video_analyzer" not in tool_names
# Verify log tool IS in bound tools (should still be there)
assert "analyze_logs" in tool_names
@patch("artemis.agents.diagnoser.diagnoser.get_llm")
@patch("artemis.tools.video_tool.VideoAnalyzer")
@pytest.mark.asyncio
async def test_diagnoser_background_job_lifecycle(
mock_video_analyzer_class, mock_get_llm, mock_state
):
# Setup context
ctx = ArtemisContext.model_construct(
device=Mock(),
llm_config=Mock(),
execution_setup=Mock(video_recording_tools_enabled=True),
)
ctx.data_engine = None
# The Mock VideoAnalyzer agent running inside the background job
mock_agent_instance = AsyncMock()
async def mock_agent_run(time_desc, purpose):
return "Video shows step successfully completed.", "success"
mock_agent_instance.run.side_effect = mock_agent_run
mock_video_analyzer_class.return_value = mock_agent_instance
# Prepare LLM responses
mock_llm = Mock()
mock_llm.bind_tools.return_value = mock_llm
# Turn 1: LLM requests tool call
mock_response_1 = AIMessage(
content="Checking the video first.",
tool_calls=[
{
"name": "video_analyzer",
"args": {
"time_description": "from 0s to 5s",
"purpose": "check UI",
},
"id": "call_video_1",
}
],
)
# Turn 2: LLM gets the initial job start message, continues, then background job finishes, and result is injected.
mock_response_2 = AIMessage(content="Final complete diagnosis.")
responses = [mock_response_1, mock_response_2]
call_count = 0
async def mock_astream(*args, **kwargs):
nonlocal call_count
if call_count < len(responses):
yield responses[call_count]
call_count += 1
mock_llm.astream.side_effect = mock_astream
mock_get_llm.return_value = mock_llm
# Mock asanitize_update on mock_state
async def mock_asanitize_update(ctx, update, agent):
return update
mock_state.asanitize_update = mock_asanitize_update
# Instantiate the Diagnoser
agent = Diagnoser(ctx)
# We want to use the actual video analyzer tool so it triggers the background task
from artemis.tools.video_tool import get_video_analyzer_tool
real_video_tool = get_video_analyzer_tool(ctx, role="diagnoser")
with patch(
"artemis.tools.log_tool.get_analyze_logs_tool",
return_value=Mock(name="analyze_logs"),
):
with patch(
"artemis.tools.video_tool.get_video_analyzer_tool",
return_value=real_video_tool,
):
result = await agent.run("Diagnose video issue", mock_state)
assert result == "Final complete diagnosis."
# Verify job registry
assert len(ctx.background_jobs) == 1
job_id = list(ctx.background_jobs.keys())[0]
job = ctx.background_jobs[job_id]
assert job["tool_call_id"] == "call_video_1"
assert job["status"] == "completed"
assert job["result"] == "Video shows step successfully completed."
assert job["consumed"] is True
@@ -0,0 +1,343 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, Mock, patch
from adbutils import AdbClient
from langchain_core.messages import AIMessage, ToolMessage
from artemis.agents.diagnoser.diagnoser import Diagnoser
from artemis.context import ArtemisContext
from artemis.graph.state import State
import pytest
@pytest.fixture(scope="session", autouse=True)
def ensure_device_connected():
try:
client = AdbClient()
if not client.device_list():
pytest.skip("No active Android device/emulator found. Skipping diagnostic tool tests.")
except Exception as e:
pytest.skip(f"ADB Client connection failed: {e}. Skipping tests.")
@pytest.fixture
def mock_context():
ctx = Mock(spec=ArtemisContext)
ctx.llm_config = Mock()
ctx.data_engine = Mock()
ctx.data_engine.base_dir = "/tmp"
ctx.data_engine.get_agent_friendly_steps.return_value = []
# Mock execution_setup
mock_setup = Mock()
mock_setup.video_recording_tools_enabled = True
ctx.execution_setup = mock_setup
return ctx
@pytest.fixture
def mock_state():
state = Mock(spec=State)
state.initial_goal = "Test goal"
state.latest_screenshot = None
state.latest_ui_hierarchy = []
return state
@patch("artemis.agents.diagnoser.diagnoser.get_llm")
@patch("artemis.tools.video_tool.get_video_analyzer_tool")
@patch("artemis.tools.log_tool.get_analyze_logs_tool")
@patch("artemis.agents.diagnoser.diagnoser.get_ui_hierarchy_tool")
@pytest.mark.asyncio
async def test_maestro_uninstall_timeout(
mock_get_ui_tool,
mock_get_log_tool,
mock_get_video_tool,
mock_get_llm,
mock_context,
mock_state,
):
"""Test Maestro Uninstall Timeout / Blocker Failure."""
# Mock LLM calls
mock_llm = Mock()
mock_llm.bind_tools.return_value = mock_llm
mock_response_1 = AIMessage(
content="I need to check the UI hierarchy.",
tool_calls=[
{
"name": "get_ui_hierarchy",
"args": {},
"id": "call_hierarchy",
}
],
)
mock_response_2 = AIMessage(
content=(
"Diagnoser identifies the blocker failure and recommends restarting"
" ADB/uninstalling conflicting tools manually."
)
)
responses = [mock_response_1, mock_response_2]
call_count = 0
async def mock_astream(*args, **kwargs):
nonlocal call_count
if call_count < len(responses):
yield responses[call_count]
call_count += 1
mock_llm.astream.side_effect = mock_astream
mock_get_llm.return_value = mock_llm
# Mock tool
mock_ui_tool = AsyncMock()
mock_ui_tool.name = "get_ui_hierarchy"
mock_ui_tool.args = {}
mock_ui_tool.ainvoke.return_value = "Error: pm list packages timed out"
mock_get_ui_tool.return_value = mock_ui_tool
# Run agent
agent = Diagnoser(mock_context)
result = await agent.run("Diagnose why the app crashed", mock_state)
assert "recommends restarting ADB" in result
assert "uninstalling conflicting tools" in result
# Assert get_ui_hierarchy was called
mock_ui_tool.ainvoke.assert_called_once()
# Assert that the ToolMessage was sent back to LLM
assert len(mock_llm.astream.call_args_list) == 2
sent_messages = mock_llm.astream.call_args_list[1][0][0]
tool_message = sent_messages[-2]
assert isinstance(tool_message, ToolMessage)
assert tool_message.tool_call_id == "call_hierarchy"
assert tool_message.content == "Error: pm list packages timed out"
assert tool_message.status == "error"
@patch("artemis.agents.diagnoser.diagnoser.get_llm")
@patch("artemis.tools.video_tool.get_video_analyzer_tool")
@patch("artemis.tools.log_tool.get_analyze_logs_tool")
@patch("artemis.agents.diagnoser.diagnoser.get_ui_hierarchy_tool")
@pytest.mark.asyncio
async def test_uiautomator2_connection_lost(
mock_get_ui_tool,
mock_get_log_tool,
mock_get_video_tool,
mock_get_llm,
mock_context,
mock_state,
):
"""Test UIAutomator2 Daemon Connection Lost."""
# Mock LLM
mock_llm = Mock()
mock_llm.bind_tools.return_value = mock_llm
mock_response_1 = AIMessage(
content="Let me check the UI hierarchy.",
tool_calls=[
{
"name": "get_ui_hierarchy",
"args": {},
"id": "call_hierarchy_lost",
}
],
)
mock_response_2 = AIMessage(content="Diagnoser reports connection loss with UIAutomator2.")
responses = [mock_response_1, mock_response_2]
call_count = 0
async def mock_astream(*args, **kwargs):
nonlocal call_count
if call_count < len(responses):
yield responses[call_count]
call_count += 1
mock_llm.astream.side_effect = mock_astream
mock_get_llm.return_value = mock_llm
# Mock tool to throw ConnectionError
mock_ui_tool = AsyncMock()
mock_ui_tool.name = "get_ui_hierarchy"
mock_ui_tool.args = {}
mock_ui_tool.ainvoke.side_effect = ConnectionError(
"UIAutomator2 connection lost, reconnecting..."
)
mock_get_ui_tool.return_value = mock_ui_tool
# Run agent
agent = Diagnoser(mock_context)
result = await agent.run("Diagnose crash", mock_state)
assert "reports connection loss" in result
# Assert that tool error is wrapped and returned in the ToolMessage
assert len(mock_llm.astream.call_args_list) == 2
sent_messages = mock_llm.astream.call_args_list[1][0][0]
tool_message = sent_messages[-2]
assert isinstance(tool_message, ToolMessage)
assert tool_message.tool_call_id == "call_hierarchy_lost"
assert "UIAutomator2 connection lost, reconnecting..." in tool_message.content
assert tool_message.status == "error"
# Verify that the trace span recorded the failure
record_trace_calls = mock_context.data_engine.record_trace.call_args_list
tool_trace_calls = [c for c in record_trace_calls if c[1].get("name") == "get_ui_hierarchy"]
failed_trace = next((c for c in tool_trace_calls if c[1].get("status") == "failed"), None)
assert failed_trace is not None
assert "UIAutomator2 connection lost, reconnecting..." in failed_trace[1]["payload"]["error"]
@patch("artemis.agents.diagnoser.diagnoser.get_llm")
@patch("artemis.tools.video_tool.get_video_analyzer_tool")
@patch("artemis.tools.log_tool.get_analyze_logs_tool")
@patch("artemis.agents.diagnoser.diagnoser.get_ui_hierarchy_tool")
@pytest.mark.asyncio
async def test_ui_hierarchy_dump_stall(
mock_get_ui_tool,
mock_get_log_tool,
mock_get_video_tool,
mock_get_llm,
mock_context,
mock_state,
):
"""Test UI Hierarchy Dump Stall (UI Thread Frozen)."""
# Mock LLM
mock_llm = Mock()
mock_llm.bind_tools.return_value = mock_llm
mock_response_1 = AIMessage(
content="Let me fetch the UI hierarchy to see what's on screen.",
tool_calls=[
{
"name": "get_ui_hierarchy",
"args": {},
"id": "call_hierarchy_stall",
}
],
)
mock_response_2 = AIMessage(
content=(
"Diagnoser identifies frozen UI and suggests rendering-safe"
" recovery actions (like ADB wake commands or a wait/retry cycle)."
)
)
responses = [mock_response_1, mock_response_2]
call_count = 0
async def mock_astream(*args, **kwargs):
nonlocal call_count
if call_count < len(responses):
yield responses[call_count]
call_count += 1
mock_llm.astream.side_effect = mock_astream
mock_get_llm.return_value = mock_llm
# Mock tool
mock_ui_tool = AsyncMock()
mock_ui_tool.name = "get_ui_hierarchy"
mock_ui_tool.args = {}
mock_ui_tool.ainvoke.return_value = (
"Error retrieving UI hierarchy: dump_hierarchy timed out after 3 seconds"
)
mock_get_ui_tool.return_value = mock_ui_tool
# Run agent
agent = Diagnoser(mock_context)
result = await agent.run("Diagnose stall", mock_state)
assert "rendering-safe recovery actions" in result
# Assert ToolMessage was passed back with correct error details
assert len(mock_llm.astream.call_args_list) == 2
sent_messages = mock_llm.astream.call_args_list[1][0][0]
tool_message = sent_messages[-2]
assert isinstance(tool_message, ToolMessage)
assert tool_message.tool_call_id == "call_hierarchy_stall"
assert "dump_hierarchy timed out after 3 seconds" in tool_message.content
assert tool_message.status == "error"
@patch("artemis.agents.diagnoser.diagnoser.get_llm")
@patch("artemis.tools.video_tool.get_video_analyzer_tool")
@patch("artemis.tools.log_tool.get_analyze_logs_tool")
@patch("artemis.agents.diagnoser.diagnoser.get_ui_hierarchy_tool")
@pytest.mark.asyncio
async def test_device_offline(
mock_get_ui_tool,
mock_get_log_tool,
mock_get_video_tool,
mock_get_llm,
mock_context,
mock_state,
):
"""Test Device Offline (ADB Transport Lost)."""
# Mock LLM
mock_llm = Mock()
mock_llm.bind_tools.return_value = mock_llm
mock_response_1 = AIMessage(
content="Let's grab the UI hierarchy.",
tool_calls=[
{
"name": "get_ui_hierarchy",
"args": {},
"id": "call_hierarchy_offline",
}
],
)
mock_response_2 = AIMessage(
content=("The device is offline. Diagnoser flags the offline device state.")
)
responses = [mock_response_1, mock_response_2]
call_count = 0
async def mock_astream(*args, **kwargs):
nonlocal call_count
if call_count < len(responses):
yield responses[call_count]
call_count += 1
mock_llm.astream.side_effect = mock_astream
mock_get_llm.return_value = mock_llm
# Mock tool
mock_ui_tool = AsyncMock()
mock_ui_tool.name = "get_ui_hierarchy"
mock_ui_tool.args = {}
mock_ui_tool.ainvoke.side_effect = RuntimeError("device offline")
mock_get_ui_tool.return_value = mock_ui_tool
# Run agent
agent = Diagnoser(mock_context)
result = await agent.run("Diagnose offline", mock_state)
assert "flags the offline device state" in result
# Assert ToolMessage was passed back with correct error details
assert len(mock_llm.astream.call_args_list) == 2
sent_messages = mock_llm.astream.call_args_list[1][0][0]
tool_message = sent_messages[-2]
assert isinstance(tool_message, ToolMessage)
assert tool_message.tool_call_id == "call_hierarchy_offline"
assert "device offline" in tool_message.content
assert tool_message.status == "error"
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,127 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for Universal Explorer and Multi-Model Grounding Engine."""
import json
from unittest.mock import AsyncMock, MagicMock, Mock, patch
import pytest
from langchain_core.messages import AIMessage
from artemis.agents.explorer.explorer import (
UNIVERSAL_EXPLORER_TOOLS,
Explorer,
)
from artemis.config import settings
from artemis.context import ArtemisContext
from artemis.graph.state import State
@pytest.fixture
def mock_context(tmp_path):
ctx = Mock(spec=ArtemisContext)
ctx.llm_config = Mock()
mock_llm_cfg = Mock()
mock_llm_cfg.model = "claude-3-7-sonnet"
mock_llm_cfg.temperature = 0.1
ctx.llm_config.explorer = mock_llm_cfg
ctx.device = Mock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
ctx.agent_config = Mock()
ctx.agent_config.blacklisted_tools = {}
ctx.data_engine = Mock()
ctx.data_engine.base_dir = tmp_path
return ctx
def test_explorer_universal_engine_detection(mock_context):
"""Test that Explorer routes to Universal Engine when non-Gemini model is configured."""
with patch.object(settings, "GOOGLE_API_KEY", None):
agent = Explorer(mock_context)
assert agent.use_native_gemini is False
assert agent.model_name == "claude-3-7-sonnet"
def test_universal_explorer_tool_schemas():
"""Verify schema integrity of Universal Explorer tool declarations."""
assert len(UNIVERSAL_EXPLORER_TOOLS) == 6
names = [t["function"]["name"] for t in UNIVERSAL_EXPLORER_TOOLS]
assert "ask_perception_tool" in names
assert "detect_objects" in names
assert "get_ocr_list" in names
assert "ask_image_processor" in names
assert "inspect_region" in names
assert "submit_answer" in names
@pytest.mark.asyncio
async def test_run_universal_explorer_success(mock_context, tmp_path):
"""Test universal Explorer tool invocation and submit_answer coordinate parsing."""
agent = Explorer(mock_context)
agent.use_native_gemini = False
fake_screenshot = tmp_path / "screenshot.jpg"
fake_screenshot.write_bytes(b"fake_image_bytes")
# Mock tool call response for submit_answer
msg_response = AIMessage(
content="Submitting grounded elements.",
tool_calls=[
{
"name": "submit_answer",
"args": {
"candidates": [
{
"label": "D1",
"coords": [500, 600],
"description": "Settings button",
}
],
"fallback_message": "",
},
"id": "exp_call_1",
}
],
)
mock_llm = MagicMock()
mock_bound = MagicMock()
mock_bound.ainvoke = AsyncMock(return_value=msg_response)
mock_llm.bind_tools.return_value = mock_bound
with patch(
"artemis.agents.explorer.explorer.get_llm",
return_value=mock_llm,
):
mock_state = Mock(spec=State)
res_str = await agent._run_universal(
query="find settings icon",
context_feedback="",
screenshot_path=str(fake_screenshot),
state=mock_state,
minimal_list="",
version="pro",
prompt_template="You are an explorer.",
max_iterations=3,
)
res = json.loads(res_str)
assert len(res["candidates"]) == 1
assert res["candidates"][0]["label"] == "D1"
assert res["candidates"][0]["coords"] == [500, 600]
+213
View File
@@ -0,0 +1,213 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for Universal Failure Analyzer."""
import base64
from unittest.mock import AsyncMock, MagicMock, Mock, patch
import pytest
from langchain_core.messages import AIMessage
from artemis.agents.validator.failure_analyzer import (
FailureAnalyzer,
PixelTargetDisappearedStrategy,
TargetDisappearedStrategy,
ValidationErrorCategory,
VALIDATOR_TOOLS_DECLARATION,
)
from artemis.context import ArtemisContext
class DummyState:
def __init__(self, latest_screenshot=None):
self.latest_screenshot = latest_screenshot
self.indexed_points = []
self.indexed_elements = []
@pytest.fixture
def mock_context(tmp_path):
ctx = Mock(spec=ArtemisContext)
ctx.llm_config = Mock()
mock_llm_cfg = Mock()
mock_llm_cfg.model = "gemini-2.5-pro"
mock_llm_cfg.temperature = 0.1
ctx.llm_config.get_agent.return_value = mock_llm_cfg
ctx.device = Mock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
ctx.data_engine = Mock()
ctx.data_engine.base_dir = tmp_path
ctx.data_engine.get_agent_friendly_steps.return_value = []
ctx.adb_client = None
ctx.driver = Mock()
return ctx
@pytest.mark.asyncio
async def test_failure_analyzer_cannot_fix(mock_context, tmp_path):
"""Test that FailureAnalyzer natively executes and handles 'cannot_fix' report."""
screenshot_file = tmp_path / "shot.jpg"
screenshot_file.write_bytes(b"dummy_bytes")
state = DummyState(latest_screenshot=str(screenshot_file))
failed_action = {"action": "click", "coordinates": [100, 200]}
error_msg = "Element not found"
analyzer = FailureAnalyzer(mock_context)
mock_llm = MagicMock()
mock_response = AIMessage(
content="I see the error.",
tool_calls=[
{
"name": "report_failure_analysis",
"args": {
"status": "cannot_fix",
"analysis": "The element was definitely not there.",
},
"id": "call_1",
}
],
)
mock_bound = MagicMock()
mock_bound.ainvoke = AsyncMock(return_value=mock_response)
mock_llm.bind_tools.return_value = mock_bound
with (
patch("artemis.agents.validator.failure_analyzer.get_llm", return_value=mock_llm),
patch(
"artemis.agents.validator.failure_analyzer.UnifiedMobileController"
) as mock_controller_cls,
):
mock_controller = Mock()
mock_controller_cls.return_value = mock_controller
mock_device_data = Mock()
mock_device_data.base64 = base64.b64encode(b"mocked_screenshot").decode()
mock_device_data.elements = []
mock_device_data.width = 1080
mock_device_data.height = 2400
mock_controller.get_screen_data = AsyncMock(return_value=mock_device_data)
mock_controller.get_ui_elements = AsyncMock(return_value=mock_device_data.elements)
result = await analyzer.analyze(
state,
failed_action,
error_msg,
pre_screenshot=base64.b64encode(b"dummy").decode(),
post_screenshot=base64.b64encode(b"dummy").decode(),
)
assert result["status"] == "cannot_fix"
assert "not there" in result["analysis"]
assert result["new_remaining_actions"] == []
def test_report_failure_analysis_tool_declaration():
"""Test that report_failure_analysis tool declaration is correct and matches."""
report_tool = next(
(t for t in VALIDATOR_TOOLS_DECLARATION if t.name == "report_failure_analysis"),
None,
)
assert report_tool is not None
props = report_tool.parameters.get("properties", {})
assert "analysis" in props
assert "single paragraph" in props["analysis"]["description"]
@pytest.mark.asyncio
async def test_failure_analyzer_strategy_routing(mock_context):
"""Test that FailureAnalyzer selects appropriate strategies based on error category."""
analyzer = FailureAnalyzer(mock_context)
strat_disappeared = analyzer._select_strategy(ValidationErrorCategory.TARGET_DISAPPEARED)
assert isinstance(strat_disappeared, TargetDisappearedStrategy)
assert "target_disappeared" in strat_disappeared.get_prompt_template_name()
strat_pixel = analyzer._select_strategy(ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED)
assert isinstance(strat_pixel, PixelTargetDisappearedStrategy)
assert "pixel_target_disappeared" in strat_pixel.get_prompt_template_name()
@pytest.mark.asyncio
async def test_failure_analyzer_click_sequence(mock_context, tmp_path):
"""Test click_sequence execution in FailureAnalyzer."""
screenshot_file = tmp_path / "shot.jpg"
screenshot_file.write_bytes(b"dummy_bytes")
state = DummyState(latest_screenshot=str(screenshot_file))
analyzer = FailureAnalyzer(mock_context)
# Turn 1: Call click_sequence tool with direct coordinates
call_step_1 = AIMessage(
content="Let me click in sequence.",
tool_calls=[
{
"name": "click_sequence",
"args": {"sequence": [[100, 200], [300, 400]]},
"id": "call_seq_1",
}
],
)
# Turn 2: Report failure fixed
call_step_2 = AIMessage(
content="Repaired successfully.",
tool_calls=[
{
"name": "report_failure_analysis",
"args": {
"status": "fixed",
"analysis": "Sequence clicked successfully.",
},
"id": "call_report_2",
}
],
)
mock_llm = MagicMock()
mock_bound = MagicMock()
mock_bound.ainvoke = AsyncMock(side_effect=[call_step_1, call_step_2])
mock_llm.bind_tools.return_value = mock_bound
with (
patch("artemis.agents.validator.failure_analyzer.get_llm", return_value=mock_llm),
patch(
"artemis.agents.validator.failure_analyzer.UnifiedMobileController"
) as mock_controller_cls,
):
mock_controller = Mock()
mock_controller_cls.return_value = mock_controller
mock_device_data = Mock()
mock_device_data.base64 = base64.b64encode(b"mocked_screenshot").decode()
mock_device_data.elements = []
mock_device_data.width = 1080
mock_device_data.height = 2400
mock_controller.get_screen_data = AsyncMock(return_value=mock_device_data)
mock_controller.get_ui_elements = AsyncMock(return_value=mock_device_data.elements)
mock_controller.tap_at = AsyncMock(return_value=Mock(error=None))
result = await analyzer.analyze(
state,
{"action": "click", "coordinates": [100, 200]},
"Element not found",
pre_screenshot=base64.b64encode(b"dummy").decode(),
post_screenshot=base64.b64encode(b"dummy").decode(),
error_category=ValidationErrorCategory.TARGET_DISAPPEARED,
)
assert result["status"] == "fixed"
assert mock_controller.tap_at.call_count == 2
@@ -0,0 +1,460 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import base64
from unittest.mock import AsyncMock, Mock, patch
from artemis.agents.validator.failure_analyzer import (
_exec_click,
_exec_click_sequence,
_exec_input_text,
_exec_long_press,
_exec_manage_app,
_exec_press_key,
_exec_swipe,
_exec_wait_for_delay,
)
from artemis.context import ArtemisContext
import pytest
class DummyState:
def __init__(self, latest_screenshot=None):
self.latest_screenshot = latest_screenshot
self.indexed_points = []
self.indexed_elements = []
@pytest.fixture
def mock_context(tmp_path):
ctx = Mock(spec=ArtemisContext)
ctx.device = Mock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
ctx.data_engine = Mock()
ctx.data_engine.base_dir = tmp_path
ctx.data_engine.get_or_create_image.return_value = "dummy_image_name"
ctx.data_engine.get_image_path.return_value = tmp_path / "dummy_image_name.jpg"
return ctx
@pytest.fixture
def mock_controller():
controller = Mock()
mock_device_data = Mock()
mock_device_data.base64 = base64.b64encode(b"mocked_screenshot").decode()
mock_device_data.elements = []
mock_device_data.width = 1080
mock_device_data.height = 2400
controller.get_screen_data = AsyncMock(return_value=mock_device_data)
controller.get_ui_elements = AsyncMock(return_value=mock_device_data.elements)
controller.tap_at = AsyncMock(return_value=Mock(error=None))
controller.type_text = AsyncMock(return_value=True)
controller.erase_text = AsyncMock(return_value=True)
controller.swipe_coords = AsyncMock(return_value=None)
controller.press_enter = AsyncMock(return_value=True)
controller.go_back = AsyncMock(return_value=True)
controller.go_home = AsyncMock(return_value=True)
controller.press_key = AsyncMock(return_value=True)
controller.terminate_app = AsyncMock(return_value=True)
return controller
# ==========================================
# SLIDE 1: click
# ==========================================
@pytest.mark.asyncio
async def test_click_happy_path(mock_context, mock_controller):
state = DummyState()
outcome, img_bytes, shot_path, xml_list = await _exec_click(
target=[500, 600],
state=state,
controller=mock_controller,
ctx=mock_context,
times=2,
delay_ms=150,
)
assert "Clicked at [500, 600]" in outcome
assert mock_controller.tap_at.call_count == 1
mock_controller.tap_at.assert_any_call(540, 1440, times=2, delay_ms=150)
@pytest.mark.asyncio
async def test_click_input_validation(mock_context, mock_controller):
state = DummyState()
# Coordinates holding non-integer values
outcome, _, _, _ = await _exec_click(
target=["invalid", "invalid"],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Error during click:" in outcome
@pytest.mark.asyncio
async def test_click_edge_cases(mock_context, mock_controller):
state = DummyState()
# Underflow/overflow coordinates should be clamped to valid boundary
outcome, _, _, _ = await _exec_click(
target=[-10, 1050],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Clicked at [-10, 1050]" in outcome
mock_controller.tap_at.assert_called_with(0, 2399, times=1, delay_ms=100)
@pytest.mark.asyncio
async def test_click_graceful_failure(mock_context, mock_controller):
state = DummyState()
mock_controller.tap_at.return_value = Mock(error="Click failed connection lost")
outcome, _, _, _ = await _exec_click(
target=[500, 600],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Error executing click: Click failed connection lost" in outcome
# ==========================================
# SLIDE 2: click_sequence
# ==========================================
@pytest.mark.asyncio
async def test_click_sequence_happy_path(mock_context, mock_controller):
state = DummyState()
state.indexed_points = [[540, 1200]] # Mapped from index 1 (1-indexed)
outcome, img_bytes, shot_path, xml_list = await _exec_click_sequence(
sequence=[1, [300, 400]],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Sequence clicked successfully" in outcome
assert mock_controller.tap_at.call_count == 2
mock_controller.tap_at.assert_any_call(540, 1200)
mock_controller.tap_at.assert_any_call(324, 960)
@pytest.mark.asyncio
async def test_click_sequence_input_validation(mock_context, mock_controller):
state = DummyState()
state.indexed_points = [[540, 1200], [270, 600]]
# Subsequent elements can also be integers (mixed array of indexes and coordinates)
outcome, _, _, _ = await _exec_click_sequence(
sequence=[1, 2, [300, 400]],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Sequence clicked successfully" in outcome
assert mock_controller.tap_at.call_count == 3
mock_controller.tap_at.assert_any_call(540, 1200)
mock_controller.tap_at.assert_any_call(270, 600)
mock_controller.tap_at.assert_any_call(324, 960)
# Reset indexed_points since _exec_click_sequence overwrites them
# via _capture_screenshot_and_parse_ui
state.indexed_points = [[540, 1200], [270, 600]]
# Index out of bounds
outcome, _, _, _ = await _exec_click_sequence(
sequence=[5, [300, 400]],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Error during click sequence: Index 5 is out of range" in outcome
@pytest.mark.asyncio
async def test_click_sequence_edge_cases(mock_context, mock_controller):
state = DummyState()
# Coordinates in sequence outside range should be clamped
outcome, _, _, _ = await _exec_click_sequence(
sequence=[[0, 0], [1200, -50]],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Sequence clicked successfully" in outcome
mock_controller.tap_at.assert_any_call(0, 0)
mock_controller.tap_at.assert_any_call(1079, 0)
@pytest.mark.asyncio
async def test_click_sequence_graceful_failure(mock_context, mock_controller):
state = DummyState()
mock_controller.tap_at.side_effect = [
Mock(error=None),
Mock(error="Connection error on step 2"),
]
outcome, _, _, _ = await _exec_click_sequence(
sequence=[[100, 200], [300, 400]],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Error executing click at" in outcome
assert "Connection error on step 2" in outcome
@pytest.mark.asyncio
async def test_click_sequence_stringified_formats(mock_context, mock_controller):
state = DummyState()
state.indexed_points = [[540, 1200]]
# Test stringified coordinate formats commonly generated by LLMs:
# 1. string representation of lists: ["[500, 280]", "[884, 362]"]
outcome, _, _, _ = await _exec_click_sequence(
sequence=["[500, 280]", "[884, 362]"],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Sequence clicked successfully" in outcome
mock_controller.tap_at.assert_any_call(540, 672)
mock_controller.tap_at.assert_any_call(954, 868)
# 2. comma-separated strings: ["500,280", "884,362"]
outcome, _, _, _ = await _exec_click_sequence(
sequence=["500,280", "884,362"],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Sequence clicked successfully" in outcome
# 3. whole sequence as a string
outcome, _, _, _ = await _exec_click_sequence(
sequence="[[500, 280], [884, 362]]",
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Sequence clicked successfully" in outcome
# ==========================================
# SLIDE 3: long_press
# ==========================================
@pytest.mark.asyncio
async def test_long_press_happy_path(mock_context, mock_controller):
state = DummyState()
outcome, img_bytes, shot_path, xml_list = await _exec_long_press(
target=[500, 500],
duration=1500,
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Long pressed at [500, 500]" in outcome
mock_controller.tap_at.assert_called_once_with(
540, 1200, long_press=True, long_press_duration=1500
)
@pytest.mark.asyncio
async def test_long_press_input_validation(mock_context, mock_controller):
state = DummyState()
# Invalid target type
outcome, _, _, _ = await _exec_long_press(
target="not-a-list",
duration=1500,
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Error during long press:" in outcome
# ==========================================
# SLIDE 4: input_text
# ==========================================
@pytest.mark.asyncio
async def test_input_text_happy_path(mock_context, mock_controller):
state = DummyState()
outcome, img_bytes, shot_path, xml_list = await _exec_input_text(
text="hello",
target=[500, 500],
clear_exist=True,
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Executed typing 'hello'." in outcome
mock_controller.tap_at.assert_called_once_with(540, 1200)
mock_controller.erase_text.assert_called_once()
mock_controller.type_text.assert_called_once_with("hello")
@pytest.mark.asyncio
async def test_input_text_input_validation(mock_context, mock_controller):
state = DummyState()
outcome, _, _, _ = await _exec_input_text(
text="hello",
target="invalid",
clear_exist=True,
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Error during input text:" in outcome
# ==========================================
# SLIDE 5: swipe
# ==========================================
@pytest.mark.asyncio
async def test_swipe_happy_path_direction(mock_context, mock_controller):
state = DummyState()
outcome, img_bytes, shot_path, xml_list = await _exec_swipe(
action="up",
duration=400,
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Swipe completed successfully" in outcome
mock_controller.swipe_coords.assert_called_once_with(648, 1920, 648, 480, 400)
@pytest.mark.asyncio
async def test_swipe_happy_path_coords(mock_context, mock_controller):
state = DummyState()
outcome, img_bytes, shot_path, xml_list = await _exec_swipe(
action=[100, 900, 100, 100],
duration=400,
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Swipe completed successfully" in outcome
mock_controller.swipe_coords.assert_called_once_with(108, 2160, 108, 240, 400)
@pytest.mark.asyncio
async def test_swipe_input_validation(mock_context, mock_controller):
state = DummyState()
outcome, _, _, _ = await _exec_swipe(
action="invalid_direction",
duration=400,
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Error during swipe: Invalid direction:" in outcome
# ==========================================
# SLIDE 6: press_key
# ==========================================
@pytest.mark.asyncio
async def test_press_key_happy_path(mock_context, mock_controller):
state = DummyState()
outcome, img_bytes, shot_path, xml_list = await _exec_press_key(
key="BACK",
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Executed key press 'BACK'." in outcome
mock_controller.go_back.assert_called_once()
@pytest.mark.asyncio
async def test_press_key_input_validation(mock_context, mock_controller):
state = DummyState()
outcome, _, _, _ = await _exec_press_key(
key="INVALID_KEY",
state=state,
controller=mock_controller,
ctx=mock_context,
)
# The current execution doesn't raise error on unknown keys, just logs error
assert "Error executing key press 'INVALID_KEY'." in outcome
# ==========================================
# SLIDE 7: manage_app
# ==========================================
@pytest.mark.asyncio
async def test_manage_app_happy_path(mock_context, mock_controller):
state = DummyState()
with (
patch("artemis.agents.validator.tool_declarations.find_package") as mock_find_package,
patch("artemis.agents.validator.tool_declarations.launch_app_with_retries") as mock_launch,
):
mock_find_package.return_value = "com.google.android.youtube"
mock_launch.return_value = (True, "")
outcome, img_bytes, shot_path, xml_list = await _exec_manage_app(
action="launch",
app_name="YouTube",
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Launched app 'YouTube' (com.google.android.youtube) successfully." in outcome
mock_find_package.assert_called_once_with(mock_context, "YouTube", use_fallback=False)
mock_launch.assert_called_once_with(mock_context, "com.google.android.youtube")
@pytest.mark.asyncio
async def test_manage_app_input_validation(mock_context, mock_controller):
state = DummyState()
with patch("artemis.agents.validator.tool_declarations.find_package") as mock_find_package:
mock_find_package.return_value = None
outcome, _, _, _ = await _exec_manage_app(
action="launch",
app_name="NonExistentApp",
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Error finding package for app: NonExistentApp" in outcome
# ==========================================
# SLIDE 8: wait_for_delay
# ==========================================
@pytest.mark.asyncio
async def test_wait_for_delay_happy_path(mock_context, mock_controller):
state = DummyState()
outcome, img_bytes, shot_path, xml_list = await _exec_wait_for_delay(
time_in_ms=100,
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Waited for 100ms successfully." in outcome
+113
View File
@@ -0,0 +1,113 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for Universal FlashRunner."""
from unittest.mock import Mock, patch
import pytest
from langchain_core.messages import HumanMessage, ToolMessage
from artemis.agents.flash.runner import FlashRunner
from artemis.context import ArtemisContext
@pytest.fixture
def mock_context():
ctx = Mock(spec=ArtemisContext)
ctx.llm_config = Mock()
mock_llm_cfg = Mock()
mock_llm_cfg.model = "gemini-2.5-flash"
mock_llm_cfg.temperature = 0.1
ctx.llm_config.get_agent.return_value = mock_llm_cfg
ctx.device = Mock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
ctx.data_engine = None
ctx.adb_client = None
ctx.driver = Mock()
return ctx
def test_flash_runner_tools_initialization(mock_context):
"""Verify FlashRunner gathers correct active tools."""
with patch("artemis.controllers.unified_controller.get_driver"):
runner = FlashRunner(mock_context, goal="Test Open Settings")
tools = runner._get_tools()
tool_names = [t.name for t in tools]
assert "click" in tool_names
assert "click_sequence" in tool_names
assert "swipe" in tool_names
assert "input_text" in tool_names
assert "ask_explorer" in tool_names
assert "report_task_status" in tool_names
# Validator's report_failure_analysis should NOT be in FlashRunner
assert "report_failure_analysis" not in tool_names
def test_flash_runner_screenshot_pruning(mock_context):
"""Verify that earlier screenshot blocks are pruned, keeping only the latest."""
with patch("artemis.controllers.unified_controller.get_driver"):
runner = FlashRunner(mock_context, goal="Test Pruning")
messages = [
HumanMessage(
content=[
{"type": "text", "text": "Turn 1 goal"},
{
"type": "image_url",
"image_url": {"url": "data:image/jpeg;base64,SCREENSHOT_1"},
},
]
),
ToolMessage(
tool_call_id="tc1",
name="click",
content=[
{"type": "text", "text": "Clicked button"},
{
"type": "image_url",
"image_url": {"url": "data:image/jpeg;base64,SCREENSHOT_2"},
},
],
),
ToolMessage(
tool_call_id="tc2",
name="swipe",
content=[
{"type": "text", "text": "Swiped up"},
{
"type": "image_url",
"image_url": {"url": "data:image/jpeg;base64,SCREENSHOT_3"},
},
],
),
]
runner._prune_intermediate_screenshots(messages)
# Check Turn 1: image should be removed
assert len(messages[0].content) == 1
assert messages[0].content[0]["type"] == "text"
# Check Turn 2 (Tool 1): image should be removed
assert len(messages[1].content) == 1
assert messages[1].content[0]["type"] == "text"
# Check Turn 3 (Tool 2 - Latest): image MUST be preserved
assert len(messages[2].content) == 2
assert messages[2].content[0]["type"] == "text"
assert messages[2].content[1]["type"] == "image_url"
assert messages[2].content[1]["image_url"]["url"] == "data:image/jpeg;base64,SCREENSHOT_3"
@@ -0,0 +1,289 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for VisualStepSummarizer and Context Compressor in Flash profile."""
import asyncio
from unittest.mock import AsyncMock, Mock
import pytest
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
from artemis.agents.flash.context_compressor import compress_flash_messages
from artemis.agents.flash.summarizer import VisualStepSummarizer
from artemis.context import ArtemisContext
from artemis.sdk.builders import Builders
@pytest.fixture
def mock_context():
ctx = Mock(spec=ArtemisContext)
ctx.data_engine = None
return ctx
@pytest.mark.asyncio
async def test_summarizer_dispatch_and_caching(mock_context):
"""Verify non-blocking dispatch and summary generation."""
summarizer = VisualStepSummarizer(mock_context)
mock_llm_response = AIMessage(
content="[Step 1 State: Tapped 'Login' button. Home view opened.]"
)
summarizer._llm = Mock()
summarizer._llm.ainvoke = AsyncMock(return_value=mock_llm_response)
summarizer.dispatch(
step_number=1,
action_name="click",
action_args={"target": [500, 600]},
pre_img_bytes=b"fake_pre",
post_img_bytes=b"fake_post",
exec_outcome="Clicked successfully",
)
# Await background task flush
await summarizer.flush()
assert summarizer.has_summary(1)
summary = summarizer.get_summary(1)
assert "[Step 1 State: Tapped 'Login' button. Home view opened.]" in summary
# Verify memory optimization: image buffers are released
assert summarizer._step_inputs[1]["pre_img_bytes"] is None
assert summarizer._step_inputs[1]["post_img_bytes"] is None
@pytest.mark.asyncio
async def test_summarizer_retrigger_stalled_steps(mock_context):
"""Verify that earlier stalled steps are re-triggered when subsequent steps arrive."""
summarizer = VisualStepSummarizer(mock_context)
attempt_count = 0
async def mock_ainvoke(messages):
nonlocal attempt_count
attempt_count += 1
# Fail the first attempt (Step 1 first call)
if attempt_count == 1:
raise asyncio.CancelledError("Simulated timeout")
# Check messages for step number
msg_str = str(messages)
if "Step 1" in msg_str:
return AIMessage(content="[Step 1 State: Recovered Step 1 summary]")
return AIMessage(content="[Step 2 State: Step 2 summary]")
summarizer._llm = Mock()
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
# Dispatch step 1 (fails on first attempt)
summarizer.dispatch(
step_number=1,
action_name="click",
action_args={"target": [100, 100]},
pre_img_bytes=b"pre1",
post_img_bytes=b"post1",
exec_outcome="Outcome 1",
)
await summarizer.flush()
assert not summarizer.has_summary(1)
# Dispatch step 2 (should detect stalled Step 1 and re-trigger)
summarizer.dispatch(
step_number=2,
action_name="swipe",
action_args={"action": "up"},
pre_img_bytes=b"pre2",
post_img_bytes=b"post2",
exec_outcome="Outcome 2",
)
await summarizer.flush()
assert summarizer.has_summary(1)
assert summarizer.has_summary(2)
assert "Recovered Step 1" in summarizer.get_summary(1)
def test_context_compressor_with_ready_summaries(mock_context):
"""Verify that ready summaries replace past image blocks and prune outdated XML."""
summarizer = VisualStepSummarizer(mock_context)
summarizer._summaries[1] = "[Step 1 State: Tapped Search bar. Keyboard appeared.]"
summarizer._summaries[2] = "[Step 2 State: Typed 'flights'. Suggestions displayed.]"
messages = [
HumanMessage(
content=[
{"type": "text", "text": "Task: Search flights"},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_1"}},
{"type": "text", "text": "--- UI Element List ---\nOld Tree 1"},
]
),
ToolMessage(
tool_call_id="tc1",
name="click",
content=[
{"type": "text", "text": "Action 'click' completed."},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_2"}},
{"type": "text", "text": "--- UI Element List ---\nOld Tree 2"},
],
),
ToolMessage(
tool_call_id="tc2",
name="input_text",
content=[
{"type": "text", "text": "Action 'input_text' completed."},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_3"}},
{"type": "text", "text": "--- UI Element List ---\nLive Latest Tree 3"},
],
),
]
compress_flash_messages(messages, summarizer=summarizer, prune_history_xml=True)
# Initial HumanMessage: Historical initial image pruned, task text preserved
assert len(messages[0].content) == 1
assert messages[0].content[0]["text"] == "Task: Search flights"
# Step 1 (ToolMessage 1): Image replaced with Step 1 summary, old XML pruned
assert len(messages[1].content) == 2
assert messages[1].content[0]["text"] == "Action 'click' completed."
assert messages[1].content[1]["text"] == "[Step 1 State: Tapped Search bar. Keyboard appeared.]"
# Step 2 (ToolMessage 2 - Latest live screen): Image and latest live XML MUST be preserved
assert len(messages[2].content) == 3
assert messages[2].content[0]["text"] == "Action 'input_text' completed."
assert messages[2].content[1]["type"] == "image_url"
assert "Live Latest Tree 3" in messages[2].content[2]["text"]
def test_context_compressor_graceful_fallback(mock_context):
"""Verify that when a summary is not ready yet, it cleanly falls back to standard pruned state."""
summarizer = VisualStepSummarizer(mock_context)
# Step 1 has no summary in cache
messages = [
HumanMessage(
content=[
{"type": "text", "text": "Task: Search"},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_1"}},
]
),
ToolMessage(
tool_call_id="tc1",
name="click",
content=[
{"type": "text", "text": "Action 'click' completed."},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_2"}},
],
),
]
compress_flash_messages(messages, summarizer=summarizer, prune_history_xml=True)
# Turn 1 should simply have image omitted (standard fallback)
assert len(messages[0].content) == 1
assert messages[0].content[0]["text"] == "Task: Search"
# Turn 2 (Latest) preserves image
assert len(messages[1].content) == 2
assert messages[1].content[1]["type"] == "image_url"
def test_flash_config_and_builder():
"""Verify Flash profile configuration model and SDK builder fluent API."""
cfg = Builders.AgentConfig.with_flash_config(
max_turns=25,
explorer_mode="flash",
step_summarizer=True,
step_summarizer_model="gemini-2.5-flash-lite",
prune_history_xml=True,
).build()
assert cfg.flash.max_turns == 25
assert cfg.flash.explorer_mode == "flash"
assert cfg.flash.step_summarizer.enabled is True
assert cfg.flash.step_summarizer.model == "gemini-2.5-flash-lite"
assert cfg.flash.step_summarizer.prune_history_xml is True
@pytest.mark.asyncio
async def test_data_engine_integration_with_step_summarizer(mock_context):
"""Verify that generated summaries sync into DataEngine and are visible to get_agent_friendly_steps."""
mock_de = Mock()
mock_de.update_step_summary = Mock()
mock_context.data_engine = mock_de
summarizer = VisualStepSummarizer(mock_context)
summarizer._llm = Mock()
summarizer._llm.ainvoke = AsyncMock(
return_value=AIMessage(content="[Step 1 State: Input text confirmed.]")
)
summarizer.dispatch(
step_number=1,
action_name="input_text",
action_args={"text": "hello"},
pre_img_bytes=b"pre",
post_img_bytes=b"post",
exec_outcome="Executed typing",
)
await summarizer.flush()
mock_de.update_step_summary.assert_called_once_with(1, "[Step 1 State: Input text confirmed.]")
def test_draw_action_overlay_rendering():
"""Verify that draw_action_overlay_on_image correctly renders markers on images."""
import io
from PIL import Image
from artemis.utils.visualization import draw_action_overlay_on_image
# Create a blank RGB image in memory
img = Image.new("RGB", (1080, 2400), color="white")
buf = io.BytesIO()
img.save(buf, format="JPEG")
raw_bytes = buf.getvalue()
# 1. Click overlay
annotated_click = draw_action_overlay_on_image(
image_bytes=raw_bytes,
action_name="click",
action_args={"target": [540, 1200]},
)
assert len(annotated_click) > 0
assert annotated_click != raw_bytes
# 2. Swipe overlay
annotated_swipe = draw_action_overlay_on_image(
image_bytes=raw_bytes,
action_name="swipe",
action_args={"start": [500, 1800], "end": [500, 600]},
)
assert len(annotated_swipe) > 0
assert annotated_swipe != raw_bytes
# 3. Click Sequence overlay
annotated_seq = draw_action_overlay_on_image(
image_bytes=raw_bytes,
action_name="click_sequence",
action_args={"sequence": [[200, 300], [400, 500], [600, 700]]},
)
assert len(annotated_seq) > 0
assert annotated_seq != raw_bytes
# 4. Fail-safe on corrupt bytes
corrupt_result = draw_action_overlay_on_image(
image_bytes=b"invalid_bytes",
action_name="click",
action_args={"target": [100, 100]},
)
assert corrupt_result == b"invalid_bytes"
+377
View File
@@ -0,0 +1,377 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from pathlib import Path
from unittest.mock import MagicMock, patch
from langchain_core.messages import ToolMessage
from artemis.agents.history_analyzer.history_analyzer import HistoryAnalyzer
from artemis.context import ArtemisContext
import pytest
@pytest.mark.asyncio
async def test_history_analyzer_no_history():
# Mock context with a data engine that has no steps
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.get_agent_friendly_steps.return_value = []
analyzer = HistoryAnalyzer(mock_ctx)
result = await analyzer.run("What happened?")
assert result == "No history recorded for this session yet."
@pytest.mark.asyncio
async def test_history_analyzer_simple_query_no_tool_call():
# Mock context and data engine
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = "/tmp/fake_traces"
steps = [
{
"step_number": 1,
"relative_time": "1.2s",
"summary": "Opened the settings app",
},
{
"step_number": 2,
"relative_time": "4.5s",
"summary": "Toggled the wifi switch",
},
]
mock_ctx.data_engine.get_agent_friendly_steps.return_value = steps
# Mock LLM to return a direct text response (no tool calls)
mock_response = MagicMock()
mock_response.content = "First you opened settings, then you toggled wifi."
mock_response.tool_calls = []
mock_llm = MagicMock()
mock_llm.bind_tools.return_value = mock_llm
async def mock_astream(*args, **kwargs):
yield mock_response
mock_llm.astream.side_effect = mock_astream
with (
patch(
"artemis.agents.history_analyzer.history_analyzer.get_llm",
return_value=mock_llm,
),
patch("pathlib.Path.exists", return_value=False), # Force system prompt fallback
):
analyzer = HistoryAnalyzer(mock_ctx)
result = await analyzer.run("Summarize the session")
assert result == "First you opened settings, then you toggled wifi."
assert mock_llm.astream.called
@pytest.mark.asyncio
async def test_history_analyzer_detailed_query_with_tool_call():
# Mock context and data engine with full details
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = "/tmp/fake_traces"
steps = [
{
"step_number": 1,
"relative_time": "1.2s",
"summary": "Opened the settings app",
"action_taken": {"action": "tap", "coordinates": [500, 600]},
"operator_raw_thinking": "Need to open settings to configure wifi",
"last_execution_result": {"status": "success"},
},
{
"step_number": 2,
"relative_time": "4.5s",
"summary": "Toggled the wifi switch",
"action_taken": {"action": "tap", "coordinates": [100, 200]},
"operator_raw_thinking": "Toggle the switch to turn wifi on",
"last_execution_result": {"status": "success"},
},
]
mock_ctx.data_engine.get_agent_friendly_steps.return_value = steps
# Turn 1 LLM response: Tool call to get_step_details for step 2
mock_response_turn_1 = MagicMock()
mock_response_turn_1.content = ""
mock_tool_call = {
"name": "get_step_details",
"args": {"start_step": 2, "end_step": 2},
"id": "call_123456",
}
mock_response_turn_1.tool_calls = [mock_tool_call]
# Turn 2 LLM response: Final answer using the details
mock_response_turn_2 = MagicMock()
mock_response_turn_2.content = (
"In step 2, the operator was thinking: 'Toggle the switch to turn wifi on'."
)
mock_response_turn_2.tool_calls = []
mock_llm = MagicMock()
mock_llm.bind_tools.return_value = mock_llm
responses = [mock_response_turn_1, mock_response_turn_2]
call_count = 0
async def mock_astream(*args, **kwargs):
nonlocal call_count
if call_count < len(responses):
yield responses[call_count]
call_count += 1
mock_llm.astream.side_effect = mock_astream
with (
patch(
"artemis.agents.history_analyzer.history_analyzer.get_llm",
return_value=mock_llm,
),
patch("pathlib.Path.exists", return_value=False), # Force fallback prompt
):
analyzer = HistoryAnalyzer(mock_ctx)
result = await analyzer.run("What was the operator thinking in step 2?")
assert (
result == "In step 2, the operator was thinking: 'Toggle the switch to turn wifi on'."
)
assert mock_llm.astream.call_count == 2
# Verify the tool was invoked with correct parameters internally
tool_msg = mock_llm.astream.call_args_list[1][0][0][
-2
] # ToolMessage is second to last after final response is appended
assert isinstance(tool_msg, ToolMessage)
assert tool_msg.tool_call_id == "call_123456"
# Parse the JSON in the ToolMessage content
tool_result = json.loads(tool_msg.content)
assert len(tool_result) == 1
assert tool_result[0]["step_number"] == 2
assert tool_result[0]["operator_raw_thinking"] == "Toggle the switch to turn wifi on"
assert tool_result[0]["action_taken"] == {
"action": "tap",
"coordinates": [100, 200],
}
@pytest.mark.asyncio
async def test_history_analyzer_read_note_tool_call():
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = "/tmp/fake_traces"
steps = [{"step_number": 1, "relative_time": "1.0s", "summary": "Initiated"}]
mock_ctx.data_engine.get_agent_friendly_steps.return_value = steps
# Turn 1 LLM response: Tool call to list_notes
mock_response_turn_1 = MagicMock()
mock_response_turn_1.content = ""
mock_tool_call_list = {"name": "list_notes", "args": {}, "id": "call_list"}
mock_response_turn_1.tool_calls = [mock_tool_call_list]
# Turn 2 LLM response: Tool call to read_note for tactical_plan
mock_response_turn_2 = MagicMock()
mock_response_turn_2.content = ""
mock_tool_call_read = {
"name": "read_note",
"args": {"key": "tactical_plan"},
"id": "call_read",
}
mock_response_turn_2.tool_calls = [mock_tool_call_read]
# Turn 3 LLM response: Final answer
mock_response_turn_3 = MagicMock()
mock_response_turn_3.content = "The plan was to toggle wifi."
mock_response_turn_3.tool_calls = []
mock_llm = MagicMock()
mock_llm.bind_tools.return_value = mock_llm
responses = [
mock_response_turn_1,
mock_response_turn_2,
mock_response_turn_3,
]
call_count = 0
async def mock_astream(*args, **kwargs):
nonlocal call_count
if call_count < len(responses):
yield responses[call_count]
call_count += 1
mock_llm.astream.side_effect = mock_astream
# Mock the file system read using patch
fake_plan_content = "Goal: Toggle wifi"
def mock_exists(path):
# Let Path("/tmp/fake_traces/notes").exists() and notes/tactical_plan.md exist
return True
def mock_read_text(self, encoding="utf-8"):
return fake_plan_content
def mock_glob(self, pattern):
return [Path("/tmp/fake_traces/notes/tactical_plan.md")]
with (
patch(
"artemis.agents.history_analyzer.history_analyzer.get_llm",
return_value=mock_llm,
),
patch("pathlib.Path.exists", mock_exists),
patch("pathlib.Path.read_text", mock_read_text),
patch("pathlib.Path.glob", mock_glob),
):
analyzer = HistoryAnalyzer(mock_ctx)
result = await analyzer.run("What is the saved tactical plan?")
assert result == "The plan was to toggle wifi."
assert mock_llm.astream.call_count == 3
# Verify list_notes response
list_msg = mock_llm.astream.call_args_list[1][0][0][
3
] # Index 3 contains the ToolMessage for list_notes
assert isinstance(list_msg, ToolMessage)
assert list_msg.tool_call_id == "call_list"
assert list_msg.content == "Here are all the notes:\n- tactical_plan (1 lines)"
# Verify read_note response
read_msg = mock_llm.astream.call_args_list[2][0][0][
5
] # Index 5 contains the ToolMessage for read_note
assert isinstance(read_msg, ToolMessage)
assert read_msg.tool_call_id == "call_read"
assert (
read_msg.content == "Successfully read note 'tactical_plan'. 'tactical_plan' note"
f" content:\n{fake_plan_content}"
)
def test_history_analyzer_robust_tools_behavior():
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = "/tmp/fake_traces"
steps = [
{
"step_number": 1,
"relative_time": "1.0s",
"summary": "Action 1",
"action_taken": {"action": "tap"},
}
]
analyzer = HistoryAnalyzer(mock_ctx)
# 1. Test get_step_details with string inputs
details_tool = analyzer._get_step_details_tool(steps)
result_details = details_tool.invoke({"start_step": "1", "end_step": "1"})
parsed_result = json.loads(result_details)
assert len(parsed_result) == 1
assert parsed_result[0]["step_number"] == 1
# 2. Test read_note with .md suffix
from artemis.tools.scratchpad import get_read_note_tool_pure
read_tool = get_read_note_tool_pure(analyzer.ctx)
def mock_exists(path):
return "tactical_plan.md" in str(path)
with (
patch("pathlib.Path.exists", mock_exists),
patch("pathlib.Path.read_text", return_value="Tactical plan content"),
):
result_read = read_tool.invoke({"key": "tactical_plan.md"})
assert (
result_read == "Successfully read note 'tactical_plan.md'. 'tactical_plan.md'"
" note content:\nTactical plan content"
)
@pytest.mark.asyncio
async def test_history_analyzer_integration_with_task_tree():
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = "/tmp/fake_traces"
steps = [
{
"step_id": "step_1",
"step_number": 1,
"relative_time": "1.0s",
"summary": "Step 1",
},
]
mock_ctx.data_engine.get_agent_friendly_steps.return_value = steps
mock_response = MagicMock()
mock_response.content = "Answer"
mock_response.tool_calls = []
mock_llm = MagicMock()
mock_llm.bind_tools.return_value = mock_llm
async def mock_astream(*args, **kwargs):
yield mock_response
mock_llm.astream.side_effect = mock_astream
fake_plan = """- [/] Active Subgoal
- [ ] Pending Subgoal"""
mock_path = MagicMock()
mock_path.exists.return_value = True
mock_path.read_text.return_value = fake_plan
# Mock build_plan_and_history
with (
patch(
"artemis.agents.history_analyzer.history_analyzer.get_llm",
return_value=mock_llm,
),
patch(
"artemis.agents.history_analyzer.history_analyzer.get_note_file_path",
return_value=mock_path,
),
patch(
"artemis.agents.history_analyzer.history_analyzer.build_plan_and_history"
) as mock_build,
):
mock_build.return_value = "Mocked operation history"
analyzer = HistoryAnalyzer(mock_ctx)
await analyzer.run("query")
# Verify build_plan_and_history call args
mock_build.assert_called_once()
args, kwargs = mock_build.call_args
assert args[0] == fake_plan
assert args[1] == steps
import hashlib
expected_hash = hashlib.md5(b"Active Subgoal").hexdigest()
assert args[2] == expected_hash
assert kwargs.get("last_n_detailed") == 1
assert kwargs.get("min_summaries") == len(steps)
File diff suppressed because it is too large. Load diff
+508
View File
@@ -0,0 +1,508 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, Mock, patch
from pydantic import BaseModel
import pytest
from artemis.agents.outputter.outputter import outputter
from artemis.config import LLM, OutputConfig # noqa: E402
from artemis.context import ArtemisContext # noqa: E402
from artemis.utils.logger import get_logger # noqa: E402
logger = get_logger(__name__)
class _CyFunctionDetectorMeta(type):
def __instancecheck__(self, instance):
name = type(instance).__name__
return (
name
in (
"cyfunction",
"cython_function_or_method",
"builtin_function_or_method",
)
or "cyfunction" in name.lower()
)
class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta):
pass
class MockPydanticSchema(BaseModel):
model_config = {"ignored_types": (CyFunctionDetector,)}
color: str
price: float
currency_symbol: str
website_url: str
mock_dict = {
"color": "green",
"price": 20,
"currency_symbol": "$",
"website_url": "http://superwebsite.fr",
}
class DummyState:
def __init__(self, messages, initial_goal, operator_raw_data=None):
self.messages = messages
self.initial_goal = initial_goal
self.operator_raw_data = operator_raw_data
@pytest.fixture
def mock_context():
"""Create a properly mocked context with all required fields."""
ctx = Mock(spec=ArtemisContext)
ctx.llm_config = {
"planner": LLM(provider="openai", model="gpt-5-nano"),
"operator": LLM(provider="openai", model="gpt-5-nano"),
"validator": LLM(provider="openai", model="gpt-5-nano"),
}
ctx.device = Mock()
ctx.data_engine = None
return ctx
@pytest.fixture
def mock_state():
"""Create a mock state with test data."""
return DummyState(
messages=[],
initial_goal="Find a green product on my website",
)
def setup_mock_llm(mock_get_llm, react_response_content="Paris", structured_response=None):
mock_llm = Mock()
mock_llm_fallback = Mock()
# Mock bind_tools
mock_llm_with_tools = Mock()
mock_llm_with_tools.ainvoke = AsyncMock()
mock_react_response = Mock()
mock_react_response.content = react_response_content
mock_react_response.tool_calls = []
mock_llm_with_tools.ainvoke.return_value = mock_react_response
mock_llm.bind_tools.return_value = mock_llm_with_tools
mock_llm_fallback.bind_tools.return_value = mock_llm_with_tools
# Mock with_structured_output
mock_structured_llm = Mock()
mock_structured_llm.ainvoke = AsyncMock()
if structured_response is not None:
mock_structured_llm.ainvoke.return_value = structured_response
mock_llm.with_structured_output.return_value = mock_structured_llm
mock_llm_fallback.with_structured_output.return_value = mock_structured_llm
def get_llm_side_effect(ctx, name, is_utils=False, use_fallback=False):
if use_fallback:
return mock_llm_fallback
return mock_llm
mock_get_llm.side_effect = get_llm_side_effect
return mock_llm, mock_llm_with_tools, mock_structured_llm
@patch("artemis.agents.outputter.outputter.get_llm")
@pytest.mark.asyncio
async def test_outputter_with_pydantic_model(mock_get_llm, mock_context, mock_state):
"""Test outputter with Pydantic model output."""
expected_structured = MockPydanticSchema(
color="green",
price=20,
currency_symbol="$",
website_url="http://superwebsite.fr",
)
setup_mock_llm(
mock_get_llm,
react_response_content="Raw details about green product",
structured_response=expected_structured,
)
config = OutputConfig(
structured_output=MockPydanticSchema,
output_description=None,
)
result = await outputter(ctx=mock_context, output_config=config, graph_output=mock_state)
assert isinstance(result, dict)
assert result.get("color") == "green"
@patch("artemis.agents.outputter.outputter.get_llm")
@pytest.mark.asyncio
async def test_outputter_with_dict(mock_get_llm, mock_context, mock_state):
"""Test outputter with dictionary output."""
expected_dict = {
"color": "green",
"price": 20,
"currency_symbol": "$",
"website_url": "http://superwebsite.fr",
}
setup_mock_llm(
mock_get_llm,
react_response_content="Raw details",
structured_response=expected_dict,
)
config = OutputConfig(
structured_output=mock_dict,
output_description=None,
)
result = await outputter(ctx=mock_context, output_config=config, graph_output=mock_state)
assert isinstance(result, dict)
assert result.get("color") == "green"
assert result.get("price") == 20
assert result.get("currency_symbol") == "$"
assert result.get("website_url") == "http://superwebsite.fr"
@patch("artemis.agents.outputter.outputter.get_llm")
@pytest.mark.asyncio
async def test_outputter_with_natural_language_output(mock_get_llm, mock_context, mock_state):
"""Test outputter with natural language description output (returns JSON string)."""
expected_json = (
'{"color": "green", "price": 20, "currency_symbol": "$", "website_url":'
' "http://superwebsite.fr"}'
)
setup_mock_llm(mock_get_llm, react_response_content=expected_json)
config = OutputConfig(
structured_output=None,
output_description=(
"A JSON object with a color, a price, a currency_symbol and a website_url key"
),
)
result = await outputter(ctx=mock_context, output_config=config, graph_output=mock_state)
assert isinstance(result, dict)
assert result.get("color") == "green"
assert result.get("price") == 20
assert result.get("currency_symbol") == "$"
assert result.get("website_url") == "http://superwebsite.fr"
@patch("artemis.agents.outputter.outputter.get_llm")
@pytest.mark.asyncio
async def test_outputter_with_history_and_image(mock_get_llm, mock_context):
"""Test outputter with explicit history and image."""
expected_json = '{"answer": "Paris"}'
_, mock_llm_with_tools, _ = setup_mock_llm(mock_get_llm, react_response_content=expected_json)
config = OutputConfig(
structured_output=None,
output_description="The capital of France",
)
class StateWithRawData:
def __init__(self, messages, initial_goal, operator_raw_data):
self.messages = messages
self.initial_goal = initial_goal
self.operator_raw_data = operator_raw_data
state = StateWithRawData(
messages=[],
initial_goal="What is the capital of France?",
operator_raw_data={"screenshot_b64": "mock_b64_data"},
)
await outputter(
ctx=mock_context,
output_config=config,
graph_output=state,
plan_and_history="Plan: Find capital\nStep 1: Opened browser",
)
call_args = mock_llm_with_tools.ainvoke.call_args
assert call_args is not None
messages = call_args[0][0]
assert (
"Your sole objective is to verify whether the user's initial goal was"
" achieved" in messages[0].content
)
human_content = messages[1].content
assert isinstance(human_content, list)
assert human_content[0]["type"] == "text"
assert "Step 1: Opened browser" in human_content[0]["text"]
assert "Plan: Find capital" in human_content[0]["text"]
assert human_content[1]["type"] == "image_url"
assert human_content[1]["image_url"]["url"] == "data:image/jpeg;base64,mock_b64_data"
@patch("artemis.agents.outputter.outputter.get_llm")
@pytest.mark.asyncio
async def test_outputter_builds_concise_history(mock_get_llm):
"""Test that the outputter dynamically builds clean Summary ➡️ Action history."""
expected_json = '{"answer": "Success"}'
_, mock_llm_with_tools, _ = setup_mock_llm(mock_get_llm, react_response_content=expected_json)
config = OutputConfig(
structured_output=None,
output_description="Test output",
)
ctx = Mock(spec=ArtemisContext)
ctx.llm_config = {
"planner": LLM(provider="openai", model="gpt-5-nano"),
"operator": LLM(provider="openai", model="gpt-5-nano"),
"validator": LLM(provider="openai", model="gpt-5-nano"),
}
ctx.device = Mock()
mock_data_engine = Mock()
steps = [
{
"step_id": "step_1",
"step_number": 1,
"relative_time": "2.5s",
"summary": "Tapped search button",
"action_taken": [{"action": "click", "target_text": "Search"}],
"last_execution_result": {"status": "success"},
}
]
mock_data_engine.get_agent_friendly_steps.return_value = steps
mock_data_engine.base_dir = "/tmp/mock_session"
ctx.data_engine = mock_data_engine
class StateWithRawData:
def __init__(self, messages, initial_goal, operator_raw_data):
self.messages = messages
self.initial_goal = initial_goal
self.operator_raw_data = operator_raw_data
state = StateWithRawData(
messages=[],
initial_goal="Search for something",
operator_raw_data=None,
)
# Test Case 1: Fallback path (no plan file)
from pathlib import Path
original_exists = Path.exists
def mock_exists_fallback(self, *args, **kwargs):
if self.name == "task_plan.md":
return False
return original_exists(self, *args, **kwargs)
with patch("pathlib.Path.exists", mock_exists_fallback):
await outputter(
ctx=ctx,
output_config=config,
graph_output=state,
)
call_args = mock_llm_with_tools.ainvoke.call_args
assert call_args is not None
messages = call_args[0][0]
human_content_fallback = messages[1].content[0]["text"]
assert "--- Execution History ---" in human_content_fallback
assert "- *Step 1 (Start: 2.5s): Tapped search button*" in human_content_fallback
assert "*Action*: Tapped 'Search' at None" in human_content_fallback
# Reset mock call history
mock_llm_with_tools.ainvoke.reset_mock()
# Test Case 2: Standard path (plan file exists)
def mock_exists_standard(self, *args, **kwargs):
if self.name == "task_plan.md":
return True
return original_exists(self, *args, **kwargs)
original_read_text = Path.read_text
def mock_read_text(self, *args, **kwargs):
if self.name == "task_plan.md":
return "- [ ] Goal 1\n - [/] Subgoal 1.1"
return original_read_text(self, *args, **kwargs)
with (
patch("pathlib.Path.exists", mock_exists_standard),
patch("pathlib.Path.read_text", mock_read_text),
):
await outputter(
ctx=ctx,
output_config=config,
graph_output=state,
)
call_args = mock_llm_with_tools.ainvoke.call_args
assert call_args is not None
messages = call_args[0][0]
human_content_standard = messages[1].content[0]["text"]
assert "--- Execution History ---" in human_content_standard
assert "- *Step 1 (Start: 2.5s): Tapped search button*" in human_content_standard
assert "*Action*: Tapped 'Search' at None" in human_content_standard
@patch("artemis.tools.video_tool.VideoAnalyzer")
@patch("artemis.agents.outputter.outputter.get_llm")
@pytest.mark.asyncio
async def test_outputter_executes_video_analyzer_tool(
mock_get_llm, mock_video_analyzer, mock_context, mock_state
):
"""Test that the outputter ReAct loop can successfully invoke the video_analyzer tool."""
# Setup mock VideoAnalyzer
mock_agent_instance = Mock()
mock_agent_instance.run = AsyncMock(
return_value=("The user played a video for 5 seconds.", "success")
)
mock_video_analyzer.return_value = mock_agent_instance
# Setup mock LLM with two turns
mock_llm = Mock()
mock_llm_fallback = Mock()
mock_llm_with_tools = Mock()
mock_llm_with_tools.ainvoke = AsyncMock()
# Turn 1: LLM calls video_analyzer tool
tool_call = {
"name": "video_analyzer",
"args": {
"time_description": "from 5s to 10s",
"purpose": "verify video played",
},
"id": "call_123",
}
msg_turn1 = Mock()
msg_turn1.content = ""
msg_turn1.tool_calls = [tool_call]
# Turn 2: LLM returns final answer
msg_turn2 = Mock()
msg_turn2.content = "Video played successfully."
msg_turn2.tool_calls = []
mock_llm_with_tools.ainvoke.side_effect = [msg_turn1, msg_turn2]
mock_llm.bind_tools.return_value = mock_llm_with_tools
mock_llm_fallback.bind_tools.return_value = mock_llm_with_tools
def get_llm_side_effect(ctx, name, is_utils=False, use_fallback=False):
if use_fallback:
return mock_llm_fallback
return mock_llm
mock_get_llm.side_effect = get_llm_side_effect
config = OutputConfig(
structured_output=None,
output_description="Verify video",
)
result = await outputter(ctx=mock_context, output_config=config, graph_output=mock_state)
# Assert VideoAnalyzer was called correctly
mock_video_analyzer.assert_called_once_with(mock_context)
mock_agent_instance.run.assert_called_once_with("from 5s to 10s", "verify video played")
# Assert LLM's ainvoke was called twice
assert mock_llm_with_tools.ainvoke.call_count == 2
# Assert tool message was appended (messages list is mutated in place,
# so it eventually has 5 elements)
sent_messages = mock_llm_with_tools.ainvoke.call_args_list[1][0][0]
assert len(sent_messages) == 5
# Verify the sequence of messages
assert sent_messages[2] == msg_turn1
tool_message = sent_messages[3]
from langchain_core.messages import ToolMessage
assert isinstance(tool_message, ToolMessage)
assert tool_message.content == "The user played a video for 5 seconds."
assert tool_message.status == "success"
assert sent_messages[4] == msg_turn2
# Assert final result
assert result == "Video played successfully."
@patch("artemis.agents.outputter.outputter.get_llm")
@pytest.mark.asyncio
async def test_outputter_executes_save_note_tool(mock_get_llm, mock_context, mock_state):
"""Test that the outputter ReAct loop can successfully invoke the save_note tool."""
# Setup mock LLM with two turns
mock_llm = Mock()
mock_llm_fallback = Mock()
mock_llm_with_tools = Mock()
mock_llm_with_tools.ainvoke = AsyncMock()
# Turn 1: LLM calls save_note tool
tool_call = {
"name": "save_note",
"args": {"key": "verification_code", "content": "123456"},
"id": "call_save_123",
}
msg_turn1 = Mock()
msg_turn1.content = ""
msg_turn1.tool_calls = [tool_call]
# Turn 2: LLM returns final answer
msg_turn2 = Mock()
msg_turn2.content = "Verification code is 123456."
msg_turn2.tool_calls = []
mock_llm_with_tools.ainvoke.side_effect = [msg_turn1, msg_turn2]
mock_llm.bind_tools.return_value = mock_llm_with_tools
mock_llm_fallback.bind_tools.return_value = mock_llm_with_tools
def get_llm_side_effect(ctx, name, is_utils=False, use_fallback=False):
if use_fallback:
return mock_llm_fallback
return mock_llm
mock_get_llm.side_effect = get_llm_side_effect
config = OutputConfig(
structured_output=None,
output_description="Save verification code to note",
)
# Setup mock data_engine on context to support saving notes
mock_data_engine = Mock()
mock_data_engine.base_dir = "/tmp/mock_session"
mock_context.data_engine = mock_data_engine
# Patch save_note_content utility to avoid writing to disk
with patch("artemis.tools.scratchpad.save_note_content") as mock_save_content:
result = await outputter(ctx=mock_context, output_config=config, graph_output=mock_state)
mock_save_content.assert_called_once_with(
"/tmp/mock_session", "verification_code", "123456"
)
# Assert final result
assert result == "Verification code is 123456."
@@ -0,0 +1,102 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from unittest.mock import MagicMock, mock_open, patch
from artemis.agents.outputter.tools import (
get_search_history_tool,
get_step_details_tool,
get_step_screenshot_tool,
)
from artemis.context import ArtemisContext
import pytest
@pytest.fixture
def mock_context():
ctx = MagicMock(spec=ArtemisContext)
ctx.data_engine = MagicMock()
ctx.data_engine.global_base_dir = "/tmp/fake_traces"
return ctx
def test_get_step_details_tool():
steps = [
{
"step_number": 1,
"relative_time": "1s",
"summary": "Step 1",
"action_taken": "tap",
},
{
"step_number": 2,
"relative_time": "2s",
"summary": "Step 2",
"action_taken": "swipe",
},
]
tool = get_step_details_tool(steps)
# Test valid range
result = tool.invoke({"start_step": 1, "end_step": 1})
parsed = json.loads(result)
assert len(parsed) == 1
assert parsed[0]["step_number"] == 1
assert parsed[0]["action_taken"] == "tap"
# Test invalid range
result_invalid = tool.invoke({"start_step": 3, "end_step": 4})
assert "No steps found" in result_invalid
def test_get_step_screenshot_tool(mock_context):
steps = [
{"step_number": 1, "pre_image_name": "img1", "post_image_name": "img2"},
]
tool = get_step_screenshot_tool(mock_context, steps)
# Mock open and exists
with (
patch("pathlib.Path.exists", return_value=True),
patch("builtins.open", mock_open(read_data=b"fake_image_bytes")),
):
result = tool.invoke({"step_number": 1})
assert isinstance(result, list)
assert result[0]["type"] == "text"
assert result[1]["type"] == "image_url"
assert "data:image/jpeg;base64" in result[1]["image_url"]["url"]
def test_search_history_tool(mock_context):
tool = get_search_history_tool(mock_context)
# Mock steps returned by storage
mock_step_1 = MagicMock()
mock_step_1.step_number = 1
mock_step_1.pre_image_name = "img1"
mock_step_1.post_image_name = None
mock_context.data_engine.storage.get_steps.return_value = [mock_step_1]
# Mock search_ui_by_hash
mock_context.data_engine.storage.search_ui_by_hash.return_value = [
{"type": "ocr", "matched_text": "verification code: 1234", "score": 0.9}
]
result = tool.invoke({"query": "verification"})
parsed = json.loads(result)
assert len(parsed) == 1
assert parsed[0]["step_number"] == 1
assert parsed[0]["matches"][0]["matched_text"] == "verification code: 1234"
+89
View File
@@ -0,0 +1,89 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import Mock, patch
from artemis.agents.planner.planner import PlannerNode
from artemis.context import DevicePlatform, ArtemisContext
import pytest
class DummyState:
def __init__(
self,
initial_goal,
needs_replan=False,
operator_raw_data=None,
operator_replan_reason=None,
):
self.initial_goal = initial_goal
self.needs_replan = needs_replan
self.operator_raw_data = operator_raw_data
self.operator_replan_reason = operator_replan_reason
async def asanitize_update(self, ctx, update, agent):
return update
@pytest.fixture
def mock_context():
ctx = Mock(spec=ArtemisContext)
ctx.device = Mock()
ctx.device.mobile_platform = DevicePlatform.ANDROID
ctx.adb_client = Mock()
ctx.ui_adb_client = Mock()
mock_screen_data = Mock()
mock_screen_data.base64 = "YmFzZTY0"
mock_screen_data.elements = []
mock_screen_data.width = 1080
mock_screen_data.height = 2400
ctx.ui_adb_client.get_screen_data.return_value = mock_screen_data
ctx.data_engine = None
return ctx
@pytest.mark.asyncio
async def test_planner_initial_plan(mock_context):
state = DummyState(
initial_goal="Test Goal",
operator_raw_data={"screenshot_b64": "dummy_base64_string"},
)
node = PlannerNode(mock_context)
mock_llm = Mock()
mock_response = Mock()
mock_response.content = "- [ ] Subgoal 1"
mock_response.tool_calls = []
async def mock_astream(*args, **kwargs):
yield mock_response
mock_llm.astream.side_effect = mock_astream
mock_llm.bind_tools = Mock(return_value=mock_llm)
with patch("artemis.agents.planner.planner.get_llm", return_value=mock_llm):
await node(state)
assert mock_llm.astream.called
args, kwargs = mock_llm.astream.call_args
messages = args[0]
system_msg = messages[0].content
assert "## Role" in system_msg
human_msg_content = messages[1].content
assert isinstance(human_msg_content, list)
assert human_msg_content[0]["type"] == "text"
assert "Goal: Test Goal" in human_msg_content[0]["text"]
+112
View File
@@ -0,0 +1,112 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.agents.operator.operator import OperatorNode
from artemis.context import ArtemisContext
from artemis.graph.graph import execution_check_node
from artemis.graph.state import State
import pytest
@pytest.mark.asyncio
async def test_snapshot_rollback_flow(tmp_path):
# Setup dummy directories
base_dir = tmp_path / "trace"
notes_dir = base_dir / "notes"
notes_dir.mkdir(parents=True)
task_plan_path = notes_dir / "task_plan.md"
task_plan_path.write_text("Initial Plan", encoding="utf-8")
# Mock Context
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = base_dir
mock_ctx.execution_setup = None
# Mock State
mock_state = MagicMock(spec=State)
mock_state.subagent_calls = []
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
mock_state.initial_goal = "Test goal"
mock_state.short_term_memory = ""
mock_state.operator_raw_data = {
"screenshot_b64": "YmFzZTY0",
"xml_hierarchy": "<hierarchy></hierarchy>",
"ocr_results": [],
}
mock_state.current_step_id = "test-step-id"
# Mock Controller and LLM to let Operator run minimally
mock_controller = MagicMock()
mock_controller.take_screenshot = AsyncMock(return_value="YmFzZTY0")
mock_device_data = MagicMock()
mock_device_data.elements = []
mock_controller.get_screen_data = AsyncMock(return_value=mock_device_data)
mock_llm = MagicMock()
mock_response = MagicMock()
mock_response.tool_calls = []
async def mock_astream(*args, **kwargs):
yield mock_response
mock_llm.astream.side_effect = mock_astream
mock_llm.bind_tools.return_value = mock_llm
# 1. Run Operator to trigger snapshot
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
node = OperatorNode(mock_ctx)
await node(mock_state)
# Verify snapshot was created
snapshot_dir = notes_dir.with_name("notes_snapshot")
assert snapshot_dir.exists()
assert (snapshot_dir / "task_plan.md").read_text(encoding="utf-8") == "Initial Plan"
assert mock_ctx.task_plan_snapshot == snapshot_dir
# 2. Simulate Operator optimistic modification
task_plan_path.write_text("Modified Plan", encoding="utf-8")
# 3. Run execution_check_node with failure to trigger rollback
future = asyncio.Future()
future.set_result({"status": "failed"})
mock_ctx.checker_task = future
await execution_check_node(mock_state, mock_ctx)
# Verify rollback restored the file
assert task_plan_path.read_text(encoding="utf-8") == "Initial Plan"
# 4. Run Operator again to create new snapshot (simulate next attempt)
# First modify it again to simulate operator working on restored plan
task_plan_path.write_text("Modified Plan 2", encoding="utf-8")
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
await node(mock_state)
assert (snapshot_dir / "task_plan.md").read_text(encoding="utf-8") == "Modified Plan 2"
# 5. Run execution_check_node with success to trigger commit (cleanup)
future_success = asyncio.Future()
future_success.set_result({"status": "success"})
mock_ctx.checker_task = future_success
await execution_check_node(mock_state, mock_ctx)
# Verify snapshot was deleted
assert not snapshot_dir.exists()
+216
View File
@@ -0,0 +1,216 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from unittest.mock import MagicMock, Mock, patch
from artemis.agents.summarizer.summarizer import SummarizerNode
from artemis.context import ArtemisContext
import pytest
class DummyState:
def __init__(
self,
structured_decisions=None,
operator_raw_thinking=None,
operator_native_thinking=None,
last_execution_result=None,
current_step_id=None,
):
self.structured_decisions = structured_decisions
self.operator_raw_thinking = operator_raw_thinking
self.operator_native_thinking = operator_native_thinking
self.last_execution_result = last_execution_result
self.current_step_id = current_step_id
@pytest.fixture
def mock_context():
ctx = Mock(spec=ArtemisContext)
ctx.device = None
ctx.data_engine = Mock()
ctx.background_tasks = []
return ctx
@pytest.mark.asyncio
async def test_summarizer_success(mock_context):
"""Test that SummarizerNode successfully generates summary from raw thinking and execution result."""
decisions = json.dumps([{"action": "tap", "coordinates": [500, 600]}])
thinking = "The operator clicked the search button."
result = {
"status": "success",
"executed_actions": [{"action": "tap", "coordinates": [500, 600]}],
}
state = DummyState(
structured_decisions=decisions,
operator_raw_thinking=thinking,
last_execution_result=result,
current_step_id="12345678-1234-5678-1234-567812345678",
)
# Mock LLM response
mock_response = Mock()
mock_response.content = "Step completed successfully."
mock_llm = MagicMock()
async def mock_astream(*args, **kwargs):
yield mock_response
mock_llm.astream.side_effect = mock_astream
with (
patch(
"artemis.agents.summarizer.summarizer.get_llm", return_value=mock_llm
) as mock_get_llm,
patch("asyncio.create_task") as mock_create_task,
):
# Run the node call
node = SummarizerNode(mock_context)
await node(state)
# Verify task was registered in context
assert len(mock_context.background_tasks) == 1
assert mock_context.background_tasks[0] == mock_create_task.return_value
# Verify background task was spawned
assert mock_create_task.called
background_coroutine = mock_create_task.call_args[0][0]
# Await the background task synchronously for test verification
await background_coroutine
# Verify LLM was invoked with proper structured messages
assert mock_llm.astream.called
invoked_messages = mock_llm.astream.call_args[0][0]
system_msg = invoked_messages[0].content
human_msg = invoked_messages[1].content
assert "Summarizer Agent" in system_msg
assert "Decisions made" in human_msg
assert "The operator clicked the search button." in human_msg
# Verify Data Engine updated the step summary
from uuid import UUID
mock_context.data_engine.update_step_summary.assert_called_once_with(
UUID("12345678-1234-5678-1234-567812345678"),
"Step completed successfully.",
)
@pytest.mark.asyncio
async def test_summarizer_with_native_thinking(mock_context):
"""Test that SummarizerNode successfully generates summary from both explicit and native thoughts."""
decisions = json.dumps([{"action": "tap", "coordinates": [500, 600]}])
raw_thinking = "Explicit monologue."
native_thinking = "Silent core reasoning."
result = {"status": "success"}
state = DummyState(
structured_decisions=decisions,
operator_raw_thinking=raw_thinking,
operator_native_thinking=native_thinking,
last_execution_result=result,
current_step_id="12345678-1234-5678-1234-567812345678",
)
mock_response = Mock()
mock_response.content = "Summary result."
mock_llm = MagicMock()
async def mock_astream(*args, **kwargs):
yield mock_response
mock_llm.astream.side_effect = mock_astream
with (
patch(
"artemis.agents.summarizer.summarizer.get_llm", return_value=mock_llm
) as mock_get_llm,
patch("asyncio.create_task") as mock_create_task,
):
node = SummarizerNode(mock_context)
await node(state)
# Verify task was registered in context
assert len(mock_context.background_tasks) == 1
assert mock_context.background_tasks[0] == mock_create_task.return_value
background_coroutine = mock_create_task.call_args[0][0]
await background_coroutine
assert mock_llm.astream.called
invoked_messages = mock_llm.astream.call_args[0][0]
human_msg = invoked_messages[1].content
assert "Operator explicit thoughts:" in human_msg
assert "Explicit monologue." in human_msg
assert "Operator native thoughts:" in human_msg
assert "Silent core reasoning." in human_msg
@pytest.mark.asyncio
async def test_summarizer_with_chronological_trace(mock_context):
"""Test that SummarizerNode generates a prompt containing the chronological step trace when available."""
step_id = "12345678-1234-5678-1234-567812345678"
state = DummyState(
current_step_id=step_id,
)
mock_step_data = {
"step_id": step_id,
"step_number": 1,
"relative_time": "5.0s",
"action_taken": [{"action": "click", "target_text": "Search"}],
"last_execution_result": {"status": "success"},
"interleaved_events": [
{"type": "thought", "content": "Let's click search."},
],
}
mock_context.data_engine.get_agent_friendly_steps.return_value = [mock_step_data]
# Mock LLM response
mock_response = Mock()
mock_response.content = "Summary using chronological trace."
mock_llm = MagicMock()
async def mock_astream(*args, **kwargs):
yield mock_response
mock_llm.astream.side_effect = mock_astream
with (
patch("artemis.agents.summarizer.summarizer.get_llm", return_value=mock_llm),
patch("asyncio.create_task") as mock_create_task,
):
node = SummarizerNode(mock_context)
await node(state)
# Verify task was registered in context
assert len(mock_context.background_tasks) == 1
assert mock_context.background_tasks[0] == mock_create_task.return_value
background_coroutine = mock_create_task.call_args[0][0]
await background_coroutine
assert mock_llm.astream.called
invoked_messages = mock_llm.astream.call_args[0][0]
human_msg = invoked_messages[1].content
assert "--- Execution History ---" in human_msg
assert "Let's click search." in human_msg
+913
View File
@@ -0,0 +1,913 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from unittest.mock import AsyncMock, MagicMock, Mock, patch
from langchain_core.messages import AIMessage
from artemis.agents.validator.failure_analyzer import ValidationErrorCategory
from artemis.agents.validator.validator import ValidatorNode
from artemis.context import ArtemisContext
import pytest
class DummyState:
def __init__(
self,
structured_decisions,
validator_messages=[],
current_step_id=None,
latest_screenshot=None,
):
self.structured_decisions = structured_decisions
self.validator_messages = validator_messages
self.current_step_id = current_step_id
self.latest_screenshot = latest_screenshot
async def asanitize_update(self, ctx, update, agent):
return update
@pytest.fixture
def mock_context(tmp_path):
ctx = Mock(spec=ArtemisContext)
ctx.llm_config = Mock()
ctx.data_engine = Mock()
ctx.data_engine.base_dir = tmp_path
ctx.data_engine.get_relative_time.return_value = 1.0
return ctx
@pytest.fixture
def temp_screenshot(tmp_path):
p = tmp_path / "screenshot.png"
p.write_bytes(b"dummy_data")
return str(p)
@pytest.fixture
def mock_mcp():
with (
patch("mcp.client.stdio.stdio_client") as mock_stdio,
patch("artemis.agents.validator.validator.ClientSession") as mock_session_cls,
):
mock_ctx = AsyncMock()
mock_ctx.__aenter__.return_value = (Mock(), Mock())
mock_stdio.return_value = mock_ctx
mock_session = AsyncMock()
mock_session.initialize = AsyncMock()
mock_session_cls.return_value = mock_session
yield mock_session
@pytest.mark.asyncio
async def test_validator_success(mock_mcp, mock_context, temp_screenshot):
"""Test that ValidatorNode executes actions successfully."""
# Mock MCP calls: screenshot, tap, screenshot
mock_mcp.call_tool.side_effect = [
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # pre-screenshot
Mock(content=[Mock(text="Success")]), # tap action
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # post-screenshot
]
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
with patch("artemis.utils.image_diff.check_ui_change", return_value=True):
node = ValidatorNode(mock_context)
result = await node(state)
assert "last_execution_result" in result
report = result["last_execution_result"]
assert "execution" in report
assert len(report["execution"]) == 1
assert report["execution"][0]["action"] == "tap"
assert "attempts" not in report["execution"][0]
@pytest.mark.asyncio
async def test_validator_failure_analysis(mock_mcp, mock_context, temp_screenshot):
"""Test that ValidatorNode triggers failure analysis on error and handles
cannot_fix via tool.
"""
tap_count = 0
def mock_call_tool(name, args):
nonlocal tap_count
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
if name == "tap":
tap_count += 1
return Mock(content=[Mock(text="Error: Element not found")])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
node = ValidatorNode(mock_context)
with (
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
):
mock_analyze.return_value = {
"status": "cannot_fix",
"analysis": "Analysis: Coordinates are likely wrong.",
}
result = await node(state)
assert "last_execution_result" in result
report = result["last_execution_result"]
assert "execution" in report
assert len(report["execution"]) == 1
assert report["execution"][0]["action"] == "tap"
assert report["execution"][0]["attempts"] == [
"Error: Element not found",
"Error: Element not found",
]
assert report["execution"][0]["repair"] == "Analysis: Coordinates are likely wrong."
@pytest.mark.asyncio
async def test_validator_repair_success(mock_mcp, mock_context, temp_screenshot):
"""Test that ValidatorNode terminates successfully and clears remaining actions
after successful repair.
"""
tap_count = 0
def mock_call_tool(name, args):
nonlocal tap_count
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
if name == "tap":
tap_count += 1
if tap_count <= 2: # First tap fails
return Mock(content=[Mock(text="Error: Element not found")])
return Mock(content=[Mock(text="Success")])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
# Two actions planned initially
decisions = json.dumps(
[
{"action": "tap", "coordinates": [105, 205]},
{"action": "tap", "coordinates": [205, 305]},
]
)
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
node = ValidatorNode(mock_context)
with (
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
):
mock_analyze.return_value = {
"status": "fixed",
"analysis": "Fixed by updating actions",
}
result = await node(state)
assert "last_execution_result" in result
report = result["last_execution_result"]
assert "execution" in report
assert len(report["execution"]) == 1 # Only the first failed action is in report
assert report["execution"][0]["coordinates"] == [105, 205]
assert report["execution"][0]["attempts"] == [
"Error: Element not found",
"Error: Element not found",
]
assert report["execution"][0]["repair"] == "Fixed by updating actions"
@pytest.mark.asyncio
async def test_validator_wait_for_delay(mock_mcp, mock_context, temp_screenshot):
"""Test that wait_for_delay executes successfully."""
def mock_call(name, args):
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
mock_mcp.call_tool.side_effect = mock_call
decisions = json.dumps([{"action": "wait_for_delay", "time_in_ms": 10}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
with patch("artemis.utils.image_diff.check_ui_change", return_value=False):
node = ValidatorNode(mock_context)
result = await node(state)
assert "last_execution_result" in result
report = result["last_execution_result"]
assert "execution" in report
assert len(report["execution"]) == 1
assert report["execution"][0]["action"] == "wait_for_delay"
@pytest.mark.asyncio
async def test_validator_focus_and_clear_text_no_ui_change(mock_mcp, mock_context, temp_screenshot):
"""Test that focus_and_clear_text executes successfully even if no UI change is detected."""
def mock_call(name, args):
if name == "focus_and_clear_text":
return Mock(content=[Mock(text="Success")])
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
mock_mcp.call_tool.side_effect = mock_call
decisions = json.dumps([{"action": "focus_and_clear_text", "coordinates": [400, 500]}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
with patch("artemis.utils.image_diff.check_ui_change", return_value=False):
node = ValidatorNode(mock_context)
result = await node(state)
assert "last_execution_result" in result
report = result["last_execution_result"]
assert "execution" in report
assert len(report["execution"]) == 1
assert report["execution"][0]["action"] == "focus_and_clear_text"
@pytest.mark.asyncio
async def test_validator_silent_failure_treated_as_success(mock_mcp, mock_context, temp_screenshot):
"""Test that silent failure (exec succeeds but no UI change) is treated as
success and does not trigger FailureAnalyzer.
"""
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
if name == "tap":
return Mock(content=[Mock(text="Success")])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
node = ValidatorNode(mock_context)
with (
patch("artemis.agents.validator.validator.VALIDATOR_POLL_TIMEOUT", 0.1),
patch("artemis.agents.validator.validator.VALIDATOR_POLL_INTERVAL", 0.01),
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=False), # No UI change
):
result = await node(state)
# Assert FailureAnalyzer was NOT called
mock_analyze.assert_not_called()
# Assert the action is considered executed successfully
assert "last_execution_result" in result
report = result["last_execution_result"]
assert "execution" in report
assert len(report["execution"]) == 1
assert report["execution"][0]["action"] == "tap"
assert "attempts" not in report["execution"][0]
@pytest.mark.asyncio
async def test_validator_records_skipped_actions_on_cannot_fix(
mock_mcp, mock_context, temp_screenshot
):
"""Test that ValidatorNode marks unexecuted actions as Skipped when repair fails."""
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
return Mock(content=[Mock(text="Error: Connection lost")])
mock_mcp.call_tool.side_effect = mock_call_tool
# Two actions planned
decisions = json.dumps(
[
{"action": "tap", "coordinates": [100, 200]},
{"action": "press_key", "keycode": "ENTER"},
]
)
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
node = ValidatorNode(mock_context)
with (
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
):
mock_analyze.return_value = {
"status": "cannot_fix",
"analysis": "Cannot fix: System crashed.",
}
result = await node(state)
assert "last_execution_result" in result
report = result["last_execution_result"]
assert "execution" in report
assert len(report["execution"]) == 2
# First action failed
assert report["execution"][0]["action"] == "tap"
assert report["execution"][0]["attempts"] == [
"Error: Connection lost",
"Error: Connection lost",
]
assert report["execution"][0]["repair"] == "Cannot fix: System crashed."
# Second action was skipped
assert report["execution"][1]["action"] == "press_key"
assert report["execution"][1]["attempts"] == ["Skipped"]
@pytest.mark.asyncio
async def test_validator_pre_execution_validation_and_repair(
mock_mcp, mock_context, temp_screenshot
):
"""Test that pre-execution validation fails when element changes, and is
repaired by FailureAnalyzer.
"""
mock_live_elements = [
{
"text": "Sign Up",
"bounds": "[100,200][300,300]",
"resource-id": "btn_signup",
}
]
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
if name == "get_ui_hierarchy":
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
if name == "tap":
return Mock(content=[Mock(text="Success")])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
decisions = json.dumps(
[
{
"action": "tap",
"coordinates": [200, 250],
"target_text": "Login",
"target_bounds": [100, 200, 300, 300],
"target_resource_id": "btn_login",
}
]
)
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
node = ValidatorNode(mock_context)
with (
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
patch.object(
ValidatorNode,
"_validate_action_precondition_pixel",
new_callable=AsyncMock,
) as mock_pixel,
):
mock_pixel.return_value = (
False,
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
"Pixel check failed",
)
mock_analyze.return_value = {
"status": "fixed",
"analysis": ("Pre-exec failure matched and resolved by switching to Sign Up button."),
}
result = await node(state)
mock_analyze.assert_called_once()
args, kwargs = mock_analyze.call_args
assert "Pre-execution validation failed" in args[2]
assert "Login" in args[2]
assert kwargs.get("error_category") == ValidationErrorCategory.TARGET_OCCUPIED
assert "last_execution_result" in result
report = result["last_execution_result"]
assert len(report["execution"]) == 1
assert report["execution"][0]["action"] == "tap"
assert "Pre-execution validation failed" in report["execution"][0]["attempts"][0]
assert (
report["execution"][0]["repair"]
== "Pre-exec failure matched and resolved by switching to Sign Up"
" button."
)
@pytest.mark.asyncio
async def test_validator_pre_execution_validation_self_healing(
mock_mcp, mock_context, temp_screenshot
):
"""Test that pre-execution validation succeeds and self-heals when element shifts slightly."""
mock_live_elements = [
{
"text": "Login",
"bounds": "[100,220][300,320]",
"resource-id": "btn_login",
}
]
tapped_coordinates = None
def mock_call_tool(name, args):
nonlocal tapped_coordinates
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
if name == "get_ui_hierarchy":
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
if name == "tap":
tapped_coordinates = args.get("coordinates")
return Mock(content=[Mock(text="Success")])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
decisions = json.dumps(
[
{
"action": "tap",
"coordinates": [200, 250],
"target_text": "Login",
"target_bounds": [100, 200, 300, 300],
"target_resource_id": "btn_login",
}
]
)
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
node = ValidatorNode(mock_context)
with patch("artemis.utils.image_diff.check_ui_change", return_value=True):
await node(state)
# Center of [100, 220][300, 320] is [200, 270]
assert tapped_coordinates == [200, 270]
@pytest.mark.asyncio
async def test_validator_pre_execution_validation_anonymous_occupant(
mock_mcp, mock_context, temp_screenshot
):
"""Test that pre-execution validation correctly flags TARGET_OCCUPIED
when coordinates are blocked by an anonymous clickable view.
"""
# The live screen contains an anonymous clickable view covering
# target coordinates [200, 250]
mock_live_elements = [
{
"text": "",
"bounds": "[150,200][250,300]",
"clickable": True,
"resource-id": "",
}
]
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
if name == "get_ui_hierarchy":
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
decisions = json.dumps(
[
{
"action": "tap",
"coordinates": [200, 250],
"target_text": "Login",
"target_bounds": [100, 200, 300, 300],
"target_resource_id": "btn_login",
}
]
)
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
node = ValidatorNode(mock_context)
with (
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
patch.object(
ValidatorNode,
"_validate_action_precondition_pixel",
new_callable=AsyncMock,
) as mock_pixel,
):
mock_pixel.return_value = (
False,
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
"Pixel check failed",
)
mock_analyze.return_value = {
"status": "fixed",
"analysis": "Pre-exec failure bypassed.",
}
await node(state)
# Assert that the validator correctly classified this as TARGET_OCCUPIED
# (due to the clickable anonymous view blocking the click)
mock_analyze.assert_called_once()
args, kwargs = mock_analyze.call_args
assert kwargs.get("error_category") == ValidationErrorCategory.TARGET_OCCUPIED
assert "occupied/intercepted by a different element: interactive anonymous element" in args[2]
@pytest.mark.asyncio
async def test_validator_pixel_validation_success(mock_mcp, mock_context, temp_screenshot):
"""Test that the pixel safety net succeeds when Gemini reports target is present."""
mock_mcp.call_tool.side_effect = [
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # live screenshot for pixel check
Mock(content=[Mock(text="Success")]), # tap action
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # post-screenshot
]
decisions = json.dumps([{"action": "tap", "coordinates": [100, 200]}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
mock_agent_cfg = Mock()
mock_agent_cfg.provider = "google"
mock_agent_cfg.model = "gemini-3.5-flash-lite"
mock_context.llm_config.get_agent.return_value = mock_agent_cfg
mock_response = Mock()
mock_response.text = json.dumps(
{
"is_present": True,
"reason": "Button is clearly visible and clickable.",
"confidence": 0.95,
}
)
mock_llm = MagicMock()
mock_llm.ainvoke = AsyncMock(return_value=AIMessage(content=mock_response.text))
# Mock crop and draw helpers to return dummy bytes
with (
patch(
"artemis.utils.visualization.crop_and_annotate_target",
return_value=b"dummy_bytes",
),
patch("artemis.agents.validator.validator.get_llm", return_value=mock_llm),
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
):
node = ValidatorNode(mock_context)
result = await node(state)
assert "last_execution_result" in result
report = result["last_execution_result"]
assert len(report["execution"]) == 1
assert "attempts" not in report["execution"][0]
@pytest.mark.asyncio
async def test_validator_pixel_validation_failure(mock_mcp, mock_context, temp_screenshot):
"""Test that the pixel safety net fails and triggers FailureAnalyzer
when Gemini reports target is missing.
"""
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
decisions = json.dumps([{"action": "tap", "coordinates": [100, 200]}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
mock_agent_cfg = Mock()
mock_agent_cfg.provider = "google"
mock_agent_cfg.model = "gemini-3.5-flash-lite"
mock_context.llm_config.get_agent.return_value = mock_agent_cfg
mock_response = Mock()
mock_response.text = json.dumps(
{
"is_present": False,
"reason": "The button has completely disappeared from the screen.",
"confidence": 0.90,
}
)
mock_llm = MagicMock()
mock_llm.ainvoke = AsyncMock(return_value=AIMessage(content=mock_response.text))
with (
patch(
"artemis.utils.visualization.crop_and_annotate_target",
return_value=b"dummy_bytes",
),
patch("artemis.agents.validator.validator.get_llm", return_value=mock_llm),
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
):
mock_analyze.return_value = {
"status": "cannot_fix",
"analysis": "Pixel safety net flagged missing target.",
}
node = ValidatorNode(mock_context)
await node(state)
mock_analyze.assert_called_once()
args, kwargs = mock_analyze.call_args
assert "Pixel-level validation failed" in args[2]
assert kwargs.get("error_category") == ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED
@pytest.mark.asyncio
@patch("artemis.agents.validator.validator.find_package")
@patch("artemis.utils.app_launch_utils.launch_app_with_retries")
async def test_validator_launch_app_passes_use_fallback_mcp(
mock_launch_app, mock_find_package, mock_mcp, mock_context, temp_screenshot
):
"""Test that ValidatorNode calls find_package with use_fallback=False
for launch_app action in MCP mode.
"""
mock_find_package.return_value = "com.example.app"
mock_launch_app.return_value = (True, None)
# Mock MCP calls: screenshot (pre), screenshot (post)
mock_mcp.call_tool.side_effect = [
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # pre-screenshot
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # post-screenshot
]
decisions = json.dumps([{"action": "launch_app", "app_name": "My App"}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
with patch("artemis.utils.image_diff.check_ui_change", return_value=True):
node = ValidatorNode(mock_context)
result = await node(state)
mock_find_package.assert_called_once_with(mock_context, "My App", use_fallback=False)
mock_launch_app.assert_called_once_with(mock_context, "com.example.app")
assert "last_execution_result" in result
@pytest.mark.asyncio
@patch("artemis.agents.validator.validator.find_package")
@patch("artemis.utils.app_launch_utils.launch_app_with_retries")
async def test_validator_launch_app_failure_no_retry(
mock_launch_app, mock_find_package, mock_mcp, mock_context, temp_screenshot
):
"""Test that ValidatorNode does NOT retry launch_app action on failure."""
mock_find_package.return_value = "com.example.app"
mock_launch_app.return_value = (False, "Error: Force close")
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
decisions = json.dumps([{"action": "launch_app", "app_name": "My App"}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
with (
patch("artemis.agents.validator.validator.VALIDATOR_POLL_TIMEOUT", 0.05),
patch("artemis.agents.validator.validator.VALIDATOR_POLL_INTERVAL", 0.01),
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
):
mock_analyze.return_value = {
"status": "cannot_fix",
"analysis": "Cannot launch app.",
}
node = ValidatorNode(mock_context)
result = await node(state)
# Verify find_package was called only once
mock_find_package.assert_called_once_with(mock_context, "My App", use_fallback=False)
# Verify launch_app_with_retries was called only once
# (no retry by ValidatorNode itself since it relies on launch_app_with_retries)
mock_launch_app.assert_called_once_with(mock_context, "com.example.app")
# Verify attempts list has only 1 element
assert "last_execution_result" in result
report = result["last_execution_result"]
assert len(report["execution"]) == 1
assert report["execution"][0]["action"] == "launch_app"
assert report["execution"][0]["attempts"] == ["Error: Force close"]
@pytest.mark.asyncio
async def test_validator_pre_execution_validation_disappeared_not_shifted(
mock_mcp, mock_context, temp_screenshot
):
"""Test that a disappeared small button matching text in a giant container
is classified as TARGET_DISAPPEARED, not TARGET_SHIFTED.
"""
# Live hierarchy only contains a giant background/container
# containing the text "Sign Up"
mock_live_elements = [
{
"text": "Sign Up",
"bounds": "[0,0][1080,2400]",
"resource-id": "root_container",
}
]
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
if name == "get_ui_hierarchy":
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
decisions = json.dumps(
[
{
"action": "tap",
"coordinates": [200, 250],
"target_text": "Sign Up",
"target_bounds": [100, 200, 300, 300],
"target_resource_id": "btn_signup",
}
]
)
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
node = ValidatorNode(mock_context)
with (
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
patch.object(
ValidatorNode,
"_validate_action_precondition_pixel",
new_callable=AsyncMock,
) as mock_pixel,
):
mock_pixel.return_value = (
False,
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
"Pixel check failed",
)
mock_analyze.return_value = {
"status": "cannot_fix",
"analysis": "Sign Up button disappeared.",
}
await node(state)
# Assert that the validator correctly classified this as TARGET_DISAPPEARED
# (since size mismatch prevents shift matching)
mock_analyze.assert_called_once()
args, kwargs = mock_analyze.call_args
assert kwargs.get("error_category") == ValidationErrorCategory.TARGET_DISAPPEARED
@pytest.mark.asyncio
async def test_validator_pre_execution_validation_ocr_direct_to_pixel(
mock_mcp, mock_context, temp_screenshot
):
"""Test that OCR-derived elements bypass XML validation and
directly use pixel-based validation.
"""
mock_mcp.call_tool.side_effect = [
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # live screenshot for pixel check
Mock(content=[Mock(text="Success")]), # tap action
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # post-screenshot
]
decisions = json.dumps(
[
{
"action": "tap",
"coordinates": [200, 250],
"target_text": "Login",
"target_bounds": [100, 200, 300, 300],
"target_class": "android.view.View [OCR]",
}
]
)
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
node = ValidatorNode(mock_context)
with (
patch.object(
ValidatorNode,
"_validate_action_precondition",
new_callable=AsyncMock,
) as mock_xml,
patch.object(
ValidatorNode,
"_validate_action_precondition_pixel",
new_callable=AsyncMock,
) as mock_pixel,
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
):
mock_pixel.return_value = (True, ValidationErrorCategory.NONE, "")
result = await node(state)
# Assert XML validation was NOT called
mock_xml.assert_not_called()
# Assert Pixel validation WAS called
mock_pixel.assert_called_once()
assert "last_execution_result" in result
report = result["last_execution_result"]
assert report["status"] == "success"
@pytest.mark.asyncio
async def test_validator_pre_execution_xml_failure_not_overridden_when_pixel_bypassed(
mock_mcp, mock_context, temp_screenshot
):
"""Test that when XML validation fails AND Pixel validation is bypassed
(PIXEL_BYPASSED), the XML failure is NOT overridden.
"""
mock_live_elements = [
{
"text": "Sign Up",
"bounds": "[100,200][300,300]",
"resource-id": "btn_signup",
}
]
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
if name == "get_ui_hierarchy":
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
decisions = json.dumps(
[
{
"action": "tap",
"coordinates": [200, 250],
"target_text": "Login",
"target_bounds": [100, 200, 300, 300],
"target_resource_id": "btn_login",
}
]
)
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
node = ValidatorNode(mock_context)
with (
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
patch.object(
ValidatorNode,
"_validate_action_precondition_pixel",
new_callable=AsyncMock,
) as mock_pixel,
):
# Pixel check was bypassed due to VLM error or low confidence
mock_pixel.return_value = (
True,
ValidationErrorCategory.PIXEL_BYPASSED,
"",
)
mock_analyze.return_value = {
"status": "cannot_fix",
"analysis": ("XML validation failed and pixel bypass did not override."),
}
await node(state)
# FailureAnalyzer SHOULD be called because XML validation failed
# and PIXEL_BYPASSED did not override it
mock_analyze.assert_called_once()
args, kwargs = mock_analyze.call_args
assert "Pre-execution validation failed" in args[2]
+456
View File
@@ -0,0 +1,456 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.agents.video_analyzer.video_analyzer import VideoAnalyzer
from artemis.context import ArtemisContext
import pytest
@pytest.mark.asyncio
async def test_video_analyzer_run():
# Mock context
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
mock_ctx.llm_config = MagicMock()
mock_ctx.llm_config.utils.video_analyzer = MagicMock()
mock_ctx.llm_config.utils.video_analyzer.model = "gemini-3.6-flash"
# Mock Gemini Client
mock_client = MagicMock()
mock_client.aio = MagicMock()
mock_client.aio.models = MagicMock()
mock_chunk = MagicMock()
mock_chunk.text = "Final analysis: The action succeeded."
mock_chunk.function_calls = []
async def mock_stream():
yield mock_chunk
mock_client.aio.models.generate_content_stream = AsyncMock(return_value=mock_stream())
# Mock file reading for prompts
mock_prompt = "# Dynamic Video Analyzer..."
with (
patch(
"artemis.agents.video_analyzer.video_analyzer.genai.Client",
return_value=mock_client,
),
patch(
"pathlib.Path.exists",
autospec=True,
side_effect=lambda self: not str(self).endswith(".jpg"),
),
patch("pathlib.Path.read_text", return_value=mock_prompt),
patch(
"artemis.agents.video_analyzer.video_analyzer.cleanup_abandoned_gemini_files",
AsyncMock(),
),
):
analyzer = VideoAnalyzer(mock_ctx)
result, status = await analyzer.run(
time_description="from 10s to 15s", purpose="Verify action"
)
assert result == "Final analysis: The action succeeded."
assert status == "success"
assert mock_client.aio.models.generate_content_stream.called
@pytest.mark.asyncio
async def test_video_analyzer_preserves_thought_signature():
# Mock context
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
mock_ctx.llm_config = MagicMock()
mock_ctx.llm_config.utils.video_analyzer = MagicMock()
mock_ctx.llm_config.utils.video_analyzer.model = "gemini-3.6-flash"
# Mock MobileDeviceController
mock_controller = MagicMock()
mock_controller.extract_segment_metadata = AsyncMock(
return_value=MagicMock(
success=True,
video_path="/tmp/video.mp4",
actual_start_relative_time=10.0,
)
)
from google.genai import types
dummy_signature = b"dummy_thought_signature_bytes_56789"
# Turn 1: Model calls spawn_sub_agent
spawn_call = types.FunctionCall(
name="spawn_sub_agent",
args={
"start_time": 10.0,
"end_time": 15.0,
"specific_query": "Verify login screen",
},
)
spawn_part = types.Part(
function_call=spawn_call,
thought_signature=dummy_signature,
)
mock_chunk_1 = MagicMock()
mock_chunk_1.text = ""
mock_chunk_1.function_calls = [spawn_call]
mock_chunk_1.candidates = [
types.Candidate(content=types.Content(role="model", parts=[spawn_part]))
]
# Turn 2: Model returns final answer
mock_chunk_2 = MagicMock()
mock_chunk_2.text = "Final analysis: The spawn succeeded and login button is visible."
mock_chunk_2.function_calls = []
mock_chunk_2.candidates = [
types.Candidate(
content=types.Content(
role="model",
parts=[types.Part.from_text(text=mock_chunk_2.text)],
)
)
]
# We mock generate_content_stream
call_count = 0
captured_contents = []
async def mock_stream_1():
yield mock_chunk_1
async def mock_stream_2():
yield mock_chunk_2
async def mock_generate_content_stream(model, contents, config):
nonlocal call_count
has_submit_answer = False
if config.tools:
for t in config.tools:
fd_list = getattr(t, "function_declarations", None)
if fd_list:
for fd in fd_list:
if fd.name == "submit_answer":
has_submit_answer = True
break
if not has_submit_answer:
captured_contents.append(list(contents)) # copy history contents passed to call
call_count += 1
if call_count == 1:
return mock_stream_1()
else:
return mock_stream_2()
else:
submit_call = types.FunctionCall(
name="submit_answer",
args={
"confidence_score": 0.9,
"summary": "Sub-agent verification finished",
"analysis": "Analysis here",
"verification_timestamp_secs": 10.0,
},
)
mock_sub_chunk = MagicMock()
mock_sub_chunk.text = ""
mock_sub_chunk.function_calls = [submit_call]
async def sub_stream():
yield mock_sub_chunk
return sub_stream()
# Mock Gemini Client
mock_client = MagicMock()
mock_client.aio = MagicMock()
mock_client.aio.files = MagicMock()
mock_file = MagicMock()
mock_file.name = "dummy_cloud_file_123"
mock_file.uri = "https://gemini-file-api/dummy_cloud_file_123"
mock_file.mime_type = "video/mp4"
mock_client.aio.files.upload = AsyncMock(return_value=mock_file)
mock_f_state = MagicMock()
mock_f_state.state.name = "ACTIVE"
mock_client.aio.files.get = AsyncMock(return_value=mock_f_state)
mock_client.aio.files.delete = AsyncMock()
mock_client.aio.models = MagicMock()
mock_client.aio.models.generate_content_stream = AsyncMock(
side_effect=mock_generate_content_stream
)
# Mock file reading for prompts
mock_prompt = "# Dynamic Video Analyzer..."
with (
patch(
"artemis.agents.video_analyzer.video_analyzer.genai.Client",
return_value=mock_client,
),
patch(
"artemis.agents.video_analyzer.video_analyzer.get_controller",
return_value=mock_controller,
),
patch(
"pathlib.Path.exists",
autospec=True,
side_effect=lambda self: not str(self).endswith(".jpg"),
),
patch(
"pathlib.Path.stat",
return_value=os.stat_result((0o100644, 0, 0, 0, 0, 0, 1024 * 1024, 0, 0, 0)),
),
patch("pathlib.Path.read_text", return_value=mock_prompt),
patch(
"artemis.agents.video_analyzer.video_analyzer.cleanup_abandoned_gemini_files",
AsyncMock(),
),
patch(
"artemis.agents.video_analyzer.video_analyzer.compress_video_for_api",
AsyncMock(return_value=Path("/tmp/video.mp4")),
),
):
analyzer = VideoAnalyzer(mock_ctx)
result, status = await analyzer.run(
time_description="from 10s to 15s", purpose="Verify login"
)
assert status == "success"
assert "spawn succeeded" in result
assert call_count == 2
# Verify that the second model call's history contains the model turn with the thought_signature
second_call_history = captured_contents[1]
assert len(second_call_history) >= 4
model_turn = second_call_history[1]
assert model_turn.role == "model"
assert len(model_turn.parts) == 1
assert model_turn.parts[0].function_call.name == "spawn_sub_agent"
assert model_turn.parts[0].thought_signature == dummy_signature
@pytest.mark.asyncio
async def test_video_analyzer_sub_agent_confidence_validation():
# Mock context
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
mock_ctx.llm_config = MagicMock()
mock_ctx.llm_config.utils.video_analyzer = MagicMock()
mock_ctx.llm_config.utils.video_analyzer.model = "gemini-3.6-flash"
# Mock MobileDeviceController
mock_controller = MagicMock()
mock_controller.extract_segment_metadata = AsyncMock(
return_value=MagicMock(
success=True,
video_path="/tmp/video.mp4",
actual_start_relative_time=10.0,
)
)
from google.genai import types
# Main agent responses
# Turn 1: Calls spawn_sub_agent
spawn_call = types.FunctionCall(
name="spawn_sub_agent",
args={
"start_time": 10.0,
"end_time": 15.0,
"specific_query": "Verify login screen",
},
)
mock_main_chunk_1 = MagicMock()
mock_main_chunk_1.text = ""
mock_main_chunk_1.function_calls = [spawn_call]
# Turn 2: Returns final answer
mock_main_chunk_2 = MagicMock()
mock_main_chunk_2.text = "Final blackboard status checked."
mock_main_chunk_2.function_calls = []
# Sub-agent responses
# Turn 1: Calls submit_answer with invalid confidence_score (string)
invalid_submit_call = types.FunctionCall(
name="submit_answer",
args={
"timeline_events": [
{
"confidence_score": "HIGH",
"transcription": "Invalid confidence score call",
"verification_timestamp_secs": 10.0,
}
],
"summary": "Invalid confidence score call",
"analysis": "Some analysis",
},
)
sub_dummy_signature = b"sub_dummy_signature_9999"
sub_part_1 = types.Part(
function_call=invalid_submit_call,
thought_signature=sub_dummy_signature,
)
mock_sub_chunk_1 = MagicMock()
mock_sub_chunk_1.text = ""
mock_sub_chunk_1.function_calls = [invalid_submit_call]
mock_sub_chunk_1.candidates = [
types.Candidate(content=types.Content(role="model", parts=[sub_part_1]))
]
# Turn 2: Calls submit_answer with valid confidence_score
valid_submit_call = types.FunctionCall(
name="submit_answer",
args={
"timeline_events": [
{
"confidence_score": 0.8,
"transcription": "Valid confidence score call",
"verification_timestamp_secs": 10.0,
}
],
"summary": "Valid confidence score call",
"analysis": "Some analysis",
},
)
mock_sub_chunk_2 = MagicMock()
mock_sub_chunk_2.text = ""
mock_sub_chunk_2.function_calls = [valid_submit_call]
mock_sub_chunk_2.candidates = [
types.Candidate(
content=types.Content(role="model", parts=[types.Part(function_call=valid_submit_call)])
)
]
# Stream generator mocks
main_calls = 0
sub_calls = 0
captured_sub_contents = []
async def mock_main_stream():
nonlocal main_calls
if main_calls == 0:
main_calls += 1
yield mock_main_chunk_1
else:
yield mock_main_chunk_2
async def mock_sub_stream():
nonlocal sub_calls
if sub_calls == 0:
sub_calls += 1
yield mock_sub_chunk_1
else:
yield mock_sub_chunk_2
async def mock_generate_content_stream(model, contents, config):
system_instruction = config.system_instruction
if "Dynamic Video Analyzer" in str(system_instruction):
return mock_main_stream()
else:
captured_sub_contents.append(list(contents))
return mock_sub_stream()
# Mock Gemini Client
mock_client = MagicMock()
mock_client.aio = MagicMock()
mock_client.aio.files = MagicMock()
mock_file = MagicMock()
mock_file.name = "dummy_cloud_file_123"
mock_file.uri = "https://gemini-file-api/dummy_cloud_file_123"
mock_file.mime_type = "video/mp4"
mock_client.aio.files.upload = AsyncMock(return_value=mock_file)
mock_f_state = MagicMock()
mock_f_state.state.name = "ACTIVE"
mock_client.aio.files.get = AsyncMock(return_value=mock_f_state)
mock_client.aio.files.delete = AsyncMock()
mock_client.aio.models = MagicMock()
mock_client.aio.models.generate_content_stream = AsyncMock(
side_effect=mock_generate_content_stream
)
# Mock file reading for prompts
mock_main_prompt = "# Dynamic Video Analyzer..."
mock_sub_prompt = "# Sub Agent..."
def mock_read_text(self, encoding="utf-8"):
if "video_analyzer.md" in str(self):
return mock_main_prompt
elif "video_sub_agent.md" in str(self):
return mock_sub_prompt
return ""
with (
patch(
"artemis.agents.video_analyzer.video_analyzer.genai.Client",
return_value=mock_client,
),
patch(
"artemis.agents.video_analyzer.video_analyzer.get_controller",
return_value=mock_controller,
),
patch(
"pathlib.Path.exists",
autospec=True,
side_effect=lambda self: not str(self).endswith(".jpg"),
),
patch(
"pathlib.Path.stat",
return_value=os.stat_result((0o100644, 0, 0, 0, 0, 0, 1024 * 1024, 0, 0, 0)),
),
patch("pathlib.Path.read_text", mock_read_text),
patch("pathlib.Path.mkdir"),
patch("pathlib.Path.touch"),
patch("pathlib.Path.is_dir", return_value=True),
patch(
"artemis.agents.video_analyzer.video_analyzer.cleanup_abandoned_gemini_files",
AsyncMock(),
),
patch(
"artemis.agents.video_analyzer.video_analyzer.compress_video_for_api",
AsyncMock(return_value=Path("/tmp/video.mp4")),
),
patch(
"asyncio.create_subprocess_exec",
AsyncMock(return_value=MagicMock(communicate=AsyncMock(return_value=(b"", b"")))),
),
):
analyzer = VideoAnalyzer(mock_ctx)
result, status = await analyzer.run(
time_description="from 10s to 15s", purpose="Verify login"
)
assert status == "success"
assert sub_calls == 1
assert len(analyzer.blackboard_entries) == 1
entry = analyzer.blackboard_entries[0]
assert entry["confidence_score"] == 0.8
assert entry["summary"] == "Valid confidence score call"
# Verify that sub-agent turn 2 received the model turn with the thought_signature
assert len(captured_sub_contents) >= 2
sub_turn_2_contents = captured_sub_contents[1]
model_turns = [c for c in sub_turn_2_contents if c.role == "model"]
assert len(model_turns) == 1
assert model_turns[0].parts[0].thought_signature == sub_dummy_signature
@@ -0,0 +1,234 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for Universal VideoAnalyzer dual-path execution engine."""
from unittest.mock import AsyncMock, MagicMock, Mock, patch
import pytest
from langchain_core.messages import AIMessage
from artemis.agents.video_analyzer.video_analyzer import (
UNIVERSAL_MAIN_TOOLS,
UNIVERSAL_SUBMIT_ANSWER_TOOL,
VideoAnalyzer,
)
from artemis.context import ArtemisContext
@pytest.fixture
def mock_context(tmp_path):
ctx = Mock(spec=ArtemisContext)
ctx.llm_config = Mock()
mock_llm_cfg = Mock()
mock_llm_cfg.model = "claude-3-7-sonnet"
mock_llm_cfg.temperature = 0.2
ctx.llm_config.utils = Mock()
ctx.llm_config.utils.video_analyzer = mock_llm_cfg
ctx.device = Mock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
ctx.data_engine = Mock()
ctx.data_engine.base_dir = tmp_path
return ctx
def test_video_analyzer_universal_engine_detection(mock_context):
"""Test that VideoAnalyzer routes to Universal Engine when non-Gemini model is set."""
from artemis.config import settings
with patch.object(settings, "GOOGLE_API_KEY", None):
agent = VideoAnalyzer(mock_context)
assert agent.use_native_gemini is False
assert agent.model_name == "claude-3-7-sonnet"
def test_universal_tool_schemas():
"""Verify schema structures for Universal tools."""
assert UNIVERSAL_SUBMIT_ANSWER_TOOL["type"] == "function"
assert UNIVERSAL_SUBMIT_ANSWER_TOOL["function"]["name"] == "submit_answer"
assert len(UNIVERSAL_MAIN_TOOLS) == 3
tool_names = [t["function"]["name"] for t in UNIVERSAL_MAIN_TOOLS]
assert "extract_segment_metadata" in tool_names
assert "spawn_sub_agent" in tool_names
assert "analyze_audio_only" in tool_names
@pytest.mark.asyncio
async def test_exec_single_chunk_universal(mock_context, tmp_path):
"""Test keyframe extraction and LangChain tool invocation in universal sub-agent."""
agent = VideoAnalyzer(mock_context)
agent.use_native_gemini = False
fake_video = tmp_path / "compressed.mp4"
fake_video.write_bytes(b"video_bytes")
raw_video = tmp_path / "raw.mp4"
raw_video.write_bytes(b"raw_bytes")
fake_keyframes = [
(0.0, b"frame0"),
(1.0, b"frame1"),
(2.0, b"frame2"),
]
mock_llm = MagicMock()
mock_response = AIMessage(
content="Sub-agent analyzed frames.",
tool_calls=[
{
"name": "submit_answer",
"args": {
"summary": "Detected settings button tap.",
"analysis": "At 1.0s user clicked Settings icon.",
"timeline_events": [
{
"start_time": 0.5,
"end_time": 1.5,
"transcription": "Tap on Settings",
"confidence_score": 0.95,
"verification_timestamp_secs": 1.0,
}
],
},
"id": "sub_call_1",
}
],
)
mock_bound = MagicMock()
mock_bound.ainvoke = AsyncMock(return_value=mock_response)
mock_llm.bind_tools.return_value = mock_bound
with (
patch(
"artemis.agents.video_analyzer.video_analyzer.extract_keyframes_from_video",
return_value=fake_keyframes,
),
patch(
"artemis.agents.video_analyzer.video_analyzer.get_llm",
return_value=mock_llm,
),
):
result = await agent._exec_single_chunk_universal(
compressed_path=fake_video,
raw_path=raw_video,
start_time=0.0,
end_time=3.0,
actual_start=0.0,
prompt_with_context="Find when settings was opened.",
specific_query="settings icon",
)
assert "Detected settings button tap" in result
assert "At 1.0s user clicked Settings icon" in result
assert len(agent.blackboard_entries) == 1
assert agent.blackboard_entries[0]["summary"] == "Tap on Settings"
assert agent.blackboard_entries[0]["confidence_score"] == 0.95
@pytest.mark.asyncio
async def test_exec_analyze_audio_universal(mock_context, tmp_path):
"""Test audio analysis in universal mode."""
agent = VideoAnalyzer(mock_context)
agent.use_native_gemini = False
fake_audio = tmp_path / "audio.aac"
fake_audio.write_bytes(b"audio_bytes")
mock_llm = MagicMock()
mock_response = AIMessage(
content="",
tool_calls=[
{
"name": "submit_answer",
"args": {
"summary": "Heard notification sound.",
"analysis": "Notification bell sounded at 2.0s.",
},
"id": "audio_call_1",
}
],
)
mock_bound = MagicMock()
mock_bound.ainvoke = AsyncMock(return_value=mock_response)
mock_llm.bind_tools.return_value = mock_bound
with patch(
"artemis.agents.video_analyzer.video_analyzer.get_llm",
return_value=mock_llm,
):
result = await agent._exec_analyze_audio_universal(
audio_path=fake_audio,
start_time=1.0,
end_time=4.0,
actual_start=1.0,
prompt_with_context="Listen for notification sound.",
specific_query="notification bell",
)
assert "Heard notification sound" in result
assert len(agent.blackboard_entries) == 1
assert agent.blackboard_entries[0]["summary"] == "Heard notification sound."
@pytest.mark.asyncio
async def test_run_universal_multi_turn_reasoning(mock_context):
"""Test main reasoning loop of VideoAnalyzer in universal mode with tool dispatch."""
agent = VideoAnalyzer(mock_context)
agent.use_native_gemini = False
# Turn 1: Call extract_segment_metadata
msg_turn1 = AIMessage(
content="Let me check the video metadata first.",
tool_calls=[
{
"name": "extract_segment_metadata",
"args": {"start_time": 0.0, "end_time": 10.0},
"id": "main_call_1",
}
],
)
# Turn 2: Final response
msg_turn2 = AIMessage(
content="The video shows the app launched successfully at 3.5s.",
tool_calls=[],
)
mock_llm = MagicMock()
mock_bound = MagicMock()
mock_bound.ainvoke = AsyncMock(side_effect=[msg_turn1, msg_turn2])
mock_llm.bind_tools.return_value = mock_bound
with (
patch(
"artemis.agents.video_analyzer.video_analyzer.get_llm",
return_value=mock_llm,
),
patch.object(
agent,
"exec_extract_segment_metadata",
new_callable=AsyncMock,
return_value='{"duration_seconds": 10.0, "file_size_mb": 2.5}',
),
):
outcome, status = await agent._run_universal(
time_description="from 0s to 10s",
purpose="Verify app launch",
system_prompt="You are a video analyzer.",
)
assert status == "success"
assert "launched successfully at 3.5s" in outcome
assert agent.exec_extract_segment_metadata.call_count == 1
@@ -0,0 +1,167 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pathlib import Path
import shutil
import tempfile
from unittest.mock import MagicMock
from uuid import uuid4
from langchain_core.messages import AIMessage, HumanMessage
from artemis.context import ArtemisContext
from artemis.data_engine.engine import DataEngine
from artemis.data_engine.trace import CURRENT_TRACE_ID, DataEngineCallbackHandler
from artemis.utils.task_tree import build_plan_and_history
import pytest
@pytest.fixture
def temp_workspace():
tmpdir = tempfile.mkdtemp()
yield Path(tmpdir)
shutil.rmtree(tmpdir)
@pytest.mark.asyncio
async def test_end_to_end_thoughts_non_duplication(temp_workspace):
# 1. Setup mock context and DataEngine
mock_ctx = MagicMock(spec=ArtemisContext)
mock_execution_setup = MagicMock()
mock_execution_setup.traces_path = str(temp_workspace / "traces")
mock_ctx.execution_setup = mock_execution_setup
mock_ctx.device = None
mock_ctx.llm_config = MagicMock()
engine = DataEngine(mock_ctx)
mock_ctx.data_engine = engine
# Start session
engine.start_session("Test initial goal")
# 2. Allocate step ID
step_id = engine.allocate_step_id()
# 3. Simulate Operator Agent run starts
operator_trace_id = uuid4()
engine.record_trace(
type="agent",
name="operator",
payload={"args": {}},
trace_id=operator_trace_id,
step_id=step_id,
status="running",
)
# Set context variables for parenting
token = CURRENT_TRACE_ID.set(operator_trace_id)
try:
# 4. Simulate LLM Call callback sequence
handler = DataEngineCallbackHandler(mock_ctx)
llm_run_id = uuid4()
# Start model
handler.on_chat_model_start(
serialized={"name": "ChatGoogleGenerativeAI"},
messages=[[HumanMessage(content="What should I do?")]],
run_id=llm_run_id,
parent_run_id=None,
)
# End model (generating thought + action)
class MockGeneration:
def __init__(self, message):
self.message = message
class MockLLMResult:
def __init__(self, generations):
self.generations = generations
response_message = AIMessage(
content=[
{
"type": "thinking",
"thinking": "I need to tap the search bar to find books.",
},
{"type": "text", "text": "Tapping search bar."},
]
)
mock_response = MockLLMResult(generations=[[MockGeneration(response_message)]])
handler.on_llm_end(mock_response, run_id=llm_run_id)
finally:
CURRENT_TRACE_ID.reset(token)
# 5. Finish Operator Agent trace
engine.record_trace(
type="agent",
name="operator",
payload={"result": "Success"},
trace_id=operator_trace_id,
step_id=step_id,
status="success",
)
# 6. Record step
engine.record_step(
pre_screenshot_bytes=b"pre_screenshot",
ui_tree=[],
ocr_result=[],
action_taken=[{"action": "tap", "coordinates": [100, 200]}],
operator_raw_thinking="I need to tap the search bar to find books.",
operator_native_thinking="I need to tap the search bar to find books.",
last_execution_result={"status": "success"},
summary="Tapped search bar",
)
# Flush any pending tasks
await engine.shutdown()
# 7. Retrieve agent-friendly steps
steps = engine.get_agent_friendly_steps()
# Assert step was retrieved
assert len(steps) == 1
step_data = steps[0]
# Assert thoughts are present in interleaved events (since we associated step_id now!)
events = step_data.get("interleaved_events", [])
thought_events = [e for e in events if e.get("type") in ("thought", "native_thought")]
assert len(thought_events) > 0
assert any(
e["content"] == "I need to tap the search bar to find books." for e in thought_events
)
# 8. Build history and check for duplications
plan = "- [ ] Search for books"
history = build_plan_and_history(
plan, steps, "default", last_n_detailed=1, for_failure_analyzer=True
)
print("\n--- Generated History Output ---")
print(history)
print("--------------------------------")
# Assert thought is printed
assert "I need to tap the search bar to find books." in history
# Assert thought is NOT printed multiple times
# Count occurrences of the thought string
count = history.count("I need to tap the search bar to find books.")
# The thought might appear in native thinking block and raw thinking block if both were stored.
# In our test we set BOTH operator_raw_thinking and operator_native_thinking to the same string,
# AND the LLM trace generated both "thinking" block and "text" block.
# Let's count how many times it was actually rendered in the Operator Decision Loop.
# It should appear at most 2 times (once as Native Thought, once as Thought/Raw Thought).
# If there was duplicate appending due to fallback failure, it would appear 4+ times.
assert count <= 2
+185
View File
@@ -0,0 +1,185 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import MagicMock
from artemis.context import ArtemisContext
from artemis.data_engine.engine import DataEngine
def test_get_or_create_image_updates_missing_data(tmp_path):
# Setup mock context
mock_ctx = MagicMock(spec=ArtemisContext)
mock_execution_setup = MagicMock()
mock_execution_setup.traces_path = str(tmp_path)
mock_ctx.execution_setup = mock_execution_setup
mock_ctx.device = None
# Initialize DataEngine
engine = DataEngine(mock_ctx)
image_bytes = b"fake_image_bytes"
# 1. Save image without OCR/UI tree
image_name1 = engine.get_or_create_image(image_bytes)
# Verify it was saved with None for OCR/UI tree
record1 = engine.storage.get_image(image_name1)
assert record1 is not None
assert record1.ocr_result is None
assert record1.ui_tree is None
# 2. Save same image with OCR/UI tree
fake_ocr = [{"text": "hello", "box": [0, 0, 10, 10]}]
fake_ui = [{"resource-id": "button1", "text": "click me"}]
image_name2 = engine.get_or_create_image(image_bytes, ui_tree=fake_ui, ocr_result=fake_ocr)
assert image_name1 == image_name2
# Verify it now has OCR/UI tree
record2 = engine.storage.get_image(image_name2)
assert record2 is not None
assert record2.ocr_result == fake_ocr
assert record2.ui_tree == fake_ui
def test_create_image_duplicate_handling(tmp_path):
from artemis.data_engine.storage import StorageManager
from artemis.data_engine.models import ImageRecord
storage = StorageManager(tmp_path / "data_engine.db", tmp_path)
image_record = ImageRecord(
image_name="test_image_hash",
timestamp=123.456,
ocr_result=None,
ui_tree=None,
extra_metadata={},
)
# First insert
storage.create_image(image_record)
# Second insert with same image_name should not raise IntegrityError due to INSERT OR IGNORE
storage.create_image(image_record)
record = storage.get_image("test_image_hash")
assert record is not None
assert record.image_name == "test_image_hash"
def test_video_recording_persistence(tmp_path):
from uuid import uuid4
from artemis.data_engine.storage import StorageManager
from artemis.data_engine.models import VideoRecordingRecord
storage = StorageManager(tmp_path / "data_engine.db", tmp_path)
video_id = uuid4()
session_id = uuid4()
record = VideoRecordingRecord(
video_id=video_id,
session_id=session_id,
device_id="emulator-5554",
start_time=100.0,
end_time=None,
local_video_path="/tmp/recording.mp4",
)
# Test insert
storage.create_video_recording(record)
persisted = storage.get_video_recording(video_id)
assert persisted is not None
assert persisted.video_id == video_id
assert persisted.session_id == session_id
assert persisted.device_id == "emulator-5554"
assert persisted.start_time == 100.0
assert persisted.end_time is None
assert persisted.local_video_path == "/tmp/recording.mp4"
# Test update
record.end_time = 150.0
record.local_video_path = "/tmp/recording_final.mp4"
storage.update_video_recording(record)
updated = storage.get_video_recording(video_id)
assert updated is not None
assert updated.end_time == 150.0
assert updated.local_video_path == "/tmp/recording_final.mp4"
def test_record_step_suppresses_identical_post_screenshot(tmp_path):
mock_ctx = MagicMock(spec=ArtemisContext)
mock_execution_setup = MagicMock()
mock_execution_setup.traces_path = str(tmp_path)
mock_ctx.execution_setup = mock_execution_setup
mock_ctx.device = None
engine = DataEngine(mock_ctx)
engine.start_session("test_goal")
same_image_bytes = b"identical_screen_bytes_123"
step_id = engine.record_step(
pre_screenshot_bytes=same_image_bytes,
post_screenshot_bytes=same_image_bytes,
summary="Test step with identical screenshot",
)
for t in list(engine._pending_threads):
t.join()
step_record = engine.storage.get_steps(engine.current_session_id)[0]
assert step_record is not None
assert step_record.pre_image_name is not None
# post_image_name must be None when no visual transition occurred
assert step_record.post_image_name is None
def test_update_step_execution_result_suppresses_identical_post_screenshot(tmp_path):
mock_ctx = MagicMock(spec=ArtemisContext)
mock_execution_setup = MagicMock()
mock_execution_setup.traces_path = str(tmp_path)
mock_ctx.execution_setup = mock_execution_setup
mock_ctx.device = None
engine = DataEngine(mock_ctx)
engine.start_session("test_goal")
image_bytes = b"pre_screen_bytes_456"
step_id = engine.record_step(
pre_screenshot_bytes=image_bytes,
summary="Initial step",
)
step_record = engine.storage.get_steps(engine.current_session_id)[0]
assert step_record.post_image_name is None
# Update with post_image_name matching pre_image_name
engine.storage.update_step_execution_result(
step_id,
{"status": "success"},
post_image_name=step_record.pre_image_name,
)
updated_step = engine.storage.get_steps(engine.current_session_id)[0]
assert updated_step.post_image_name is None
# Update with distinct post_image_name
engine.storage.update_step_execution_result(
step_id,
{"status": "success"},
post_image_name="distinct_new_hash_789",
)
updated_step_distinct = engine.storage.get_steps(engine.current_session_id)[0]
assert updated_step_distinct.post_image_name == "distinct_new_hash_789"
@@ -0,0 +1,566 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import base64
import hashlib
import json
import logging
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch
from langchain_core.messages import HumanMessage
from artemis.context import ArtemisContext
from artemis.data_engine.engine import DataEngine
from artemis.graph.state import State
from artemis.utils.logger import DataEngineHandler
import pytest
@pytest.fixture
def mock_artemis_ctx(tmp_path):
mock_ctx = MagicMock(spec=ArtemisContext)
mock_execution_setup = MagicMock()
mock_execution_setup.traces_path = str(tmp_path)
mock_ctx.execution_setup = mock_execution_setup
mock_ctx.device = None
mock_ctx.task_plan_snapshot = None
return mock_ctx
@pytest.mark.asyncio
async def test_data_engine_handler_queue_buffering_and_background_worker(
mock_artemis_ctx,
):
"""Verify DataEngineHandler.emit buffers log records via in-memory queue without spawning OS threads per log line."""
engine = DataEngine(mock_artemis_ctx)
session_id = engine.start_session("Test Log Queue")
handler = DataEngineHandler()
with patch("sys.modules") as mock_modules:
mock_engine_mod = MagicMock()
mock_engine_mod._CURRENT_DATA_ENGINE = engine
mock_modules.get.return_value = mock_engine_mod
# Emit 50 rapid log records
for i in range(50):
record = logging.LogRecord(
name=f"test.logger.{i}",
level=logging.INFO,
pathname="test.py",
lineno=10 + i,
msg=f"Async queue test log message {i}",
args=(),
exc_info=None,
)
handler.emit(record)
# Wait for the background worker thread of DataEngineHandler and DataEngine tasks to drain
await asyncio.sleep(0.4)
if engine._pending_tasks:
await asyncio.gather(*engine._pending_tasks)
# Verify that traces were recorded and persisted accurately via direct SQLite query
with engine.storage._get_connection() as conn:
cursor = conn.execute(
"SELECT * FROM traces WHERE session_id = ? AND type = 'log'",
(str(session_id),),
)
log_rows = cursor.fetchall()
assert len(log_rows) >= 50
messages = {json.loads(row["payload"])["message"] for row in log_rows}
for i in range(50):
assert f"Async queue test log message {i}" in messages
@pytest.mark.asyncio
async def test_update_step_action_and_metadata_background_offloading(mock_artemis_ctx, tmp_path):
"""Verify that update_step_* methods publish SSE events instantly in memory and offload SQLite & disk I/O to background."""
engine = DataEngine(mock_artemis_ctx)
session_id = engine.start_session("Test Step Offloading")
step_id = engine.allocate_step_id()
# Create the initial step
engine.record_step(summary="Initial step summary")
if engine._pending_tasks:
await asyncio.gather(*engine._pending_tasks)
# Mock _publish to verify immediate real-time SSE delivery
published_events = []
engine._publish = lambda event_type, payload: published_events.append((event_type, payload))
fake_action = {"action": "tap", "coordinates": [100, 200]}
fake_post_jpg = b"fake_post_screenshot_jpg_bytes_12345"
# Call update_step_action with post screenshot bytes
engine.update_step_action(fake_action, post_screenshot_bytes=fake_post_jpg)
# Verify SSE was published immediately before/during task execution
assert len(published_events) == 1
assert published_events[0][0] == "step_updated"
assert published_events[0][1]["action_taken"] == fake_action
# Call other update_step_* methods
engine.update_step_summary(step_id, "Updated concise summary")
engine.update_step_thinking(step_id, "Raw thinking process...")
engine.update_step_native_thinking(step_id, "Native thinking process...")
engine.update_step_execution_result(
step_id, {"status": "success"}, post_image_name="hash_post_123"
)
# Now await all background pending tasks
if engine._pending_tasks:
await asyncio.gather(*engine._pending_tasks)
# Verify SQLite record has been correctly updated by background tasks
steps = engine.storage.get_steps(session_id)
step_record = next((s for s in steps if s.step_id == step_id), None)
assert step_record is not None
assert step_record.action_taken == fake_action
assert step_record.summary == "Updated concise summary"
assert step_record.operator_raw_thinking == "Raw thinking process..."
assert step_record.operator_native_thinking == "Native thinking process..."
assert step_record.last_execution_result == {"status": "success"}
assert step_record.post_image_name == "hash_post_123"
# Verify post.jpg was written to disk inside step directory
post_jpg_path = engine.current_step_dir / "post.jpg"
assert post_jpg_path.exists()
assert post_jpg_path.read_bytes() == fake_post_jpg
@pytest.mark.asyncio
async def test_record_step_background_image_and_step_persistence(
mock_artemis_ctx,
):
"""Verify record_step immediately calculates SHA-256 hashes and offloads get_or_create_image + create_step to background."""
engine = DataEngine(mock_artemis_ctx)
session_id = engine.start_session("Test Record Step Image Offloading")
step_id = engine.allocate_step_id()
pre_img_bytes = b"fake_pre_screenshot_data_001"
post_img_bytes = b"fake_post_screenshot_data_002"
expected_pre_hash = hashlib.sha256(pre_img_bytes).hexdigest()
expected_post_hash = hashlib.sha256(post_img_bytes).hexdigest()
returned_step_id = engine.record_step(
pre_screenshot_bytes=pre_img_bytes,
post_screenshot_bytes=post_img_bytes,
summary="Testing background image persistence",
)
assert returned_step_id == step_id
# Await background storage tasks
if engine._pending_tasks:
await asyncio.gather(*engine._pending_tasks)
# Verify image records were created in SQLite
pre_record = engine.storage.get_image(expected_pre_hash)
post_record = engine.storage.get_image(expected_post_hash)
assert pre_record is not None
assert pre_record.image_name == expected_pre_hash
assert post_record is not None
assert post_record.image_name == expected_post_hash
# Verify image files exist on disk
pre_path = engine.get_image_path(expected_pre_hash)
post_path = engine.get_image_path(expected_post_hash)
assert pre_path.exists() and pre_path.read_bytes() == pre_img_bytes
assert post_path.exists() and post_path.read_bytes() == post_img_bytes
# Verify StepRecord has correct hashes and summary
steps = engine.storage.get_steps(session_id)
step_record = next((s for s in steps if s.step_id == step_id), None)
assert step_record is not None
assert step_record.pre_image_name == expected_pre_hash
assert step_record.post_image_name == expected_post_hash
assert step_record.summary == "Testing background image persistence"
@pytest.mark.asyncio
async def test_perception_node_async_offloading(mock_artemis_ctx, tmp_path):
"""Verify perception_node offloads injected_instruction reading/unlinking and get_or_create_image to background."""
from artemis.graph.perception import perception_node
engine = DataEngine(mock_artemis_ctx)
engine.start_session("Test Perception Node Offloading")
mock_artemis_ctx.data_engine = engine
# Create an injected_instruction.json file
instruction_file = Path(engine.base_dir) / "injected_instruction.json"
instruction_file.write_text(json.dumps({"instruction": "Avoid clicking Ads"}), encoding="utf-8")
# Prepare mock controller
fake_screenshot_bytes = b"fake_screenshot_for_perception_node_123"
fake_screenshot_b64 = base64.b64encode(fake_screenshot_bytes).decode("utf-8")
expected_hash = hashlib.sha256(fake_screenshot_bytes).hexdigest()
mock_screen_data = MagicMock()
mock_screen_data.base64 = fake_screenshot_b64
mock_screen_data.elements = [{"resource-id": "app_bar", "bounds": "[0,0][1080,200]"}]
mock_screen_data.width = 1080
mock_screen_data.height = 2400
with (
patch("artemis.graph.perception.UnifiedMobileController") as mock_controller_cls,
patch("artemis.graph.perception.perform_ocr", new_callable=AsyncMock) as mock_ocr,
patch("artemis.graph.perception._detect_status_bar_height", return_value=0),
patch("artemis.graph.perception._should_skip_settling", return_value=True),
):
mock_controller = mock_controller_cls.return_value
mock_controller.get_screen_data = AsyncMock(return_value=mock_screen_data)
mock_ocr.return_value = [{"text": "Home", "box": [100, 100, 200, 200]}]
state = State(
messages=[HumanMessage(content="test")],
initial_goal="test goal",
complete_subgoals_by_ids=[],
validator_messages=[],
latest_ui_hierarchy=[],
latest_screenshot=None,
focused_app_info=None,
device_date=None,
structured_decisions=None,
)
update = await perception_node(state, mock_artemis_ctx)
# Verify perception_node returned expected update immediately
assert update["injected_instruction"] == "Avoid clicking Ads"
assert update["latest_screenshot"] == str(engine.get_image_path(expected_hash))
assert update["operator_raw_data"]["width"] == 1080
# Verify injected_instruction.json was unlinked
assert not instruction_file.exists()
# Await background tasks triggered by perception_node
await asyncio.sleep(0.1)
if engine._pending_tasks:
await asyncio.gather(*engine._pending_tasks)
# Verify image record and file were created in background by perception_node
img_record = engine.storage.get_image(expected_hash)
assert img_record is not None
assert img_record.image_name == expected_hash
assert engine.get_image_path(expected_hash).exists()
@pytest.mark.asyncio
async def test_snapshot_copy_and_restore_offloading(mock_artemis_ctx, tmp_path):
"""Verify operator create_snapshot and graph rollback/restore execute asynchronously without blocking."""
from artemis.utils.file import create_snapshot, restore_snapshot
notes_dir = tmp_path / "notes"
notes_dir.mkdir(parents=True, exist_ok=True)
(notes_dir / "task_plan.md").write_text("# Plan\n- Step 1", encoding="utf-8")
(notes_dir / "verification_chat_1.json").write_text('{"messages": ["hello"]}', encoding="utf-8")
snapshot_dir = tmp_path / "notes_snapshot"
# Test async offloading of create_snapshot (operator pattern)
await asyncio.to_thread(create_snapshot, notes_dir, snapshot_dir)
assert snapshot_dir.exists()
assert (snapshot_dir / "task_plan.md").read_text(encoding="utf-8") == "# Plan\n- Step 1"
# Modify live task_plan.md
(notes_dir / "task_plan.md").write_text("# Plan\n- Step 1 (Modified)", encoding="utf-8")
# Test async offloading of restore_snapshot (graph pattern when checker fails)
def _restore_and_preserve():
chat_contents = {}
for p in notes_dir.glob("verification_chat_*.json"):
chat_contents[p.name] = p.read_text(encoding="utf-8")
restore_snapshot(snapshot_dir, notes_dir)
for name, content in chat_contents.items():
(notes_dir / name).write_text(content, encoding="utf-8")
await asyncio.to_thread(_restore_and_preserve)
assert (notes_dir / "task_plan.md").read_text(encoding="utf-8") == "# Plan\n- Step 1"
assert (notes_dir / "verification_chat_1.json").read_text(
encoding="utf-8"
) == '{"messages": ["hello"]}'
@pytest.mark.asyncio
async def test_ocr_api_persistent_http_client_singleton_and_tls_reuse():
"""Verify get_http_client returns a persistent singleton httpx.AsyncClient that reuses TLS connections across OCR calls."""
import httpx
from artemis.utils import ocr_api
# Reset singleton to test fresh creation & persistence
ocr_api._HTTP_CLIENT = None
client1 = ocr_api.get_http_client()
client2 = ocr_api.get_http_client()
assert isinstance(client1, httpx.AsyncClient)
assert client1 is client2
# Verify perform_ocr uses this exact persistent client
fake_ocr_resp = MagicMock()
fake_ocr_resp.status_code = 200
fake_ocr_resp.json.return_value = {
"responses": [
{
"textAnnotations": [
{
"description": "Combined text block",
"boundingPoly": {"vertices": []},
},
{
"description": "Hello World",
"boundingPoly": {"vertices": [{"x": 10, "y": 20}, {"x": 100, "y": 200}]},
},
]
}
]
}
with patch.object(client1, "post", new_callable=AsyncMock) as mock_post:
mock_post.return_value = fake_ocr_resp
result = await ocr_api.perform_ocr(screenshot_b64="ZHVtbXlfYjY0")
mock_post.assert_called_once()
assert len(result) == 1
assert result[0]["text"] == "Hello World"
@pytest.mark.asyncio
async def test_ui_filter_and_ui_automator_client_pre_parsed_bounds_o1_lookup():
"""Verify XML parsing pre-populates parsed_bounds and _parse_bounds hits this cache in O(1) without regex matches."""
import re
from artemis.clients.ui_automator_client import _parse_hierarchy_xml_to_elements
from artemis.utils.ui_filter import _parse_bounds
# 1. Test ingestion pre-parsing in _parse_hierarchy_xml_to_elements
xml_data = '<hierarchy><node text="Login Btn" bounds="[15,25][350,120]"/></hierarchy>'
elements = _parse_hierarchy_xml_to_elements(xml_data)
assert len(elements) == 1
assert elements[0]["parsed_bounds"] == {
"left": 15,
"top": 25,
"right": 350,
"bottom": 120,
}
# 2. Test _parse_bounds O(1) hit vs fallback
with patch.object(re, "match", wraps=re.match) as spy_match:
# Hit pre-parsed bounds -> re.match MUST NOT be called
hit_result = _parse_bounds(elements[0])
assert hit_result == {
"left": 15,
"top": 25,
"right": 350,
"bottom": 120,
}
spy_match.assert_not_called()
# Fallback raw string -> re.match is called exactly once
fallback_result = _parse_bounds("[50,60][150,160]")
assert fallback_result == {
"left": 50,
"top": 60,
"right": 150,
"bottom": 160,
}
assert spy_match.call_count == 1
@pytest.mark.asyncio
async def test_validator_pre_execution_loop_reverted_to_exact_safety_contract(
mock_artemis_ctx, tmp_path
):
"""Verify Validator pre-execution check runs VLM on XML failure, but retains XML error if VLM fails (reverted contract)."""
from artemis.agents.validator.validator import ValidatorNode, ValidationErrorCategory
# Set mock context properties required by tracing decorators
mock_artemis_ctx.data_engine = MagicMock()
node = ValidatorNode(mock_artemis_ctx)
session = MagicMock()
session.call_tool = AsyncMock(
return_value=MagicMock(content=[MagicMock(text="ZHVtbXlfZGF0YQ==")])
)
decisions = json.dumps(
[
{
"action": "tap",
"coordinates": [200, 250],
"target_text": "Login",
"target_bounds": [100, 200, 300, 300],
"target_resource_id": "btn_login",
}
]
)
state = State(
messages=[],
initial_goal="test",
complete_subgoals_by_ids=[],
validator_messages=[],
latest_ui_hierarchy=[],
latest_screenshot="dummy.png",
structured_decisions=decisions,
focused_app_info=None,
device_date=None,
)
with (
patch.object(
ValidatorNode, "_get_initial_screenshot", new_callable=AsyncMock
) as mock_get_screen,
patch.object(
ValidatorNode, "_validate_action_precondition", new_callable=AsyncMock
) as mock_xml,
patch.object(
ValidatorNode,
"_validate_action_precondition_pixel",
new_callable=AsyncMock,
) as mock_pixel,
patch(
"artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze",
new_callable=AsyncMock,
) as mock_analyze,
):
mock_get_screen.return_value = ("dummy_b64", "dummy_stem")
# 1. XML check returns failure (TARGET_OCCUPIED by 'Sign Up' button)
mock_xml.return_value = (
False,
ValidationErrorCategory.TARGET_OCCUPIED,
"Occupied by Sign Up button",
)
# 2. VLM check returns failure (PIXEL_TARGET_DISAPPEARED)
mock_pixel.return_value = (
False,
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
"VLM target missing",
)
mock_analyze.return_value = {
"status": "cannot_fix",
"analysis": "Unrepairable drift",
}
result = await node(state)
# Verify both XML check and VLM check were called (reverted double-check contract)
mock_xml.assert_called_once()
mock_pixel.assert_called_once()
# Verify FailureAnalyzer received the EXACT XML error (TARGET_OCCUPIED), NOT the VLM error
mock_analyze.assert_called_once()
args, kwargs = mock_analyze.call_args
assert kwargs.get("error_category") == ValidationErrorCategory.TARGET_OCCUPIED
assert "Occupied by Sign Up button" in args[2]
@pytest.mark.asyncio
async def test_failure_analyzer_prune_intermediate_screenshots(mock_artemis_ctx):
"""Verify FailureAnalyzer._prune_intermediate_screenshots keeps initial reference images and the latest live image, while pruning intermediate images."""
from google.genai import types
from artemis.agents.validator.failure_analyzer import FailureAnalyzer
analyzer = FailureAnalyzer(mock_artemis_ctx)
# Build simulated contents history:
# 0: User (Initial reference screenshots + prompt)
# 1: Model
# 2: User (Intermediate tool results)
# 3: User (Intermediate observation screenshot - SHOULD BE PRUNED)
# 4: Model
# 5: User (Intermediate tool results 2)
# 6: User (Latest observation screenshot - SHOULD BE PRESERVED)
contents = [
types.Content(
role="user",
parts=[
types.Part.from_text(text="Reference screenshot 1"),
types.Part.from_bytes(data=b"initial_pre_image_bytes", mime_type="image/jpeg"),
types.Part.from_bytes(data=b"initial_post_image_bytes", mime_type="image/jpeg"),
],
),
types.Content(role="model", parts=[types.Part.from_text(text="Clicked something")]),
types.Content(
role="user",
parts=[types.Part.from_text(text="Result text")],
),
types.Content(
role="user",
parts=[
types.Part.from_text(text="Intermediate observation"),
types.Part.from_bytes(
data=b"intermediate_image_bytes_to_prune",
mime_type="image/jpeg",
),
],
),
types.Content(role="model", parts=[types.Part.from_text(text="Clicking again")]),
types.Content(
role="user",
parts=[types.Part.from_text(text="Result text 2")],
),
types.Content(
role="user",
parts=[
types.Part.from_text(text="Latest live observation"),
types.Part.from_bytes(
data=b"latest_live_image_bytes_to_keep",
mime_type="image/jpeg",
),
],
),
]
analyzer._prune_intermediate_screenshots(contents)
# 1. Verify index 0 images are untouched
assert contents[0].parts[1].inline_data.data == b"initial_pre_image_bytes"
assert contents[0].parts[2].inline_data.data == b"initial_post_image_bytes"
# 2. Verify index 3 (intermediate image) is replaced by text placeholder
assert getattr(contents[3].parts[1], "inline_data", None) is None
assert "[Screenshot of intermediate step omitted for performance]" in contents[3].parts[1].text
# 3. Verify index 6 (latest image) is fully preserved
assert contents[6].parts[1].inline_data.data == b"latest_live_image_bytes_to_keep"
@pytest.mark.asyncio
async def test_record_step_pre_allocated_id_reset(mock_artemis_ctx):
"""Verify that after record_step is called, the pre-allocated current_step_id is reset to None so subsequent steps generate distinct IDs."""
from artemis.data_engine.engine import DataEngine
engine = DataEngine(mock_artemis_ctx)
engine.start_session("Pre-allocated ID Reset Test")
# 1. Pre-allocate ID for step 1
step_id_1 = engine.allocate_step_id()
assert engine.current_step_id == step_id_1
# 2. Record step 1 (should use step_id_1 and reset current_step_id to None)
returned_id_1 = engine.record_step(summary="Step 1")
assert returned_id_1 == step_id_1
assert engine.current_step_id is None
# 3. Pre-allocate ID for step 2 (should get a distinct new ID)
step_id_2 = engine.allocate_step_id()
assert step_id_2 != step_id_1
assert engine.current_step_id == step_id_2
# 4. Record step 2 (should use step_id_2 and reset current_step_id to None)
returned_id_2 = engine.record_step(summary="Step 2")
assert returned_id_2 == step_id_2
assert returned_id_2 != returned_id_1
assert engine.current_step_id is None
@@ -0,0 +1,189 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from unittest.mock import MagicMock
from artemis.context import ArtemisContext
from artemis.data_engine.trace import (
TraceSpan,
smart_serialize,
)
import pytest
def test_smart_serialize():
# Test dict
d = {"a": 1, "b": "string"}
assert smart_serialize(d) == json.dumps(d, ensure_ascii=False)
# Test list
items_list = [1, 2, "three"]
assert smart_serialize(items_list) == json.dumps(items_list, ensure_ascii=False)
# Test bytes
b = b"hello world"
import hashlib
expected_sha = hashlib.sha256(b).hexdigest()[:8]
assert smart_serialize(b) == f"<Bytes length={len(b)} sha256={expected_sha}>"
# Test other types
assert smart_serialize(123) == "123"
assert smart_serialize("just a string") == "just a string"
@pytest.mark.asyncio
async def test_trace_span():
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
with TraceSpan(name="test_span", ctx=mock_ctx) as span:
span.result = "span_result"
# Simulate work
# TraceSpan writes twice: start (running) and end (success)
assert mock_ctx.data_engine.record_trace.call_count == 2
# Verify the final success call (which is the last call)
args, kwargs = mock_ctx.data_engine.record_trace.call_args
assert kwargs["type"] == "span"
assert kwargs["name"] == "test_span"
assert kwargs["status"] == "success"
assert kwargs["payload"]["result"] == "span_result"
@pytest.mark.asyncio
async def test_trace_span_failure():
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
try:
with TraceSpan(name="failed_span", ctx=mock_ctx):
raise ValueError("Span error")
except ValueError:
pass
# TraceSpan writes twice: start (running) and end (failed)
assert mock_ctx.data_engine.record_trace.call_count == 2
# Verify the final failed call (which is the last call)
args, kwargs = mock_ctx.data_engine.record_trace.call_args
assert kwargs["type"] == "span"
assert kwargs["name"] == "failed_span"
assert kwargs["status"] == "failed"
assert "error" in kwargs["payload"]
assert "Span error" in kwargs["payload"]["error"]
def test_data_engine_callback_handler():
from uuid import uuid4
from langchain_core.messages import HumanMessage, AIMessage
from artemis.data_engine.trace import DataEngineCallbackHandler
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_step_id = uuid4()
mock_ctx.data_engine.current_step_id = mock_step_id
handler = DataEngineCallbackHandler(mock_ctx)
run_id = uuid4()
# 1. Simulate on_chat_model_start
serialized = {"name": "test_model"}
messages = [[HumanMessage(content="Hello LLM")]]
handler.on_chat_model_start(serialized, messages, run_id=run_id)
assert mock_ctx.data_engine.record_trace.call_count == 1
call1_args, call1_kwargs = mock_ctx.data_engine.record_trace.call_args_list[0]
assert call1_kwargs["type"] == "llm_call"
assert call1_kwargs["name"] == "test_model"
assert call1_kwargs["status"] == "running"
assert call1_kwargs["step_id"] == mock_step_id
assert call1_kwargs["payload"]["messages"][0]["content"] == "Hello LLM"
# 2. Simulate on_llm_end (successful complete response)
class MockGeneration:
def __init__(self, message):
self.message = message
class MockLLMResult:
def __init__(self, generations):
self.generations = generations
response_message = AIMessage(content="Hello User")
mock_response = MockLLMResult(generations=[[MockGeneration(response_message)]])
handler.on_llm_end(mock_response, run_id=run_id)
assert mock_ctx.data_engine.record_trace.call_count == 3
call2_args, call2_kwargs = mock_ctx.data_engine.record_trace.call_args_list[1]
assert call2_kwargs["type"] == "llm_call"
assert call2_kwargs["status"] == "success"
assert call2_kwargs["step_id"] == mock_step_id
assert call2_kwargs["payload"]["messages"][0]["content"] == "Hello LLM"
assert call2_kwargs["payload"]["response"][0]["content"] == "Hello User"
assert isinstance(call2_kwargs.get("duration"), float)
assert call2_kwargs["duration"] >= 0.0
call3_args, call3_kwargs = mock_ctx.data_engine.record_trace.call_args_list[2]
assert call3_kwargs["type"] == "raw_thinking"
assert call3_kwargs["parent_trace_id"] == run_id
assert call3_kwargs["payload"]["thought"] == "Hello User"
def test_callback_handler_chain_parenting():
from uuid import uuid4
from langchain_core.messages import HumanMessage
from artemis.data_engine.trace import DataEngineCallbackHandler, CURRENT_TRACE_ID
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
handler = DataEngineCallbackHandler(mock_ctx)
agent_run_id = uuid4()
chain1_run_id = uuid4()
chain2_run_id = uuid4()
llm_run_id = uuid4()
token = CURRENT_TRACE_ID.set(agent_run_id)
try:
# Simulate chain start (top level, parent is agent_run_id)
handler.on_chain_start({}, {}, run_id=chain1_run_id, parent_run_id=None)
# Simulate nested chain start (parent is chain1_run_id)
handler.on_chain_start({}, {}, run_id=chain2_run_id, parent_run_id=chain1_run_id)
# Simulate LLM start inside nested chain (parent is chain2_run_id)
messages = [[HumanMessage(content="Hello Nested LLM")]]
handler.on_chat_model_start(
{"name": "test_nested_model"},
messages,
run_id=llm_run_id,
parent_run_id=chain2_run_id,
)
# Verify parent trace ID resolves to agent_run_id (since chains are not recorded)
assert mock_ctx.data_engine.record_trace.call_count == 1
args, kwargs = mock_ctx.data_engine.record_trace.call_args
assert kwargs["parent_trace_id"] == agent_run_id
# Cleanup
handler.on_chain_end({}, run_id=chain2_run_id)
handler.on_chain_end({}, run_id=chain1_run_id)
assert len(handler.chain_parents) == 0
finally:
CURRENT_TRACE_ID.reset(token)
+297
View File
@@ -0,0 +1,297 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.context import ArtemisContext
from artemis.graph.graph import wrap_note_tool, wrap_update_note_tool
import pytest
@pytest.mark.asyncio
async def test_wrap_update_note_tool_valid_status_change(tmp_path):
base_dir = tmp_path
notes_dir = base_dir / "notes"
notes_dir.mkdir(parents=True, exist_ok=True)
task_plan_path = notes_dir / "task_plan.md"
original_plan = """# Test Plan
- [ ] Milestone 1
- [ ] Substep 1.1
- [ ] Milestone 2
"""
task_plan_path.write_text(original_plan, encoding="utf-8")
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = str(base_dir)
mock_ctx.execution_setup = None
mock_original_tool = MagicMock()
mock_original_tool.name = "update_note"
mock_original_tool.description = "Update a note"
mock_original_tool.ainvoke = AsyncMock(return_value="Success")
async def fake_invoke_tool(tool, args, tool_call_id, state, record_trace=None):
content = task_plan_path.read_text(encoding="utf-8")
updated = content.replace(args["target"], args["replacement"])
task_plan_path.write_text(updated, encoding="utf-8")
return "Success"
target = "- [ ] Milestone 1"
replacement = "- [x] Milestone 1"
with patch(
"artemis.graph.graph.invoke_tool_with_injection",
side_effect=fake_invoke_tool,
):
# Instantiate inside patch context to capture the mocked import locally
wrapped_tool = wrap_update_note_tool(mock_ctx, mock_original_tool)
result = await wrapped_tool.ainvoke(
{
"key": "task_plan",
"target": target,
"replacement": replacement,
"tool_call_id": "test_call_id",
}
)
assert result == "Success"
updated_content = task_plan_path.read_text(encoding="utf-8")
assert "- [x] Milestone 1" in updated_content
assert "- [ ] Milestone 2" in updated_content
@pytest.mark.asyncio
async def test_wrap_update_note_tool_invalid_milestone_text_change(tmp_path):
base_dir = tmp_path
notes_dir = base_dir / "notes"
notes_dir.mkdir(parents=True, exist_ok=True)
task_plan_path = notes_dir / "task_plan.md"
original_plan = """# Test Plan
- [ ] Milestone 1
- [ ] Substep 1.1
- [ ] Milestone 2
"""
task_plan_path.write_text(original_plan, encoding="utf-8")
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = str(base_dir)
mock_ctx.execution_setup = None
mock_original_tool = MagicMock()
mock_original_tool.name = "update_note"
mock_original_tool.ainvoke = AsyncMock(return_value="Success")
async def fake_invoke_tool(tool, args, tool_call_id, state, record_trace=None):
content = task_plan_path.read_text(encoding="utf-8")
updated = content.replace(args["target"], args["replacement"])
task_plan_path.write_text(updated, encoding="utf-8")
return "Success"
target = "- [ ] Milestone 1"
replacement = "- [ ] Milestone Renamed"
mock_validation = AsyncMock(return_value={"status": "failed", "feedback": "rejected"})
with (
patch(
"artemis.graph.graph.invoke_tool_with_injection",
side_effect=fake_invoke_tool,
),
patch(
"artemis.agents.planner.planner.run_async_planner_validation",
mock_validation,
),
):
wrapped_tool = wrap_update_note_tool(mock_ctx, mock_original_tool)
result = await wrapped_tool.ainvoke(
{
"key": "task_plan",
"target": target,
"replacement": replacement,
"tool_call_id": "test_call_id",
}
)
assert result == "Success"
assert mock_ctx.planner_task is not None
@pytest.mark.asyncio
async def test_wrap_update_note_tool_allow_subgoal_changes(tmp_path):
base_dir = tmp_path
notes_dir = base_dir / "notes"
notes_dir.mkdir(parents=True, exist_ok=True)
task_plan_path = notes_dir / "task_plan.md"
original_plan = """# Test Plan
- [ ] Milestone 1
- [ ] Substep 1.1
- [ ] Milestone 2
"""
task_plan_path.write_text(original_plan, encoding="utf-8")
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = str(base_dir)
mock_ctx.execution_setup = None
mock_original_tool = MagicMock()
mock_original_tool.name = "update_note"
mock_original_tool.ainvoke = AsyncMock(return_value="Success")
async def fake_invoke_tool(tool, args, tool_call_id, state, record_trace=None):
content = task_plan_path.read_text(encoding="utf-8")
updated = content.replace(args["target"], args["replacement"])
task_plan_path.write_text(updated, encoding="utf-8")
return "Success"
target = " - [ ] Substep 1.1"
replacement = " - [x] Substep 1.1 Completed"
with patch(
"artemis.graph.graph.invoke_tool_with_injection",
side_effect=fake_invoke_tool,
):
wrapped_tool = wrap_update_note_tool(mock_ctx, mock_original_tool)
result = await wrapped_tool.ainvoke(
{
"key": "task_plan",
"target": target,
"replacement": replacement,
"tool_call_id": "test_call_id",
}
)
assert result == "Success"
updated_content = task_plan_path.read_text(encoding="utf-8")
assert "Substep 1.1 Completed" in updated_content
assert "- [ ] Milestone 1" in updated_content
@pytest.mark.asyncio
async def test_wrap_note_tool_invalid_rewrite(tmp_path):
base_dir = tmp_path
notes_dir = base_dir / "notes"
notes_dir.mkdir(parents=True, exist_ok=True)
task_plan_path = notes_dir / "task_plan.md"
original_plan = """# Test Plan
- [ ] Milestone 1
- [ ] Substep 1.1
- [ ] Milestone 2
"""
task_plan_path.write_text(original_plan, encoding="utf-8")
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = str(base_dir)
mock_ctx.execution_setup = None
mock_original_tool = MagicMock()
mock_original_tool.name = "save_note"
mock_original_tool.ainvoke = AsyncMock(return_value="Success")
async def fake_invoke_tool(tool, args, tool_call_id, state, record_trace=None):
task_plan_path.write_text(args["content"], encoding="utf-8")
return "Success"
corrupted_plan = """# Test Plan
- [ ] Milestone 1 (Altered)
- [ ] New Milestone Added
"""
mock_validation = AsyncMock(return_value={"status": "failed", "feedback": "rejected"})
with (
patch(
"artemis.graph.graph.invoke_tool_with_injection",
side_effect=fake_invoke_tool,
),
patch(
"artemis.agents.planner.planner.run_async_planner_validation",
mock_validation,
),
):
wrapped_tool = wrap_note_tool(mock_ctx, mock_original_tool)
result = await wrapped_tool.ainvoke(
{
"key": "task_plan",
"content": corrupted_plan,
"tool_call_id": "test_call_id",
}
)
assert result == "Success"
assert mock_ctx.planner_task is not None
@pytest.mark.asyncio
async def test_wrap_note_tool_valid_lexical_drift(tmp_path):
from langgraph.types import Command
from langchain_core.messages import ToolMessage
from artemis.constants import VALIDATOR_MESSAGES_KEY
base_dir = tmp_path
notes_dir = base_dir / "notes"
notes_dir.mkdir(parents=True, exist_ok=True)
task_plan_path = notes_dir / "task_plan.md"
original_plan = """# Test Plan
- [ ] Open the Chrome browser and navigate to youtube.com
"""
task_plan_path.write_text(original_plan, encoding="utf-8")
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = str(base_dir)
mock_ctx.execution_setup = None
# Create mock Command result
tool_msg = ToolMessage(tool_call_id="test_call_id", content="Success outcome", status="success")
mock_command = Command(update={VALIDATOR_MESSAGES_KEY: [tool_msg]})
mock_original_tool = MagicMock()
mock_original_tool.name = "save_note"
mock_original_tool.ainvoke = AsyncMock(return_value=mock_command)
async def fake_invoke_tool(tool, args, tool_call_id, state, record_trace=None):
task_plan_path.write_text(args["content"], encoding="utf-8")
return mock_command
# Minor wording change that is semantically valid (similarity ~92%)
minor_drift_plan = """# Test Plan
- [ ] Open Chrome browser and navigate to youtube.com
"""
with patch(
"artemis.graph.graph.invoke_tool_with_injection",
side_effect=fake_invoke_tool,
):
wrapped_tool = wrap_note_tool(mock_ctx, mock_original_tool)
result = await wrapped_tool.ainvoke(
{
"key": "task_plan",
"content": minor_drift_plan,
"tool_call_id": "test_call_id",
}
)
assert isinstance(result, Command)
assert not hasattr(mock_ctx, "planner_task") or mock_ctx.planner_task is None
# Verify that the drift plan is actually saved
saved_content = task_plan_path.read_text(encoding="utf-8")
assert "Open Chrome browser" in saved_content
+146
View File
@@ -0,0 +1,146 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for Base Agent Protocol & Lifecycle Registry."""
import pytest
from typing import Any
from artemis.agents.base import AgentConfig, AgentRegistry, BaseAgent
from artemis.agents.validator.failure_analyzer_agent import FailureAnalyzerAgent
from artemis.agents.validator.recovery_strategies import (
RecoveryStrategyRegistry,
)
from artemis.drivers.mock.mock_driver import MockDeviceDriver
class DummyTestAgent(BaseAgent):
"""Simple test agent implementing BaseAgent protocol."""
def __init__(self, config: AgentConfig | None = None, ctx: Any | None = None, **kwargs: Any):
super().__init__(
name="dummy_test_agent",
role="Test Worker",
description="Agent used for testing lifecycle transitions.",
config=config,
ctx=ctx,
)
async def process(self, state: Any) -> dict[str, Any]:
return {"status": "success", "processed_state": True}
@pytest.mark.asyncio
async def test_base_agent_lifecycle():
"""Verify BaseAgent initialization, execution, and cleanup lifecycle."""
agent = DummyTestAgent()
assert agent.name == "dummy_test_agent"
assert not agent._is_initialized
# 1. Initialize
mock_ctx = object()
await agent.initialize(mock_ctx)
assert agent._is_initialized
assert agent.ctx is mock_ctx
# 2. Process
result = await agent.process({})
assert result.get("status") == "success"
assert result.get("processed_state") is True
# 3. Cleanup
await agent.cleanup()
assert not agent._is_initialized
def test_agent_registry():
"""Verify AgentRegistry registers and instantiates agents correctly."""
registered = AgentRegistry.list_agents()
assert "planner" in registered
assert "summarizer" in registered
assert "operator" in registered
assert "failure_analyzer" in registered
# Instantiate PlannerAgent via registry
planner = AgentRegistry.create_agent("planner", ctx=None)
assert planner.name == "planner"
assert planner.role == "Strategic Task Planner"
# Instantiate SummarizerAgent via registry
summarizer = AgentRegistry.create_agent("summarizer", ctx=None)
assert summarizer.name == "summarizer"
assert summarizer.role == "Execution History Summarizer"
# Instantiate OperatorAgent via registry
operator = AgentRegistry.create_agent("operator", ctx=None)
assert operator.name == "operator"
assert operator.role == "Mobile UI Interaction Operator"
# Instantiate FailureAnalyzerAgent via registry
fa = AgentRegistry.create_agent("failure_analyzer", ctx=None)
assert fa.name == "failure_analyzer"
assert fa.role == "Autonomous Self-Healing Failure Analyzer"
def test_agent_config():
"""Verify AgentConfig parameters can be customized."""
config = AgentConfig(
model_name="gemini-2.5-pro",
temperature=0.7,
max_tokens=2048,
enabled_tools=["click", "swipe"],
)
agent = DummyTestAgent(config=config)
assert agent.config.model_name == "gemini-2.5-pro"
assert agent.config.temperature == 0.7
assert agent.config.enabled_tools == ["click", "swipe"]
@pytest.mark.asyncio
async def test_recovery_strategies_and_failure_analyzer():
"""Verify self-healing recovery strategies and FailureAnalyzerAgent execution."""
driver = MockDeviceDriver(device_id="mock-rec-1")
# 1. Test dialog dismissal strategy
dialog_ctx = {"category": "ui_obstruction", "error_message": "Unexpected permission popup"}
dialog_res = await RecoveryStrategyRegistry.attempt_recovery(driver, dialog_ctx)
assert dialog_res is not None
assert dialog_res.success
assert dialog_res.action_type.value == "dismiss_dialog"
assert driver.action_history[-1].get("key") == "back"
# 2. Test navigate back strategy
nav_ctx = {"category": "wrong_screen"}
nav_res = await RecoveryStrategyRegistry.attempt_recovery(driver, nav_ctx)
assert nav_res is not None
assert nav_res.success
assert nav_res.action_type.value == "navigate_back"
# 3. Test scroll and search strategy
scroll_ctx = {"category": "element_not_found", "scroll_direction": "down"}
scroll_res = await RecoveryStrategyRegistry.attempt_recovery(driver, scroll_ctx)
assert scroll_res is not None
assert scroll_res.success
assert scroll_res.action_type.value == "scroll_and_retry"
# 4. Test FailureAnalyzerAgent process
class MockContext:
pass
ctx = MockContext()
setattr(ctx, "_active_driver", driver)
fa_agent = FailureAnalyzerAgent(ctx=ctx)
fa_result = await fa_agent.process({"failure_context": {"category": "wrong_screen"}})
assert fa_result["status"] == "recovered"
assert fa_result["recovery_action"] == "navigate_back"
+30
View File
@@ -0,0 +1,30 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
async def f():
return [1, 2]
async def m():
try:
val = await f()[-1]
print(val)
except Exception as e:
print(repr(e))
asyncio.run(m())
+65
View File
@@ -0,0 +1,65 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for ARTEMIS Unified CLI application."""
from typer.testing import CliRunner
from artemis.interfaces.cli.main import app
runner = CliRunner()
def test_cli_help():
"""Verify top-level CLI help returns status 0 and lists core subcommands."""
result = runner.invoke(app, ["--help"])
assert result.exit_code == 0
assert "Artemis: Autonomous Multimodal Android Agent" in result.output
assert "run" in result.output
assert "batch" in result.output
assert "bench" in result.output
assert "server" in result.output
assert "trace" in result.output
assert "mcp" in result.output
def test_cli_version():
"""Verify --version returns version banner."""
result = runner.invoke(app, ["--version"])
assert result.exit_code == 0
assert "Artemis Agent Platform" in result.output
def test_cli_run_help():
"""Verify 'artemis run --help' displays execution options."""
result = runner.invoke(app, ["run", "--help"])
assert result.exit_code == 0
assert "--profile" in result.output
assert "--locked-app" in result.output
assert "--traces-path" in result.output
def test_cli_batch_help():
"""Verify 'artemis batch --help' displays batch options."""
result = runner.invoke(app, ["batch", "--help"])
assert result.exit_code == 0
assert "--file" in result.output
assert "--delay" in result.output
def test_cli_trace_help():
"""Verify 'artemis trace --help' lists trace subcommands."""
result = runner.invoke(app, ["trace", "--help"])
assert result.exit_code == 0
assert "list" in result.output
assert "view" in result.output
+541
View File
@@ -0,0 +1,541 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for the unified artemis.config package."""
from pathlib import Path
import tempfile
from unittest.mock import MagicMock
import pytest
from pydantic import SecretStr
from artemis.config import (
CONFIG_DIR,
ROOT_DIR,
AgentGlobalConfig,
LLMConfig,
OutputConfig,
Settings,
cleanup_temp_dir,
clear_ipc_port,
clear_ls_address,
deep_merge_llm_config,
get_app_dir,
get_config_path,
get_data_engine_db_path,
get_default_llm_config,
get_default_traces_path,
get_temp_dir,
get_traces_dir,
load_agent_config,
read_ipc_port,
read_ls_address,
record_events,
write_ipc_port,
write_ls_address,
)
def test_paths_and_directories():
"""Verify central path resolution."""
assert ROOT_DIR.exists()
assert (ROOT_DIR / "artemis").exists()
assert CONFIG_DIR.exists()
assert (CONFIG_DIR / "artemis.jsonc").exists()
app_dir = get_app_dir()
assert isinstance(app_dir, Path)
traces_path = get_default_traces_path()
assert isinstance(traces_path, Path)
assert get_traces_dir() == traces_path
db_path = get_data_engine_db_path()
assert db_path.name == "data_engine.db"
temp_dir = get_temp_dir("test_subfolder")
assert temp_dir.exists()
assert temp_dir.name == "test_subfolder"
def test_config_file_resolver():
"""Test get_config_path for unified artemis.jsonc."""
artemis_path = get_config_path("artemis.jsonc")
assert artemis_path.exists()
with pytest.raises(FileNotFoundError):
get_config_path("non_existent_config_12345.json")
def test_settings_and_api_key_fallbacks(monkeypatch):
"""Test Settings model, API key fallbacks, and get/set methods."""
monkeypatch.delenv("GOOGLE_API_KEY", raising=False)
monkeypatch.delenv("GEMINI_API_KEY", raising=False)
monkeypatch.delenv("GCP_API_KEY", raising=False)
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
custom_settings = Settings(
_env_file=None,
GOOGLE_API_KEY=None,
GEMINI_API_KEY=SecretStr("gemini_key_123"),
OPENAI_API_KEY=SecretStr("openai_key_456"),
OCR_API_KEY=SecretStr("ocr_key_789"),
)
# Automatic fallback should populate GOOGLE_API_KEY from GEMINI_API_KEY
assert custom_settings.GOOGLE_API_KEY is not None
assert custom_settings.GOOGLE_API_KEY.get_secret_value() == "gemini_key_123"
assert custom_settings.get_api_key("ocr").get_secret_value() == "ocr_key_789"
assert custom_settings.get_api_key("google").get_secret_value() == "gemini_key_123"
assert custom_settings.get_api_key("gemini").get_secret_value() == "gemini_key_123"
assert custom_settings.get_api_key("openai").get_secret_value() == "openai_key_456"
assert custom_settings.get_api_key("nonexistent") is None
# Test dynamic set_api_key
custom_settings.set_api_key("xai", "xai_test_key", persist_to_env=False)
assert custom_settings.get_api_key("xai").get_secret_value() == "xai_test_key"
def test_llm_config_parsing_and_merging():
"""Test LLMConfig parsing, agent querying, and deep merging."""
llm_cfg = get_default_llm_config()
assert isinstance(llm_cfg, LLMConfig)
assert llm_cfg.planner.provider in ("google", "openai", "openrouter", "xai", "vertexai")
assert llm_cfg.get_agent("planner") is not None
assert llm_cfg.get_utils("hopper") is not None
# Deep merge overrides
overrides = {
"planner": {
"model": "custom-model-v1",
"temperature": 0.7,
}
}
merged = deep_merge_llm_config(llm_cfg, overrides)
assert merged.planner.model == "custom-model-v1"
assert merged.planner.temperature == 0.7
def test_agent_config_loading():
"""Test AgentGlobalConfig parsing from agent_config.json / artemis.jsonc."""
agent_cfg = load_agent_config()
assert isinstance(agent_cfg, AgentGlobalConfig)
assert "operator" in agent_cfg.explorer_versions
assert agent_cfg.explorer.default_version == "flash"
assert agent_cfg.explorer.flash_mode == "flash"
assert agent_cfg.explorer.pro_mode == "flash"
assert agent_cfg.explorer.caching is True
assert "explorer" in agent_cfg.blacklisted_tools
assert agent_cfg.video_analyzer.enable_ledger is True
assert agent_cfg.planner_validation.enabled is False
assert agent_cfg.planner_validation.similarity_threshold == 0.85
assert agent_cfg.committee.enabled is False
assert agent_cfg.committee.debate_rounds == 2
assert agent_cfg.checker.enabled is False
assert agent_cfg.checker.max_iterations == 20
assert agent_cfg.checker.max_chat_rounds == 4
assert agent_cfg.outputter.enabled is True
assert agent_cfg.outputter.force_synthesis is False
assert agent_cfg.flash.max_turns == 30
assert agent_cfg.flash.explorer_mode == "flash"
assert agent_cfg.pro.explorer.mode == "flash"
assert agent_cfg.pro.checker.enabled is False
assert agent_cfg.pro.committee.enabled is False
assert agent_cfg.pro.planner_validation.enabled is False
assert agent_cfg.pro.video_analyzer.enable_ledger is True
def test_output_config_and_recording():
"""Test OutputConfig behavior and event recording."""
cfg = OutputConfig(output_description="Output a valid JSON summary")
assert cfg.needs_structured_format() is True
cfg_disabled = OutputConfig(output_description="Output JSON", enable_outputter=False)
assert cfg_disabled.needs_structured_format() is False
cfg_forced = OutputConfig(force_synthesis=True)
assert cfg_forced.needs_structured_format() is True
with tempfile.TemporaryDirectory() as tmpdir:
output_file = Path(tmpdir) / "events.json"
events_data = [{"event": "test_event", "status": "ok"}]
record_events(output_file, events_data)
assert output_file.exists()
assert "test_event" in output_file.read_text(encoding="utf-8")
def test_runtime_state_and_ipc():
"""Test IPC port and LS address state helpers."""
# Test IPC port
write_ipc_port(49152)
assert read_ipc_port() == 49152
clear_ipc_port()
# Test LS address
write_ls_address("localhost:12345")
assert read_ls_address() == "localhost:12345"
clear_ls_address()
# Test temp dir cleanup
test_temp_sub = get_temp_dir("test_cleanup_dir")
test_temp_file = test_temp_sub / "temp_marker.txt"
test_temp_file.write_text("temporary data")
assert test_temp_file.exists()
deleted = cleanup_temp_dir("test_cleanup_dir")
assert deleted >= 1
assert not test_temp_file.exists()
def test_planner_validation_builder_and_milestones():
"""Test AgentConfigBuilder methods for planner validation and validate_milestones threshold."""
from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder
from artemis.graph.graph import validate_milestones
# Default builder inherits from artemis.jsonc (enabled=False, similarity_threshold=0.85)
builder = AgentConfigBuilder()
cfg = builder.build()
assert cfg.disable_planner_validation is True
assert cfg.planner_validation_threshold == 0.85
# Fluent enabling
cfg_enabled = (
AgentConfigBuilder()
.with_planner_validation(enabled=True, similarity_threshold=0.90)
.build()
)
assert cfg_enabled.disable_planner_validation is False
assert cfg_enabled.planner_validation_threshold == 0.90
# Fluent disabling
cfg_disabled = AgentConfigBuilder().with_disable_planner_validation(True).build()
assert cfg_disabled.disable_planner_validation is True
# Test validate_milestones with custom thresholds
before = "- [ ] Tap Login button\n- [ ] Enter password"
after_minor = "- [ ] Tap the Login button\n- [ ] Enter password"
# Minor edit has ~0.94 similarity
assert validate_milestones(before, after_minor, similarity_threshold=0.85) is False
assert validate_milestones(before, after_minor, similarity_threshold=0.98) is True
@pytest.mark.asyncio
async def test_committee_builder_and_graph_mounting():
"""Test AgentConfigBuilder committee methods and graph mounting."""
from artemis.context import ArtemisContext, DeviceContext, DevicePlatform, ExecutionSetup
from artemis.graph.graph import get_graph
from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder
# Default: disabled
cfg_default = AgentConfigBuilder().build()
assert cfg_default.enable_committee is False
assert cfg_default.committee_debate_rounds == 2
# Enabled via builder
cfg_enabled = AgentConfigBuilder().with_committee(enabled=True, debate_rounds=3).build()
assert cfg_enabled.enable_committee is True
assert cfg_enabled.committee_debate_rounds == 3
# Test mounting in graph
device = DeviceContext(
host_platform="LINUX",
mobile_platform=DevicePlatform.ANDROID,
device_id="dummy",
device_width=1080,
device_height=2400,
)
ctx_disabled = ArtemisContext(
device=device,
execution_setup=ExecutionSetup(enable_committee=False),
)
graph_disabled = await get_graph(ctx_disabled)
op_node_disabled = graph_disabled.nodes.get("operator")
assert op_node_disabled is not None
# Verify ask_committee is not mounted when disabled
op_tools_disabled = [t.name for t in op_node_disabled.bound.afunc.tools]
assert "ask_committee" not in op_tools_disabled
ctx_enabled = ArtemisContext(
device=device,
execution_setup=ExecutionSetup(enable_committee=True, committee_debate_rounds=3),
)
graph_enabled = await get_graph(ctx_enabled)
op_node_enabled = graph_enabled.nodes.get("operator")
assert op_node_enabled is not None
# Verify ask_committee is mounted when enabled
op_tools_enabled = [t.name for t in op_node_enabled.bound.afunc.tools]
assert "ask_committee" in op_tools_enabled
def test_checker_builder_and_context_propagation():
"""Test AgentConfigBuilder methods and context propagation for checker."""
from unittest.mock import MagicMock
from artemis.context import ArtemisContext, DeviceContext, DevicePlatform
from artemis.sdk.agent import Agent
from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder
# Default builder inherits from artemis.jsonc (enabled=False, max_iterations=20, max_chat_rounds=4)
builder = AgentConfigBuilder()
cfg = builder.build()
assert cfg.disable_checker is True
assert cfg.checker_max_iterations == 20
assert cfg.checker_max_chat_rounds == 4
# Fluent enabling
cfg_enabled = (
AgentConfigBuilder()
.with_checker(enabled=True, max_iterations=25, max_chat_rounds=5)
.build()
)
assert cfg_enabled.disable_checker is False
assert cfg_enabled.checker_max_iterations == 25
assert cfg_enabled.checker_max_chat_rounds == 5
# Fluent disabling
cfg_disabled = AgentConfigBuilder().with_disable_checker(True).build()
assert cfg_disabled.disable_checker is True
# Test propagation to ExecutionSetup via Agent._prepare_tracing
agent = Agent(config=cfg_enabled)
device = DeviceContext(
host_platform="LINUX",
mobile_platform=DevicePlatform.ANDROID,
device_id="dummy",
device_width=1080,
device_height=2400,
)
ctx = ArtemisContext(device=device)
mock_task = MagicMock()
mock_task.get_name.return_value = "test_task"
mock_task.request.record_trace = False
mock_task.request.name = "test_task"
mock_task.request.profile = None
mock_task.request.goal = "Test goal"
agent._prepare_tracing(mock_task, ctx)
assert ctx.execution_setup is not None
assert ctx.execution_setup.disable_checker is False
assert ctx.execution_setup.checker_max_iterations == 25
assert ctx.execution_setup.checker_max_chat_rounds == 5
def test_explorer_builder_and_resolution(monkeypatch):
"""Test AgentConfigBuilder explorer methods and multi-tier resolution logic."""
from artemis.context import ArtemisContext, DeviceContext, DevicePlatform
from artemis.sdk.agent import Agent
from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder
from artemis.tools.explorer_tool import resolve_explorer_version
# Default builder inherits from artemis.jsonc (default="flash", flash_mode="flash", pro_mode="flash", caching=True)
builder = AgentConfigBuilder()
cfg = builder.build()
assert cfg.explorer.default_version == "flash"
assert cfg.explorer.flash_mode == "flash"
assert cfg.explorer.pro_mode == "flash"
assert cfg.explorer.caching is True
# Fluent configuration with with_explorer
cfg_custom = (
AgentConfigBuilder()
.with_explorer(
version="pro",
flash_mode="flash",
pro_mode="ultra",
caching=False,
versions={"operator": "ultra", "validator": "pro"},
)
.build()
)
assert cfg_custom.explorer.default_version == "pro"
assert cfg_custom.explorer.flash_mode == "flash"
assert cfg_custom.explorer.pro_mode == "ultra"
assert cfg_custom.explorer.caching is False
assert cfg_custom.explorer_versions["operator"] == "ultra"
assert cfg_custom.explorer_versions["validator"] == "pro"
# Fluent configuration with with_explorer_version shorthand
cfg_shorthand = AgentConfigBuilder().with_explorer_version("ultra").build()
assert cfg_shorthand.explorer.default_version == "ultra"
# Context setup for resolution tests
device = DeviceContext(
host_platform="LINUX",
mobile_platform=DevicePlatform.ANDROID,
device_id="dummy",
device_width=1080,
device_height=2400,
)
ctx = ArtemisContext(device=device, agent_config=cfg_custom)
# 1. Explicit override takes highest priority
assert resolve_explorer_version(ctx, explicit_version="flash") == "flash"
assert resolve_explorer_version(ctx, explicit_version="pro") == "pro"
assert resolve_explorer_version(ctx, explicit_version="ultra") == "ultra"
# 2. Environment variable override
monkeypatch.setenv("ARTEMIS_EXPLORER_VERSION", "ultra")
assert resolve_explorer_version(ctx) == "ultra"
monkeypatch.delenv("ARTEMIS_EXPLORER_VERSION", raising=False)
# 3. Per-agent mapping in explorer_versions
assert resolve_explorer_version(ctx, agent_or_profile_name="operator") == "ultra"
assert resolve_explorer_version(ctx, agent_or_profile_name="validator") == "pro"
# 4. Pro profile vs Flash profile resolution
cfg_profiled = (
AgentConfigBuilder()
.with_explorer(flash_mode="flash", pro_mode="pro", default_version="flash", versions={})
.build()
)
ctx_profiled = ArtemisContext(device=device, agent_config=cfg_profiled)
assert resolve_explorer_version(ctx_profiled, agent_or_profile_name="flash") == "flash"
assert resolve_explorer_version(ctx_profiled, agent_or_profile_name="flash_runner") == "flash"
assert resolve_explorer_version(ctx_profiled, agent_or_profile_name="pro") == "pro"
assert resolve_explorer_version(ctx_profiled, agent_or_profile_name="operator") == "pro"
# 5. Propagation to ExecutionSetup via Agent._prepare_tracing
agent = Agent(config=cfg_custom)
mock_task = MagicMock()
mock_task.get_name.return_value = "explorer_test_task"
mock_task.request.record_trace = False
mock_task.request.name = "explorer_test_task"
mock_task.request.profile = None
mock_task.request.goal = "Test explorer propagation"
agent._prepare_tracing(mock_task, ctx)
assert ctx.execution_setup is not None
assert ctx.execution_setup.explorer_version == "pro"
assert ctx.execution_setup.explorer_flash_mode == "flash"
assert ctx.execution_setup.explorer_pro_mode == "ultra"
assert ctx.execution_setup.explorer_caching is False
def test_outputter_builder_and_context_propagation():
"""Test AgentConfigBuilder outputter methods and propagation to ExecutionSetup."""
from artemis.context import ArtemisContext, DeviceContext, DevicePlatform
from artemis.sdk.agent import Agent
from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder
# Default builder inherits from artemis.jsonc (enabled=True, force_synthesis=False)
builder = AgentConfigBuilder()
cfg = builder.build()
assert cfg.disable_outputter is False
assert cfg.outputter.enabled is True
assert cfg.outputter.force_synthesis is False
# Fluent configuration with with_outputter
cfg_custom = AgentConfigBuilder().with_outputter(enabled=True, force_synthesis=True).build()
assert cfg_custom.disable_outputter is False
assert cfg_custom.outputter.enabled is True
assert cfg_custom.outputter.force_synthesis is True
# Fluent disabling with with_disable_outputter
cfg_disabled = AgentConfigBuilder().with_disable_outputter(True).build()
assert cfg_disabled.disable_outputter is True
assert cfg_disabled.outputter.enabled is False
# Test propagation to ExecutionSetup via Agent._prepare_tracing
device = DeviceContext(
host_platform="LINUX",
mobile_platform=DevicePlatform.ANDROID,
device_id="dummy",
device_width=1080,
device_height=2400,
)
ctx = ArtemisContext(device=device)
agent = Agent(config=cfg_custom)
mock_task = MagicMock()
mock_task.get_name.return_value = "outputter_test_task"
mock_task.request.record_trace = False
mock_task.request.name = "outputter_test_task"
mock_task.request.profile = None
mock_task.request.goal = "Test outputter propagation"
agent._prepare_tracing(mock_task, ctx)
assert ctx.execution_setup is not None
assert ctx.execution_setup.disable_outputter is False
assert ctx.execution_setup.outputter.enabled is True
assert ctx.execution_setup.outputter.force_synthesis is True
def test_categorized_flash_and_pro_profile_builders():
"""Test with_flash_config and with_pro_config fluent builders and bidirectional sync."""
from artemis.config.agent import AgentGlobalConfig
from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder
# 1. Test with_flash_config
cfg_flash = AgentConfigBuilder().with_flash_config(max_turns=15, explorer_mode="flash").build()
assert cfg_flash.flash.max_turns == 15
assert cfg_flash.flash.explorer_mode == "flash"
assert cfg_flash.explorer.flash_mode == "flash"
# 2. Test with_pro_config
cfg_pro = (
AgentConfigBuilder()
.with_pro_config(
explorer_mode="ultra",
planner_validation=True,
committee=True,
checker=True,
video_ledger=False,
)
.build()
)
assert cfg_pro.pro.explorer.mode == "ultra"
assert cfg_pro.explorer.pro_mode == "ultra"
assert cfg_pro.disable_planner_validation is False
assert cfg_pro.enable_committee is True
assert cfg_pro.disable_checker is False
assert cfg_pro.enable_video_ledger is False
# 3. Test AgentGlobalConfig bidirectional synchronization
raw_dict = {
"flash": {"max_turns": 25, "explorer_mode": "flash"},
"pro": {
"explorer": {"mode": "pro", "caching": False},
"checker": {"enabled": True, "max_iterations": 10},
"committee": {"enabled": True, "debate_rounds": 4},
},
}
synced = AgentGlobalConfig.model_validate(raw_dict)
assert synced.flash.max_turns == 25
assert synced.explorer.flash_mode == "flash"
assert synced.explorer.pro_mode == "pro"
assert synced.explorer.caching is False
assert synced.checker.enabled is True
assert synced.checker.max_iterations == 10
assert synced.committee.enabled is True
assert synced.committee.debate_rounds == 4
def test_agent_config_environment_variable_overrides(monkeypatch):
"""Test environment variable overrides for AgentGlobalConfig switches."""
from artemis.config.agent import load_agent_config
monkeypatch.setenv("ARTEMIS_CHECKER_ENABLED", "true")
monkeypatch.setenv("ARTEMIS_COMMITTEE_ENABLED", "1")
monkeypatch.setenv("ARTEMIS_PLANNER_VALIDATION_ENABLED", "yes")
monkeypatch.setenv("ARTEMIS_OUTPUTTER_ENABLED", "false")
monkeypatch.setenv("ARTEMIS_VIDEO_LEDGER_ENABLED", "0")
cfg = load_agent_config()
assert cfg.checker.enabled is True
assert cfg.pro.checker.enabled is True
assert cfg.committee.enabled is True
assert cfg.pro.committee.enabled is True
assert cfg.planner_validation.enabled is True
assert cfg.pro.planner_validation.enabled is True
assert cfg.outputter.enabled is False
assert cfg.video_analyzer.enable_ledger is False
assert cfg.pro.video_analyzer.enable_ledger is False
+30
View File
@@ -0,0 +1,30 @@
import asyncio
import pytest
from artemis.context import ArtemisContext, DeviceContext, DevicePlatform
@pytest.mark.asyncio
async def test_artemis_context_waits_for_background_tasks():
device = DeviceContext(
host_platform="LINUX",
mobile_platform=DevicePlatform.ANDROID,
device_id="dummy",
device_width=1080,
device_height=2400,
)
context = ArtemisContext(device=device)
task_run = False
async def dummy_task():
nonlocal task_run
await asyncio.sleep(0.1)
task_run = True
async with context:
task = asyncio.create_task(dummy_task())
context.background_tasks.append(task)
# After exiting the context, the task should have run to completion
assert task_run is True
+91
View File
@@ -0,0 +1,91 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for Unified Base Device Driver and implementations."""
import pytest
from unittest.mock import MagicMock
from artemis.drivers.base import KeyCode, ScreenData, SwipeDirection
from artemis.drivers.mock.mock_driver import MockDeviceDriver
from artemis.drivers.android.adb_driver import AndroidAdbDriver
@pytest.mark.asyncio
async def test_mock_driver_actions():
"""Verify MockDeviceDriver records and performs all core actions."""
driver = MockDeviceDriver(device_id="test-mock", width=1080, height=2400)
assert driver.device_id == "test-mock"
assert driver.screen_size == (1080, 2400)
# 1. Screen data
screen_data = await driver.get_screen_data()
assert isinstance(screen_data, ScreenData)
assert screen_data.width == 1080
assert screen_data.height == 2400
assert screen_data.screenshot_base64 is not None
# 2. Tap
assert await driver.tap(500, 1000) is True
assert driver.action_history[-1]["action"] == "tap"
assert driver.action_history[-1]["x"] == 500
# 3. Swipe
assert await driver.swipe(100, 200, 300, 400) is True
assert driver.action_history[-1]["action"] == "swipe"
# 4. Swipe direction
assert await driver.swipe_direction(SwipeDirection.UP) is True
assert driver.action_history[-1]["direction"] == "up"
# 5. Input text
assert await driver.input_text("Hello Artemis") is True
assert driver.action_history[-1]["text"] == "Hello Artemis"
# 6. Press key
assert await driver.press_key(KeyCode.HOME) is True
assert driver.action_history[-1]["key"] == "home"
# 7. App lifecycle
assert await driver.launch_app("com.android.settings") is True
assert await driver.get_current_package() == "com.android.settings"
assert await driver.stop_app("com.android.settings") is True
@pytest.mark.asyncio
async def test_android_driver_with_mock_adb():
"""Verify AndroidAdbDriver formats shell commands properly."""
mock_adb_client = MagicMock()
mock_adb_device = MagicMock()
mock_adb_client.device.return_value = mock_adb_device
driver = AndroidAdbDriver(
device_id="emulator-5554",
adb_client=mock_adb_client,
width=1080,
height=2400,
)
# Test tap
await driver.tap(540, 1200)
mock_adb_device.shell.assert_called_with("input tap 540 1200")
# Test multi-tap
await driver.tap(540, 1200, times=3, delay_ms=100)
expected_multi = "input tap 540 1200 && sleep 0.100 && input tap 540 1200 && sleep 0.100 && input tap 540 1200"
mock_adb_device.shell.assert_called_with(expected_multi)
# Test press key
await driver.press_key(KeyCode.BACK)
mock_adb_device.shell.assert_called_with("input keyevent 4")
+67
View File
@@ -0,0 +1,67 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from google import genai
from google.genai import types
from google.genai.errors import APIError
import pytest
@pytest.fixture
def client():
"""Initializes the Gemini client. Requires GEMINI_API_KEY in environment."""
try:
return genai.Client()
except Exception as e:
pytest.skip(f"Could not initialize genai.Client: {e}")
def test_gemini_role_tool_fails(client):
"""Confirm that using role='tool' results in a 400 APIError from Gemini."""
contents = [
types.Content(
role="tool",
parts=[types.Part.from_text(text="This is a test payload for tool execution.")],
)
]
with pytest.raises(APIError) as exc_info:
client.models.generate_content(
model="gemini-3.6-flash",
contents=contents,
)
# Assert it failed specifically due to the invalid role string
error_msg = str(exc_info.value)
assert "Invalid role string" in error_msg or "400" in error_msg
def test_gemini_role_user_succeeds(client):
"""Confirm that using role='user' (which is the correct fix) succeeds."""
contents = [
types.Content(
role="user",
parts=[
types.Part.from_text(text="Acknowledge this message with 'ok' and nothing else.")
],
)
]
# This should not raise any exceptions
response = client.models.generate_content(
model="gemini-3.6-flash",
contents=contents,
)
assert response.text is not None
+211
View File
@@ -0,0 +1,211 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for Dynamic Model Router."""
from artemis.llm.router import ModelEndpoint, ModelProvider, ModelRouter
def test_model_router_default_routes():
"""Verify default role routing returns expected models."""
router = ModelRouter()
# Planner route
planner_ep = router.get_endpoint("planner")
assert planner_ep.provider == ModelProvider.GEMINI
assert "pro" in planner_ep.model_name
# Operator route
operator_ep = router.get_endpoint("operator")
assert operator_ep.provider == ModelProvider.GEMINI
assert "flash" in operator_ep.model_name
def test_model_router_custom_route_override():
"""Verify dynamic role route override."""
router = ModelRouter()
custom_ep = ModelEndpoint(
provider=ModelProvider.OPENAI,
model_name="gpt-4o",
api_base="https://custom.api.endpoint/v1",
)
router.set_role_route("operator", custom_ep)
ep = router.get_endpoint("operator")
assert ep.provider == ModelProvider.OPENAI
assert ep.model_name == "gpt-4o"
assert ep.api_base == "https://custom.api.endpoint/v1"
def test_model_router_fallbacks():
"""Verify fallback endpoint configuration."""
router = ModelRouter()
fallbacks = [
ModelEndpoint(provider=ModelProvider.GEMINI, model_name="gemini-2.5-flash"),
ModelEndpoint(provider=ModelProvider.OPENAI, model_name="gpt-4o-mini"),
]
router.set_role_route("custom_worker", "gemini-2.5-pro", fallbacks=fallbacks)
fb = router.get_fallbacks("custom_worker")
assert len(fb) == 2
assert fb[0].model_name == "gemini-2.5-flash"
assert fb[1].model_name == "gpt-4o-mini"
def test_model_router_anthropic_instantiation():
"""Verify Anthropic Claude model instantiation with thinking budget."""
router = ModelRouter()
anth_ep = ModelEndpoint(
provider=ModelProvider.ANTHROPIC,
model_name="claude-3-7-sonnet-20250219",
api_key="sk-ant-test-key",
reasoning_effort="high",
)
model = router.instantiate_model(anth_ep)
assert model.model == "claude-3-7-sonnet-20250219"
assert getattr(model, "thinking", None) == {"type": "enabled", "budget_tokens": 32768}
def test_model_router_openai_instantiation():
"""Verify OpenAI model instantiation with reasoning effort."""
router = ModelRouter()
oai_ep = ModelEndpoint(
provider=ModelProvider.OPENAI,
model_name="o3-mini",
api_key="sk-test-key",
reasoning_effort="medium",
)
model = router.instantiate_model(oai_ep)
assert model.model_name == "o3-mini"
assert getattr(model, "reasoning_effort", None) == "medium"
def test_robust_chat_model_wrapper_grounding_google():
"""Verify RobustChatModelWrapper auto-injects google_search and server-side tool config for Gemini."""
from artemis.services.llm import RobustChatModelWrapper
from langchain_core.tools import tool
@tool
def dummy_tool(query: str) -> str:
"""A test tool."""
return query
class MockGeminiModel:
def __init__(self):
self.bound_tools = []
self.bound_kwargs = {}
def bind_tools(self, tools, *args, **kwargs):
self.bound_tools = tools
self.bound_kwargs = kwargs
return self
# Case 1: enable_grounding=True automatically attaches google_search and tool_config
ep_google = ModelEndpoint(
provider=ModelProvider.GOOGLE,
model_name="gemini-2.5-flash",
enable_grounding=True,
)
wrapper = RobustChatModelWrapper(MockGeminiModel(), endpoint=ep_google)
bound = wrapper.bind_tools([dummy_tool])
assert any(isinstance(t, dict) and "google_search" in t for t in bound.base_model.bound_tools)
assert bound.base_model.bound_kwargs.get("tool_config") == {
"include_server_side_tool_invocations": True
}
# Case 2: Explicit google_search in tools automatically ensures tool_config
ep_google_no_ground = ModelEndpoint(
provider=ModelProvider.GOOGLE,
model_name="gemini-2.5-flash",
enable_grounding=False,
)
wrapper_explicit = RobustChatModelWrapper(MockGeminiModel(), endpoint=ep_google_no_ground)
bound_explicit = wrapper_explicit.bind_tools([dummy_tool, {"google_search": {}}])
assert any(
isinstance(t, dict) and "google_search" in t for t in bound_explicit.base_model.bound_tools
)
assert bound_explicit.base_model.bound_kwargs.get("tool_config") == {
"include_server_side_tool_invocations": True
}
def test_robust_chat_model_wrapper_grounding_non_google():
"""Verify RobustChatModelWrapper ignores grounding and strips google_search for non-Google providers."""
from artemis.services.llm import RobustChatModelWrapper
from langchain_core.tools import tool
@tool
def dummy_tool(query: str) -> str:
"""A test tool."""
return query
class MockOpenAIModel:
def __init__(self):
self.bound_tools = []
self.bound_kwargs = {}
def bind_tools(self, tools, *args, **kwargs):
self.bound_tools = tools
self.bound_kwargs = kwargs
return self
ep_openai = ModelEndpoint(
provider=ModelProvider.OPENAI,
model_name="gpt-4o",
enable_grounding=True,
)
wrapper = RobustChatModelWrapper(MockOpenAIModel(), endpoint=ep_openai)
bound = wrapper.bind_tools([dummy_tool, {"google_search": {}}])
# Should ignore grounding and strip {"google_search": {}}
assert len(bound.base_model.bound_tools) == 1
assert bound.base_model.bound_tools[0].name == "dummy_tool"
assert "tool_config" not in bound.base_model.bound_kwargs
def test_chat_google_generative_ai_process_tool_config_patch():
"""Verify patched ChatGoogleGenerativeAI preserves include_server_side_tool_invocations even with tool_choice."""
from artemis.llm.router import ModelFactory
from langchain_google_genai import ChatGoogleGenerativeAI
from langchain_core.tools import tool
@tool
def dummy_tool(query: str) -> str:
"""A test tool."""
return query
# Trigger patching via Google model creation
ep = ModelEndpoint(provider=ModelProvider.GOOGLE, model_name="gemini-2.5-flash", api_key="fake")
model = ModelFactory.create_model(ep)
assert isinstance(model, ChatGoogleGenerativeAI)
formatted_tools = model._format_tools([dummy_tool, {"google_search": {}}], None)
# 1. With tool_choice="auto"
config_auto = model._process_tool_config("auto", None, formatted_tools)
assert config_auto is not None
assert config_auto.include_server_side_tool_invocations is True
# 2. With tool_choice="any"
config_any = model._process_tool_config("any", None, formatted_tools)
assert config_any is not None
assert config_any.include_server_side_tool_invocations is True
# 3. With explicit tool_config
config_explicit = model._process_tool_config(
None, {"include_server_side_tool_invocations": True}, formatted_tools
)
assert config_explicit is not None
assert config_explicit.include_server_side_tool_invocations is True
+46
View File
@@ -0,0 +1,46 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
from pathlib import Path
from artemis.agents.planner.planner import run_async_planner_validation
class DummyDataEngine:
def get_agent_friendly_steps(self):
return []
class DummyCtx:
data_engine = DummyDataEngine()
project_dir = str(Path(__file__).resolve().parent.parent.parent)
async def test():
content_before = "- [/] Click the button\n- [ ] Complete task"
content_after = (
"- [/] Click the button\n- [ ] Complete task\n- [ ] New subgoal appended at bottom"
)
res = await run_async_planner_validation(
ctx=DummyCtx(),
initial_goal="Do the task",
content_before=content_before,
content_after=content_after,
operator_raw_thinking="I need to add a new subgoal",
operator_native_thinking="Thinking...",
)
print(res)
asyncio.run(test())
+56
View File
@@ -0,0 +1,56 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for OperatorPromptBuilder and LoopDetector."""
from artemis.agents.operator.loop_detector import LoopDetector
from artemis.agents.operator.prompt_builder import OperatorPromptBuilder
def test_operator_prompt_builder():
"""Verify prompt rendering formats variables correctly."""
system_msg = OperatorPromptBuilder.build_system_message()
assert "ARTEMIS Operator" in system_msg
assert "click" in system_msg
human_msg = OperatorPromptBuilder.build_human_message(
goal="Open Settings",
sub_goal="Tap Network & internet",
current_turn=2,
history="1. Tapped Settings icon",
)
assert "Open Settings" in human_msg
assert "Tap Network & internet" in human_msg
assert "Current Turn: 2" in human_msg
def test_loop_detector():
"""Verify loop detector flags repeated action sequences."""
detector = LoopDetector(repetition_threshold=3)
assert not detector.is_loop_detected()
# Record distinct actions
detector.record_action("click", {"target": [100, 200]})
detector.record_action("swipe", {"action": "up"})
assert not detector.is_loop_detected()
# Record 3 identical actions
detector.record_action("click", {"target": [500, 500]})
detector.record_action("click", {"target": [500, 500]})
detector.record_action("click", {"target": [500, 500]})
assert detector.is_loop_detected()
# Reset
detector.reset()
assert not detector.is_loop_detected()
+66
View File
@@ -0,0 +1,66 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for the official Python SDK (ArtemisClient, Task, TaskResult)."""
import pytest
from artemis import ArtemisClient, Task, TaskResult
@pytest.mark.asyncio
async def test_artemis_client_run_mock():
"""Verify ArtemisClient runs a task through the execution pipeline."""
client = ArtemisClient(device_id="mock-test-dev", default_profile="pro")
assert client.device_id == "mock-test-dev"
assert client.default_profile == "pro"
result = await client.run(goal="Open Settings and enable Dark Theme")
assert isinstance(result, TaskResult)
assert result.status in ("completed", "running", "success")
assert result.trace_id.startswith("trace_") or len(result.trace_id) > 0
def test_task_model_validation():
"""Verify Task validation and default arguments."""
task = Task(goal="Search for flight prices", profile="flash", device_id="emulator-5554")
assert task.goal == "Search for flight prices"
assert task.profile == "flash"
assert task.device_id == "emulator-5554"
assert task.max_turns == 30
assert task.locked_package is None
@pytest.mark.asyncio
async def test_artemis_client_stream_run():
"""Verify ArtemisClient.stream_run yields real-time StreamEvents."""
from artemis import StreamEvent, StreamEventType
client = ArtemisClient(device_id="mock-stream-dev", default_profile="flash")
collected_events: list[StreamEvent] = []
def on_event(ev: StreamEvent):
collected_events.append(ev)
events: list[StreamEvent] = []
async for event in client.stream_run(
goal="Test streaming UI action flow",
profile="flash",
callbacks=[on_event],
):
events.append(event)
assert len(events) >= 2
assert events[0].event_type == StreamEventType.STATUS
assert events[0].payload.get("status") == "starting"
assert len(collected_events) == len(events)
+55
View File
@@ -0,0 +1,55 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for Telemetry & Span Lifecycle Tracing."""
import pytest
from pathlib import Path
from artemis.telemetry.models import SpanType
from artemis.telemetry.tracer import TelemetryTracer
@pytest.mark.asyncio
async def test_telemetry_span_lifecycle(tmp_path: Path):
"""Verify TelemetryTracer properly records and persists nested spans."""
tracer = TelemetryTracer(trace_id="test-session-001", traces_dir=tmp_path)
async with tracer.start_span("parent_agent_step", span_type=SpanType.AGENT) as parent_span:
assert parent_span.status == "running"
assert parent_span.name == "parent_agent_step"
async with tracer.start_span(
"child_device_action", span_type=SpanType.DEVICE_ACTION, payload={"action": "click"}
) as child_span:
assert child_span.parent_id == parent_span.span_id
assert child_span.payload["action"] == "click"
# Spans should be persisted in JSONL
assert len(tracer.spans) == 2
child_recorded = tracer.spans[0]
parent_recorded = tracer.spans[1]
assert child_recorded.name == "child_device_action"
assert child_recorded.status == "success"
assert child_recorded.duration_ms is not None
assert parent_recorded.name == "parent_agent_step"
assert parent_recorded.status == "success"
# File existence
log_file = tmp_path / "telemetry_spans.jsonl"
assert log_file.exists()
content = log_file.read_text(encoding="utf-8")
assert "child_device_action" in content
assert "parent_agent_step" in content
+70
View File
@@ -0,0 +1,70 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for Universal Tool Protocol and Actions."""
import pytest
from artemis.drivers.mock.mock_driver import MockDeviceDriver
from artemis.tools.base import ToolRegistry
import artemis.tools.actions.device_actions # noqa: F401 - Register action tools
@pytest.mark.asyncio
async def test_tool_registry_registration():
"""Verify standard device actions are registered in ToolRegistry."""
click_tool = ToolRegistry.get("click")
assert click_tool is not None
assert click_tool.name == "click"
assert click_tool.category == "action"
swipe_tool = ToolRegistry.get("swipe")
assert swipe_tool is not None
assert swipe_tool.name == "swipe"
@pytest.mark.asyncio
async def test_tool_execution_with_driver():
"""Verify tool execution correctly dispatches to MockDeviceDriver."""
driver = MockDeviceDriver(width=1000, height=2000)
# Click normalized [500, 500] -> pixel (500, 1000)
result = await ToolRegistry.execute("click", {"target": [500, 500]}, driver=driver, ctx=None)
assert "Click" in result or "Clicked" in result
assert driver.action_history[-1]["action"] == "tap"
assert driver.action_history[-1]["x"] == 500
assert driver.action_history[-1]["y"] == 1000
# Swipe cardinal "up"
result = await ToolRegistry.execute("swipe", {"action": "up"}, driver=driver, ctx=None)
assert "Swipe" in result or "Swiped" in result
assert driver.action_history[-1]["action"] == "swipe_direction"
def test_tool_to_genai_declaration():
"""Verify tools export to Google GenAI FunctionDeclaration properly."""
declarations = ToolRegistry.get_genai_declarations(["click", "swipe", "input_text"])
assert len(declarations) == 3
decl_names = [d.name for d in declarations]
assert "click" in decl_names
assert "swipe" in decl_names
assert "input_text" in decl_names
def test_tool_to_langchain_tool():
"""Verify tools export to LangChain BaseTool."""
lc_tools = ToolRegistry.get_langchain_tools(ctx=None, names=["click", "press_key"])
assert len(lc_tools) == 2
assert lc_tools[0].name == "click"
assert lc_tools[1].name == "press_key"
+279
View File
@@ -0,0 +1,279 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for UnifiedMobileController video recording and playback features."""
import time
from unittest.mock import AsyncMock, MagicMock, patch
from uuid import uuid4
import pytest
from artemis.context import ArtemisContext
from artemis.controllers.unified_controller import UnifiedMobileController
from artemis.drivers.mock.mock_driver import MockDeviceDriver
from artemis.utils.video import (
RecordingSession,
get_active_session,
remove_active_session,
set_active_session,
)
@pytest.fixture
def mock_ctx(tmp_path):
ctx = MagicMock(spec=ArtemisContext)
ctx.device = MagicMock()
ctx.device.device_id = "emulator-5554"
ctx.device.mobile_platform = "android"
# DataEngine mock
ctx.data_engine = MagicMock()
ctx.data_engine.current_session_id = uuid4()
ctx.data_engine.session_start_time = time.time()
ctx.data_engine.storage = MagicMock()
# Mock driver
mock_driver = MockDeviceDriver(device_id="emulator-5554")
ctx._active_driver = mock_driver
return ctx
@pytest.mark.asyncio
async def test_unified_controller_start_recording(mock_ctx, tmp_path):
controller = UnifiedMobileController(mock_ctx)
remove_active_session("emulator-5554")
# Mock scrcpy subprocess
mock_proc = MagicMock()
mock_proc.returncode = None
with patch("asyncio.create_subprocess_exec", AsyncMock(return_value=mock_proc)):
with patch("asyncio.sleep", AsyncMock()):
res = await controller.start_video_recording(output_dir=tmp_path)
assert res.success is True
assert get_active_session("emulator-5554") is not None
assert mock_ctx.data_engine.record_video_start.called
# Calling start again should report already in progress
res2 = await controller.start_video_recording(output_dir=tmp_path)
assert res2.success is False
assert "already in progress" in res2.message
remove_active_session("emulator-5554")
@pytest.mark.asyncio
async def test_unified_controller_stop_recording(mock_ctx, tmp_path):
controller = UnifiedMobileController(mock_ctx)
remove_active_session("emulator-5554")
# Create dummy recording.mkv
mkv_path = tmp_path / "recording.mkv"
mkv_path.write_bytes(b"dummy mkv content")
mock_proc = MagicMock()
mock_proc.returncode = None
mock_proc.terminate = MagicMock()
mock_proc.wait = AsyncMock()
session = RecordingSession(
video_id=uuid4(),
device_id="emulator-5554",
start_time=time.time() - 10.0,
data_engine_start_time=time.time() - 10.0,
local_video_path=mkv_path,
process=mock_proc,
)
set_active_session("emulator-5554", session)
with patch.object(
controller,
"_convert_mkv_to_mp4",
AsyncMock(side_effect=lambda src, dst: dst.write_bytes(b"mp4 content") or True),
):
res = await controller.stop_video_recording()
assert res.success is True
assert res.video_path is not None
assert str(res.video_path).endswith("recording.mp4")
assert get_active_session("emulator-5554") is None
assert mock_ctx.data_engine.record_video_stop.called
@pytest.mark.asyncio
async def test_unified_controller_extract_segment_metadata(mock_ctx, tmp_path):
controller = UnifiedMobileController(mock_ctx)
remove_active_session("emulator-5554")
mkv_path = tmp_path / "recording.mkv"
mkv_path.write_bytes(b"dummy mkv content")
mock_proc = MagicMock()
mock_proc.returncode = None
session = RecordingSession(
video_id=uuid4(),
device_id="emulator-5554",
start_time=time.time() - 20.0,
data_engine_start_time=time.time() - 20.0,
local_video_path=mkv_path,
process=mock_proc,
)
set_active_session("emulator-5554", session)
with patch(
"artemis.controllers.unified_controller.trim_video",
AsyncMock(side_effect=lambda src, s, e, dst: dst.write_bytes(b"segment mp4") or True),
):
res = await controller.extract_segment_metadata(start_time=2.0, end_time=8.0)
assert res.success is True
assert res.video_path is not None
assert res.video_path.exists()
# Ensure session is still active (not stopped by extraction!)
assert get_active_session("emulator-5554") is not None
remove_active_session("emulator-5554")
def test_data_engine_video_lifecycle(tmp_path):
from artemis.data_engine.engine import DataEngine
from artemis.context import ArtemisContext
mock_c = MagicMock(spec=ArtemisContext)
mock_c.execution_setup = MagicMock(traces_path=str(tmp_path / "traces"))
mock_c.device = None
engine = DataEngine(mock_c)
session_id = engine.start_session(goal="Test Video Engine")
vid = uuid4()
vpath = tmp_path / "recording.mp4"
vpath.write_bytes(b"video bytes")
# Start video recording
engine.record_video_start(vid, "device-1", vpath)
rec = engine.storage.get_video_recording(vid)
assert rec is not None
assert rec.device_id == "device-1"
# Stop video recording
engine.record_video_stop(vid, "device-1", vpath, time.time() - 10.0, time.time())
sess = engine.storage.get_session(session_id)
assert sess.video_filepath == str(vpath)
# Move video path on finalize
new_vpath = tmp_path / "archived" / "recording.mp4"
new_vpath.parent.mkdir(parents=True, exist_ok=True)
new_vpath.write_bytes(b"video bytes")
engine.update_video_path(new_vpath)
sess_updated = engine.storage.get_session(session_id)
assert sess_updated.video_filepath == str(new_vpath)
@pytest.mark.asyncio
async def test_unified_controller_crash_recovery_and_multi_segment(mock_ctx, tmp_path):
"""Verify that if recording is interrupted, segments are auto-recovered and concatenated seamlessly."""
controller = UnifiedMobileController(mock_ctx)
remove_active_session("emulator-5554")
seg1 = tmp_path / "recording_0.mkv"
seg1.write_bytes(b"segment 0")
seg2 = tmp_path / "recording_1.mkv"
seg2.write_bytes(b"segment 1")
mock_proc = MagicMock()
mock_proc.returncode = 1 # Simulated crash
session = RecordingSession(
video_id=uuid4(),
device_id="emulator-5554",
start_time=time.time() - 30.0,
data_engine_start_time=time.time() - 30.0,
local_video_path=seg2,
android_video_segments=[seg1],
android_segment_index=1,
process=mock_proc,
)
set_active_session("emulator-5554", session)
with (
patch(
"artemis.controllers.unified_controller.concatenate_videos",
AsyncMock(side_effect=lambda segs, out: out.write_bytes(b"combined mkv") or True),
),
patch.object(
controller,
"_convert_mkv_to_mp4",
AsyncMock(side_effect=lambda src, dst: dst.write_bytes(b"final mp4") or True),
),
):
res = await controller.stop_video_recording()
assert res.success is True
assert res.video_path is not None
assert res.video_path.exists()
assert str(res.video_path).endswith("recording.mp4")
assert get_active_session("emulator-5554") is None
@pytest.mark.asyncio
async def test_unified_controller_timeline_alignment(mock_ctx, tmp_path):
"""Verify that relative time offset between DataEngine T0 and video start is precisely calculated."""
controller = UnifiedMobileController(mock_ctx)
remove_active_session("emulator-5554")
mkv_path = tmp_path / "recording.mkv"
mkv_path.write_bytes(b"dummy mkv content")
now = time.time()
# Assume session started 15s ago, and video started 12s ago (offset = 3.0s)
session_start_t0 = now - 15.0
video_start_t = now - 12.0
session = RecordingSession(
video_id=uuid4(),
device_id="emulator-5554",
start_time=video_start_t,
data_engine_start_time=session_start_t0,
local_video_path=mkv_path,
process=MagicMock(returncode=None),
)
set_active_session("emulator-5554", session)
trimmed_ranges = []
async def mock_trim(src, s_time, e_time, dst):
trimmed_ranges.append((s_time, e_time))
dst.write_bytes(b"trimmed")
return True
with patch(
"artemis.controllers.unified_controller.trim_video", AsyncMock(side_effect=mock_trim)
):
# Agent asks for system time range [5.0s, 10.0s]
res = await controller.extract_segment_metadata(start_time=5.0, end_time=10.0)
assert res.success is True
# Since offset = 3.0s, video_start_relative_time should be 5.0 - 3.0 = 2.0s
# and video_end_relative_time should be 10.0 - 3.0 = 7.0s
assert len(trimmed_ranges) == 1
s_time, e_time = trimmed_ranges[0]
assert abs(s_time - 2.0) < 0.1
assert abs(e_time - 7.0) < 0.1
# actual_start_relative_time reported back to agent must align with system T0 (5.0s)
assert abs(res.actual_start_relative_time - 5.0) < 0.1
remove_active_session("emulator-5554")
+36
View File
@@ -0,0 +1,36 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from artemis.utils.notes import update_note_content
base_dir = "/tmp/artemis_test"
os.makedirs(f"{base_dir}/notes", exist_ok=True)
task_plan = """- [x] Initial task
- [/] Active task
- [ ] Future task"""
with open(f"{base_dir}/notes/task_plan.md", "w") as f:
f.write(task_plan)
try:
res = update_note_content(
base_dir,
"task_plan",
"- [/] Active task",
"- [/] Active task\n - [ ] New subgoal",
)
print("Update successful!")
print(open(f"{base_dir}/notes/task_plan.md").read())
except Exception as e:
print("Error:", e)
@@ -0,0 +1,132 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pathlib import Path
from unittest.mock import MagicMock, patch
from artemis.context import ArtemisContext
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.scratchpad import (
AppendNoteArgs,
AppendNotePure,
AppendNotePureTool,
append_note_pure,
get_append_note_tool_pure,
)
import pytest
@pytest.fixture
def mock_ctx(tmp_path):
ctx = MagicMock(spec=ArtemisContext)
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = str(tmp_path)
return ctx
def test_append_note_pure_tool_subclass_and_registry():
"""Verify AppendNotePureTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(AppendNotePureTool, ArtemisTool)
assert issubclass(AppendNotePure, ArtemisTool)
assert isinstance(append_note_pure, ArtemisTool)
assert isinstance(append_note_pure, AppendNotePureTool)
assert append_note_pure.name == "append_note_pure"
assert append_note_pure.category == "memory"
assert append_note_pure.args_schema == AppendNoteArgs
# Registry lookup
reg_tool = ToolRegistry.get("append_note_pure")
assert reg_tool is not None
assert isinstance(reg_tool, AppendNotePureTool)
# GenAI FunctionDeclaration export
declaration = append_note_pure.to_genai_declaration()
assert declaration.name == "append_note_pure"
assert "key" in declaration.parameters.properties
assert "content" in declaration.parameters.properties
@pytest.mark.asyncio
async def test_append_note_pure_direct_execution_success(mock_ctx, tmp_path):
"""Verify direct execution of AppendNotePureTool.execute appends to note file."""
# First append creates note
result1 = await append_note_pure.execute(
ctx=mock_ctx,
key="pure_append",
content="First line",
)
assert result1 == "Successfully appended note to pure_append.md."
note_path = Path(tmp_path) / "notes" / "pure_append.md"
assert note_path.exists()
assert note_path.read_text(encoding="utf-8") == "First line\n"
# Second append appends line
result2 = await append_note_pure.execute(
ctx=mock_ctx,
key="pure_append",
content="Second line",
)
assert result2 == "Successfully appended note to pure_append.md."
assert note_path.read_text(encoding="utf-8") == "First line\nSecond line\n"
@pytest.mark.asyncio
async def test_append_note_pure_callable_execution(mock_ctx, tmp_path):
"""Verify invoking append_note_pure directly as a callable."""
result = await append_note_pure(
ctx=mock_ctx,
key="callable_pure_append",
content="Pure callable append",
)
assert result == "Successfully appended note to callable_pure_append.md."
note_path = Path(tmp_path) / "notes" / "callable_pure_append.md"
assert note_path.exists()
assert note_path.read_text(encoding="utf-8") == "Pure callable append\n"
@pytest.mark.asyncio
async def test_append_note_pure_execution_failure(mock_ctx):
"""Verify error handling when append_note_content fails."""
with patch(
"artemis.tools.scratchpad.append_note_content",
side_effect=PermissionError("Permission denied"),
):
result = await append_note_pure.execute(
ctx=mock_ctx,
key="fail_append",
content="Some content",
)
assert "Failed to append note fail_append.md: Permission denied" in result
@pytest.mark.asyncio
async def test_get_append_note_tool_pure_langchain_ainvoke(mock_ctx, tmp_path):
"""Verify get_append_note_tool_pure exports a LangChain tool named 'append_note'."""
pure_tool = get_append_note_tool_pure(mock_ctx)
assert pure_tool.name == "append_note"
result = await pure_tool.ainvoke(
{
"key": "lc_pure_append",
"content": "Saved via pure append tool",
}
)
assert result == "Successfully appended note to lc_pure_append.md."
note_path = Path(tmp_path) / "notes" / "lc_pure_append.md"
assert note_path.exists()
assert note_path.read_text(encoding="utf-8") == "Saved via pure append tool\n"
+190
View File
@@ -0,0 +1,190 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.scratchpad import (
AppendNote,
AppendNoteArgs,
AppendNoteTool,
get_append_note_tool,
get_append_note_tool_pure,
append_note,
append_note_wrapper,
)
from langgraph.types import Command
import pytest
@pytest.fixture
def mock_ctx(tmp_path):
ctx = MagicMock(spec=ArtemisContext)
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = str(tmp_path)
return ctx
def test_append_note_tool_subclass_and_registry():
"""Verify AppendNoteTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(AppendNoteTool, ArtemisTool)
assert issubclass(AppendNote, ArtemisTool)
assert isinstance(append_note, ArtemisTool)
assert isinstance(append_note, AppendNoteTool)
assert append_note.name == "append_note"
assert append_note.category == "memory"
assert append_note.args_schema == AppendNoteArgs
# Registry lookup
reg_tool = ToolRegistry.get("append_note")
assert reg_tool is not None
assert isinstance(reg_tool, AppendNoteTool)
# GenAI FunctionDeclaration export
declaration = append_note.to_genai_declaration()
assert declaration.name == "append_note"
assert "key" in declaration.parameters.properties
assert "content" in declaration.parameters.properties
# Wrapper check
assert append_note_wrapper is not None
assert append_note_wrapper.tool_fn_getter == get_append_note_tool
@pytest.mark.asyncio
async def test_append_note_direct_execution_success(mock_ctx, tmp_path):
"""Verify direct execution of AppendNoteTool.execute appends to note file."""
# First append creates the note
result1 = await append_note.execute(
ctx=mock_ctx,
key="test_append",
content="First line",
)
assert result1 == "Successfully appended note to test_append.md."
note_path = Path(tmp_path) / "notes" / "test_append.md"
assert note_path.exists()
assert note_path.read_text(encoding="utf-8") == "First line\n"
# Second append appends with newline
result2 = await append_note.execute(
ctx=mock_ctx,
key="test_append",
content="Second line",
)
assert result2 == "Successfully appended note to test_append.md."
assert note_path.read_text(encoding="utf-8") == "First line\nSecond line\n"
@pytest.mark.asyncio
async def test_append_note_callable_execution(mock_ctx, tmp_path):
"""Verify invoking append_note directly as a callable."""
result = await append_note(
ctx=mock_ctx,
key="callable_append",
content="Callable append test",
)
assert result == "Successfully appended note to callable_append.md."
note_path = Path(tmp_path) / "notes" / "callable_append.md"
assert note_path.exists()
assert note_path.read_text(encoding="utf-8") == "Callable append test\n"
@pytest.mark.asyncio
async def test_append_note_with_state_command(mock_ctx, tmp_path):
"""Verify AppendNoteTool.execute with state returns Command updating VALIDATOR_MESSAGES_KEY."""
mock_state = MagicMock(spec=State)
mock_state.current_agent = "operator"
mock_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update)
result = await append_note.execute(
ctx=mock_ctx,
key="state_append_note",
content="Appended with state",
state=mock_state,
tool_call_id="call_append_999",
)
assert isinstance(result, Command)
assert VALIDATOR_MESSAGES_KEY in result.update
tool_messages = result.update[VALIDATOR_MESSAGES_KEY]
assert len(tool_messages) == 1
assert tool_messages[0].tool_call_id == "call_append_999"
assert tool_messages[0].content == "Successfully appended note to state_append_note.md."
assert tool_messages[0].status == "success"
mock_state.asanitize_update.assert_called_once()
note_path = Path(tmp_path) / "notes" / "state_append_note.md"
assert note_path.exists()
assert note_path.read_text(encoding="utf-8") == "Appended with state\n"
@pytest.mark.asyncio
async def test_append_note_execution_failure(mock_ctx):
"""Verify error handling when appending note fails."""
with patch(
"artemis.tools.scratchpad.append_note_content",
side_effect=PermissionError("Permission denied"),
):
result = await append_note.execute(
ctx=mock_ctx,
key="fail_append_note",
content="Fail content",
)
assert "Failed to append note fail_append_note.md: Permission denied" in result
@pytest.mark.asyncio
async def test_get_append_note_tool_langchain_ainvoke(mock_ctx, tmp_path):
"""Verify get_append_note_tool exports a LangChain tool that works with ainvoke."""
lc_tool = get_append_note_tool(mock_ctx)
assert lc_tool.name == "append_note"
result = await lc_tool.ainvoke(
{
"key": "lc_append_note",
"content": "Appended via LangChain BaseTool",
}
)
assert result == "Successfully appended note to lc_append_note.md."
note_path = Path(tmp_path) / "notes" / "lc_append_note.md"
assert note_path.exists()
assert note_path.read_text(encoding="utf-8") == "Appended via LangChain BaseTool\n"
@pytest.mark.asyncio
async def test_get_append_note_tool_pure(mock_ctx, tmp_path):
"""Verify get_append_note_tool_pure returns a pure LangChain tool."""
pure_tool = get_append_note_tool_pure(mock_ctx)
assert pure_tool.name == "append_note"
result = await pure_tool.ainvoke(
{
"key": "pure_append_note",
"content": "Appended via pure tool",
}
)
assert result == "Successfully appended note to pure_append_note.md."
note_path = Path(tmp_path) / "notes" / "pure_append_note.md"
assert note_path.exists()
assert note_path.read_text(encoding="utf-8") == "Appended via pure tool\n"
+158
View File
@@ -0,0 +1,158 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.mobile.exec_tools import (
Click,
ClickArgs,
ClickTool,
click,
click_wrapper,
get_click_tool,
)
from langgraph.types import Command
import pytest
@pytest.fixture
def mock_ctx():
ctx = MagicMock(spec=ArtemisContext)
ctx.device = MagicMock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
return ctx
@pytest.fixture
def mock_driver():
driver = MagicMock(spec=BaseDeviceDriver)
driver.screen_size = (1080, 2400)
driver.tap = AsyncMock(return_value=True)
return driver
@pytest.fixture
def mock_state():
state = MagicMock(spec=State)
async def _mock_asanitize_update(ctx, update, agent):
return update
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
return state
def test_click_tool_subclass_and_registry():
"""Verify ClickTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(ClickTool, ArtemisTool)
assert issubclass(Click, ArtemisTool)
assert isinstance(click, ArtemisTool)
assert isinstance(click, ClickTool)
assert click.name == "click"
assert click.category == "action"
assert click.args_schema == ClickArgs
# Registry lookup
reg_tool = ToolRegistry.get("click")
assert reg_tool is not None
assert isinstance(reg_tool, ClickTool)
# GenAI FunctionDeclaration export
declaration = click.to_genai_declaration()
assert declaration.name == "click"
assert "target" in declaration.parameters.properties
# Wrapper check
assert click_wrapper is not None
assert click_wrapper.tool_fn_getter == get_click_tool
@pytest.mark.asyncio
async def test_click_direct_execution_with_driver(mock_driver):
"""Verify direct execution of ClickTool with BaseDeviceDriver."""
result = await click.execute(driver=mock_driver, target=[500, 500])
mock_driver.tap.assert_called_once_with(540, 1200)
assert "Clicked at [500, 500] (normalized) successfully." in result
@pytest.mark.asyncio
async def test_click_direct_execution_with_ctx(mock_ctx):
"""Verify direct execution of ClickTool with ArtemisContext."""
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
mock_controller_cls.return_value = mock_controller_inst
result = await click.execute(ctx=mock_ctx, target=[250, 750])
mock_controller_inst.tap_at.assert_called_once_with(270, 1800)
assert "Clicked at [250, 750] (normalized) successfully." in result
@pytest.mark.asyncio
async def test_click_with_state_command(mock_ctx, mock_state):
"""Verify ClickTool returns Command when state is provided."""
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
mock_controller_cls.return_value = mock_controller_inst
cmd = await click.execute(
ctx=mock_ctx,
target=[100, 200],
tool_call_id="call_click_1",
state=mock_state,
)
assert isinstance(cmd, Command)
assert VALIDATOR_MESSAGES_KEY in cmd.update
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
assert len(messages) == 1
assert messages[0].tool_call_id == "call_click_1"
assert messages[0].status == "success"
assert "Clicked at [100, 200]" in messages[0].content
@pytest.mark.asyncio
async def test_click_callable_execution(mock_driver):
"""Verify invoking click directly as a callable."""
result = await click(driver=mock_driver, target=[100, 100])
assert "Clicked at [100, 100] (normalized) successfully." in result
@pytest.mark.asyncio
async def test_click_invalid_target(mock_ctx):
"""Verify error handling on invalid target coordinates."""
result = await click.execute(ctx=mock_ctx, target=[])
assert "Error during click: Invalid target coordinates" in result
@pytest.mark.asyncio
async def test_get_click_tool_langchain_ainvoke(mock_ctx):
"""Verify get_click_tool exports a LangChain tool that works with ainvoke."""
click_tool = get_click_tool(mock_ctx)
assert click_tool.name == "click"
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
mock_controller_cls.return_value = mock_controller_inst
result = await click_tool.ainvoke({"target": [500, 500]})
assert "Clicked at [500, 500] (normalized) successfully." in result
+603
View File
@@ -0,0 +1,603 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.context import ArtemisContext
from artemis.tools.command_tool import (
_BACKGROUND_TASKS,
_PERSISTENT_ENVIRONMENTS,
get_manage_task_tool,
get_run_adb_command_tool,
get_run_short_adb_command_tool,
)
import pytest
class MockProcess:
def __init__(
self,
output_bytes: bytes,
exit_code: int = 0,
pid: int = 12345,
delay: float = 0,
):
self.output_bytes = output_bytes
self.exit_code = exit_code
self.pid = pid
self.returncode = None
self._read_offset = 0
self.delay = delay
self.stdin = MagicMock()
self.stdin.write = MagicMock()
self.stdin.drain = AsyncMock()
self.stdin.close = MagicMock()
self.stdout = MagicMock()
self.stdout.readline = AsyncMock(side_effect=self._readline)
async def _readline(self) -> bytes:
if self.delay > 0:
await asyncio.sleep(self.delay)
self.delay = 0 # Only delay the first read
if self._read_offset >= len(self.output_bytes):
self.returncode = self.exit_code
return b""
idx = self.output_bytes.find(b"\n", self._read_offset)
if idx == -1:
line = self.output_bytes[self._read_offset :]
self._read_offset = len(self.output_bytes)
else:
line = self.output_bytes[self._read_offset : idx + 1]
self._read_offset = idx + 1
return line
async def wait(self) -> int:
self.returncode = self.exit_code
return self.exit_code
def terminate(self):
self.returncode = -15
self.exit_code = -15
async def communicate(self) -> tuple[bytes, bytes]:
lines = []
while True:
line = await self._readline()
if not line:
break
lines.append(line)
self.returncode = self.exit_code
return b"".join(lines), b""
@pytest.fixture
def mock_ctx():
ctx = MagicMock(spec=ArtemisContext)
ctx.data_engine = None
ctx.device = MagicMock()
ctx.device.device_id = "test_device_1234"
return ctx
@pytest.mark.asyncio
@patch("asyncio.create_subprocess_exec")
async def test_run_command_sync(mock_exec, mock_ctx):
# Mock subprocess output
mock_process = MockProcess(
output_bytes=b"hello world\n===EXIT_CODE===0\n===ENV_START===\n",
exit_code=0,
)
mock_exec.return_value = mock_process
run_command = get_run_adb_command_tool(mock_ctx)
result = await run_command.ainvoke(
{
"CommandLine": "echo 'hello world'",
"Cwd": "/data/local/tmp",
"RunPersistent": False,
"WaitMsBeforeAsync": 500,
}
)
assert "completed with exit code 0" in result
assert "hello world" in result
mock_exec.assert_called_once()
# Check that adb was called with device ID and "shell"
args, kwargs = mock_exec.call_args
assert args[0] == "adb"
assert args[1] == "-s"
assert args[2] == "test_device_1234"
assert args[3] == "shell"
@pytest.mark.asyncio
@patch("asyncio.create_subprocess_exec")
async def test_run_command_persistent_env(mock_exec, mock_ctx):
run_command = get_run_adb_command_tool(mock_ctx)
terminal_id = "test_term_android"
# 1. Set environment variable in persistent terminal
mock_process_1 = MockProcess(
output_bytes=(b"===EXIT_CODE===0\n===ENV_START===\nARTEMIS_TEST_ENV=coffee\n"),
exit_code=0,
)
mock_exec.return_value = mock_process_1
res1 = await run_command.ainvoke(
{
"CommandLine": "export ARTEMIS_TEST_ENV=coffee",
"Cwd": "/data/local/tmp",
"RunPersistent": True,
"RequestedTerminalID": terminal_id,
"WaitMsBeforeAsync": 500,
}
)
assert "completed with exit code 0" in res1
assert terminal_id in _PERSISTENT_ENVIRONMENTS
assert _PERSISTENT_ENVIRONMENTS[terminal_id].get("ARTEMIS_TEST_ENV") == "coffee"
# 2. Query environment variable in subsequent command on same terminal
mock_process_2 = MockProcess(
output_bytes=b"coffee\n===EXIT_CODE===0\n===ENV_START===\nARTEMIS_TEST_ENV=coffee\n",
exit_code=0,
)
mock_exec.return_value = mock_process_2
res2 = await run_command.ainvoke(
{
"CommandLine": "echo $ARTEMIS_TEST_ENV",
"Cwd": "/data/local/tmp",
"RunPersistent": True,
"RequestedTerminalID": terminal_id,
"WaitMsBeforeAsync": 500,
}
)
assert "completed with exit code 0" in res2
assert "coffee" in res2
@pytest.mark.asyncio
@patch("asyncio.create_subprocess_exec")
async def test_run_command_async_and_status(mock_exec, mock_ctx):
# Mock a long running command by delaying stdout response
mock_process = MockProcess(
output_bytes=(b"background task finished\n===EXIT_CODE===0\n===ENV_START===\n"),
exit_code=0,
delay=1.0, # 1.0s delay will trigger timeout
)
mock_exec.return_value = mock_process
run_command = get_run_adb_command_tool(mock_ctx)
manage_task = get_manage_task_tool(mock_ctx)
_BACKGROUND_TASKS.clear()
res = await run_command.ainvoke(
{
"CommandLine": "sleep 2 && echo 'background task finished'",
"Cwd": "/data/local/tmp",
"RunPersistent": False,
"WaitMsBeforeAsync": 100, # 100ms wait threshold
}
)
assert "sent to the background as a task" in res
assert "TaskId: task_" in res
lines = res.splitlines()
task_id_line = [line for line in lines if line.startswith("TaskId: ")][0]
task_id = task_id_line.split(": ")[1]
assert task_id in _BACKGROUND_TASKS
# Check status (should be running)
status_res = await manage_task.ainvoke(
{
"Action": "status",
"TaskId": task_id,
}
)
assert "Status: running" in status_res
# Wait for the background listener task to finish reading delayed output
await asyncio.sleep(1.2)
# Status should be completed (and task removed from active map)
assert task_id not in _BACKGROUND_TASKS
@pytest.mark.asyncio
@patch("asyncio.create_subprocess_exec")
async def test_run_command_kill(mock_exec, mock_ctx):
mock_process = MockProcess(output_bytes=b"", exit_code=0, delay=5.0)
mock_exec.return_value = mock_process
run_command = get_run_adb_command_tool(mock_ctx)
manage_task = get_manage_task_tool(mock_ctx)
_BACKGROUND_TASKS.clear()
res = await run_command.ainvoke(
{
"CommandLine": "sleep 10",
"Cwd": "/data/local/tmp",
"RunPersistent": False,
"WaitMsBeforeAsync": 100,
}
)
lines = res.splitlines()
task_id_line = [line for line in lines if line.startswith("TaskId: ")][0]
task_id = task_id_line.split(": ")[1]
# Kill the task
kill_res = await manage_task.ainvoke(
{
"Action": "kill",
"TaskId": task_id,
}
)
assert f"Task {task_id} successfully terminated" in kill_res
assert task_id not in _BACKGROUND_TASKS
@pytest.mark.asyncio
@patch("asyncio.create_subprocess_exec")
async def test_run_short_command_success(mock_exec, mock_ctx):
mock_process = MockProcess(output_bytes=b"short command success output", exit_code=0)
mock_exec.return_value = mock_process
run_short_command = get_run_short_adb_command_tool(mock_ctx)
result = await run_short_command.ainvoke(
{
"CommandLine": "echo 'short command'",
}
)
assert "completed with exit code 0" in result
assert "short command success output" in result
mock_exec.assert_called_once()
args, kwargs = mock_exec.call_args
assert args[0] == "adb"
assert args[1] == "-s"
assert args[2] == "test_device_1234"
assert args[3] == "shell"
# Verify it cd'd to /data/local/tmp by default
assert "cd /data/local/tmp" in args[4]
@pytest.mark.asyncio
@patch("asyncio.create_subprocess_exec")
async def test_run_short_command_timeout(mock_exec, mock_ctx):
mock_process = MockProcess(output_bytes=b"some output", exit_code=0, delay=5.0)
mock_exec.return_value = mock_process
run_short_command = get_run_short_adb_command_tool(mock_ctx)
result = await run_short_command.ainvoke(
{
"CommandLine": "sleep 5",
}
)
assert "Error: Command timed out after 3 seconds." in result
@pytest.mark.asyncio
@patch("asyncio.create_subprocess_exec")
async def test_run_command_sync_long_output(mock_exec, mock_ctx):
# Generate long output (> 500 lines)
long_output = "\n".join(f"Output line {i}" for i in range(1, 600))
mock_process = MockProcess(
output_bytes=long_output.encode() + b"\n===EXIT_CODE===0\n===ENV_START===\n",
exit_code=0,
)
mock_exec.return_value = mock_process
run_command = get_run_adb_command_tool(mock_ctx)
result = await run_command.ainvoke(
{
"CommandLine": "generate_long_output",
"Cwd": "/data/local/tmp",
"RunPersistent": False,
"WaitMsBeforeAsync": 500,
}
)
assert "ADB command completed with exit code 0." in result
assert "has been truncated" in result
assert "Output line 599" in result
assert "Output line 1" not in result # since it was truncated to last 200 lines
assert "analyze_task_output" in result
assert "TaskId: task_sync_" in result
@pytest.mark.asyncio
@patch("artemis.agents.log_analyzer.output_analyzer.get_llm")
async def test_analyze_task_output_tool(mock_get_llm, mock_ctx):
# Set up mock LLM
mock_llm = MagicMock()
mock_response = MagicMock()
mock_response.content = "This is the analysis result: everything looks good."
mock_response.tool_calls = []
mock_llm.bind_tools.return_value = mock_llm
mock_llm.ainvoke = AsyncMock(return_value=mock_response)
mock_get_llm.return_value = mock_llm
# Set up finished tasks logs cache manually
from artemis.tools.command_tool import (
_FINISHED_TASKS_LOGS,
_register_finished_task,
get_analyze_task_output_tool,
)
task_id = "test_task_long_output"
_register_finished_task(
task_id=task_id,
command="cat long_log.txt",
cwd="/data/local/tmp",
terminal_id=None,
status="completed",
exit_code=0,
output="line1\nline2\n" + "\n".join(f"line{i}" for i in range(3, 50)),
)
analyze_tool = get_analyze_task_output_tool(mock_ctx)
result = await analyze_tool.ainvoke({"TaskId": task_id, "Query": "Summarize the log"})
assert "Analysis result for Task test_task_long_output:" in result
assert "This is the analysis result: everything looks good." in result
mock_get_llm.assert_called_once_with(ctx=mock_ctx, name="output_analyzer")
# Clean up registry
if task_id in _FINISHED_TASKS_LOGS:
del _FINISHED_TASKS_LOGS[task_id]
def test_run_adb_command_tool_subclass_and_registry():
"""Verify RunAdbCommandTool is a subclass of ArtemisTool and properly registered."""
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.command_tool import (
RunAdbCommand,
RunAdbCommandArgs,
RunAdbCommandTool,
run_adb_command,
)
assert issubclass(RunAdbCommandTool, ArtemisTool)
assert issubclass(RunAdbCommand, ArtemisTool)
assert isinstance(run_adb_command, ArtemisTool)
assert isinstance(run_adb_command, RunAdbCommandTool)
assert run_adb_command.name == "run_adb_command"
assert run_adb_command.category == "system"
assert run_adb_command.args_schema == RunAdbCommandArgs
# Registry lookup
reg_tool = ToolRegistry.get("run_adb_command")
assert reg_tool is not None
assert isinstance(reg_tool, RunAdbCommandTool)
# GenAI FunctionDeclaration export
declaration = run_adb_command.to_genai_declaration()
assert declaration.name == "run_adb_command"
assert "CommandLine" in declaration.parameters.properties
@pytest.mark.asyncio
async def test_run_adb_command_tool_mock_driver_execute():
"""Verify RunAdbCommandTool.execute dispatches to MockDeviceDriver when ctx is None."""
from artemis.drivers.mock.mock_driver import MockDeviceDriver
from artemis.tools.command_tool import RunAdbCommandTool
driver = MockDeviceDriver(width=1080, height=2400)
tool_inst = RunAdbCommandTool()
result = await tool_inst.execute(
driver=driver,
ctx=None,
CommandLine="pm list packages",
)
assert "mock_output: pm list packages" in result
assert "pm list packages" in driver.action_history[-1]["command"]
def test_manage_task_tool_subclass_and_registry():
"""Verify ManageTaskTool is a subclass of ArtemisTool and properly registered."""
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.command_tool import (
ManageTask,
ManageTaskArgs,
ManageTaskTool,
manage_task,
)
assert issubclass(ManageTaskTool, ArtemisTool)
assert issubclass(ManageTask, ArtemisTool)
assert isinstance(manage_task, ArtemisTool)
assert isinstance(manage_task, ManageTaskTool)
assert manage_task.name == "manage_task"
assert manage_task.category == "system"
assert manage_task.args_schema == ManageTaskArgs
# Registry lookup
reg_tool = ToolRegistry.get("manage_task")
assert reg_tool is not None
assert isinstance(reg_tool, ManageTaskTool)
# GenAI FunctionDeclaration export
declaration = manage_task.to_genai_declaration()
assert declaration.name == "manage_task"
assert "Action" in declaration.parameters.properties
@pytest.mark.asyncio
async def test_manage_task_tool_direct_execute(mock_ctx):
"""Verify direct ManageTaskTool.execute execution."""
from artemis.tools.command_tool import (
_BACKGROUND_TASKS,
BackgroundTask,
manage_task,
)
_BACKGROUND_TASKS.clear()
list_res = await manage_task.execute(ctx=mock_ctx, Action="list")
assert "No active background tasks" in list_res
# Create dummy task
dummy_proc = MagicMock()
bg_task = BackgroundTask(
task_id="dummy_task_99",
command="sleep 100",
process=dummy_proc,
terminal_id="term_1",
cwd="/data/local/tmp",
)
_BACKGROUND_TASKS["dummy_task_99"] = bg_task
list_res2 = await manage_task.execute(ctx=mock_ctx, Action="list")
assert "dummy_task_99" in list_res2
assert "sleep 100" in list_res2
# Status check
status_res = await manage_task.execute(ctx=mock_ctx, Action="status", TaskId="dummy_task_99")
assert "Status: running" in status_res
assert "dummy_task_99" in status_res
_BACKGROUND_TASKS.clear()
def test_analyze_task_output_tool_subclass_and_registry():
"""Verify AnalyzeTaskOutputTool is a subclass of ArtemisTool and properly registered."""
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.command_tool import (
AnalyzeTaskOutput,
AnalyzeTaskOutputArgs,
AnalyzeTaskOutputTool,
analyze_task_output,
)
assert issubclass(AnalyzeTaskOutputTool, ArtemisTool)
assert issubclass(AnalyzeTaskOutput, ArtemisTool)
assert isinstance(analyze_task_output, ArtemisTool)
assert isinstance(analyze_task_output, AnalyzeTaskOutputTool)
assert analyze_task_output.name == "analyze_task_output"
assert analyze_task_output.category == "system"
assert analyze_task_output.args_schema == AnalyzeTaskOutputArgs
# Registry lookup
reg_tool = ToolRegistry.get("analyze_task_output")
assert reg_tool is not None
assert isinstance(reg_tool, AnalyzeTaskOutputTool)
# GenAI FunctionDeclaration export
declaration = analyze_task_output.to_genai_declaration()
assert declaration.name == "analyze_task_output"
assert "TaskId" in declaration.parameters.properties
assert "Query" in declaration.parameters.properties
@pytest.mark.asyncio
@patch("artemis.agents.log_analyzer.output_analyzer.get_llm")
async def test_analyze_task_output_tool_direct_execute(mock_get_llm, mock_ctx):
"""Verify direct AnalyzeTaskOutputTool.execute execution."""
from artemis.tools.command_tool import (
_FINISHED_TASKS_LOGS,
_register_finished_task,
analyze_task_output,
)
mock_llm = MagicMock()
mock_response = MagicMock()
mock_response.content = "Summary of task output."
mock_response.tool_calls = []
mock_llm.bind_tools.return_value = mock_llm
mock_llm.ainvoke = AsyncMock(return_value=mock_response)
mock_get_llm.return_value = mock_llm
task_id = "test_direct_analyze"
_register_finished_task(
task_id=task_id,
command="cat file.txt",
cwd="/data/local/tmp",
terminal_id=None,
status="completed",
exit_code=0,
output="Sample log content",
)
result = await analyze_task_output.execute(
ctx=mock_ctx, TaskId=task_id, Query="What is the content?"
)
assert "Analysis result for Task test_direct_analyze:" in result
assert "Summary of task output." in result
if task_id in _FINISHED_TASKS_LOGS:
del _FINISHED_TASKS_LOGS[task_id]
def test_run_short_adb_command_tool_subclass_and_registry():
"""Verify RunShortAdbCommandTool is a subclass of ArtemisTool and properly registered."""
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.command_tool import (
RunShortAdbCommand,
RunShortAdbCommandArgs,
RunShortAdbCommandTool,
run_short_adb_command,
)
assert issubclass(RunShortAdbCommandTool, ArtemisTool)
assert issubclass(RunShortAdbCommand, ArtemisTool)
assert isinstance(run_short_adb_command, ArtemisTool)
assert isinstance(run_short_adb_command, RunShortAdbCommandTool)
assert run_short_adb_command.name == "run_short_adb_command"
assert run_short_adb_command.category == "system"
assert run_short_adb_command.args_schema == RunShortAdbCommandArgs
# Registry lookup
reg_tool = ToolRegistry.get("run_short_adb_command")
assert reg_tool is not None
assert isinstance(reg_tool, RunShortAdbCommandTool)
# GenAI FunctionDeclaration export
declaration = run_short_adb_command.to_genai_declaration()
assert declaration.name == "run_short_adb_command"
assert "CommandLine" in declaration.parameters.properties
@pytest.mark.asyncio
async def test_run_short_adb_command_tool_mock_driver_execute():
"""Verify RunShortAdbCommandTool.execute dispatches to MockDeviceDriver when ctx is None."""
from artemis.drivers.mock.mock_driver import MockDeviceDriver
from artemis.tools.command_tool import RunShortAdbCommandTool
driver = MockDeviceDriver(width=1080, height=2400)
tool_inst = RunShortAdbCommandTool()
result = await tool_inst.execute(
driver=driver,
ctx=None,
CommandLine="getprop ro.build.version.release",
)
assert "mock_output: getprop ro.build.version.release" in result
assert "getprop ro.build.version.release" in driver.action_history[-1]["command"]
+222
View File
@@ -0,0 +1,222 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pathlib import Path
import shutil
import tempfile
import unittest
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.context import ArtemisContext
from artemis.tools.committee_tool import get_ask_committee_tool
class TestCommitteeTool(unittest.IsolatedAsyncioTestCase):
async def test_ask_committee_tool_with_directive(self):
# Create a temp directory for testing
temp_dir = tempfile.mkdtemp()
try:
notes_dir = Path(temp_dir) / "notes"
notes_dir.mkdir(parents=True, exist_ok=True)
# Create a mock screenshot file
mock_screenshot_path = Path(temp_dir) / "mock_screenshot.jpg"
mock_screenshot_path.write_bytes(b"fake image bytes")
# Create a mock failed plans history file
mock_failed_plans_history_path = notes_dir / "failed_plans_history.md"
mock_failed_plans_history_path.write_text(
"## Failed Plan Attempt #1\n### Plan:\n- [ ] Step 1\n###"
" Failure Reason:\nTimeout waiting for element",
encoding="utf-8",
)
# Mock context and state
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.device = MagicMock()
mock_ctx.device.device_width = 1080
mock_ctx.device.device_height = 2400
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = temp_dir
mock_ctx.data_engine.get_agent_friendly_steps.return_value = [
{"pre_image_name": "test_image", "step_number": 1}
]
mock_state = MagicMock()
mock_state.initial_goal = "Test Goal"
mock_state.messages = []
mock_state.latest_ui_hierarchy = None
mock_state.latest_screenshot = str(mock_screenshot_path)
mock_state.focused_app_info = None
mock_state.device_date = None
mock_state.structured_decisions = None
mock_state.complete_subgoals_by_ids = []
mock_state.validator_messages = []
mock_state.operator_replan_reason = "Mock Immediate Replan Reason"
# Mock LLM responses for the members
mock_llm_pl = MagicMock()
mock_llm_pl.bind_tools.return_value = mock_llm_pl
m1 = MagicMock(content="Op: I think we should tap.")
m1.tool_calls = []
m2 = MagicMock(content="Op: I agree with Exp.")
m2.tool_calls = []
m3 = MagicMock(content="Op: Final conclusion: Tap at [50, 50]")
m3.tool_calls = []
mock_llm_pl.ainvoke = AsyncMock(side_effect=[m1, m2, m3])
mock_llm_diag = MagicMock()
mock_llm_diag.bind_tools.return_value = mock_llm_diag
m_diag = MagicMock(content="Diag: I see an error.")
m_diag.tool_calls = []
mock_llm_diag.ainvoke = AsyncMock(return_value=m_diag)
# Mock History Analyzer to call a tool first
mock_llm_hist = MagicMock()
mock_llm_hist.bind_tools.return_value = mock_llm_hist
mock_response_tool_call = MagicMock()
mock_response_tool_call.tool_calls = [
{
"name": "get_step_details",
"args": {"start_step": 1, "end_step": 1},
"id": "call_1",
}
]
mock_response_tool_call.content = ""
mock_response_final = MagicMock()
mock_response_final.tool_calls = []
mock_response_final.content = "Hist: I found step 1 details."
m_hist_agree = MagicMock(content="Hist: Still agree.")
m_hist_agree.tool_calls = []
mock_llm_hist.ainvoke = AsyncMock(
side_effect=[
mock_response_tool_call,
mock_response_final,
# For second round
m_hist_agree,
]
)
def mock_get_llm(ctx, name, temperature=None):
if name == "planner_avatar":
return mock_llm_pl
elif name == "diagnoser_expert":
return mock_llm_diag
elif name == "history_analyzer_expert":
return mock_llm_hist
return MagicMock()
# Patch get_llm and the actual tool functions to avoid real calls
with (
patch(
"artemis.tools.committee_tool.get_llm",
side_effect=mock_get_llm,
),
patch(
"artemis.tools.committee_tool.trace_langchain_tool",
side_effect=lambda t, ctx: t,
),
):
tool = get_ask_committee_tool(mock_ctx)
# Invoke tool with avatar_directive
from artemis.data_engine.trace import CURRENT_TRACE_ID
token = CURRENT_TRACE_ID.set("test_trace_id")
try:
result = await tool.ainvoke(
{
"avatar_directive": "Advocate for paying",
"state": mock_state,
}
)
finally:
CURRENT_TRACE_ID.reset(token)
# Verify result
self.assertIn("Final conclusion: Tap at [50, 50]", result)
# Verify file was created and contains content
notes_dir = Path(temp_dir) / "notes"
blackboard_files = list(notes_dir.glob("blackboard_*.md"))
self.assertTrue(len(blackboard_files) > 0, "No blackboard file created")
blackboard_path = blackboard_files[0]
content = blackboard_path.read_text(encoding="utf-8")
self.assertIn("Advocate for paying", content) # Check directive is in blackboard
self.assertIn("Hist: I found step 1 details.", content)
self.assertIn(
"Timeout waiting for element", content
) # Check failed history is in blackboard
self.assertIn("Test Goal", content) # Check initial goal is in blackboard
self.assertIn(
"Mock Immediate Replan Reason", content
) # Check current replan reason is in blackboard
finally:
# Clean up temp directory
shutil.rmtree(temp_dir)
def test_ask_committee_tool_subclass_and_registry(self):
"""Verify AskCommitteeTool is a subclass of ArtemisTool and properly registered."""
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.committee_tool import (
AskCommittee,
AskCommitteeArgs,
AskCommitteeTool,
AskCommitteeToolAlias,
ask_committee,
ask_committee_wrapper,
get_ask_committee_tool,
)
self.assertTrue(issubclass(AskCommitteeTool, ArtemisTool))
self.assertTrue(issubclass(AskCommittee, ArtemisTool))
self.assertTrue(issubclass(AskCommitteeToolAlias, ArtemisTool))
self.assertIsInstance(ask_committee, ArtemisTool)
self.assertIsInstance(ask_committee, AskCommitteeTool)
self.assertEqual(ask_committee.name, "ask_committee")
self.assertEqual(ask_committee.category, "custom")
self.assertEqual(ask_committee.args_schema, AskCommitteeArgs)
# Registry lookup
reg_tool = ToolRegistry.get("ask_committee")
self.assertIsNotNone(reg_tool)
self.assertIsInstance(reg_tool, AskCommitteeTool)
# GenAI FunctionDeclaration export
declaration = ask_committee.to_genai_declaration()
self.assertEqual(declaration.name, "ask_committee")
self.assertIn("avatar_directive", declaration.parameters.properties)
# Wrapper check
self.assertIsNotNone(ask_committee_wrapper)
self.assertEqual(ask_committee_wrapper.tool_fn_getter, get_ask_committee_tool)
async def test_ask_committee_no_ctx(self):
"""Verify executing without ctx returns an error message."""
from artemis.tools.committee_tool import ask_committee
result = await ask_committee.execute(avatar_directive="Test")
self.assertEqual(result, "Error: ArtemisContext is required for ask_committee.")
if __name__ == "__main__":
unittest.main()
+375
View File
@@ -0,0 +1,375 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from unittest.mock import AsyncMock, MagicMock, mock_open, patch
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.explorer_tool import (
AskExplorer,
AskExplorerArgs,
AskExplorerTool,
AskExplorerToolAlias,
ask_explorer,
ask_explorer_wrapper,
get_ask_explorer_tool,
)
import pytest
def test_ask_explorer_tool_subclass_and_registry():
"""Verify AskExplorerTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(AskExplorerTool, ArtemisTool)
assert issubclass(AskExplorer, ArtemisTool)
assert issubclass(AskExplorerToolAlias, ArtemisTool)
assert isinstance(ask_explorer, ArtemisTool)
assert isinstance(ask_explorer, AskExplorerTool)
assert ask_explorer.name == "ask_explorer"
assert ask_explorer.category == "explorer"
assert ask_explorer.args_schema == AskExplorerArgs
# Registry lookup
reg_tool = ToolRegistry.get("ask_explorer")
assert reg_tool is not None
assert isinstance(reg_tool, AskExplorerTool)
# GenAI FunctionDeclaration export
declaration = ask_explorer.to_genai_declaration()
assert declaration.name == "ask_explorer"
assert "query" in declaration.parameters.properties
# Wrapper check
assert ask_explorer_wrapper is not None
assert ask_explorer_wrapper.tool_fn_getter == get_ask_explorer_tool
@pytest.mark.asyncio
async def test_ask_explorer_multiple_candidates():
# Mock Context and State
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
mock_ctx.agent_config = None
mock_ctx.llm_config = None
mock_ctx.execution_setup = None
mock_ctx.device = MagicMock()
mock_ctx.device.device_width = 1080
mock_ctx.device.device_height = 2400
mock_state = MagicMock(spec=State)
mock_state.latest_screenshot = "/tmp/test.jpg"
mock_state.indexed_points = [[100, 200]] # Pre-existing point
mock_state.indexed_elements = [
{
"index": 1,
"center": [100, 200],
"text": "Pre-existing",
"bounds": None,
"class": "Test",
"resource_id": None,
"is_ocr": False,
}
]
# Mock Explorer Agent Outcome with multiple candidates and fallback message
explorer_outcome = {
"candidates": [
{
"label": "S1",
"coords": [500, 500],
"description": "First Button",
},
{
"label": "S2",
"coords": [250, 750],
"description": "Second Button",
},
],
"fallback_message": "Please choose carefully.",
}
explorer_output_str = json.dumps(explorer_outcome)
# We need to patch the Explorer class inside the tool
mock_explorer_instance = MagicMock()
mock_explorer_instance.run = AsyncMock(return_value=explorer_output_str)
with patch(
"artemis.tools.explorer_tool.Explorer",
return_value=mock_explorer_instance,
):
# Retrieve the tool
ask_explorer_tool = get_ask_explorer_tool(mock_ctx, version="pro")
result = await ask_explorer_tool.ainvoke(
{
"query": "Find buttons",
"context_feedback": "Attempt 1",
"state": mock_state,
}
)
assert mock_state.indexed_points == [
[100, 200],
[540, 1200],
[270, 1800],
]
assert len(mock_state.indexed_elements) == 3
assert mock_state.indexed_elements[1]["index"] == 2
assert mock_state.indexed_elements[1]["center"] == [540, 1200]
assert mock_state.indexed_elements[1]["text"] == "First Button"
assert mock_state.indexed_elements[2]["index"] == 3
assert mock_state.indexed_elements[2]["center"] == [270, 1800]
assert mock_state.indexed_elements[2]["text"] == "Second Button"
assert isinstance(result, str)
assert "Explorer successfully located the following candidate(s):" in result
assert "- [2] 'First Button' at coordinate [500, 500]" in result
assert "- [3] 'Second Button' at coordinate [250, 750]" in result
assert (
"You can click/act on them directly by calling perform_action with"
" their respective index." in result
)
assert "Additional Notes from Explorer: Please choose carefully." in result
@pytest.mark.asyncio
async def test_ask_explorer_only_fallback_message():
# Mock Context and State
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
mock_ctx.agent_config = None
mock_ctx.llm_config = None
mock_ctx.execution_setup = None
mock_state = MagicMock(spec=State)
mock_state.latest_screenshot = "/tmp/test.jpg"
mock_state.indexed_points = []
mock_state.indexed_elements = []
explorer_outcome = {
"candidates": [],
"fallback_message": ("Element not found because it is covered by a keyboard."),
}
explorer_output_str = json.dumps(explorer_outcome)
mock_explorer_instance = MagicMock()
mock_explorer_instance.run = AsyncMock(return_value=explorer_output_str)
with patch(
"artemis.tools.explorer_tool.Explorer",
return_value=mock_explorer_instance,
):
ask_explorer_tool = get_ask_explorer_tool(mock_ctx, version="pro")
result = await ask_explorer_tool.ainvoke(
{
"query": "Find hidden button",
"context_feedback": "",
"state": mock_state,
}
)
assert mock_state.indexed_points == []
assert mock_state.indexed_elements == []
assert (
"Explorer could not locate the element. Message: Element not found"
" because it is covered by a keyboard." in result
)
@pytest.mark.asyncio
async def test_ask_explorer_multimodal_success():
# Mock Context and State
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
mock_ctx.agent_config = None
mock_ctx.llm_config = None
mock_ctx.execution_setup = None
mock_ctx.device = MagicMock()
mock_ctx.device.device_width = 1080
mock_ctx.device.device_height = 2400
mock_state = MagicMock(spec=State)
mock_state.latest_screenshot = "/tmp/test.jpg"
mock_state.indexed_points = [[100, 200]]
mock_state.indexed_elements = [
{
"index": 1,
"center": [100, 200],
"text": "Pre-existing",
"bounds": None,
"class": "Test",
"resource_id": None,
"is_ocr": False,
}
]
explorer_outcome = {
"candidates": [
{
"label": "S1",
"coords": [500, 500],
"description": "First Button",
},
],
"fallback_message": "",
}
explorer_output_str = json.dumps(explorer_outcome)
mock_explorer_instance = MagicMock()
mock_explorer_instance.run = AsyncMock(return_value=explorer_output_str)
m_open = mock_open(read_data=b"fake_annotated_image_bytes")
with (
patch(
"artemis.tools.explorer_tool.Explorer",
return_value=mock_explorer_instance,
),
patch("artemis.tools.explorer_tool.draw_dots") as mock_draw_dots,
patch("builtins.open", m_open),
patch("pathlib.Path.mkdir"),
):
ask_explorer_tool = get_ask_explorer_tool(mock_ctx, version="pro")
result = await ask_explorer_tool.ainvoke(
{
"query": "Find buttons",
"context_feedback": "",
"state": mock_state,
}
)
assert isinstance(result, list)
assert len(result) == 2
assert result[0]["type"] == "text"
assert "Explorer successfully located" in result[0]["text"]
assert result[1]["type"] == "image_url"
assert result[1]["image_url"]["url"].startswith("data:image/jpeg;base64,")
mock_draw_dots.assert_called_once()
points_arg = mock_draw_dots.call_args[0][1]
assert points_arg == [[100, 200], [540, 1200]]
labels_arg = mock_draw_dots.call_args[0][2]
assert labels_arg == ["1", "2"]
@pytest.mark.asyncio
async def test_ask_explorer_direct_execution():
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
mock_ctx.agent_config = None
mock_ctx.llm_config = None
mock_ctx.execution_setup = None
mock_ctx.device = None
mock_state = MagicMock(spec=State)
mock_state.latest_screenshot = "/tmp/test.jpg"
mock_state.indexed_points = []
mock_state.indexed_elements = []
mock_explorer_instance = MagicMock()
mock_explorer_instance.run = AsyncMock(
return_value=json.dumps(
{
"candidates": [],
"fallback_message": "Direct execution not found.",
}
)
)
with patch(
"artemis.tools.explorer_tool.Explorer",
return_value=mock_explorer_instance,
):
result = await ask_explorer.execute(
ctx=mock_ctx, state=mock_state, query="Find icon", version="pro"
)
assert "Direct execution not found." in result
@pytest.mark.asyncio
async def test_ask_explorer_callable_execution():
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
mock_ctx.agent_config = None
mock_ctx.llm_config = None
mock_ctx.execution_setup = None
mock_ctx.device = None
mock_state = MagicMock(spec=State)
mock_state.latest_screenshot = "/tmp/test.jpg"
mock_state.indexed_points = []
mock_state.indexed_elements = []
mock_explorer_instance = MagicMock()
mock_explorer_instance.run = AsyncMock(
return_value=json.dumps(
{
"candidates": [],
"fallback_message": "Callable test message.",
}
)
)
with patch(
"artemis.tools.explorer_tool.Explorer",
return_value=mock_explorer_instance,
):
result = await ask_explorer(
ctx=mock_ctx, state=mock_state, query="Search bar", version="pro"
)
assert "Callable test message." in result
@pytest.mark.asyncio
async def test_ask_explorer_flash_mode():
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
mock_ctx.agent_config = None
mock_ctx.llm_config = None
mock_ctx.execution_setup = None
mock_ctx.device = MagicMock()
mock_ctx.device.device_width = 1080
mock_ctx.device.device_height = 2400
mock_state = MagicMock(spec=State)
mock_state.latest_screenshot = "/tmp/test.jpg"
mock_state.indexed_points = []
mock_state.indexed_elements = []
with (
patch(
"artemis.tools.explorer_tool._run_object_detection",
new_callable=AsyncMock,
return_value={
"detected": [
{"label": "Icon", "point": [500, 500]},
],
"failed": [],
},
),
patch(
"builtins.open",
mock_open(read_data=json.dumps({"templates": [], "instructions": ""})),
),
):
result = await ask_explorer.execute(
ctx=mock_ctx, state=mock_state, query="Icon", version="flash"
)
assert "Explorer successfully located" in result
assert "Icon" in result
@@ -0,0 +1,99 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.tools.image_processor_tool import get_ask_image_processor_tool
import pytest
@pytest.mark.asyncio
async def test_ask_vision_coder_success():
# Mock Context and State
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
mock_state = MagicMock(spec=State)
mock_state.latest_screenshot = "/tmp/test_screen.jpg"
# Mock VisionCoder Agent Outcome
vision_coder_outcome = {
"new_image_path": "/tmp/coder_output.jpg",
"summary": "Found red button at pixel coords (100, 200)",
}
# We need to patch the ImageProcessor class inside the tool
mock_coder_instance = MagicMock()
mock_coder_instance.run = AsyncMock(return_value=vision_coder_outcome)
with patch(
"artemis.agents.image_processor.image_processor.ImageProcessor",
return_value=mock_coder_instance,
):
# Retrieve the tool
ask_vision_coder_tool = get_ask_image_processor_tool(mock_ctx)
# Invoke the tool
result = await ask_vision_coder_tool.ainvoke(
{
"instruction": "Find the red button",
"target_image_id": "img_0",
"state": mock_state,
}
)
# Verify the agent was called with correct arguments
mock_coder_instance.run.assert_called_once_with(
"Find the red button", "/tmp/test_screen.jpg"
)
# Verify the output matches
assert result == vision_coder_outcome
def test_ask_image_processor_tool_subclass_and_registry():
"""Verify AskImageProcessorTool is an ArtemisTool subclass and registered in ToolRegistry."""
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.image_processor_tool import (
AskImageProcessor,
AskImageProcessorTool,
AskVisionCoderArgs,
AskVisionCoderTool,
ImageProcessorTool,
ask_image_processor,
)
assert issubclass(AskImageProcessorTool, ArtemisTool)
assert issubclass(AskImageProcessor, ArtemisTool)
assert issubclass(AskVisionCoderTool, ArtemisTool)
assert issubclass(ImageProcessorTool, ArtemisTool)
assert isinstance(ask_image_processor, ArtemisTool)
assert isinstance(ask_image_processor, AskImageProcessorTool)
assert ask_image_processor.name == "ask_image_processor"
assert ask_image_processor.category == "custom"
assert ask_image_processor.args_schema == AskVisionCoderArgs
# Registry lookup
reg_tool = ToolRegistry.get("ask_image_processor")
assert reg_tool is not None
assert isinstance(reg_tool, AskImageProcessorTool)
# GenAI FunctionDeclaration export
declaration = ask_image_processor.to_genai_declaration()
assert declaration.name == "ask_image_processor"
assert "instruction" in declaration.parameters.properties
assert "target_image_id" in declaration.parameters.properties
+182
View File
@@ -0,0 +1,182 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.mobile.exec_tools import (
InputText,
InputTextArgs,
InputTextTool,
get_input_text_tool,
input_text,
input_text_wrapper,
)
from langgraph.types import Command
import pytest
@pytest.fixture
def mock_ctx():
ctx = MagicMock(spec=ArtemisContext)
ctx.device = MagicMock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
return ctx
@pytest.fixture
def mock_driver():
driver = MagicMock(spec=BaseDeviceDriver)
driver.screen_size = (1080, 2400)
driver.tap = AsyncMock(return_value=True)
driver.input_text = AsyncMock(return_value=True)
return driver
@pytest.fixture
def mock_state():
state = MagicMock(spec=State)
async def _mock_asanitize_update(ctx, update, agent):
return update
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
return state
def test_input_text_tool_subclass_and_registry():
"""Verify InputTextTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(InputTextTool, ArtemisTool)
assert issubclass(InputText, ArtemisTool)
assert isinstance(input_text, ArtemisTool)
assert isinstance(input_text, InputTextTool)
assert input_text.name == "input_text"
assert input_text.category == "action"
assert input_text.args_schema == InputTextArgs
# Registry lookup
reg_tool = ToolRegistry.get("input_text")
assert reg_tool is not None
assert isinstance(reg_tool, InputTextTool)
# GenAI FunctionDeclaration export
declaration = input_text.to_genai_declaration()
assert declaration.name == "input_text"
assert "text" in declaration.parameters.properties
assert "target" in declaration.parameters.properties
assert "clear_exist" in declaration.parameters.properties
# Wrapper check
assert input_text_wrapper is not None
assert input_text_wrapper.tool_fn_getter == get_input_text_tool
@pytest.mark.asyncio
async def test_input_text_direct_execution_with_driver(mock_driver):
"""Verify direct execution of InputTextTool with BaseDeviceDriver."""
result = await input_text.execute(
driver=mock_driver, text="hello world", target=[500, 500], clear_exist=True
)
mock_driver.tap.assert_called_once_with(540, 1200)
mock_driver.input_text.assert_called_once_with("hello world", clear_existing=True)
assert "Typed 'hello world' successfully." in result
@pytest.mark.asyncio
async def test_input_text_direct_execution_with_ctx(mock_ctx):
"""Verify direct execution of InputTextTool with ArtemisContext."""
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
mock_controller_inst.erase_text = AsyncMock(return_value=True)
mock_controller_inst.type_text = AsyncMock(return_value=True)
mock_controller_cls.return_value = mock_controller_inst
result = await input_text.execute(
ctx=mock_ctx, text="test input", target=[250, 750], clear_exist=True
)
mock_controller_inst.tap_at.assert_called_once_with(270, 1800)
mock_controller_inst.erase_text.assert_called_once()
mock_controller_inst.type_text.assert_called_once_with("test input", clear_existing=False)
assert "Typed 'test input' successfully." in result
@pytest.mark.asyncio
async def test_input_text_with_state_command(mock_ctx, mock_state):
"""Verify InputTextTool returns Command when state is provided."""
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
mock_controller_inst.erase_text = AsyncMock(return_value=True)
mock_controller_inst.type_text = AsyncMock(return_value=True)
mock_controller_cls.return_value = mock_controller_inst
cmd = await input_text.execute(
ctx=mock_ctx,
text="query",
target=[100, 200],
tool_call_id="call_it_1",
state=mock_state,
)
assert isinstance(cmd, Command)
assert VALIDATOR_MESSAGES_KEY in cmd.update
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
assert len(messages) == 1
assert messages[0].tool_call_id == "call_it_1"
assert messages[0].status == "success"
assert "Typed 'query'" in messages[0].content
@pytest.mark.asyncio
async def test_input_text_callable_execution(mock_driver):
"""Verify invoking input_text directly as a callable."""
result = await input_text(driver=mock_driver, text="call text", target=[100, 100])
assert "Typed 'call text' successfully." in result
@pytest.mark.asyncio
async def test_input_text_invalid_target(mock_ctx):
"""Verify error handling on invalid target coordinates."""
result = await input_text.execute(ctx=mock_ctx, text="bad", target=[])
assert "Error during input text: Invalid target coordinates" in result
@pytest.mark.asyncio
async def test_get_input_text_tool_langchain_ainvoke(mock_ctx):
"""Verify get_input_text_tool exports a LangChain tool that works with ainvoke."""
it_tool = get_input_text_tool(mock_ctx)
assert it_tool.name == "input_text"
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
mock_controller_inst.press_key = AsyncMock(return_value=True)
mock_controller_inst.type_text = AsyncMock(return_value=True)
mock_controller_cls.return_value = mock_controller_inst
result = await it_tool.ainvoke(
{
"text": "abc",
"target": [500, 500],
"clear_exist": False,
}
)
mock_controller_inst.press_key.assert_called_once_with("123")
assert "Typed 'abc' successfully." in result
+92
View File
@@ -0,0 +1,92 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import MagicMock, patch
from artemis.agents.hopper.hopper import HopperOutput
from artemis.context import ArtemisContext
from artemis.tools.mobile.launch_app import find_package
import pytest
@pytest.fixture
def mock_context():
ctx = MagicMock(spec=ArtemisContext)
ctx.package_cache = {}
return ctx
@pytest.mark.asyncio
@patch("artemis.tools.mobile.launch_app.list_packages_async")
@patch("artemis.tools.mobile.launch_app.hopper")
async def test_find_package_cache_hit(mock_hopper, mock_list_packages, mock_context):
mock_list_packages.return_value = "com.example.app1\ncom.example.app2"
# First call: not in cache, calls hopper
mock_hopper.return_value = HopperOutput(
found=True, output="com.example.app1", reason="Found it"
)
pkg1 = await find_package(mock_context, "App 1")
assert pkg1 == "com.example.app1"
assert mock_hopper.call_count == 1
assert mock_context.package_cache["App 1"] == "com.example.app1"
# Second call: cache hit, does not call hopper or list_packages again
mock_hopper.reset_mock()
mock_list_packages.reset_mock()
pkg2 = await find_package(mock_context, "App 1")
assert pkg2 == "com.example.app1"
mock_hopper.assert_not_called()
mock_list_packages.assert_not_called()
@pytest.mark.asyncio
@patch("artemis.tools.mobile.launch_app.list_packages_async")
@patch("artemis.tools.mobile.launch_app.hopper")
async def test_find_package_not_found_caches_none(mock_hopper, mock_list_packages, mock_context):
mock_list_packages.return_value = "com.example.app1\ncom.example.app2"
# Hopper doesn't find it
mock_hopper.return_value = HopperOutput(found=False, output=None, reason="Not found")
pkg1 = await find_package(mock_context, "App 3")
assert pkg1 is None
assert mock_hopper.call_count == 1
assert mock_context.package_cache["App 3"] is None
# Second call: cache hit (None), does not call hopper again
mock_hopper.reset_mock()
pkg2 = await find_package(mock_context, "App 3")
assert pkg2 is None
mock_hopper.assert_not_called()
@pytest.mark.asyncio
@patch("artemis.tools.mobile.launch_app.list_packages_async")
@patch("artemis.tools.mobile.launch_app.hopper")
async def test_find_package_passes_use_fallback(mock_hopper, mock_list_packages, mock_context):
mock_list_packages.return_value = "com.example.app1\ncom.example.app2"
mock_hopper.return_value = HopperOutput(
found=True, output="com.example.app1", reason="Found it"
)
# Pass use_fallback=False
await find_package(mock_context, "App 1", use_fallback=False)
mock_hopper.assert_called_once_with(
ctx=mock_context,
request=("I'm looking for the package name of the following app: 'App 1'"),
data="com.example.app1\ncom.example.app2",
use_fallback=False,
)
+208
View File
@@ -0,0 +1,208 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.mobile.launch_app import (
LaunchApp,
LaunchAppArgs,
LaunchAppTool,
get_launch_app_tool,
launch_app,
launch_app_wrapper,
)
from langgraph.types import Command
import pytest
@pytest.fixture
def mock_ctx():
ctx = MagicMock(spec=ArtemisContext)
ctx.package_cache = {}
return ctx
@pytest.fixture
def mock_driver():
driver = MagicMock(spec=BaseDeviceDriver)
driver.launch_app = AsyncMock(return_value=True)
return driver
@pytest.fixture
def mock_state():
state = MagicMock(spec=State)
async def _mock_asanitize_update(ctx, update, agent):
return update
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
return state
def test_launch_app_tool_subclass_and_registry():
"""Verify LaunchAppTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(LaunchAppTool, ArtemisTool)
assert issubclass(LaunchApp, ArtemisTool)
assert isinstance(launch_app, ArtemisTool)
assert isinstance(launch_app, LaunchAppTool)
assert launch_app.name == "launch_app"
assert launch_app.category == "action"
assert launch_app.args_schema == LaunchAppArgs
# Registry lookup
reg_tool = ToolRegistry.get("launch_app")
assert reg_tool is not None
assert isinstance(reg_tool, LaunchAppTool)
# GenAI FunctionDeclaration export
declaration = launch_app.to_genai_declaration()
assert declaration.name == "launch_app"
assert "app_name" in declaration.parameters.properties
# Wrapper check
assert launch_app_wrapper is not None
assert launch_app_wrapper.tool_fn_getter == get_launch_app_tool
@pytest.mark.asyncio
async def test_launch_app_direct_execution_with_driver(mock_driver):
"""Verify direct execution with BaseDeviceDriver."""
result = await launch_app.execute(driver=mock_driver, app_name="com.android.settings")
mock_driver.launch_app.assert_called_once_with("com.android.settings")
assert "App 'com.android.settings' launched successfully." in result
@pytest.mark.asyncio
async def test_launch_app_direct_execution_with_ctx(mock_ctx):
"""Verify direct execution with ArtemisContext."""
with (
patch(
"artemis.tools.mobile.launch_app.find_package",
new_callable=AsyncMock,
return_value="com.example.myapp",
) as mock_find,
patch(
"artemis.tools.mobile.launch_app.launch_app_with_retries",
new_callable=AsyncMock,
return_value=(True, None),
) as mock_launch,
):
result = await launch_app.execute(ctx=mock_ctx, app_name="My App")
mock_find.assert_called_once_with(ctx=mock_ctx, app_name="My App")
mock_launch.assert_called_once_with(ctx=mock_ctx, app_package="com.example.myapp")
assert "App 'My App' launched successfully." in result
@pytest.mark.asyncio
async def test_launch_app_package_not_found(mock_ctx):
"""Verify outcome when package is not found."""
with patch(
"artemis.tools.mobile.launch_app.find_package",
new_callable=AsyncMock,
return_value=None,
):
result = await launch_app.execute(ctx=mock_ctx, app_name="Nonexistent App")
assert "Failed to launch app 'Nonexistent App': Package not found." in result
@pytest.mark.asyncio
async def test_launch_app_execution_failure(mock_ctx):
"""Verify outcome when launch_app_with_retries fails."""
with (
patch(
"artemis.tools.mobile.launch_app.find_package",
new_callable=AsyncMock,
return_value="com.example.failapp",
),
patch(
"artemis.tools.mobile.launch_app.launch_app_with_retries",
new_callable=AsyncMock,
return_value=(False, "App crashed on startup"),
),
):
result = await launch_app.execute(ctx=mock_ctx, app_name="Crash App")
assert "Failed to launch app 'Crash App': App crashed on startup" in result
@pytest.mark.asyncio
async def test_launch_app_with_state_command(mock_ctx, mock_state):
"""Verify LaunchAppTool returns Command when state is provided."""
with (
patch(
"artemis.tools.mobile.launch_app.find_package",
new_callable=AsyncMock,
return_value="com.example.stateapp",
),
patch(
"artemis.tools.mobile.launch_app.launch_app_with_retries",
new_callable=AsyncMock,
return_value=(True, None),
),
):
cmd = await launch_app.execute(
ctx=mock_ctx,
app_name="State App",
tool_call_id="call_la_1",
state=mock_state,
)
assert isinstance(cmd, Command)
assert VALIDATOR_MESSAGES_KEY in cmd.update
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
assert len(messages) == 1
assert messages[0].tool_call_id == "call_la_1"
assert messages[0].status == "success"
assert "App 'State App' launched successfully." in messages[0].content
@pytest.mark.asyncio
async def test_launch_app_callable_execution(mock_driver):
"""Verify invoking launch_app directly as a callable."""
result = await launch_app(driver=mock_driver, app_name="com.android.chrome")
assert "App 'com.android.chrome' launched successfully." in result
@pytest.mark.asyncio
async def test_launch_app_missing_name(mock_ctx):
"""Verify error handling when app_name is empty."""
result = await launch_app.execute(ctx=mock_ctx, app_name="")
assert "Error during launch app: app_name parameter is required." in result
@pytest.mark.asyncio
async def test_get_launch_app_tool_langchain_ainvoke(mock_ctx):
"""Verify get_launch_app_tool exports a LangChain tool that works with ainvoke."""
la_tool = get_launch_app_tool(mock_ctx)
assert la_tool.name == "launch_app"
with (
patch(
"artemis.tools.mobile.launch_app.find_package",
new_callable=AsyncMock,
return_value="com.example.ainvoke",
),
patch(
"artemis.tools.mobile.launch_app.launch_app_with_retries",
new_callable=AsyncMock,
return_value=(True, None),
),
):
result = await la_tool.ainvoke({"app_name": "Invoke App"})
assert "App 'Invoke App' launched successfully." in result
@@ -0,0 +1,109 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import MagicMock, patch
from artemis.context import ArtemisContext
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.scratchpad import (
ListNotesArgs,
ListNotesPure,
ListNotesPureTool,
get_list_notes_tool_pure,
list_notes_pure,
)
from artemis.utils.notes import save_note_content
import pytest
@pytest.fixture
def mock_ctx(tmp_path):
ctx = MagicMock(spec=ArtemisContext)
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = str(tmp_path)
return ctx
def test_list_notes_pure_tool_subclass_and_registry():
"""Verify ListNotesPureTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(ListNotesPureTool, ArtemisTool)
assert issubclass(ListNotesPure, ArtemisTool)
assert isinstance(list_notes_pure, ArtemisTool)
assert isinstance(list_notes_pure, ListNotesPureTool)
assert list_notes_pure.name == "list_notes_pure"
assert list_notes_pure.category == "memory"
assert list_notes_pure.args_schema == ListNotesArgs
# Registry lookup
reg_tool = ToolRegistry.get("list_notes_pure")
assert reg_tool is not None
assert isinstance(reg_tool, ListNotesPureTool)
# GenAI FunctionDeclaration export
declaration = list_notes_pure.to_genai_declaration()
assert declaration.name == "list_notes_pure"
@pytest.mark.asyncio
async def test_list_notes_pure_direct_execution_empty(mock_ctx):
"""Verify listing notes when no notes are stored."""
result = await list_notes_pure.execute(ctx=mock_ctx)
assert "No notes saved yet." in result
@pytest.mark.asyncio
async def test_list_notes_pure_direct_execution_success(mock_ctx, tmp_path):
"""Verify direct execution of ListNotesPureTool.execute lists stored notes."""
save_note_content(tmp_path, "note_1", "line 1\nline 2")
save_note_content(tmp_path, "note_2", "single line")
result = await list_notes_pure.execute(ctx=mock_ctx)
assert "Here are all the notes:" in result
assert "- note_1 (2 lines)" in result
assert "- note_2 (1 lines)" in result
@pytest.mark.asyncio
async def test_list_notes_pure_callable_execution(mock_ctx, tmp_path):
"""Verify invoking list_notes_pure directly as a callable."""
save_note_content(tmp_path, "callable_note", "line A\nline B")
result = await list_notes_pure(ctx=mock_ctx)
assert "Here are all the notes:" in result
assert "- callable_note (2 lines)" in result
@pytest.mark.asyncio
async def test_list_notes_pure_execution_failure(mock_ctx):
"""Verify error handling when list_notes_pure fails."""
with patch(
"artemis.tools.scratchpad.list_notes_info",
side_effect=RuntimeError("Disk failure"),
):
result = await list_notes_pure.execute(ctx=mock_ctx)
assert "Failed to list notes: Disk failure" in result
@pytest.mark.asyncio
async def test_get_list_notes_tool_pure_langchain_ainvoke(mock_ctx, tmp_path):
"""Verify get_list_notes_tool_pure exports a LangChain tool named 'list_notes'."""
save_note_content(tmp_path, "pure_export_note", "content")
pure_tool = get_list_notes_tool_pure(mock_ctx)
assert pure_tool.name == "list_notes"
result = await pure_tool.ainvoke({})
assert "Here are all the notes:" in result
assert "- pure_export_note (1 lines)" in result
+158
View File
@@ -0,0 +1,158 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.scratchpad import (
ListNotes,
ListNotesArgs,
ListNotesTool,
get_list_notes_tool,
get_list_notes_tool_pure,
list_notes,
list_notes_wrapper,
)
from artemis.utils.notes import save_note_content
from langgraph.types import Command
import pytest
@pytest.fixture
def mock_ctx(tmp_path):
ctx = MagicMock(spec=ArtemisContext)
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = str(tmp_path)
return ctx
def test_list_notes_tool_subclass_and_registry():
"""Verify ListNotesTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(ListNotesTool, ArtemisTool)
assert issubclass(ListNotes, ArtemisTool)
assert isinstance(list_notes, ArtemisTool)
assert isinstance(list_notes, ListNotesTool)
assert list_notes.name == "list_notes"
assert list_notes.category == "memory"
assert list_notes.args_schema == ListNotesArgs
# Registry lookup
reg_tool = ToolRegistry.get("list_notes")
assert reg_tool is not None
assert isinstance(reg_tool, ListNotesTool)
# GenAI FunctionDeclaration export
declaration = list_notes.to_genai_declaration()
assert declaration.name == "list_notes"
# Wrapper check
assert list_notes_wrapper is not None
assert list_notes_wrapper.tool_fn_getter == get_list_notes_tool
@pytest.mark.asyncio
async def test_list_notes_direct_execution_empty(mock_ctx):
"""Verify listing notes when no notes are stored."""
result = await list_notes.execute(ctx=mock_ctx)
assert "No notes saved yet." in result
@pytest.mark.asyncio
async def test_list_notes_direct_execution_success(mock_ctx, tmp_path):
"""Verify direct execution of ListNotesTool.execute lists stored notes."""
save_note_content(tmp_path, "note_alpha", "line 1\nline 2")
save_note_content(tmp_path, "note_beta", "single line")
result = await list_notes.execute(ctx=mock_ctx)
assert "Here are all the notes:" in result
assert "- note_alpha (2 lines)" in result
assert "- note_beta (1 lines)" in result
@pytest.mark.asyncio
async def test_list_notes_callable_execution(mock_ctx, tmp_path):
"""Verify invoking list_notes directly as a callable."""
save_note_content(tmp_path, "note_gamma", "line A\nline B\nline C")
result = await list_notes(ctx=mock_ctx)
assert "Here are all the notes:" in result
assert "- note_gamma (3 lines)" in result
@pytest.mark.asyncio
async def test_list_notes_with_state_command(mock_ctx, tmp_path):
"""Verify ListNotesTool.execute with state returns Command updating VALIDATOR_MESSAGES_KEY."""
save_note_content(tmp_path, "state_note", "content")
mock_state = MagicMock(spec=State)
mock_state.current_agent = "operator"
mock_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update)
result = await list_notes.execute(
ctx=mock_ctx,
state=mock_state,
tool_call_id="call_list_999",
)
assert isinstance(result, Command)
assert VALIDATOR_MESSAGES_KEY in result.update
tool_messages = result.update[VALIDATOR_MESSAGES_KEY]
assert len(tool_messages) == 1
assert tool_messages[0].tool_call_id == "call_list_999"
assert "Here are all the notes:" in tool_messages[0].content
assert "- state_note (1 lines)" in tool_messages[0].content
assert tool_messages[0].status == "success"
mock_state.asanitize_update.assert_called_once()
@pytest.mark.asyncio
async def test_list_notes_execution_failure(mock_ctx):
"""Verify error handling when list_notes fails."""
with patch(
"artemis.tools.scratchpad.list_notes_info",
side_effect=RuntimeError("Disk failure"),
):
result = await list_notes.execute(ctx=mock_ctx)
assert "Failed to list notes: Disk failure" in result
@pytest.mark.asyncio
async def test_get_list_notes_tool_langchain_ainvoke(mock_ctx, tmp_path):
"""Verify get_list_notes_tool exports a LangChain tool that works with ainvoke."""
save_note_content(tmp_path, "lc_note", "hello")
lc_tool = get_list_notes_tool(mock_ctx)
assert lc_tool.name == "list_notes"
result = await lc_tool.ainvoke({})
assert "Here are all the notes:" in result
assert "- lc_note (1 lines)" in result
@pytest.mark.asyncio
async def test_get_list_notes_tool_pure(mock_ctx, tmp_path):
"""Verify get_list_notes_tool_pure returns a pure LangChain tool."""
save_note_content(tmp_path, "pure_note", "world")
pure_tool = get_list_notes_tool_pure(mock_ctx)
assert pure_tool.name == "list_notes"
result = await pure_tool.ainvoke({})
assert "Here are all the notes:" in result
assert "- pure_note (1 lines)" in result
+115
View File
@@ -0,0 +1,115 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.log_tool import (
AnalyzeLogs,
AnalyzeLogsArgs,
AnalyzeLogsTool,
LogTool,
analyze_logs,
analyze_logs_wrapper,
get_analyze_logs_tool,
)
import pytest
@pytest.mark.asyncio
async def test_analyze_logs_tool_execution():
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
mock_state = MagicMock(spec=State)
expected_output = "Found 2 errors in system log: NullPointerException at line 42."
mock_analyst = MagicMock()
mock_analyst.run = AsyncMock(return_value=expected_output)
with patch(
"artemis.tools.log_tool.LogAnalyzerNode",
return_value=mock_analyst,
):
tool = get_analyze_logs_tool(mock_ctx)
result = await tool.ainvoke(
{
"specific_query": "Check for crash exceptions",
"state": mock_state,
}
)
mock_analyst.run.assert_called_once_with("Check for crash exceptions", mock_state)
assert result == expected_output
@pytest.mark.asyncio
async def test_analyze_logs_tool_direct_call():
mock_ctx = MagicMock(spec=ArtemisContext)
mock_state = MagicMock(spec=State)
mock_analyst = MagicMock()
mock_analyst.run = AsyncMock(return_value="Log scan complete.")
with patch(
"artemis.tools.log_tool.LogAnalyzerNode",
return_value=mock_analyst,
):
result = await analyze_logs(
ctx=mock_ctx,
specific_query="Find ANR",
state=mock_state,
)
mock_analyst.run.assert_called_once_with("Find ANR", mock_state)
assert result == "Log scan complete."
@pytest.mark.asyncio
async def test_analyze_logs_tool_no_context():
result = await analyze_logs(
ctx=None,
specific_query="Find crash",
)
assert result == "Error: ArtemisContext is required for LogAnalyzer."
def test_analyze_logs_tool_subclass_and_registry():
"""Verify AnalyzeLogsTool is an ArtemisTool subclass and registered in ToolRegistry."""
assert issubclass(AnalyzeLogsTool, ArtemisTool)
assert issubclass(AnalyzeLogs, ArtemisTool)
assert issubclass(LogTool, ArtemisTool)
assert isinstance(analyze_logs, ArtemisTool)
assert isinstance(analyze_logs, AnalyzeLogsTool)
assert analyze_logs.name == "analyze_logs"
assert analyze_logs.category == "custom"
assert analyze_logs.args_schema == AnalyzeLogsArgs
# Registry lookup
reg_tool = ToolRegistry.get("analyze_logs")
assert reg_tool is not None
assert isinstance(reg_tool, AnalyzeLogsTool)
# GenAI FunctionDeclaration export
declaration = analyze_logs.to_genai_declaration()
assert declaration.name == "analyze_logs"
assert "specific_query" in declaration.parameters.properties
# Wrapper check
assert analyze_logs_wrapper is not None
assert analyze_logs_wrapper.tool_fn_getter == get_analyze_logs_tool
+116
View File
@@ -0,0 +1,116 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from datetime import datetime
from unittest.mock import Mock, patch
from artemis.context import ArtemisContext
from artemis.tools.mobile.log_utils import fetch_and_filter_logs
import pytest
@pytest.fixture
def mock_context():
ctx = Mock(spec=ArtemisContext)
ctx.data_engine = Mock()
ctx.data_engine.session_start_time = 1700000000.0 # Example timestamp
return ctx
@pytest.fixture
def mock_device():
return Mock()
@patch("artemis.tools.mobile.log_utils.get_adb_device")
def test_fetch_and_filter_logs_basic(mock_get_adb_device, mock_context, mock_device):
mock_get_adb_device.return_value = mock_device
mock_device.shell.return_value = "line 1\nline 2\n"
result = fetch_and_filter_logs(mock_context, lines=10)
mock_device.shell.assert_called_once_with("logcat -v threadtime -t 10")
assert result == "line 1\nline 2\n"
@patch("artemis.tools.mobile.log_utils.get_adb_device")
@patch("artemis.tools.mobile.log_utils.resolve_time")
def test_fetch_and_filter_logs_since_time_success(
mock_resolve_time, mock_get_adb_device, mock_context, mock_device
):
mock_get_adb_device.return_value = mock_device
since_dt = datetime(datetime.now().year, 5, 3, 11, 11, 11, 111000)
mock_resolve_time.side_effect = [
("05-03 11:11:11.111", since_dt), # for since_time
(None, None), # for until_time
]
mock_device.shell.return_value = "05-03 11:11:12.000 line 1\n05-03 11:11:13.000 line 2\n"
result = fetch_and_filter_logs(mock_context, since_time="05-03 11:11:11.111")
mock_device.shell.assert_called_once_with('logcat -v threadtime -t "05-03 11:11:11.111"')
assert "line 1" in result
assert "line 2" in result
@patch("artemis.tools.mobile.log_utils.get_adb_device")
@patch("artemis.tools.mobile.log_utils.resolve_time")
def test_fetch_and_filter_logs_since_time_fallback(
mock_resolve_time, mock_get_adb_device, mock_context, mock_device
):
mock_get_adb_device.return_value = mock_device
current_year = datetime.now().year
since_dt = datetime(current_year, 5, 4, 12, 0, 0)
mock_resolve_time.side_effect = [
("05-04 12:00:00.000", since_dt), # for since_time
(None, None), # for until_time
]
# Direct fetch fails
mock_device.shell.side_effect = [
Exception("logcat failed"), # first call to direct fetch
"05-04 11:59:00.000 line old\n05-04 12:00:01.000 line new\n", # second call (fallback)
]
result = fetch_and_filter_logs(mock_context, lines=100, since_time="05-04 12:00:00.000")
assert mock_device.shell.call_count == 2
mock_device.shell.assert_any_call('logcat -v threadtime -t "05-04 12:00:00.000"')
mock_device.shell.assert_any_call("logcat -v threadtime -t 100")
# Should filter out "line old" because it's before since_dt
assert "line old" not in result
assert "line new" in result
@patch("artemis.tools.mobile.log_utils.get_adb_device")
@patch("artemis.tools.mobile.log_utils.resolve_time")
def test_fetch_and_filter_logs_until_time(
mock_resolve_time, mock_get_adb_device, mock_context, mock_device
):
mock_get_adb_device.return_value = mock_device
current_year = datetime.now().year
until_dt = datetime(current_year, 5, 4, 12, 0, 0)
mock_resolve_time.side_effect = [
(None, None), # for since_time
("05-04 12:00:00.000", until_dt), # for until_time
]
mock_device.shell.return_value = "05-04 11:59:00.000 line old\n05-04 12:00:01.000 line new\n"
result = fetch_and_filter_logs(mock_context, lines=100, until_time="05-04 12:00:00.000")
mock_device.shell.assert_called_once_with("logcat -v threadtime -t 100")
assert "line old" in result
assert "line new" not in result
+162
View File
@@ -0,0 +1,162 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.mobile.exec_tools import (
LongPress,
LongPressArgs,
LongPressTool,
get_long_press_tool,
long_press,
long_press_wrapper,
)
from langgraph.types import Command
import pytest
@pytest.fixture
def mock_ctx():
ctx = MagicMock(spec=ArtemisContext)
ctx.device = MagicMock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
return ctx
@pytest.fixture
def mock_driver():
driver = MagicMock(spec=BaseDeviceDriver)
driver.screen_size = (1080, 2400)
driver.long_press = AsyncMock(return_value=True)
return driver
@pytest.fixture
def mock_state():
state = MagicMock(spec=State)
async def _mock_asanitize_update(ctx, update, agent):
return update
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
return state
def test_long_press_tool_subclass_and_registry():
"""Verify LongPressTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(LongPressTool, ArtemisTool)
assert issubclass(LongPress, ArtemisTool)
assert isinstance(long_press, ArtemisTool)
assert isinstance(long_press, LongPressTool)
assert long_press.name == "long_press"
assert long_press.category == "action"
assert long_press.args_schema == LongPressArgs
# Registry lookup
reg_tool = ToolRegistry.get("long_press")
assert reg_tool is not None
assert isinstance(reg_tool, LongPressTool)
# GenAI FunctionDeclaration export
declaration = long_press.to_genai_declaration()
assert declaration.name == "long_press"
assert "target" in declaration.parameters.properties
assert "duration" in declaration.parameters.properties
# Wrapper check
assert long_press_wrapper is not None
assert long_press_wrapper.tool_fn_getter == get_long_press_tool
@pytest.mark.asyncio
async def test_long_press_direct_execution_with_driver(mock_driver):
"""Verify direct execution of LongPressTool with BaseDeviceDriver."""
result = await long_press.execute(driver=mock_driver, target=[500, 500], duration=1500)
mock_driver.long_press.assert_called_once_with(540, 1200, duration_ms=1500)
assert "Long pressed at [500, 500] (normalized) successfully." in result
@pytest.mark.asyncio
async def test_long_press_direct_execution_with_ctx(mock_ctx):
"""Verify direct execution of LongPressTool with ArtemisContext."""
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
mock_controller_cls.return_value = mock_controller_inst
result = await long_press.execute(ctx=mock_ctx, target=[250, 750], duration=2000)
mock_controller_inst.tap_at.assert_called_once_with(
270, 1800, long_press=True, long_press_duration=2000
)
assert "Long pressed at [250, 750] (normalized) successfully." in result
@pytest.mark.asyncio
async def test_long_press_with_state_command(mock_ctx, mock_state):
"""Verify LongPressTool returns Command when state is provided."""
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
mock_controller_cls.return_value = mock_controller_inst
cmd = await long_press.execute(
ctx=mock_ctx,
target=[100, 200],
duration=1200,
tool_call_id="call_lp_1",
state=mock_state,
)
assert isinstance(cmd, Command)
assert VALIDATOR_MESSAGES_KEY in cmd.update
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
assert len(messages) == 1
assert messages[0].tool_call_id == "call_lp_1"
assert messages[0].status == "success"
assert "Long pressed at [100, 200]" in messages[0].content
@pytest.mark.asyncio
async def test_long_press_callable_execution(mock_driver):
"""Verify invoking long_press directly as a callable."""
result = await long_press(driver=mock_driver, target=[100, 100])
assert "Long pressed at [100, 100] (normalized) successfully." in result
@pytest.mark.asyncio
async def test_long_press_invalid_target(mock_ctx):
"""Verify error handling on invalid target coordinates."""
result = await long_press.execute(ctx=mock_ctx, target=[])
assert "Error during long press: Invalid target coordinates" in result
@pytest.mark.asyncio
async def test_get_long_press_tool_langchain_ainvoke(mock_ctx):
"""Verify get_long_press_tool exports a LangChain tool that works with ainvoke."""
lp_tool = get_long_press_tool(mock_ctx)
assert lp_tool.name == "long_press"
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
mock_controller_cls.return_value = mock_controller_inst
result = await lp_tool.ainvoke({"target": [500, 500], "duration": 1500})
assert "Long pressed at [500, 500] (normalized) successfully." in result
@@ -0,0 +1,186 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for ObjectDetectionTool and OperatorObjectDetectionTool."""
import json
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.object_detection_tool import (
ObjectDetection,
ObjectDetectionArgs,
ObjectDetectionTool,
ObjectDetectorTool,
OperatorObjectDetection,
OperatorObjectDetectionArgs,
OperatorObjectDetectionTool,
OperatorObjectDetectorTool,
get_object_detector_tool,
get_operator_object_detector_tool,
object_detection,
object_detection_wrapper,
operator_object_detection,
operator_object_detection_wrapper,
)
import pytest
def test_object_detection_tool_subclass_and_registry():
"""Verify ObjectDetectionTool and OperatorObjectDetectionTool subclass ArtemisTool and register."""
assert issubclass(ObjectDetectionTool, ArtemisTool)
assert issubclass(ObjectDetection, ArtemisTool)
assert issubclass(ObjectDetectorTool, ArtemisTool)
assert issubclass(OperatorObjectDetectionTool, ArtemisTool)
assert issubclass(OperatorObjectDetectionTool, ObjectDetectionTool)
assert issubclass(OperatorObjectDetection, ArtemisTool)
assert issubclass(OperatorObjectDetectorTool, ArtemisTool)
assert isinstance(object_detection, ArtemisTool)
assert isinstance(object_detection, ObjectDetectionTool)
assert isinstance(operator_object_detection, ArtemisTool)
assert isinstance(operator_object_detection, OperatorObjectDetectionTool)
# Properties
assert object_detection.name == "object_detection"
assert object_detection.category == "perception"
assert object_detection.args_schema == ObjectDetectionArgs
assert operator_object_detection.name == "object_detection"
assert operator_object_detection.category == "perception"
assert operator_object_detection.args_schema == OperatorObjectDetectionArgs
# Registry lookup
reg_tool = ToolRegistry.get("object_detection")
assert reg_tool is not None
assert isinstance(reg_tool, ObjectDetectionTool)
# GenAI FunctionDeclaration export
declaration = object_detection.to_genai_declaration()
assert declaration.name == "object_detection"
assert "image_path" in declaration.parameters.properties
assert "queries" in declaration.parameters.properties
op_declaration = operator_object_detection.to_genai_declaration()
assert op_declaration.name == "object_detection"
assert "queries" in op_declaration.parameters.properties
# Wrappers
assert object_detection_wrapper.tool_fn_getter == get_object_detector_tool
assert operator_object_detection_wrapper.tool_fn_getter == get_operator_object_detector_tool
@pytest.mark.asyncio
async def test_object_detection_direct_execution_success():
"""Verify direct async call to object_detection with image_path."""
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
expected_result = {
"detected": [{"label": "button", "point": [100, 200]}],
"failed": [],
}
with patch(
"artemis.tools.object_detection_tool._run_object_detection",
AsyncMock(return_value=expected_result),
) as mock_run:
result = await object_detection(
ctx=mock_ctx,
image_path="/path/to/screenshot.jpg",
queries=["button"],
)
mock_run.assert_called_once()
assert json.loads(result) == expected_result
@pytest.mark.asyncio
async def test_operator_object_detection_direct_execution_with_state():
"""Verify operator_object_detection reads screenshot from state."""
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
mock_state = MagicMock(spec=State)
mock_state.latest_screenshot = "/path/to/state_screen.jpg"
mock_state.asanitize_update = None # Non-langgraph direct call
expected_result = {
"detected": [{"label": "search bar", "point": [500, 100]}],
"failed": [],
}
with patch(
"artemis.tools.object_detection_tool._run_object_detection",
AsyncMock(return_value=expected_result),
) as mock_run:
result = await operator_object_detection(
ctx=mock_ctx,
state=mock_state,
queries=["search bar"],
)
mock_run.assert_called_once()
# Verify it used the latest_screenshot from state
call_args = mock_run.call_args[0]
assert call_args[1] == "/path/to/state_screen.jpg"
assert json.loads(result) == expected_result
@pytest.mark.asyncio
async def test_object_detection_no_image_error():
"""Verify error when no image path is given and no state screenshot is available."""
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
result = await object_detection(
ctx=mock_ctx,
image_path=None,
queries=["icon"],
)
assert "Object Detection failed" in result
assert "No image path provided" in result
@pytest.mark.asyncio
async def test_object_detection_exception_handling():
"""Verify graceful handling when _run_object_detection raises an exception."""
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
with patch(
"artemis.tools.object_detection_tool._run_object_detection",
AsyncMock(side_effect=RuntimeError("Model inference timeout")),
):
result = await object_detection(
ctx=mock_ctx,
image_path="/path/to/screen.jpg",
queries=["icon"],
)
assert "Object Detection failed: Model inference timeout" in result
def test_get_langchain_tools():
"""Verify LangChain BaseTool exports."""
mock_ctx = MagicMock(spec=ArtemisContext)
tool = get_object_detector_tool(mock_ctx)
op_tool = get_operator_object_detector_tool(mock_ctx)
assert tool.name == "object_detection"
assert tool.args_schema == ObjectDetectionArgs
assert op_tool.name == "object_detection"
assert op_tool.args_schema == OperatorObjectDetectionArgs
@@ -0,0 +1,70 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.tools.object_detection_tool import get_operator_object_detector_tool
import pytest
@pytest.mark.asyncio
async def test_object_detection_trace():
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_state = MagicMock(spec=State)
mock_state.latest_screenshot = "/path/to/screenshot.jpg"
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
# We need to mock the decorator in object_detection_tool.py if we want to avoid it calling real get_llm
# Or we can just mock get_llm.
# In the tool definition: @trace(type="tool", ctx=ctx)
# This calls record_trace immediately on entry/exit.
mock_tool = get_operator_object_detector_tool(mock_ctx)
# Mock the internal function called by the tool
with patch(
"artemis.tools.object_detection_tool._run_object_detection",
AsyncMock(return_value="{}"),
) as mock_run:
# We need to handle the fact that ainvoke might fail if LangChain internals are not fully mocked.
# But let's try it.
await mock_tool.ainvoke(
{
"type": "tool_call",
"name": "object_detection",
"id": "call_123",
"args": {"queries": ["test_label"], "state": mock_state},
}
)
# Verify _run_object_detection was called
mock_run.assert_called_once()
# Verify record_trace was called for the span
assert mock_ctx.data_engine.record_trace.call_count >= 1
calls = mock_ctx.data_engine.record_trace.call_args_list
span_called = False
for call in calls:
kwargs = call.kwargs
if kwargs.get("type") == "span" and kwargs.get("name") == "run_object_detection":
span_called = True
break
assert span_called, "TraceSpan 'run_object_detection' was not called"
+238
View File
@@ -0,0 +1,238 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import base64
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.mobile.ocr import (
OCRRecognition,
OcrArgs,
OcrRecognition,
OcrRecognitionTool,
OcrTool,
get_ocr_tool,
ocr_recognition,
ocr_recognition_wrapper,
)
from langgraph.types import Command
import pytest
@pytest.fixture
def mock_ctx():
ctx = MagicMock(spec=ArtemisContext)
ctx.device = MagicMock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
return ctx
@pytest.fixture
def mock_driver():
driver = MagicMock(spec=BaseDeviceDriver)
driver.screen_size = (1080, 2400)
screen_data = MagicMock()
screen_data.screenshot_base64 = base64.b64encode(b"dummy_image_data").decode("utf-8")
screen_data.screenshot_bytes = b"dummy_image_data"
driver.get_screen_data = AsyncMock(return_value=screen_data)
return driver
@pytest.fixture
def mock_state(tmp_path):
screenshot_file = tmp_path / "test_screenshot.png"
screenshot_file.write_bytes(b"dummy_image_content")
state = MagicMock(spec=State)
state.latest_screenshot = str(screenshot_file)
async def _mock_asanitize_update(ctx, update, agent):
return update
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
return state
def test_ocr_tool_subclass_and_registry():
"""Verify OcrRecognitionTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(OcrRecognitionTool, ArtemisTool)
assert issubclass(OcrRecognition, ArtemisTool)
assert issubclass(OCRRecognition, ArtemisTool)
assert issubclass(OcrTool, ArtemisTool)
assert isinstance(ocr_recognition, ArtemisTool)
assert isinstance(ocr_recognition, OcrRecognitionTool)
assert ocr_recognition.name == "ocr_recognition"
assert ocr_recognition.category == "explorer"
assert ocr_recognition.args_schema == OcrArgs
# Registry lookup
reg_tool = ToolRegistry.get("ocr_recognition")
assert reg_tool is not None
assert isinstance(reg_tool, OcrRecognitionTool)
# GenAI FunctionDeclaration export
declaration = ocr_recognition.to_genai_declaration()
assert declaration.name == "ocr_recognition"
# Wrapper check
assert ocr_recognition_wrapper is not None
assert ocr_recognition_wrapper.tool_fn_getter == get_ocr_tool
class SimpleState:
def __init__(self, screenshot_path: str):
self.latest_screenshot = screenshot_path
@pytest.mark.asyncio
async def test_ocr_direct_execution_with_state(mock_ctx, tmp_path):
"""Verify direct execution with state and screenshot file."""
screenshot_file = tmp_path / "test_screenshot.png"
screenshot_file.write_bytes(b"dummy_image_content")
simple_state = SimpleState(str(screenshot_file))
mock_results = [
{
"text": "Submit",
"position": [
{"x": 100, "y": 200},
{"x": 200, "y": 200},
{"x": 200, "y": 300},
{"x": 100, "y": 300},
],
}
]
with patch(
"artemis.tools.mobile.ocr.perform_ocr",
new_callable=AsyncMock,
return_value=mock_results,
):
result = await ocr_recognition.execute(ctx=mock_ctx, state=simple_state)
assert "OCR Recognition successful." in result
assert "Submit" in result
@pytest.mark.asyncio
async def test_ocr_direct_execution_with_driver(mock_ctx, mock_driver):
"""Verify direct execution with driver."""
mock_results = [
{
"text": "Settings",
"position": [
{"x": 500, "y": 500},
{"x": 600, "y": 600},
],
}
]
with patch(
"artemis.tools.mobile.ocr.perform_ocr",
new_callable=AsyncMock,
return_value=mock_results,
):
result = await ocr_recognition.execute(ctx=mock_ctx, driver=mock_driver)
assert "OCR Recognition successful." in result
assert "Settings" in result
@pytest.mark.asyncio
async def test_ocr_no_text_detected(mock_ctx, tmp_path):
"""Verify outcome when no text is detected."""
screenshot_file = tmp_path / "test_empty_screenshot.png"
screenshot_file.write_bytes(b"dummy_empty_content")
simple_state = SimpleState(str(screenshot_file))
with patch(
"artemis.tools.mobile.ocr.perform_ocr",
new_callable=AsyncMock,
return_value=[],
):
result = await ocr_recognition.execute(ctx=mock_ctx, state=simple_state)
assert "No text detected on the screen." in result
@pytest.mark.asyncio
async def test_ocr_with_state_command(mock_ctx, mock_state):
"""Verify OcrRecognitionTool returns Command when state is provided."""
mock_results = [{"text": "Hello", "position": [{"x": 100, "y": 100}]}]
with patch(
"artemis.tools.mobile.ocr.perform_ocr",
new_callable=AsyncMock,
return_value=mock_results,
):
cmd = await ocr_recognition.execute(
ctx=mock_ctx,
tool_call_id="call_ocr_1",
state=mock_state,
)
assert isinstance(cmd, Command)
assert VALIDATOR_MESSAGES_KEY in cmd.update
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
assert len(messages) == 1
assert messages[0].tool_call_id == "call_ocr_1"
assert messages[0].status == "success"
assert "Hello" in messages[0].content
@pytest.mark.asyncio
async def test_ocr_failure_missing_screenshot(mock_ctx):
"""Verify error handling when screenshot path is missing."""
empty_state = MagicMock(spec=State)
empty_state.latest_screenshot = None
empty_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update)
cmd = await ocr_recognition.execute(
ctx=mock_ctx,
tool_call_id="call_ocr_err",
state=empty_state,
)
assert isinstance(cmd, Command)
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
assert messages[0].status == "error"
assert "OCR Recognition failed" in messages[0].content
@pytest.mark.asyncio
async def test_ocr_callable_execution(mock_ctx, mock_driver):
"""Verify invoking ocr_recognition directly as a callable."""
with patch(
"artemis.tools.mobile.ocr.perform_ocr",
new_callable=AsyncMock,
return_value=[{"text": "Callable", "position": []}],
):
result = await ocr_recognition(ctx=mock_ctx, driver=mock_driver)
assert "OCR Recognition successful." in result
assert "Callable" in result
@pytest.mark.asyncio
async def test_get_ocr_tool_langchain_ainvoke(mock_ctx, mock_driver):
"""Verify get_ocr_tool exports a LangChain tool that works with ainvoke."""
mock_ctx._active_driver = mock_driver
ocr_tool = get_ocr_tool(mock_ctx)
assert ocr_tool.name == "ocr_recognition"
with patch(
"artemis.tools.mobile.ocr.perform_ocr",
new_callable=AsyncMock,
return_value=[{"text": "LangChain", "position": []}],
):
result = await ocr_tool.ainvoke({})
assert "OCR Recognition successful." in result
assert "LangChain" in result
+174
View File
@@ -0,0 +1,174 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.mobile.exec_tools import (
PressKey,
PressKeyArgs,
PressKeyTool,
get_press_key_tool,
press_key,
press_key_wrapper,
)
from langgraph.types import Command
import pytest
@pytest.fixture
def mock_ctx():
ctx = MagicMock(spec=ArtemisContext)
ctx.device = MagicMock()
return ctx
@pytest.fixture
def mock_driver():
driver = MagicMock(spec=BaseDeviceDriver)
driver.press_key = AsyncMock(return_value=True)
return driver
@pytest.fixture
def mock_state():
state = MagicMock(spec=State)
async def _mock_asanitize_update(ctx, update, agent):
return update
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
return state
def test_press_key_tool_subclass_and_registry():
"""Verify PressKeyTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(PressKeyTool, ArtemisTool)
assert issubclass(PressKey, ArtemisTool)
assert isinstance(press_key, ArtemisTool)
assert isinstance(press_key, PressKeyTool)
assert press_key.name == "press_key"
assert press_key.category == "action"
assert press_key.args_schema == PressKeyArgs
# Registry lookup
reg_tool = ToolRegistry.get("press_key")
assert reg_tool is not None
assert isinstance(reg_tool, PressKeyTool)
# GenAI FunctionDeclaration export
declaration = press_key.to_genai_declaration()
assert declaration.name == "press_key"
assert "key" in declaration.parameters.properties
# Wrapper check
assert press_key_wrapper is not None
assert press_key_wrapper.tool_fn_getter == get_press_key_tool
@pytest.mark.asyncio
async def test_press_key_direct_execution_with_driver(mock_driver):
"""Verify direct execution with BaseDeviceDriver."""
result = await press_key.execute(driver=mock_driver, key="BACK")
mock_driver.press_key.assert_called_once_with("BACK")
assert "Pressed key 'BACK' successfully." in result
@pytest.mark.asyncio
async def test_press_key_direct_execution_with_ctx(mock_ctx):
"""Verify direct execution with ArtemisContext for various keys."""
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.press_enter = AsyncMock(return_value=True)
mock_controller_inst.go_back = AsyncMock(return_value=True)
mock_controller_inst.go_home = AsyncMock(return_value=True)
mock_controller_inst.press_key = AsyncMock(return_value=True)
mock_controller_cls.return_value = mock_controller_inst
# Test ENTER
res_enter = await press_key.execute(ctx=mock_ctx, key="ENTER")
mock_controller_inst.press_enter.assert_called_once()
assert "Pressed key 'ENTER' successfully." in res_enter
# Test BACK
res_back = await press_key.execute(ctx=mock_ctx, key="BACK")
mock_controller_inst.go_back.assert_called_once()
assert "Pressed key 'BACK' successfully." in res_back
# Test HOME
res_home = await press_key.execute(ctx=mock_ctx, key="HOME")
mock_controller_inst.go_home.assert_called_once()
assert "Pressed key 'HOME' successfully." in res_home
# Test APP_SWITCH
res_app_switch = await press_key.execute(ctx=mock_ctx, key="APP_SWITCH")
mock_controller_inst.press_key.assert_called_with("KEYCODE_APP_SWITCH")
assert "Pressed key 'APP_SWITCH' successfully." in res_app_switch
@pytest.mark.asyncio
async def test_press_key_with_state_command(mock_ctx, mock_state):
"""Verify PressKeyTool returns Command when state is provided."""
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.go_home = AsyncMock(return_value=True)
mock_controller_cls.return_value = mock_controller_inst
cmd = await press_key.execute(
ctx=mock_ctx,
key="HOME",
tool_call_id="call_pk_1",
state=mock_state,
)
assert isinstance(cmd, Command)
assert VALIDATOR_MESSAGES_KEY in cmd.update
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
assert len(messages) == 1
assert messages[0].tool_call_id == "call_pk_1"
assert messages[0].status == "success"
assert "Pressed key 'HOME' successfully." in messages[0].content
@pytest.mark.asyncio
async def test_press_key_callable_execution(mock_driver):
"""Verify invoking press_key directly as a callable."""
result = await press_key(driver=mock_driver, key="ENTER")
assert "Pressed key 'ENTER' successfully." in result
@pytest.mark.asyncio
async def test_press_key_missing_key(mock_ctx):
"""Verify error handling when key is empty or missing."""
result = await press_key.execute(ctx=mock_ctx, key="")
assert "Error during press key: Key parameter is required." in result
@pytest.mark.asyncio
async def test_get_press_key_tool_langchain_ainvoke(mock_ctx):
"""Verify get_press_key_tool exports a LangChain tool that works with ainvoke."""
pk_tool = get_press_key_tool(mock_ctx)
assert pk_tool.name == "press_key"
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.go_back = AsyncMock(return_value=True)
mock_controller_cls.return_value = mock_controller_inst
result = await pk_tool.ainvoke({"key": "BACK"})
assert "Pressed key 'BACK' successfully." in result
@@ -0,0 +1,145 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, MagicMock
from artemis.context import ArtemisContext
from artemis.drivers.base import BaseDeviceDriver
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.mobile.read_hierarchy import (
GetUIHierarchy,
GetUiHierarchy,
GetUiHierarchyArgs,
GetUiHierarchyTool,
get_ui_hierarchy,
get_ui_hierarchy_tool,
ui_hierarchy_wrapper,
)
import pytest
@pytest.fixture
def mock_ctx():
ctx = MagicMock(spec=ArtemisContext)
ctx.ui_adb_client = MagicMock()
ctx.ui_adb_client.get_hierarchy.return_value = "<hierarchy><node text='Home'/></hierarchy>"
return ctx
@pytest.fixture
def mock_driver():
driver = MagicMock(spec=BaseDeviceDriver)
screen_data = MagicMock()
screen_data.ui_hierarchy_xml = "<hierarchy><node text='Settings'/></hierarchy>"
driver.get_screen_data = AsyncMock(return_value=screen_data)
return driver
def test_get_ui_hierarchy_tool_subclass_and_registry():
"""Verify GetUiHierarchyTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(GetUiHierarchyTool, ArtemisTool)
assert issubclass(GetUiHierarchy, ArtemisTool)
assert issubclass(GetUIHierarchy, ArtemisTool)
assert isinstance(get_ui_hierarchy, ArtemisTool)
assert isinstance(get_ui_hierarchy, GetUiHierarchyTool)
assert get_ui_hierarchy.name == "get_ui_hierarchy"
assert get_ui_hierarchy.category == "diagnostic"
assert get_ui_hierarchy.args_schema == GetUiHierarchyArgs
# Registry lookup
reg_tool = ToolRegistry.get("get_ui_hierarchy")
assert reg_tool is not None
assert isinstance(reg_tool, GetUiHierarchyTool)
# GenAI FunctionDeclaration export
declaration = get_ui_hierarchy.to_genai_declaration()
assert declaration.name == "get_ui_hierarchy"
# Wrapper check
assert ui_hierarchy_wrapper is not None
assert ui_hierarchy_wrapper.tool_fn_getter == get_ui_hierarchy_tool
@pytest.mark.asyncio
async def test_get_ui_hierarchy_direct_execution_with_ctx(mock_ctx):
"""Verify direct execution with ArtemisContext."""
result = await get_ui_hierarchy.execute(ctx=mock_ctx)
mock_ctx.ui_adb_client.get_hierarchy.assert_called_once()
assert "<hierarchy><node text='Home'/></hierarchy>" in result
@pytest.mark.asyncio
async def test_get_ui_hierarchy_direct_execution_with_driver_screen_data(mock_driver):
"""Verify direct execution with driver's get_screen_data."""
result = await get_ui_hierarchy.execute(driver=mock_driver)
mock_driver.get_screen_data.assert_called_once()
assert "<hierarchy><node text='Settings'/></hierarchy>" in result
@pytest.mark.asyncio
async def test_get_ui_hierarchy_direct_execution_with_driver_get_hierarchy():
"""Verify direct execution with driver's get_ui_hierarchy."""
driver = MagicMock(spec=BaseDeviceDriver)
del driver.get_screen_data
driver.get_ui_hierarchy = AsyncMock(
return_value="<hierarchy><node text='DirectDriver'/></hierarchy>"
)
result = await get_ui_hierarchy.execute(driver=driver)
driver.get_ui_hierarchy.assert_called_once()
assert "<hierarchy><node text='DirectDriver'/></hierarchy>" in result
@pytest.mark.asyncio
async def test_get_ui_hierarchy_with_state(mock_ctx):
"""Verify GetUiHierarchyTool returns string when state is provided."""
state = MagicMock()
result = await get_ui_hierarchy.execute(
ctx=mock_ctx,
tool_call_id="call_hier_1",
state=state,
)
assert "<hierarchy><node text='Home'/></hierarchy>" in result
@pytest.mark.asyncio
async def test_get_ui_hierarchy_no_client_or_driver():
"""Verify error handling when no client or driver is provided."""
result = await get_ui_hierarchy.execute()
assert "Error retrieving UI hierarchy: No UI Automator client or driver provided." in result
@pytest.mark.asyncio
async def test_get_ui_hierarchy_exception_handling(mock_ctx):
"""Verify error handling when ui_adb_client raises an exception."""
mock_ctx.ui_adb_client.get_hierarchy.side_effect = RuntimeError("ADB server died")
result = await get_ui_hierarchy.execute(ctx=mock_ctx)
assert "Error retrieving UI hierarchy: ADB server died" in result
@pytest.mark.asyncio
async def test_get_ui_hierarchy_callable_execution(mock_ctx):
"""Verify invoking get_ui_hierarchy directly as a callable."""
result = await get_ui_hierarchy(ctx=mock_ctx)
assert "<node text='Home'/>" in result
@pytest.mark.asyncio
async def test_get_ui_hierarchy_tool_langchain_ainvoke(mock_ctx):
"""Verify get_ui_hierarchy_tool exports a LangChain tool that works with ainvoke."""
tool = get_ui_hierarchy_tool(mock_ctx)
assert tool.name == "get_ui_hierarchy"
result = await tool.ainvoke({})
assert "<node text='Home'/>" in result
+140
View File
@@ -0,0 +1,140 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import MagicMock, patch
from artemis.context import ArtemisContext
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.mobile.read_logs import (
ReadLogs,
ReadLogsArgs,
ReadLogsTool,
ReadLogsToolAlias,
get_read_logs_tool,
read_logs,
)
import pytest
@pytest.fixture
def mock_ctx():
ctx = MagicMock(spec=ArtemisContext)
return ctx
def test_read_logs_tool_subclass_and_registry():
"""Verify ReadLogsTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(ReadLogsTool, ArtemisTool)
assert issubclass(ReadLogs, ArtemisTool)
assert issubclass(ReadLogsToolAlias, ArtemisTool)
assert isinstance(read_logs, ArtemisTool)
assert isinstance(read_logs, ReadLogsTool)
assert read_logs.name == "read_logs"
assert read_logs.category == "diagnostic"
assert read_logs.args_schema == ReadLogsArgs
# Registry lookup
reg_tool = ToolRegistry.get("read_logs")
assert reg_tool is not None
assert isinstance(reg_tool, ReadLogsTool)
# GenAI FunctionDeclaration export
declaration = read_logs.to_genai_declaration()
assert declaration.name == "read_logs"
assert "lines" in declaration.parameters.properties
assert "since_time" in declaration.parameters.properties
assert "until_time" in declaration.parameters.properties
@pytest.mark.asyncio
async def test_read_logs_direct_execution(mock_ctx):
"""Verify direct execution of ReadLogsTool."""
mock_log_output = "08-12 10:00:00.000 1000 1000 I Test: Hello Logs\n"
with patch(
"artemis.tools.mobile.read_logs.fetch_and_filter_logs",
return_value=mock_log_output,
) as mock_fetch:
result = await read_logs.execute(
ctx=mock_ctx,
lines=100,
since_time="08-12 09:59:00",
until_time="08-12 10:01:00",
)
mock_fetch.assert_called_once_with(
ctx=mock_ctx,
lines=100,
since_time="08-12 09:59:00",
until_time="08-12 10:01:00",
)
assert result == mock_log_output
@pytest.mark.asyncio
async def test_read_logs_default_lines(mock_ctx):
"""Verify read_logs defaults lines to 200 when None."""
with patch(
"artemis.tools.mobile.read_logs.fetch_and_filter_logs",
return_value="logs",
) as mock_fetch:
result = await read_logs.execute(ctx=mock_ctx, lines=None)
mock_fetch.assert_called_once_with(
ctx=mock_ctx,
lines=200,
since_time=None,
until_time=None,
)
assert result == "logs"
@pytest.mark.asyncio
async def test_read_logs_exception_handling(mock_ctx):
"""Verify error handling when fetching logs raises an exception."""
with patch(
"artemis.tools.mobile.read_logs.fetch_and_filter_logs",
side_effect=RuntimeError("Device disconnected"),
):
result = await read_logs.execute(ctx=mock_ctx)
assert "Failed to read logs: Device disconnected" in result
@pytest.mark.asyncio
async def test_read_logs_callable_execution(mock_ctx):
"""Verify invoking read_logs directly as a callable."""
with patch(
"artemis.tools.mobile.read_logs.fetch_and_filter_logs",
return_value="sample log stream",
):
result = await read_logs(ctx=mock_ctx, lines=50)
assert result == "sample log stream"
@pytest.mark.asyncio
async def test_get_read_logs_tool_langchain_ainvoke(mock_ctx):
"""Verify get_read_logs_tool exports a LangChain tool that works with ainvoke."""
tool = get_read_logs_tool(mock_ctx)
assert tool.name == "read_logs"
with patch(
"artemis.tools.mobile.read_logs.fetch_and_filter_logs",
return_value="langchain log content",
) as mock_fetch:
result = await tool.ainvoke({"lines": 300})
mock_fetch.assert_called_once_with(
ctx=mock_ctx,
lines=300,
since_time=None,
until_time=None,
)
assert result == "langchain log content"
@@ -0,0 +1,127 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import MagicMock, patch
from artemis.context import ArtemisContext
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.scratchpad import (
ReadNoteArgs,
ReadNotePure,
ReadNotePureTool,
get_read_note_tool_pure,
read_note_pure,
)
from artemis.utils.notes import save_note_content
import pytest
@pytest.fixture
def mock_ctx(tmp_path):
ctx = MagicMock(spec=ArtemisContext)
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = str(tmp_path)
return ctx
def test_read_note_pure_tool_subclass_and_registry():
"""Verify ReadNotePureTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(ReadNotePureTool, ArtemisTool)
assert issubclass(ReadNotePure, ArtemisTool)
assert isinstance(read_note_pure, ArtemisTool)
assert isinstance(read_note_pure, ReadNotePureTool)
assert read_note_pure.name == "read_note_pure"
assert read_note_pure.category == "memory"
assert read_note_pure.args_schema == ReadNoteArgs
# Registry lookup
reg_tool = ToolRegistry.get("read_note_pure")
assert reg_tool is not None
assert isinstance(reg_tool, ReadNotePureTool)
# GenAI FunctionDeclaration export
declaration = read_note_pure.to_genai_declaration()
assert declaration.name == "read_note_pure"
assert "key" in declaration.parameters.properties
assert "start_line" in declaration.parameters.properties
assert "end_line" in declaration.parameters.properties
@pytest.mark.asyncio
async def test_read_note_pure_direct_execution_success(mock_ctx, tmp_path):
"""Verify direct execution of ReadNotePureTool.execute reads saved note."""
save_note_content(tmp_path, "sample_pure", "line 1\nline 2\nline 3")
result = await read_note_pure.execute(ctx=mock_ctx, key="sample_pure")
assert "Successfully read note 'sample_pure'." in result
assert "line 1\nline 2\nline 3" in result
@pytest.mark.asyncio
async def test_read_note_pure_slice(mock_ctx, tmp_path):
"""Verify reading a line slice with start_line and end_line via ReadNotePureTool."""
save_note_content(tmp_path, "slice_pure", "line 1\nline 2\nline 3\nline 4")
result = await read_note_pure.execute(
ctx=mock_ctx,
key="slice_pure",
start_line=2,
end_line=3,
)
assert "Successfully read note 'slice_pure' (lines 2 to 3)." in result
assert "line 1" not in result
assert "line 2\nline 3" in result
assert "line 4" not in result
@pytest.mark.asyncio
async def test_read_note_pure_callable_execution(mock_ctx, tmp_path):
"""Verify invoking read_note_pure directly as a callable."""
save_note_content(tmp_path, "callable_pure", "hello from pure callable")
result = await read_note_pure(ctx=mock_ctx, key="callable_pure")
assert "Successfully read note 'callable_pure'." in result
assert "hello from pure callable" in result
@pytest.mark.asyncio
async def test_read_note_pure_not_found(mock_ctx):
"""Verify error handling when note is missing."""
result = await read_note_pure.execute(ctx=mock_ctx, key="nonexistent_pure")
assert "Note 'nonexistent_pure' not found" in result
@pytest.mark.asyncio
async def test_read_note_pure_other_error(mock_ctx):
"""Verify error handling when read_note_content raises an exception."""
with patch(
"artemis.tools.scratchpad.read_note_content",
side_effect=OSError("Disk error"),
):
result = await read_note_pure.execute(ctx=mock_ctx, key="err_note")
assert "Failed to read note 'err_note': Disk error" in result
@pytest.mark.asyncio
async def test_get_read_note_tool_pure_langchain_ainvoke(mock_ctx, tmp_path):
"""Verify get_read_note_tool_pure exports a LangChain tool named 'read_note'."""
save_note_content(tmp_path, "pure_export", "exported pure note")
pure_tool = get_read_note_tool_pure(mock_ctx)
assert pure_tool.name == "read_note"
result = await pure_tool.ainvoke({"key": "pure_export"})
assert "Successfully read note 'pure_export'." in result
assert "exported pure note" in result
+177
View File
@@ -0,0 +1,177 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.scratchpad import (
ReadNote,
ReadNoteArgs,
ReadNoteTool,
get_read_note_tool,
get_read_note_tool_pure,
read_note,
read_note_wrapper,
)
from artemis.utils.notes import save_note_content
from langgraph.types import Command
import pytest
@pytest.fixture
def mock_ctx(tmp_path):
ctx = MagicMock(spec=ArtemisContext)
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = str(tmp_path)
return ctx
def test_read_note_tool_subclass_and_registry():
"""Verify ReadNoteTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(ReadNoteTool, ArtemisTool)
assert issubclass(ReadNote, ArtemisTool)
assert isinstance(read_note, ArtemisTool)
assert isinstance(read_note, ReadNoteTool)
assert read_note.name == "read_note"
assert read_note.category == "memory"
assert read_note.args_schema == ReadNoteArgs
# Registry lookup
reg_tool = ToolRegistry.get("read_note")
assert reg_tool is not None
assert isinstance(reg_tool, ReadNoteTool)
# GenAI FunctionDeclaration export
declaration = read_note.to_genai_declaration()
assert declaration.name == "read_note"
assert "key" in declaration.parameters.properties
assert "start_line" in declaration.parameters.properties
assert "end_line" in declaration.parameters.properties
# Wrapper check
assert read_note_wrapper is not None
assert read_note_wrapper.tool_fn_getter == get_read_note_tool
@pytest.mark.asyncio
async def test_read_note_direct_execution_success(mock_ctx, tmp_path):
"""Verify direct execution of ReadNoteTool.execute reads saved note."""
save_note_content(tmp_path, "sample", "line 1\nline 2\nline 3\nline 4\nline 5")
result = await read_note.execute(ctx=mock_ctx, key="sample")
assert "Successfully read note 'sample'." in result
assert "line 1\nline 2\nline 3\nline 4\nline 5" in result
@pytest.mark.asyncio
async def test_read_note_slice(mock_ctx, tmp_path):
"""Verify reading a line slice with start_line and end_line."""
save_note_content(tmp_path, "sample", "line 1\nline 2\nline 3\nline 4\nline 5")
result = await read_note.execute(
ctx=mock_ctx,
key="sample",
start_line=2,
end_line=4,
)
assert "Successfully read note 'sample' (lines 2 to 4)." in result
assert "line 1" not in result
assert "line 2\nline 3\nline 4" in result
assert "line 5" not in result
@pytest.mark.asyncio
async def test_read_note_callable_execution(mock_ctx, tmp_path):
"""Verify invoking read_note directly as a callable."""
save_note_content(tmp_path, "callable_sample", "hello from callable")
result = await read_note(ctx=mock_ctx, key="callable_sample")
assert "Successfully read note 'callable_sample'." in result
assert "hello from callable" in result
@pytest.mark.asyncio
async def test_read_note_with_state_command(mock_ctx, tmp_path):
"""Verify ReadNoteTool.execute with state returns Command updating VALIDATOR_MESSAGES_KEY."""
save_note_content(tmp_path, "state_sample", "state note content")
mock_state = MagicMock(spec=State)
mock_state.current_agent = "operator"
mock_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update)
result = await read_note.execute(
ctx=mock_ctx,
key="state_sample",
state=mock_state,
tool_call_id="call_read_999",
)
assert isinstance(result, Command)
assert VALIDATOR_MESSAGES_KEY in result.update
tool_messages = result.update[VALIDATOR_MESSAGES_KEY]
assert len(tool_messages) == 1
assert tool_messages[0].tool_call_id == "call_read_999"
assert "Successfully read note 'state_sample'." in tool_messages[0].content
assert "state note content" in tool_messages[0].content
assert tool_messages[0].status == "success"
mock_state.asanitize_update.assert_called_once()
@pytest.mark.asyncio
async def test_read_note_not_found(mock_ctx):
"""Verify error handling when a note is not found."""
result = await read_note.execute(ctx=mock_ctx, key="nonexistent")
assert "Note 'nonexistent' not found" in result
@pytest.mark.asyncio
async def test_read_note_other_error(mock_ctx):
"""Verify error handling for unexpected exceptions."""
with patch(
"artemis.tools.scratchpad.read_note_content",
side_effect=ValueError("Corrupted data"),
):
result = await read_note.execute(ctx=mock_ctx, key="bad_note")
assert "Failed to read note 'bad_note': Corrupted data" in result
@pytest.mark.asyncio
async def test_get_read_note_tool_langchain_ainvoke(mock_ctx, tmp_path):
"""Verify get_read_note_tool exports a LangChain tool that works with ainvoke."""
save_note_content(tmp_path, "lc_sample", "langchain content")
lc_tool = get_read_note_tool(mock_ctx)
assert lc_tool.name == "read_note"
result = await lc_tool.ainvoke({"key": "lc_sample"})
assert "Successfully read note 'lc_sample'." in result
assert "langchain content" in result
@pytest.mark.asyncio
async def test_get_read_note_tool_pure(mock_ctx, tmp_path):
"""Verify get_read_note_tool_pure returns a pure LangChain tool."""
save_note_content(tmp_path, "pure_sample", "pure content")
pure_tool = get_read_note_tool_pure(mock_ctx)
assert pure_tool.name == "read_note"
result = await pure_tool.ainvoke({"key": "pure_sample"})
assert "Successfully read note 'pure_sample'." in result
assert "pure content" in result
@@ -0,0 +1,122 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pathlib import Path
from unittest.mock import MagicMock, patch
from artemis.context import ArtemisContext
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.scratchpad import (
SaveNoteArgs,
SaveNotePure,
SaveNotePureTool,
get_save_note_tool_pure,
save_note_pure,
)
import pytest
@pytest.fixture
def mock_ctx(tmp_path):
ctx = MagicMock(spec=ArtemisContext)
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = str(tmp_path)
return ctx
def test_save_note_pure_tool_subclass_and_registry():
"""Verify SaveNotePureTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(SaveNotePureTool, ArtemisTool)
assert issubclass(SaveNotePure, ArtemisTool)
assert isinstance(save_note_pure, ArtemisTool)
assert isinstance(save_note_pure, SaveNotePureTool)
assert save_note_pure.name == "save_note_pure"
assert save_note_pure.category == "memory"
assert save_note_pure.args_schema == SaveNoteArgs
# Registry lookup
reg_tool = ToolRegistry.get("save_note_pure")
assert reg_tool is not None
assert isinstance(reg_tool, SaveNotePureTool)
# GenAI FunctionDeclaration export
declaration = save_note_pure.to_genai_declaration()
assert declaration.name == "save_note_pure"
assert "key" in declaration.parameters.properties
assert "content" in declaration.parameters.properties
@pytest.mark.asyncio
async def test_save_note_pure_direct_execution_success(mock_ctx, tmp_path):
"""Verify direct execution of SaveNotePureTool.execute creates note file."""
result = await save_note_pure.execute(
ctx=mock_ctx,
key="test_pure_save",
content="Pure saved content",
)
assert result == "Successfully saved note to test_pure_save.md."
note_path = Path(tmp_path) / "notes" / "test_pure_save.md"
assert note_path.exists()
assert note_path.read_text(encoding="utf-8") == "Pure saved content"
@pytest.mark.asyncio
async def test_save_note_pure_callable_execution(mock_ctx, tmp_path):
"""Verify invoking save_note_pure directly as a callable."""
result = await save_note_pure(
ctx=mock_ctx,
key="callable_pure_save",
content="Callable pure content",
)
assert result == "Successfully saved note to callable_pure_save.md."
note_path = Path(tmp_path) / "notes" / "callable_pure_save.md"
assert note_path.exists()
assert note_path.read_text(encoding="utf-8") == "Callable pure content"
@pytest.mark.asyncio
async def test_save_note_pure_execution_failure(mock_ctx):
"""Verify error handling when save_note_content fails."""
with patch(
"artemis.tools.scratchpad.save_note_content",
side_effect=PermissionError("Permission denied"),
):
result = await save_note_pure.execute(
ctx=mock_ctx,
key="fail_save",
content="Some content",
)
assert "Failed to save note fail_save.md: Permission denied" in result
@pytest.mark.asyncio
async def test_get_save_note_tool_pure_langchain_ainvoke(mock_ctx, tmp_path):
"""Verify get_save_note_tool_pure exports a LangChain tool named 'save_note'."""
pure_tool = get_save_note_tool_pure(mock_ctx)
assert pure_tool.name == "save_note"
result = await pure_tool.ainvoke(
{
"key": "lc_pure_save",
"content": "Saved via LangChain pure tool",
}
)
assert result == "Successfully saved note to lc_pure_save.md."
note_path = Path(tmp_path) / "notes" / "lc_pure_save.md"
assert note_path.exists()
assert note_path.read_text(encoding="utf-8") == "Saved via LangChain pure tool"
+180
View File
@@ -0,0 +1,180 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.scratchpad import (
SaveNote,
SaveNoteArgs,
SaveNoteTool,
get_save_note_tool,
get_save_note_tool_pure,
save_note,
save_note_wrapper,
)
from langgraph.types import Command
import pytest
@pytest.fixture
def mock_ctx(tmp_path):
ctx = MagicMock(spec=ArtemisContext)
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = str(tmp_path)
return ctx
def test_save_note_tool_subclass_and_registry():
"""Verify SaveNoteTool is a subclass of ArtemisTool and properly registered in ToolRegistry."""
assert issubclass(SaveNoteTool, ArtemisTool)
assert issubclass(SaveNote, ArtemisTool)
assert isinstance(save_note, ArtemisTool)
assert isinstance(save_note, SaveNoteTool)
assert save_note.name == "save_note"
assert save_note.category == "memory"
assert save_note.args_schema == SaveNoteArgs
# Registry lookup
reg_tool = ToolRegistry.get("save_note")
assert reg_tool is not None
assert isinstance(reg_tool, SaveNoteTool)
# GenAI FunctionDeclaration export
declaration = save_note.to_genai_declaration()
assert declaration.name == "save_note"
assert "key" in declaration.parameters.properties
assert "content" in declaration.parameters.properties
# Wrapper check
assert save_note_wrapper is not None
assert save_note_wrapper.tool_fn_getter == get_save_note_tool
@pytest.mark.asyncio
async def test_save_note_direct_execution_success(mock_ctx, tmp_path):
"""Verify direct execution of SaveNoteTool.execute creates note file."""
result = await save_note.execute(
ctx=mock_ctx,
key="test_note",
content="Hello Artemis memory",
)
assert result == "Successfully saved note to test_note.md."
note_path = Path(tmp_path) / "notes" / "test_note.md"
assert note_path.exists()
assert note_path.read_text(encoding="utf-8") == "Hello Artemis memory"
@pytest.mark.asyncio
async def test_save_note_callable_execution(mock_ctx, tmp_path):
"""Verify invoking save_note directly as a callable."""
result = await save_note(
ctx=mock_ctx,
key="callable_note",
content="Callable invocation test",
)
assert result == "Successfully saved note to callable_note.md."
note_path = Path(tmp_path) / "notes" / "callable_note.md"
assert note_path.exists()
assert note_path.read_text(encoding="utf-8") == "Callable invocation test"
@pytest.mark.asyncio
async def test_save_note_with_state_command(mock_ctx, tmp_path):
"""Verify SaveNoteTool.execute with state returns Command updating VALIDATOR_MESSAGES_KEY."""
mock_state = MagicMock(spec=State)
mock_state.current_agent = "operator"
mock_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update)
result = await save_note.execute(
ctx=mock_ctx,
key="state_note",
content="Note with state",
state=mock_state,
tool_call_id="call_save_999",
)
assert isinstance(result, Command)
assert VALIDATOR_MESSAGES_KEY in result.update
tool_messages = result.update[VALIDATOR_MESSAGES_KEY]
assert len(tool_messages) == 1
assert tool_messages[0].tool_call_id == "call_save_999"
assert tool_messages[0].content == "Successfully saved note to state_note.md."
assert tool_messages[0].status == "success"
mock_state.asanitize_update.assert_called_once()
note_path = Path(tmp_path) / "notes" / "state_note.md"
assert note_path.exists()
assert note_path.read_text(encoding="utf-8") == "Note with state"
@pytest.mark.asyncio
async def test_save_note_execution_failure(mock_ctx):
"""Verify error handling when saving note fails."""
with patch(
"artemis.tools.scratchpad.save_note_content",
side_effect=PermissionError("Permission denied"),
):
result = await save_note.execute(
ctx=mock_ctx,
key="fail_note",
content="Fail content",
)
assert "Failed to save note fail_note.md: Permission denied" in result
@pytest.mark.asyncio
async def test_get_save_note_tool_langchain_ainvoke(mock_ctx, tmp_path):
"""Verify get_save_note_tool exports a LangChain tool that works with ainvoke."""
lc_tool = get_save_note_tool(mock_ctx)
assert lc_tool.name == "save_note"
result = await lc_tool.ainvoke(
{
"key": "lc_note",
"content": "Saved via LangChain BaseTool",
}
)
assert result == "Successfully saved note to lc_note.md."
note_path = Path(tmp_path) / "notes" / "lc_note.md"
assert note_path.exists()
assert note_path.read_text(encoding="utf-8") == "Saved via LangChain BaseTool"
@pytest.mark.asyncio
async def test_get_save_note_tool_pure(mock_ctx, tmp_path):
"""Verify get_save_note_tool_pure returns a pure LangChain tool."""
pure_tool = get_save_note_tool_pure(mock_ctx)
assert pure_tool.name == "save_note"
result = await pure_tool.ainvoke(
{
"key": "pure_note",
"content": "Saved via pure tool",
}
)
assert result == "Successfully saved note to pure_note.md."
note_path = Path(tmp_path) / "notes" / "pure_note.md"
assert note_path.exists()
assert note_path.read_text(encoding="utf-8") == "Saved via pure tool"
+262
View File
@@ -0,0 +1,262 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import MagicMock, patch
from artemis.context import ArtemisContext
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.mobile.search_logs import (
SearchLogs,
SearchLogsArgs,
SearchLogsTool,
SearchLogsToolAlias,
get_search_logs_tool,
search_and_merge_logs,
search_logs,
)
import pytest
def test_search_and_merge_logs_no_matches():
logs = "line 0: info\nline 1: info\nline 2: info"
result = search_and_merge_logs(logs, "ERROR", context_lines=2)
assert result == "No matches found for 'ERROR'."
def test_search_and_merge_logs_single_match():
logs = "\n".join([f"line {i}" if i != 5 else "ERROR: failure" for i in range(10)])
result = search_and_merge_logs(logs, "ERROR", context_lines=2)
expected = (
"--- Match at line(s) 5 ---\n"
" 3: line 3\n"
" 4: line 4\n"
" 5: [MATCH] ERROR: failure\n"
" 6: line 6\n"
" 7: line 7\n"
"--------------------"
)
assert result == expected
def test_search_and_merge_logs_overlapping_matches():
logs = "\n".join([f"line {i}" if i not in [4, 6] else "ERROR: failure" for i in range(10)])
result = search_and_merge_logs(logs, "ERROR", context_lines=2)
# Match at 4 (range 2 to 6 inclusive)
# Match at 6 (range 4 to 8 inclusive)
# Merged range: 2 to 8 inclusive
expected = (
"--- Match at line(s) 4, 6 ---\n"
" 2: line 2\n"
" 3: line 3\n"
" 4: [MATCH] ERROR: failure\n"
" 5: line 5\n"
" 6: [MATCH] ERROR: failure\n"
" 7: line 7\n"
" 8: line 8\n"
"--------------------"
)
assert result == expected
def test_search_and_merge_logs_adjacent_matches():
# Match at 3 (range 1 to 5 inclusive)
# Match at 6 (range 4 to 8 inclusive)
# Overlap is at lines 4-5. They are adjacent/overlapping and should merge.
logs = "\n".join([f"line {i}" if i not in [3, 6] else "ERROR: failure" for i in range(10)])
result = search_and_merge_logs(logs, "ERROR", context_lines=2)
expected = (
"--- Match at line(s) 3, 6 ---\n"
" 1: line 1\n"
" 2: line 2\n"
" 3: [MATCH] ERROR: failure\n"
" 4: line 4\n"
" 5: line 5\n"
" 6: [MATCH] ERROR: failure\n"
" 7: line 7\n"
" 8: line 8\n"
"--------------------"
)
assert result == expected
def test_search_and_merge_logs_multiple_separate_blocks():
# Match at 2 (range 0 to 4 inclusive)
# Match at 8 (range 6 to 9 inclusive since len is 10)
# Non-overlapping since end of first is 4, start of second is 6.
logs = "\n".join([f"line {i}" if i not in [2, 8] else "ERROR: failure" for i in range(10)])
result = search_and_merge_logs(logs, "ERROR", context_lines=2)
expected = (
"--- Match at line(s) 2 ---\n"
" 0: line 0\n"
" 1: line 1\n"
" 2: [MATCH] ERROR: failure\n"
" 3: line 3\n"
" 4: line 4\n"
"--------------------\n"
"--- Match at line(s) 8 ---\n"
" 6: line 6\n"
" 7: line 7\n"
" 8: [MATCH] ERROR: failure\n"
" 9: line 9\n"
"--------------------"
)
assert result == expected
def test_search_and_merge_logs_regex():
logs = "line 0\nERR_123: failed\nline 2\nERR_456: critical"
result = search_and_merge_logs(logs, r"ERR_\d+", context_lines=1, is_regex=True)
expected = (
"--- Match at line(s) 1, 3 ---\n"
" 0: line 0\n"
" 1: [MATCH] ERR_123: failed\n"
" 2: line 2\n"
" 3: [MATCH] ERR_456: critical\n"
"--------------------"
)
assert result == expected
def test_search_and_merge_logs_invalid_regex():
logs = "line 0"
result = search_and_merge_logs(logs, "[invalid", is_regex=True)
assert result.startswith("Error: Invalid regex:")
# ---------------------------------------------------------------------------
# SearchLogsTool Unit Tests
# ---------------------------------------------------------------------------
@pytest.fixture
def mock_ctx():
return MagicMock(spec=ArtemisContext)
def test_search_logs_tool_subclass_and_registry():
"""Verify SearchLogsTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(SearchLogsTool, ArtemisTool)
assert issubclass(SearchLogs, ArtemisTool)
assert issubclass(SearchLogsToolAlias, ArtemisTool)
assert isinstance(search_logs, ArtemisTool)
assert isinstance(search_logs, SearchLogsTool)
assert search_logs.name == "search_logs"
assert search_logs.category == "diagnostic"
assert search_logs.args_schema == SearchLogsArgs
# Registry lookup
reg_tool = ToolRegistry.get("search_logs")
assert reg_tool is not None
assert isinstance(reg_tool, SearchLogsTool)
# GenAI FunctionDeclaration export
declaration = search_logs.to_genai_declaration()
assert declaration.name == "search_logs"
assert "keyword" in declaration.parameters.properties
assert "is_regex" in declaration.parameters.properties
assert "lines" in declaration.parameters.properties
assert "since_time" in declaration.parameters.properties
assert "until_time" in declaration.parameters.properties
assert "context_lines" in declaration.parameters.properties
@pytest.mark.asyncio
async def test_search_logs_direct_execution(mock_ctx):
"""Verify direct execution of SearchLogsTool."""
mock_log_output = "line 1: FATAL: crash occurred\nline 2: normal"
with patch(
"artemis.tools.mobile.search_logs.fetch_and_filter_logs",
return_value=mock_log_output,
) as mock_fetch:
result = await search_logs.execute(
ctx=mock_ctx,
keyword="FATAL",
is_regex=False,
lines=5000,
since_time="08-12 10:00:00",
until_time="08-12 10:05:00",
context_lines=1,
)
mock_fetch.assert_called_once_with(
ctx=mock_ctx,
lines=5000,
since_time="08-12 10:00:00",
until_time="08-12 10:05:00",
)
assert "[MATCH] line 1: FATAL: crash occurred" in result
@pytest.mark.asyncio
async def test_search_logs_default_args(mock_ctx):
"""Verify search_logs defaults lines to 10000 and context_lines to 0."""
with patch(
"artemis.tools.mobile.search_logs.fetch_and_filter_logs",
return_value="info line\nerror line",
) as mock_fetch:
result = await search_logs.execute(
ctx=mock_ctx, keyword="error", lines=None, context_lines=None
)
mock_fetch.assert_called_once_with(
ctx=mock_ctx,
lines=10000,
since_time=None,
until_time=None,
)
assert "[MATCH] error line" in result
@pytest.mark.asyncio
async def test_search_logs_exception_handling(mock_ctx):
"""Verify error handling when searching logs raises an exception."""
with patch(
"artemis.tools.mobile.search_logs.fetch_and_filter_logs",
side_effect=RuntimeError("Device unreachable"),
):
result = await search_logs.execute(ctx=mock_ctx, keyword="test")
assert "Error searching logs: Device unreachable" in result
@pytest.mark.asyncio
async def test_search_logs_callable_execution(mock_ctx):
"""Verify invoking search_logs directly as a callable."""
with patch(
"artemis.tools.mobile.search_logs.fetch_and_filter_logs",
return_value="NullPointerException in activity",
):
result = await search_logs(ctx=mock_ctx, keyword="NullPointer", context_lines=0)
assert "[MATCH] NullPointerException in activity" in result
@pytest.mark.asyncio
async def test_get_search_logs_tool_langchain_ainvoke(mock_ctx):
"""Verify get_search_logs_tool exports a LangChain tool that works with ainvoke."""
tool = get_search_logs_tool(mock_ctx)
assert tool.name == "search_logs"
with patch(
"artemis.tools.mobile.search_logs.fetch_and_filter_logs",
return_value="critical error detected",
) as mock_fetch:
result = await tool.ainvoke({"keyword": "critical", "lines": 2000})
mock_fetch.assert_called_once_with(
ctx=mock_ctx,
lines=2000,
since_time=None,
until_time=None,
)
assert "[MATCH] critical error detected" in result
+255
View File
@@ -0,0 +1,255 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.mobile.exec_tools import (
Swipe,
SwipeArgs,
SwipeTool,
get_swipe_tool,
swipe,
swipe_wrapper,
)
from langgraph.types import Command
import pytest
@pytest.fixture
def mock_ctx():
ctx = MagicMock(spec=ArtemisContext)
ctx.device = MagicMock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
return ctx
@pytest.fixture
def mock_driver():
driver = MagicMock(spec=BaseDeviceDriver)
driver.screen_size = (1080, 2400)
driver.swipe_direction = AsyncMock(return_value=True)
driver.swipe = AsyncMock(return_value=True)
return driver
@pytest.fixture
def mock_state():
state = MagicMock(spec=State)
async def _mock_asanitize_update(ctx, update, agent):
return update
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
return state
def test_swipe_tool_subclass_and_registry():
"""Verify SwipeTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(SwipeTool, ArtemisTool)
assert issubclass(Swipe, ArtemisTool)
assert isinstance(swipe, ArtemisTool)
assert isinstance(swipe, SwipeTool)
assert swipe.name == "swipe"
assert swipe.category == "action"
assert swipe.args_schema == SwipeArgs
# Registry lookup
reg_tool = ToolRegistry.get("swipe")
assert reg_tool is not None
assert isinstance(reg_tool, ArtemisTool)
# GenAI FunctionDeclaration export
declaration = swipe.to_genai_declaration()
assert declaration.name == "swipe"
assert "action" in declaration.parameters.properties
assert "duration" in declaration.parameters.properties
# Wrapper check
assert swipe_wrapper is not None
assert swipe_wrapper.tool_fn_getter == get_swipe_tool
@pytest.mark.asyncio
async def test_swipe_direct_execution_with_driver_direction(mock_driver):
"""Verify direct execution with direction on BaseDeviceDriver."""
result = await swipe.execute(driver=mock_driver, action="up", duration=500)
mock_driver.swipe_direction.assert_called_once_with("up", duration_ms=500)
assert "Swipe completed successfully." in result
@pytest.mark.asyncio
async def test_swipe_direct_execution_with_driver_coords(mock_driver):
"""Verify direct execution with coordinates on BaseDeviceDriver."""
result = await swipe.execute(driver=mock_driver, action=[100, 200, 300, 400], duration=600)
mock_driver.swipe.assert_called_once_with(108, 480, 324, 960, duration_ms=600)
assert "Swipe completed successfully." in result
@pytest.mark.asyncio
async def test_swipe_direct_execution_with_ctx_direction(mock_ctx):
"""Verify direct execution with direction on ArtemisContext."""
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.swipe_coords = AsyncMock(return_value=None)
mock_controller_cls.return_value = mock_controller_inst
result = await swipe.execute(ctx=mock_ctx, action="down", duration=400)
mock_controller_inst.swipe_coords.assert_called_once_with(540, 480, 540, 1920, 400)
assert "Swipe completed successfully." in result
@pytest.mark.asyncio
async def test_swipe_direct_execution_with_ctx_coords(mock_ctx):
"""Verify direct execution with coordinates on ArtemisContext."""
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.swipe_coords = AsyncMock(return_value=None)
mock_controller_cls.return_value = mock_controller_inst
result = await swipe.execute(ctx=mock_ctx, action=[200, 300, 800, 900], duration=800)
mock_controller_inst.swipe_coords.assert_called_once_with(216, 720, 864, 2160, 800)
assert "Swipe completed successfully." in result
@pytest.mark.asyncio
async def test_swipe_with_state_command(mock_ctx, mock_state):
"""Verify SwipeTool returns Command when state is provided."""
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.swipe_coords = AsyncMock(return_value=None)
mock_controller_cls.return_value = mock_controller_inst
cmd = await swipe.execute(
ctx=mock_ctx,
action="left",
tool_call_id="call_sw_1",
state=mock_state,
)
assert isinstance(cmd, Command)
assert VALIDATOR_MESSAGES_KEY in cmd.update
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
assert len(messages) == 1
assert messages[0].tool_call_id == "call_sw_1"
assert messages[0].status == "success"
assert "Swipe completed successfully." in messages[0].content
@pytest.mark.asyncio
async def test_swipe_callable_execution(mock_driver):
"""Verify invoking swipe directly as a callable."""
result = await swipe(driver=mock_driver, action="right")
assert "Swipe completed successfully." in result
@pytest.mark.asyncio
async def test_swipe_invalid_action(mock_ctx):
"""Verify error handling on invalid action."""
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController"):
result = await swipe.execute(ctx=mock_ctx, action="diagonal")
assert "Error during swipe: Invalid" in result
@pytest.mark.asyncio
async def test_get_swipe_tool_langchain_ainvoke(mock_ctx):
"""Verify get_swipe_tool exports a LangChain tool that works with ainvoke."""
sw_tool = get_swipe_tool(mock_ctx)
assert sw_tool.name == "swipe"
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.swipe_coords = AsyncMock(return_value=None)
mock_controller_cls.return_value = mock_controller_inst
result = await sw_tool.ainvoke({"action": "up"})
assert "Swipe completed successfully." in result
@pytest.mark.asyncio
async def test_swipe_with_direction_and_start_end(mock_driver):
"""Verify swipe works with explicit direction and start/end parameters."""
# Direction parameter
result_dir = await swipe.execute(driver=mock_driver, direction="left", duration=500)
mock_driver.swipe_direction.assert_called_with("left", duration_ms=500)
assert "Swipe completed successfully." in result_dir
# Start and End parameters
result_drag = await swipe.execute(
driver=mock_driver, start=[100, 200], end=[300, 400], duration=1500
)
mock_driver.swipe.assert_called_with(108, 480, 324, 960, duration_ms=1500)
assert "Swipe completed successfully." in result_drag
@pytest.mark.asyncio
async def test_swipe_with_coordinates_alias(mock_driver):
"""Verify swipe works with coordinates alias parameter."""
result = await swipe.execute(driver=mock_driver, coordinates=[200, 300, 400, 500], duration=800)
mock_driver.swipe.assert_called_with(216, 720, 432, 1200, duration_ms=800)
assert "Swipe completed successfully." in result
@pytest.mark.asyncio
async def test_swipe_with_string_coordinates(mock_driver):
"""Verify swipe works with string-formatted coordinates."""
# List format in string
result1 = await swipe.execute(driver=mock_driver, action="[920, 290, 920, 180]", duration=1000)
mock_driver.swipe.assert_called_with(993, 696, 993, 432, duration_ms=1000)
assert "Swipe completed successfully." in result1
# Space-separated format in string
result2 = await swipe.execute(driver=mock_driver, action="920 290 920 180", duration=1000)
assert "Swipe completed successfully." in result2
def test_parse_swipe_parameters_direct():
"""Verify parse_swipe_parameters with various parameter formats."""
from artemis.utils.coordinates import parse_swipe_parameters
# List
k, t, d = parse_swipe_parameters([100, 200, 300, 400])
assert k == "coords"
assert t == [100, 200, 300, 400]
# String list
k, t, d = parse_swipe_parameters("[920, 290, 920, 180]")
assert k == "coords"
assert t == [920, 290, 920, 180]
# String space separated
k, t, d = parse_swipe_parameters("920 290 920 180")
assert k == "coords"
assert t == [920, 290, 920, 180]
# Dict with action string
k, t, d = parse_swipe_parameters({"action": "[920, 290, 920, 180]", "duration": 1000})
assert k == "coords"
assert t == [920, 290, 920, 180]
assert d == 1000
# Dict with start and end
k, t, d = parse_swipe_parameters({"start": [100, 200], "end": [300, 400]})
assert k == "coords"
assert t == [100, 200, 300, 400]
# Direction string
k, t, d = parse_swipe_parameters({"action": "up"})
assert k == "direction"
assert t == "up"
@@ -0,0 +1,146 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pathlib import Path
from unittest.mock import MagicMock, patch
from artemis.context import ArtemisContext
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.scratchpad import (
UpdateNoteArgs,
UpdateNotePure,
UpdateNotePureTool,
get_update_note_tool_pure,
update_note_pure,
)
from artemis.utils.notes import save_note_content
import pytest
@pytest.fixture
def mock_ctx(tmp_path):
ctx = MagicMock(spec=ArtemisContext)
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = str(tmp_path)
return ctx
def test_update_note_pure_tool_subclass_and_registry():
"""Verify UpdateNotePureTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(UpdateNotePureTool, ArtemisTool)
assert issubclass(UpdateNotePure, ArtemisTool)
assert isinstance(update_note_pure, ArtemisTool)
assert isinstance(update_note_pure, UpdateNotePureTool)
assert update_note_pure.name == "update_note_pure"
assert update_note_pure.category == "memory"
assert update_note_pure.args_schema == UpdateNoteArgs
# Registry lookup
reg_tool = ToolRegistry.get("update_note_pure")
assert reg_tool is not None
assert isinstance(reg_tool, UpdateNotePureTool)
# GenAI FunctionDeclaration export
declaration = update_note_pure.to_genai_declaration()
assert declaration.name == "update_note_pure"
assert "key" in declaration.parameters.properties
assert "target" in declaration.parameters.properties
assert "replacement" in declaration.parameters.properties
@pytest.mark.asyncio
async def test_update_note_pure_direct_execution_success(mock_ctx, tmp_path):
"""Verify direct execution of UpdateNotePureTool.execute updates existing note."""
save_note_content(tmp_path, "plan", "Step 1: Pending\nStep 2: Done")
result = await update_note_pure.execute(
ctx=mock_ctx,
key="plan",
target="Step 1: Pending",
replacement="Step 1: Completed",
)
assert result == "Successfully updated note 'plan'."
note_path = Path(tmp_path) / "notes" / "plan.md"
assert note_path.read_text(encoding="utf-8") == "Step 1: Completed\nStep 2: Done"
@pytest.mark.asyncio
async def test_update_note_pure_with_warning(mock_ctx, tmp_path):
"""Verify update note pure handles warning from fuzzy matching."""
save_note_content(tmp_path, "fuzzy_note", "Target String")
with patch(
"artemis.tools.scratchpad.update_note_content",
return_value="Fuzzy match applied",
):
result = await update_note_pure.execute(
ctx=mock_ctx,
key="fuzzy_note",
target="target string",
replacement="New String",
)
assert "Successfully updated note 'fuzzy_note'." in result
assert "WARNING: Fuzzy match applied" in result
@pytest.mark.asyncio
async def test_update_note_pure_callable_execution(mock_ctx, tmp_path):
"""Verify invoking update_note_pure directly as a callable."""
save_note_content(tmp_path, "callable_plan", "alpha beta")
result = await update_note_pure(
ctx=mock_ctx,
key="callable_plan",
target="beta",
replacement="gamma",
)
assert result == "Successfully updated note 'callable_plan'."
note_path = Path(tmp_path) / "notes" / "callable_plan.md"
assert note_path.read_text(encoding="utf-8") == "alpha gamma"
@pytest.mark.asyncio
async def test_update_note_pure_execution_failure(mock_ctx):
"""Verify error handling when note update fails."""
result = await update_note_pure.execute(
ctx=mock_ctx,
key="missing_note",
target="old",
replacement="new",
)
assert "Failed to update note 'missing_note':" in result
@pytest.mark.asyncio
async def test_get_update_note_tool_pure_langchain_ainvoke(mock_ctx, tmp_path):
"""Verify get_update_note_tool_pure exports a LangChain tool named 'update_note'."""
save_note_content(tmp_path, "export_plan", "initial value")
pure_tool = get_update_note_tool_pure(mock_ctx)
assert pure_tool.name == "update_note"
result = await pure_tool.ainvoke(
{
"key": "export_plan",
"target": "initial",
"replacement": "updated",
}
)
assert result == "Successfully updated note 'export_plan'."
note_path = Path(tmp_path) / "notes" / "export_plan.md"
assert note_path.read_text(encoding="utf-8") == "updated value"
+208
View File
@@ -0,0 +1,208 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.scratchpad import (
UpdateNote,
UpdateNoteArgs,
UpdateNoteTool,
get_update_note_tool,
get_update_note_tool_pure,
update_note,
update_note_wrapper,
)
from artemis.utils.notes import save_note_content
from langgraph.types import Command
import pytest
@pytest.fixture
def mock_ctx(tmp_path):
ctx = MagicMock(spec=ArtemisContext)
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = str(tmp_path)
return ctx
def test_update_note_tool_subclass_and_registry():
"""Verify UpdateNoteTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(UpdateNoteTool, ArtemisTool)
assert issubclass(UpdateNote, ArtemisTool)
assert isinstance(update_note, ArtemisTool)
assert isinstance(update_note, UpdateNoteTool)
assert update_note.name == "update_note"
assert update_note.category == "memory"
assert update_note.args_schema == UpdateNoteArgs
# Registry lookup
reg_tool = ToolRegistry.get("update_note")
assert reg_tool is not None
assert isinstance(reg_tool, UpdateNoteTool)
# GenAI FunctionDeclaration export
declaration = update_note.to_genai_declaration()
assert declaration.name == "update_note"
assert "key" in declaration.parameters.properties
assert "target" in declaration.parameters.properties
assert "replacement" in declaration.parameters.properties
# Wrapper check
assert update_note_wrapper is not None
assert update_note_wrapper.tool_fn_getter == get_update_note_tool
@pytest.mark.asyncio
async def test_update_note_direct_execution_success(mock_ctx, tmp_path):
"""Verify direct execution of UpdateNoteTool.execute updates existing note."""
save_note_content(tmp_path, "plan", "- [ ] Step 1\n- [ ] Step 2")
result = await update_note.execute(
ctx=mock_ctx,
key="plan",
target="- [ ] Step 1",
replacement="- [x] Step 1",
)
assert result == "Successfully updated note 'plan'."
note_path = Path(tmp_path) / "notes" / "plan.md"
assert note_path.read_text(encoding="utf-8") == "- [x] Step 1\n- [ ] Step 2"
@pytest.mark.asyncio
async def test_update_note_with_warning(mock_ctx, tmp_path):
"""Verify update note handles warning from fuzzy/relaxed matching."""
save_note_content(tmp_path, "fuzzy_plan", "- [ ] Step 1")
with patch(
"artemis.tools.scratchpad.update_note_content",
return_value="Fuzzy match applied",
):
result = await update_note.execute(
ctx=mock_ctx,
key="fuzzy_plan",
target="Step 1",
replacement="Step 1 Done",
)
assert "Successfully updated note 'fuzzy_plan'." in result
assert "WARNING: Fuzzy match applied" in result
@pytest.mark.asyncio
async def test_update_note_callable_execution(mock_ctx, tmp_path):
"""Verify invoking update_note directly as a callable."""
save_note_content(tmp_path, "callable_plan", "- [ ] Task 1\n- [ ] Task 2")
result = await update_note(
ctx=mock_ctx,
key="callable_plan",
target="- [ ] Task 2",
replacement="- [x] Task 2",
)
assert result == "Successfully updated note 'callable_plan'."
note_path = Path(tmp_path) / "notes" / "callable_plan.md"
assert note_path.read_text(encoding="utf-8") == "- [ ] Task 1\n- [x] Task 2"
@pytest.mark.asyncio
async def test_update_note_with_state_command(mock_ctx, tmp_path):
"""Verify UpdateNoteTool.execute with state returns Command updating VALIDATOR_MESSAGES_KEY."""
save_note_content(tmp_path, "state_plan", "- [ ] Step 1")
mock_state = MagicMock(spec=State)
mock_state.current_agent = "operator"
mock_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update)
result = await update_note.execute(
ctx=mock_ctx,
key="state_plan",
target="- [ ] Step 1",
replacement="- [x] Step 1",
state=mock_state,
tool_call_id="call_update_999",
)
assert isinstance(result, Command)
assert VALIDATOR_MESSAGES_KEY in result.update
tool_messages = result.update[VALIDATOR_MESSAGES_KEY]
assert len(tool_messages) == 1
assert tool_messages[0].tool_call_id == "call_update_999"
assert tool_messages[0].content == "Successfully updated note 'state_plan'."
assert tool_messages[0].status == "success"
mock_state.asanitize_update.assert_called_once()
note_path = Path(tmp_path) / "notes" / "state_plan.md"
assert note_path.read_text(encoding="utf-8") == "- [x] Step 1"
@pytest.mark.asyncio
async def test_update_note_execution_failure(mock_ctx):
"""Verify error handling when updating a note fails."""
result = await update_note.execute(
ctx=mock_ctx,
key="non_existent_note",
target="old",
replacement="new",
)
assert "Failed to update note 'non_existent_note':" in result
@pytest.mark.asyncio
async def test_get_update_note_tool_langchain_ainvoke(mock_ctx, tmp_path):
"""Verify get_update_note_tool exports a LangChain tool that works with ainvoke."""
save_note_content(tmp_path, "lc_plan", "alpha beta gamma")
lc_tool = get_update_note_tool(mock_ctx)
assert lc_tool.name == "update_note"
result = await lc_tool.ainvoke(
{
"key": "lc_plan",
"target": "beta",
"replacement": "delta",
}
)
assert result == "Successfully updated note 'lc_plan'."
note_path = Path(tmp_path) / "notes" / "lc_plan.md"
assert note_path.read_text(encoding="utf-8") == "alpha delta gamma"
@pytest.mark.asyncio
async def test_get_update_note_tool_pure(mock_ctx, tmp_path):
"""Verify get_update_note_tool_pure returns a pure LangChain tool."""
save_note_content(tmp_path, "pure_plan", "hello world")
pure_tool = get_update_note_tool_pure(mock_ctx)
assert pure_tool.name == "update_note"
result = await pure_tool.ainvoke(
{
"key": "pure_plan",
"target": "world",
"replacement": "Artemis",
}
)
assert result == "Successfully updated note 'pure_plan'."
note_path = Path(tmp_path) / "notes" / "pure_plan.md"
assert note_path.read_text(encoding="utf-8") == "hello Artemis"
+457
View File
@@ -0,0 +1,457 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import sys
from unittest.mock import Mock, patch
import pytest
# Mock the problematic langgraph import at module level
_orig_chat_agent_executor = sys.modules.get("langgraph.prebuilt.chat_agent_executor")
_orig_state = sys.modules.get("artemis.graph.state")
sys.modules["langgraph.prebuilt.chat_agent_executor"] = Mock()
sys.modules["artemis.graph.state"] = Mock()
from artemis.context import DeviceContext, DevicePlatform, ArtemisContext # noqa: E402
from artemis.tools.types import Target # noqa: E402
from artemis.tools.utils import ( # noqa: E402
IdSelectorRequest,
SelectorRequestWithCoordinates,
focus_element_if_needed,
move_cursor_to_end_if_bounds,
)
from artemis.utils.ui_hierarchy import ElementBounds # noqa: E402
# Restore original modules to prevent contaminating other tests
if _orig_chat_agent_executor is not None:
sys.modules["langgraph.prebuilt.chat_agent_executor"] = _orig_chat_agent_executor
else:
del sys.modules["langgraph.prebuilt.chat_agent_executor"]
if _orig_state is not None:
sys.modules["artemis.graph.state"] = _orig_state
else:
del sys.modules["artemis.graph.state"]
@pytest.fixture
def mock_context():
"""Create a mock ArtemisContext for testing."""
ctx = Mock(spec=ArtemisContext)
# Create device context with necessary attributes
ctx.device = Mock(spec=DeviceContext)
ctx.device.mobile_platform = DevicePlatform.ANDROID
ctx.device.device_id = "test_device_123"
ctx.device.device_width = 1080
ctx.device.device_height = 2340
ctx.device.host_platform = "LINUX"
ctx.ui_adb_client = Mock()
# Mock the ADB client for Android
ctx.adb_client = Mock()
mock_device = Mock()
mock_device.shell = Mock(return_value="")
ctx.adb_client.device = Mock(return_value=mock_device)
# Mock the ADB client for Android
mock_response = Mock()
mock_response.json.return_value = {"elements": []}
ctx.ui_adb_client.get_screen_data = Mock(return_value=mock_response)
ctx.ui_adb_client.get_hierarchy = Mock(
return_value=(
b'<hierarchy><node bounds="[0,0][1000,1000]"><node'
b' bounds="[0,0][100,100]"/></node></hierarchy>'
)
)
return ctx
@pytest.fixture
def mock_state():
"""Create a mock State for testing."""
state = Mock()
state.latest_ui_hierarchy = []
return state
@pytest.fixture
def sample_element():
"""Create a sample UI element for testing."""
return {
"resourceId": "com.example:id/text_input",
"text": "Sample text",
"bounds": {"x": 100, "y": 200, "width": 300, "height": 50},
"focused": "false",
}
@pytest.fixture
def sample_rich_element():
"""Create a sample rich UI element for testing."""
return {
"attributes": {
"resource-id": "com.example:id/text_input",
"focused": "false",
"text": "Sample text",
"bounds": {"x": 100, "y": 200, "width": 300, "height": 50},
},
"children": [],
}
class TestMoveCursorToEndIfBounds:
"""Test cases for move_cursor_to_end_if_bounds function."""
@patch("artemis.tools.utils.tap")
@patch("artemis.tools.utils.find_element_by_resource_id")
def test_move_cursor_with_resource_id(
self,
mock_find_element,
mock_tap,
mock_context,
mock_state,
sample_element,
):
"""Test moving cursor using resource_id (highest priority)."""
mock_state.latest_ui_hierarchy = [sample_element]
mock_find_element.return_value = sample_element
target = Target(
resource_id="com.example:id/text_input",
resource_id_index=None,
text=None,
text_index=None,
bounds=None,
)
result = asyncio.run(
move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target)
)
mock_find_element.assert_called_once_with(
ui_hierarchy=[sample_element],
resource_id="com.example:id/text_input",
index=0,
)
mock_tap.assert_called_once()
call_args = mock_tap.call_args[1]
selector_request = call_args["selector_request"]
assert isinstance(selector_request, SelectorRequestWithCoordinates)
coords = selector_request.coordinates
assert coords.x == 397 # 100 + 300 * 0.99
assert coords.y == 249 # 200 + 50 * 0.99
assert result == sample_element
@patch("artemis.tools.utils.tap")
@patch("artemis.tools.utils.find_element_by_resource_id")
def test_move_cursor_with_coordinates_only(
self, mock_find_element, mock_tap, mock_context, mock_state
):
"""Test moving cursor when only coordinates are provided."""
bounds = ElementBounds(x=50, y=150, width=200, height=40)
target = Target(
resource_id=None,
resource_id_index=None,
text=None,
text_index=None,
bounds=bounds,
)
result = asyncio.run(
move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target)
)
mock_find_element.assert_not_called()
mock_tap.assert_called_once()
call_args = mock_tap.call_args[1]
selector_request = call_args["selector_request"]
coords = selector_request.coordinates
assert coords.x == 248 # 50 + 200 * 0.99
assert coords.y == 189 # 150 + 40 * 0.99
assert result is None # No element is returned when using coords directly
@patch("artemis.tools.utils.tap")
@patch("artemis.tools.utils.find_element_by_text")
def test_move_cursor_with_text_only_success(
self, mock_find_text, mock_tap, mock_context, mock_state, sample_element
):
"""Test moving cursor when only text is provided and succeeds."""
mock_state.latest_ui_hierarchy = [sample_element]
mock_find_text.return_value = sample_element
target = Target(
resource_id=None,
resource_id_index=None,
text="Sample text",
text_index=0,
bounds=None,
)
result = asyncio.run(
move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target)
)
mock_find_text.assert_called_once_with([sample_element], "Sample text", index=0)
mock_tap.assert_called_once()
assert result == sample_element
@patch("artemis.tools.utils.tap")
@patch("artemis.tools.utils.find_element_by_text")
def test_move_cursor_with_text_only_element_not_found(
self, mock_find_text, mock_tap, mock_context, mock_state
):
"""Test when searching by text finds no element."""
mock_state.latest_ui_hierarchy = []
mock_find_text.return_value = None
target = Target(
resource_id=None,
resource_id_index=None,
text="Nonexistent text",
text_index=None,
bounds=None,
)
result = asyncio.run(
move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target)
)
mock_tap.assert_not_called()
assert result is None
@patch("artemis.tools.utils.tap")
@patch("artemis.tools.utils.find_element_by_text")
def test_move_cursor_with_text_only_no_bounds(
self, mock_find_text, mock_tap, mock_context, mock_state
):
"""Test when element is found by text but has no bounds."""
element_no_bounds = {"text": "Text without bounds"}
mock_state.latest_ui_hierarchy = [element_no_bounds]
mock_find_text.return_value = element_no_bounds
target = Target(
resource_id=None,
resource_id_index=None,
text="Text without bounds",
text_index=None,
bounds=None,
)
result = asyncio.run(
move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target)
)
mock_tap.assert_not_called()
assert result is None # Should return None as no action was taken
@patch("artemis.tools.utils.find_element_by_resource_id")
def test_move_cursor_element_not_found_by_id(self, mock_find_element, mock_context, mock_state):
"""Test when element is not found by resource_id."""
mock_find_element.return_value = None
target = Target(
resource_id="com.example:id/nonexistent",
resource_id_index=None,
text=None,
text_index=None,
bounds=None,
)
result = asyncio.run(
move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target)
)
assert result is None
class TestFocusElementIfNeeded:
"""Test cases for focus_element_if_needed function."""
@patch("artemis.tools.utils.tap")
@patch("artemis.tools.utils.find_element_by_resource_id")
def test_focus_element_already_focused(
self, mock_find_element, mock_tap, mock_context, sample_rich_element
):
"""Test when element is already focused."""
focused_element = sample_rich_element.copy()
focused_element["attributes"]["focused"] = "true"
mock_response = Mock()
mock_response.json.return_value = {"elements": [focused_element]}
mock_context.ui_adb_client.get_screen_data = Mock(return_value=mock_response)
mock_find_element.return_value = focused_element["attributes"]
target = Target(
resource_id="com.example:id/text_input",
resource_id_index=None,
text=None,
text_index=None,
bounds=None,
)
result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target))
mock_tap.assert_not_called()
assert result == "resource_id"
mock_context.ui_adb_client.get_hierarchy.assert_called_once()
@patch("artemis.tools.utils.tap")
@patch("artemis.tools.utils.find_element_by_resource_id")
def test_focus_element_needs_focus_success(
self, mock_find_element, mock_tap, mock_context, sample_rich_element
):
"""Test when element needs focus and focusing succeeds."""
unfocused_element = sample_rich_element
focused_element = {
"attributes": {
"resource-id": "com.example:id/text_input",
"focused": "true",
},
"children": [],
}
mock_find_element.side_effect = [
unfocused_element["attributes"],
focused_element["attributes"],
]
target = Target(
resource_id="com.example:id/text_input",
resource_id_index=None,
text=None,
text_index=None,
bounds=None,
)
result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target))
mock_tap.assert_called_once_with(
ctx=mock_context,
selector_request=IdSelectorRequest(id="com.example:id/text_input"),
index=0,
)
assert mock_context.ui_adb_client.get_hierarchy.call_count == 2
assert result == "resource_id"
@patch("artemis.tools.utils.tap")
@patch("artemis.tools.utils.logger")
@patch("artemis.tools.utils.find_element_by_resource_id")
def test_focus_id_and_text_mismatch_fallback_to_text(
self,
mock_find_id,
mock_logger,
mock_tap,
mock_context,
sample_rich_element,
):
"""Test fallback when resource_id and text point to different elements."""
element_from_id = sample_rich_element["attributes"].copy()
element_from_id["text"] = "Different text"
element_from_text = sample_rich_element.copy()
element_from_text["attributes"]["bounds"] = {
"x": 10,
"y": 20,
"width": 100,
"height": 30,
}
mock_response = Mock()
mock_response.json.return_value = {"elements": [element_from_text]}
mock_context.ui_adb_client.get_screen_data = Mock(return_value=mock_response)
mock_find_id.return_value = element_from_id
with patch("artemis.tools.utils.find_element_by_text") as mock_find_text:
mock_find_text.return_value = element_from_text["attributes"]
target = Target(
resource_id="com.example:id/text_input",
resource_id_index=None,
text="Sample text",
text_index=None,
bounds=None,
)
result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target))
mock_logger.warning.assert_called_once()
mock_tap.assert_called_once()
assert result == "text"
@patch("artemis.tools.utils.tap")
@patch("artemis.tools.utils.find_element_by_text")
def test_focus_fallback_to_text(
self, mock_find_text, mock_tap, mock_context, sample_rich_element
):
"""Test fallback to focusing using text."""
element_with_bounds = sample_rich_element.copy()
element_with_bounds["attributes"]["bounds"] = {
"x": 10,
"y": 20,
"width": 100,
"height": 30,
}
mock_response = Mock()
mock_response.json.return_value = {"elements": [element_with_bounds]}
mock_context.ui_adb_client.get_screen_data = Mock(return_value=mock_response)
mock_find_text.return_value = element_with_bounds["attributes"]
target = Target(
resource_id=None,
resource_id_index=None,
text="Sample text",
text_index=None,
bounds=None,
)
result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target))
mock_find_text.assert_called_once()
mock_tap.assert_called_once()
call_args = mock_tap.call_args[1]
selector = call_args["selector_request"]
assert isinstance(selector, SelectorRequestWithCoordinates)
assert selector.coordinates.x == 60 # 10 + 100/2
assert selector.coordinates.y == 35 # 20 + 30/2
assert result == "text"
@patch("artemis.tools.utils.logger")
def test_focus_all_locators_fail(self, mock_logger, mock_context):
"""Test failure when no locator can find an element."""
mock_response = Mock()
mock_response.json.return_value = {"elements": []}
mock_context.ui_adb_client.get_screen_data = Mock(return_value=mock_response)
with (
patch("artemis.tools.utils.find_element_by_resource_id") as mock_find_id,
patch("artemis.tools.utils.find_element_by_text") as mock_find_text,
):
mock_find_id.return_value = None
mock_find_text.return_value = None
target = Target(
resource_id="nonexistent",
resource_id_index=None,
text="nonexistent",
text_index=None,
bounds=None,
)
result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target))
mock_logger.error.assert_called_once_with(
"Failed to focus element."
+ " No valid locator (resource_id, coordinates, or text) succeeded."
)
assert result is None
if __name__ == "__main__":
pytest.main([__file__])
@@ -0,0 +1,139 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.context import ArtemisContext
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.video_tool import (
VideoAnalyzerArgs,
VideoAnalyzerPure,
VideoAnalyzerPureTool,
get_video_analyzer_tool_pure,
video_analyzer_pure,
)
import pytest
@pytest.fixture
def mock_ctx():
ctx = MagicMock(spec=ArtemisContext)
return ctx
def test_video_analyzer_pure_tool_subclass_and_registry():
"""Verify VideoAnalyzerPureTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(VideoAnalyzerPureTool, ArtemisTool)
assert issubclass(VideoAnalyzerPure, ArtemisTool)
assert isinstance(video_analyzer_pure, ArtemisTool)
assert isinstance(video_analyzer_pure, VideoAnalyzerPureTool)
assert video_analyzer_pure.name == "video_analyzer_pure"
assert video_analyzer_pure.category == "perception"
assert video_analyzer_pure.args_schema == VideoAnalyzerArgs
# Registry lookup
reg_tool = ToolRegistry.get("video_analyzer_pure")
assert reg_tool is not None
assert isinstance(reg_tool, VideoAnalyzerPureTool)
# GenAI FunctionDeclaration export
declaration = video_analyzer_pure.to_genai_declaration()
assert declaration.name == "video_analyzer_pure"
assert "time_description" in declaration.parameters.properties
assert "purpose" in declaration.parameters.properties
@pytest.mark.asyncio
async def test_video_analyzer_pure_direct_execution_success(mock_ctx):
"""Verify direct execution of VideoAnalyzerPureTool.execute returns outcome directly."""
with patch(
"artemis.tools.video_tool.VideoAnalyzer.run",
new_callable=AsyncMock,
return_value=("Pure video analysis success", "success"),
):
result = await video_analyzer_pure.execute(
ctx=mock_ctx,
time_description="from 0s to 5s",
purpose="Verify video playback",
)
assert result == "Pure video analysis success"
@pytest.mark.asyncio
async def test_video_analyzer_pure_direct_execution_failed(mock_ctx):
"""Verify direct execution handles failed status properly."""
with patch(
"artemis.tools.video_tool.VideoAnalyzer.run",
new_callable=AsyncMock,
return_value=("Pure could not find element", "failed"),
):
result = await video_analyzer_pure.execute(
ctx=mock_ctx,
time_description="from 10s to 15s",
purpose="Check button click",
)
assert result == "Video analysis failed: Pure could not find element"
@pytest.mark.asyncio
async def test_video_analyzer_pure_callable_execution(mock_ctx):
"""Verify invoking video_analyzer_pure directly as a callable."""
with patch(
"artemis.tools.video_tool.VideoAnalyzer.run",
new_callable=AsyncMock,
return_value=("Pure callable output", "success"),
):
result = await video_analyzer_pure(
ctx=mock_ctx,
time_description="from 1s to 2s",
purpose="Callable check",
)
assert result == "Pure callable output"
@pytest.mark.asyncio
async def test_video_analyzer_pure_exception_handling(mock_ctx):
"""Verify VideoAnalyzerPureTool gracefully handles exceptions."""
with patch(
"artemis.tools.video_tool.VideoAnalyzer.run",
new_callable=AsyncMock,
side_effect=RuntimeError("Video processing error"),
):
result = await video_analyzer_pure.execute(
ctx=mock_ctx,
time_description="from 0s to 5s",
purpose="Test error",
)
assert "Error running video analyzer: Video processing error" in result
@pytest.mark.asyncio
async def test_get_video_analyzer_tool_pure_ainvoke(mock_ctx):
"""Verify get_video_analyzer_tool_pure exports a LangChain tool named 'video_analyzer'."""
pure_tool = get_video_analyzer_tool_pure(mock_ctx)
assert pure_tool.name == "video_analyzer"
with patch(
"artemis.tools.video_tool.VideoAnalyzer.run",
new_callable=AsyncMock,
return_value=("Exported pure tool output", "success"),
):
result = await pure_tool.ainvoke(
{
"time_description": "from 2s to 4s",
"purpose": "Pure invoke check",
}
)
assert result == "Exported pure tool output"
@@ -0,0 +1,204 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.video_tool import (
DIAGNOSER_VIDEO_ANALYZER_DOCSTRING,
OPERATOR_VIDEO_ANALYZER_DOCSTRING,
VideoAnalyzerArgs,
VideoAnalyzerTool,
get_video_analyzer_tool,
get_video_analyzer_tool_pure,
video_analyzer,
)
from langgraph.types import Command
import pytest
@pytest.fixture
def mock_ctx():
ctx = MagicMock(spec=ArtemisContext)
return ctx
@pytest.fixture
def mock_state(mock_ctx):
state = MagicMock(spec=State)
async def _mock_asanitize_update(ctx, update, agent):
return update
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
return state
def test_video_analyzer_tool_subclass_and_registry():
"""Verify VideoAnalyzerTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(VideoAnalyzerTool, ArtemisTool)
assert isinstance(video_analyzer, ArtemisTool)
assert isinstance(video_analyzer, VideoAnalyzerTool)
assert video_analyzer.name == "video_analyzer"
assert video_analyzer.category == "perception"
assert video_analyzer.args_schema == VideoAnalyzerArgs
# Registry lookup
reg_tool = ToolRegistry.get("video_analyzer")
assert reg_tool is not None
assert isinstance(reg_tool, VideoAnalyzerTool)
# GenAI FunctionDeclaration export
declaration = video_analyzer.to_genai_declaration()
assert declaration.name == "video_analyzer"
assert "time_description" in declaration.parameters.properties
assert "purpose" in declaration.parameters.properties
@pytest.mark.asyncio
async def test_video_analyzer_direct_execution_success(mock_ctx):
"""Verify direct execution of VideoAnalyzerTool.execute returns outcome directly."""
with patch(
"artemis.tools.video_tool.VideoAnalyzer.run",
new_callable=AsyncMock,
return_value=("Video analysis success details", "success"),
):
result = await video_analyzer.execute(
ctx=mock_ctx,
time_description="from 0s to 5s",
purpose="Verify video playback",
)
assert result == "Video analysis success details"
@pytest.mark.asyncio
async def test_video_analyzer_direct_execution_failed(mock_ctx):
"""Verify direct execution handles failed status properly."""
with patch(
"artemis.tools.video_tool.VideoAnalyzer.run",
new_callable=AsyncMock,
return_value=("Could not find target element", "failed"),
):
result = await video_analyzer.execute(
ctx=mock_ctx,
time_description="from 10s to 15s",
purpose="Check button click",
)
assert result == "Video analysis failed: Could not find target element"
@pytest.mark.asyncio
async def test_video_analyzer_with_state_command(mock_ctx, mock_state):
"""Verify execute returns a Command with ToolMessage when state is provided."""
with patch(
"artemis.tools.video_tool.VideoAnalyzer.run",
new_callable=AsyncMock,
return_value=("Action verified in video", "success"),
):
cmd = await video_analyzer.execute(
ctx=mock_ctx,
time_description="from 5s to 10s",
purpose="Verify action",
tool_call_id="call_video_123",
state=mock_state,
)
assert isinstance(cmd, Command)
assert VALIDATOR_MESSAGES_KEY in cmd.update
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
assert len(messages) == 1
assert messages[0].tool_call_id == "call_video_123"
assert messages[0].content == "Action verified in video"
assert messages[0].status == "success"
@pytest.mark.asyncio
async def test_video_analyzer_exception_handling(mock_ctx, mock_state):
"""Verify execute gracefully handles exceptions."""
with patch(
"artemis.tools.video_tool.VideoAnalyzer.run",
new_callable=AsyncMock,
side_effect=RuntimeError("Video processing error"),
):
# Without state
res = await video_analyzer.execute(
ctx=mock_ctx,
time_description="from 0s to 5s",
purpose="Test error",
)
assert "Error running video analyzer: Video processing error" in res
# With state
cmd = await video_analyzer.execute(
ctx=mock_ctx,
time_description="from 0s to 5s",
purpose="Test error",
tool_call_id="call_err",
state=mock_state,
)
assert isinstance(cmd, Command)
assert VALIDATOR_MESSAGES_KEY in cmd.update
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
assert messages[0].status == "error"
assert "Error running video analyzer" in messages[0].content
@pytest.mark.asyncio
async def test_video_analyzer_callable_execution(mock_ctx):
"""Verify invoking video_analyzer directly as a callable."""
with patch(
"artemis.tools.video_tool.VideoAnalyzer.run",
new_callable=AsyncMock,
return_value=("Callable output", "success"),
):
result = await video_analyzer(
ctx=mock_ctx,
time_description="from 1s to 2s",
purpose="Callable check",
)
assert result == "Callable output"
def test_get_video_analyzer_tool_roles(mock_ctx):
"""Verify get_video_analyzer_tool returns tool with role-specific description."""
op_tool = get_video_analyzer_tool(mock_ctx, role="operator")
assert op_tool.name == "video_analyzer"
assert op_tool.description.strip() == OPERATOR_VIDEO_ANALYZER_DOCSTRING.strip()
diag_tool = get_video_analyzer_tool(mock_ctx, role="diagnoser")
assert diag_tool.name == "video_analyzer"
assert diag_tool.description.strip() == DIAGNOSER_VIDEO_ANALYZER_DOCSTRING.strip()
@pytest.mark.asyncio
async def test_get_video_analyzer_tool_pure_ainvoke(mock_ctx):
"""Verify get_video_analyzer_tool_pure exports a LangChain tool."""
pure_tool = get_video_analyzer_tool_pure(mock_ctx)
assert pure_tool.name == "video_analyzer"
with patch(
"artemis.tools.video_tool.VideoAnalyzer.run",
new_callable=AsyncMock,
return_value=("Pure tool output", "success"),
):
result = await pure_tool.ainvoke(
{
"time_description": "from 2s to 4s",
"purpose": "Pure invoke check",
}
)
assert result == "Pure tool output"
+114
View File
@@ -0,0 +1,114 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import time
from unittest.mock import AsyncMock, MagicMock
from artemis.context import ArtemisContext
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.wait_tool import (
Wait,
WaitArgs,
WaitTool,
get_wait_tool,
wait,
wait_wrapper,
)
import pytest
@pytest.fixture
def mock_ctx():
ctx = MagicMock(spec=ArtemisContext)
return ctx
def test_wait_tool_subclass_and_registry():
"""Verify WaitTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(WaitTool, ArtemisTool)
assert issubclass(Wait, ArtemisTool)
assert isinstance(wait, ArtemisTool)
assert isinstance(wait, WaitTool)
assert wait.name == "wait"
assert wait.category == "system"
assert wait.args_schema == WaitArgs
# Registry lookup
reg_tool = ToolRegistry.get("wait")
assert reg_tool is not None
assert isinstance(reg_tool, WaitTool)
# GenAI FunctionDeclaration export
declaration = wait.to_genai_declaration()
assert declaration.name == "wait"
assert "seconds" in declaration.parameters.properties
# Wrapper check
assert wait_wrapper is not None
assert wait_wrapper.tool_fn_getter == get_wait_tool
@pytest.mark.asyncio
async def test_wait_direct_execution(mock_ctx):
"""Verify direct execution of WaitTool.execute."""
with pytest.MonkeyPatch.context() as mp:
mock_sleep = AsyncMock()
mp.setattr(asyncio, "sleep", mock_sleep)
result = await wait.execute(ctx=mock_ctx, seconds=5)
mock_sleep.assert_called_once_with(5)
assert result == "Successfully waited for 5 seconds."
@pytest.mark.asyncio
async def test_wait_callable_execution(mock_ctx):
"""Verify invoking wait directly as a callable."""
with pytest.MonkeyPatch.context() as mp:
mock_sleep = AsyncMock()
mp.setattr(asyncio, "sleep", mock_sleep)
result = await wait(ctx=mock_ctx, seconds=3)
mock_sleep.assert_called_once_with(3)
assert result == "Successfully waited for 3 seconds."
@pytest.mark.asyncio
async def test_wait_tool(mock_ctx):
wait_tool = get_wait_tool(mock_ctx)
start_time = time.time()
result = await wait_tool.ainvoke({"seconds": 2})
end_time = time.time()
duration = end_time - start_time
assert duration >= 1.9 # Allow slight timing variations
assert "Successfully waited for 2 seconds" in result
@pytest.mark.asyncio
async def test_wait_tool_bounds(mock_ctx):
wait_tool = get_wait_tool(mock_ctx)
with pytest.MonkeyPatch.context() as mp:
mock_sleep = AsyncMock()
mp.setattr(asyncio, "sleep", mock_sleep)
await wait_tool.ainvoke({"seconds": 120})
mock_sleep.assert_called_once_with(60)
mock_sleep.reset_mock()
await wait_tool.ainvoke({"seconds": -5})
mock_sleep.assert_called_once_with(1)
+148
View File
@@ -0,0 +1,148 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, Mock, patch
from artemis.context import ArtemisContext
from artemis.utils.app_launch_utils import launch_app_with_retries
import pytest
@pytest.fixture
def mock_context():
ctx = Mock(spec=ArtemisContext)
ctx.device = Mock()
ctx.device.mobile_platform = "android"
ctx.device.device_id = "emulator-5554"
ctx.data_engine = None
return ctx
@pytest.mark.asyncio
@patch("artemis.utils.app_launch_utils.UnifiedMobileController")
@patch("artemis.utils.app_launch_utils.get_current_foreground_package_async")
async def test_launch_app_success_immediate(mock_get_foreground, mock_controller_cls, mock_context):
"""Test successful immediate app launch."""
mock_controller = Mock()
mock_controller.launch_app = AsyncMock(return_value=True)
mock_controller_cls.return_value = mock_controller
# App immediately in foreground
mock_get_foreground.return_value = "com.google.android.youtube"
success, error_msg = await launch_app_with_retries(
mock_context,
"com.google.android.youtube",
max_retries=1,
max_poll_seconds=2,
)
assert success is True
assert error_msg is None
mock_controller.launch_app.assert_called_once_with("com.google.android.youtube")
@pytest.mark.asyncio
@patch("artemis.utils.app_launch_utils.get_focused_task_package")
@patch("artemis.utils.app_launch_utils.UnifiedMobileController")
@patch("artemis.utils.app_launch_utils.get_current_foreground_package_async")
async def test_launch_app_success_permission_overlay(
mock_get_foreground,
mock_controller_cls,
mock_get_focused_task,
mock_context,
):
"""Test successful launch if overlayed by system permission controller and task is matching."""
mock_controller = Mock()
mock_controller.launch_app = AsyncMock(return_value=True)
mock_controller_cls.return_value = mock_controller
# System permission controller is focused foreground
mock_get_foreground.return_value = "com.google.android.permissioncontroller"
# Second layer check: Task stack shows com.google.android.youtube is focused
mock_get_focused_task.return_value = "com.google.android.youtube"
success, error_msg = await launch_app_with_retries(
mock_context,
"com.google.android.youtube",
max_retries=1,
max_poll_seconds=2,
)
assert success is True
assert error_msg is None
mock_controller.launch_app.assert_called_once_with("com.google.android.youtube")
mock_get_focused_task.assert_called_once_with(mock_context)
@pytest.mark.asyncio
@patch("artemis.utils.app_launch_utils.get_focused_task_package")
@patch("artemis.utils.app_launch_utils.UnifiedMobileController")
@patch("artemis.utils.app_launch_utils.get_current_foreground_package_async")
async def test_launch_app_failure_permission_overlay_wrong_task(
mock_get_foreground,
mock_controller_cls,
mock_get_focused_task,
mock_context,
):
"""Test launch failure if overlayed by system permission controller but task does not match."""
mock_controller = Mock()
mock_controller.launch_app = AsyncMock(return_value=True)
mock_controller.terminate_app = AsyncMock()
mock_controller_cls.return_value = mock_controller
# System permission controller is focused foreground
mock_get_foreground.return_value = "com.google.android.permissioncontroller"
# Second layer check: Task stack is some other app (e.g. com.android.settings)
mock_get_focused_task.return_value = "com.android.settings"
success, error_msg = await launch_app_with_retries(
mock_context,
"com.google.android.youtube",
max_retries=1,
max_poll_seconds=1,
)
assert success is False
assert "Failed to launch com.google.android.youtube" in error_msg
mock_get_focused_task.assert_called_once_with(mock_context)
@pytest.mark.asyncio
@patch("artemis.utils.app_launch_utils.UnifiedMobileController")
@patch("artemis.utils.app_launch_utils.get_current_foreground_package_async")
async def test_launch_app_failure_wrong_app(mock_get_foreground, mock_controller_cls, mock_context):
"""Test launch failure and retry when wrong app remains in foreground."""
mock_controller = Mock()
mock_controller.launch_app = AsyncMock(return_value=True)
mock_controller.terminate_app = AsyncMock()
mock_controller_cls.return_value = mock_controller
# A different app (e.g., settings or home screen) is in foreground
mock_get_foreground.return_value = "com.android.settings"
success, error_msg = await launch_app_with_retries(
mock_context,
"com.google.android.youtube",
max_retries=2,
max_poll_seconds=1,
)
assert success is False
assert "Failed to launch com.google.android.youtube" in error_msg
# Should have attempted launching twice
assert mock_controller.launch_app.call_count == 2
# Should have attempted force-stopping once before retry
assert mock_controller.terminate_app.call_count == 1
+178
View File
@@ -0,0 +1,178 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import base64
import cv2
from artemis.utils.image_diff import check_ui_change
import numpy as np
def test_no_change():
img = np.zeros((1000, 1000), dtype=np.uint8)
_, img_bytes = cv2.imencode(".jpg", img)
action = {"action": "tap", "coordinates": [500, 500]}
assert not check_ui_change(img_bytes.tobytes(), img_bytes.tobytes(), action)
print("test_no_change passed")
def test_global_change():
img1 = np.zeros((1000, 1000), dtype=np.uint8)
img2 = np.ones((1000, 1000), dtype=np.uint8) * 255
_, img1_bytes = cv2.imencode(".jpg", img1)
_, img2_bytes = cv2.imencode(".jpg", img2)
action = {"action": "swipe", "coordinates": [500, 500, 500, 200]}
assert check_ui_change(img1_bytes.tobytes(), img2_bytes.tobytes(), action)
print("test_global_change passed")
def test_roi_change():
img1 = np.zeros((1000, 1000), dtype=np.uint8)
img2 = img1.copy()
img2[480:520, 480:520] = 255
_, img1_bytes = cv2.imencode(".jpg", img1)
_, img2_bytes = cv2.imencode(".jpg", img2)
action = {"action": "tap", "coordinates": [500, 500]}
assert check_ui_change(img1_bytes.tobytes(), img2_bytes.tobytes(), action)
print("test_roi_change passed")
def test_status_bar_ignore():
img1 = np.zeros((1000, 1000), dtype=np.uint8)
img2 = img1.copy()
img2[0:50, 0:1000] = 255
_, img1_bytes = cv2.imencode(".jpg", img1)
_, img2_bytes = cv2.imencode(".jpg", img2)
action = {"action": "tap", "coordinates": [500, 500]}
assert not check_ui_change(img1_bytes.tobytes(), img2_bytes.tobytes(), action)
print("test_status_bar_ignore passed")
def test_roi_threshold():
# Use standard 1080x2400 resolution to keep scale_factor = 1.0 (roi_size = 50)
img1 = np.zeros((2400, 1080), dtype=np.uint8)
img2 = img1.copy()
# Center of 1080x2400 is (540, 1200). Draw 10x10 square at center.
img2[1195:1205, 535:545] = 255
_, img1_bytes = cv2.imencode(".jpg", img1)
_, img2_bytes = cv2.imencode(".jpg", img2)
action = {"action": "tap", "coordinates": [540, 1200]}
assert not check_ui_change(
img1_bytes.tobytes(),
img2_bytes.tobytes(),
action,
full_screen_threshold=0.001,
roi_threshold=0.05,
)
print("test_roi_threshold passed")
def test_wait_for_screen_stability():
from artemis.utils.image_diff import wait_for_screen_stability
img_static = np.zeros((1000, 1000), dtype=np.uint8)
_, static_bytes = cv2.imencode(".jpg", img_static)
static_b64 = base64.b64encode(static_bytes).decode("utf-8")
# 1. Test immediate stability (already static)
calls_immediate = 0
async def mock_screenshot_immediate():
nonlocal calls_immediate
calls_immediate += 1
return static_b64
res_immediate = asyncio.run(
wait_for_screen_stability(mock_screenshot_immediate, max_timeout=1.0, interval=0.05)
)
assert res_immediate == static_b64
assert calls_immediate == 2
print("test_wait_for_screen_stability: immediate stability subtest passed")
# 2. Test gradual stability (settles after 3 moving frames)
calls_gradual = 0
async def mock_screenshot_gradual():
nonlocal calls_gradual
calls_gradual += 1
if calls_gradual <= 3:
img = np.zeros((1000, 1000), dtype=np.uint8)
img[500:, :] = calls_gradual * 50
_, bytes_data = cv2.imencode(".jpg", img)
return base64.b64encode(bytes_data).decode("utf-8")
return static_b64
res_gradual = asyncio.run(
wait_for_screen_stability(mock_screenshot_gradual, max_timeout=1.0, interval=0.05)
)
assert res_gradual == static_b64
assert calls_gradual > 3
print("test_wait_for_screen_stability: gradual stability subtest passed")
print("test_wait_for_screen_stability passed")
def test_wait_for_screen_data_stability():
from artemis.utils.image_diff import wait_for_screen_data_stability
from artemis.controllers.device_controller import ScreenDataResponse
img_static = np.zeros((1000, 1000), dtype=np.uint8)
_, static_bytes = cv2.imencode(".jpg", img_static)
static_b64 = base64.b64encode(static_bytes).decode("utf-8")
# Test stability returning ScreenDataResponse
calls = 0
async def mock_get_screen_data():
nonlocal calls
calls += 1
return ScreenDataResponse(
base64=static_b64,
elements=[],
width=1000,
height=1000,
platform="android",
)
res = asyncio.run(
wait_for_screen_data_stability(mock_get_screen_data, max_timeout=1.0, interval=0.05)
)
assert res.base64 == static_b64
assert res.width == 1000
assert calls == 2
print("test_wait_for_screen_data_stability passed")
if __name__ == "__main__":
try:
test_no_change()
test_global_change()
test_roi_change()
test_status_bar_ignore()
test_roi_threshold()
test_wait_for_screen_stability()
test_wait_for_screen_data_stability()
print("All tests passed successfully!")
except AssertionError as e:
print(f"Test failed: {e}")
except Exception as e:
print(f"Error running tests: {e}")
+165
View File
@@ -0,0 +1,165 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from artemis.utils.notes import (
append_note_content,
list_notes_info,
read_note_content,
save_note_content,
update_note_content,
)
import pytest
def test_list_notes_info(tmp_path):
base_dir = tmp_path
# Create some notes
save_note_content(base_dir, "note1", "line1\nline2\nline3")
save_note_content(base_dir, "note2", "line1\nline2")
info = list_notes_info(base_dir)
assert info == {"note1": 3, "note2": 2}
def test_read_note_content_range(tmp_path):
base_dir = tmp_path
save_note_content(base_dir, "note1", "line1\nline2\nline3\nline4")
# Read full
assert read_note_content(base_dir, "note1") == "line1\nline2\nline3\nline4"
# Read range
assert read_note_content(base_dir, "note1", start_line=2, end_line=3) == "line2\nline3"
# Read from start to line
assert read_note_content(base_dir, "note1", end_line=2) == "line1\nline2"
# Read from line to end
assert read_note_content(base_dir, "note1", start_line=3) == "line3\nline4"
def test_update_note_exact_match(tmp_path):
base_dir = tmp_path
original_content = 'def demo():\n print("Hello, World!")\n return True\n'
save_note_content(base_dir, "task_plan", original_content)
# Exact match
warning = update_note_content(
base_dir,
"task_plan",
' print("Hello, World!")',
' print("Hello, Jetski!")',
)
assert warning is None
updated_content = read_note_content(base_dir, "task_plan")
assert 'print("Hello, Jetski!")' in updated_content
assert 'print("Hello, World!")' not in updated_content
def test_update_note_relaxed_match(tmp_path):
base_dir = tmp_path
# Multi-line content with checkboxes and indentation
original_content = (
"- [ ] Step 1: Research the issue\n - [ ] Step 2: Write code\n - [ ] Step 3: Run tests\n"
)
save_note_content(base_dir, "task_plan", original_content)
# Relaxed match: case difference, quote difference, checkbox bullet difference, trailing whitespace
target = " * [ ] step 2: write code "
replacement = " - [/] Step 2: In progress"
warning = update_note_content(base_dir, "task_plan", target, replacement)
# Warning should be returned
assert warning is not None
assert "We did our best to apply changes despite some inaccuracies" in warning
updated_content = read_note_content(base_dir, "task_plan")
assert "Step 2: In progress" in updated_content
assert "Step 2: Write code" not in updated_content
# Check indentation is preserved from replacement
assert " - [/] Step 2: In progress\n" in updated_content
def test_update_note_fuzzy_match(tmp_path):
base_dir = tmp_path
original_content = 'print("Alpha message")\nprint("Gamma message")\nprint("Delta message")\n'
save_note_content(base_dir, "task_plan", original_content)
# Fuzzy match: typo (Gemma vs Gamma)
target = 'print("Gemma message")'
replacement = 'print("Gamma updated!")'
warning = update_note_content(base_dir, "task_plan", target, replacement)
assert warning is not None
assert "We did our best to apply changes despite some inaccuracies" in warning
updated_content = read_note_content(base_dir, "task_plan")
assert 'print("Gamma updated!")' in updated_content
assert 'print("Gamma message")' not in updated_content
def test_update_note_ambiguous_match(tmp_path):
base_dir = tmp_path
# Multiple lines that could match target
original_content = 'print("Beta message")\nprint("Zeta message")\nprint("Eta message")\n'
save_note_content(base_dir, "task_plan", original_content)
# Target is highly ambiguous (Xeta is almost equally close to Beta and Zeta)
target = 'print("Xeta message")'
replacement = 'print("Ambiguous match!")'
# Should raise ValueError due to ambiguity or inability to find unique match
with pytest.raises(ValueError) as exc_info:
update_note_content(base_dir, "task_plan", target, replacement)
assert "Target string 'print(\"Xeta message\")' not found" in str(
exc_info.value
) or "not unique" in str(exc_info.value)
def test_update_note_not_found(tmp_path):
base_dir = tmp_path
original_content = 'print("Alpha message")\n'
save_note_content(base_dir, "task_plan", original_content)
# Totally different target
target = 'print("Something totally different")'
replacement = 'print("Hello")'
with pytest.raises(ValueError) as exc_info:
update_note_content(base_dir, "task_plan", target, replacement)
assert "not found" in str(exc_info.value)
def test_append_note_content(tmp_path):
base_dir = tmp_path
# Case 1: File does not exist
append_note_content(base_dir, "new_note", "hello")
assert read_note_content(base_dir, "new_note") == "hello\n"
# Case 2: File exists and ends with a newline
append_note_content(base_dir, "new_note", "world")
assert read_note_content(base_dir, "new_note") == "hello\nworld\n"
# Case 3: File exists and does not end with a newline
file_path = base_dir / "notes" / "no_newline.md"
file_path.parent.mkdir(parents=True, exist_ok=True)
file_path.write_text("no newline", encoding="utf-8")
append_note_content(base_dir, "no_newline", "added text")
assert read_note_content(base_dir, "no_newline") == "no newline\nadded text\n"
+248
View File
@@ -0,0 +1,248 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import base64
from io import BytesIO
import unittest
from artemis.utils.ocr_xml_fusion import (
_calculate_overlap_ratio,
_crop_image_remove_status_bar,
_detect_status_bar_height,
_is_low_value_text,
_map_coordinates_back,
_parse_ocr_position,
fuse_ocr_with_xml,
)
from PIL import Image
class TestOCRFusion(unittest.TestCase):
def test_is_low_value_text(self):
# Test length 1
self.assertTrue(_is_low_value_text("a"))
self.assertFalse(_is_low_value_text("Yes"))
self.assertFalse(_is_low_value_text("1"))
# Test placeholders
self.assertTrue(_is_low_value_text("search"))
# Test default markers
self.assertTrue(_is_low_value_text("[Icon]"))
# Test valid text
self.assertFalse(_is_low_value_text("Click Me"))
def test_parse_ocr_position(self):
position = [
{"x": 10, "y": 20},
{"x": 50, "y": 20},
{"x": 50, "y": 60},
{"x": 10, "y": 60},
]
bounds = _parse_ocr_position(position)
self.assertEqual(bounds, {"left": 10, "top": 20, "right": 50, "bottom": 60})
def test_calculate_overlap_ratio(self):
ocr_bounds = {"left": 10, "top": 10, "right": 30, "bottom": 30}
xml_bounds = {"left": 0, "top": 0, "right": 40, "bottom": 40}
# OCR is fully inside XML
self.assertEqual(_calculate_overlap_ratio(ocr_bounds, xml_bounds), 1.0)
xml_bounds = {"left": 20, "top": 20, "right": 40, "bottom": 40}
# Intersection is 10x10 = 100. OCR area is 20x20 = 400. Ratio = 0.25
self.assertEqual(_calculate_overlap_ratio(ocr_bounds, xml_bounds), 0.25)
def test_detect_status_bar_height(self):
xml = [
{"package": "com.android.systemui", "bounds": "[0,0][1080,120]"},
{"package": "com.example.app", "bounds": "[0,120][1080,2400]"},
]
# Pass screen_height=2400, should find 120 from XML
self.assertEqual(_detect_status_bar_height(xml, screen_height=2400), 120)
# Fallback: should be 4% of 2400 = 96
xml = [{"package": "com.example.app", "bounds": "[0,0][1080,2400]"}]
self.assertEqual(_detect_status_bar_height(xml, screen_height=2400), 96)
def test_crop_image_remove_status_bar(self):
# Create a dummy image
img = Image.new("RGB", (100, 100), color="red")
buffered = BytesIO()
img.save(buffered, format="PNG")
img_b64 = base64.b64encode(buffered.getvalue()).decode("utf-8")
cropped_b64, w, h = _crop_image_remove_status_bar(img_b64, crop_height=20)
self.assertEqual(w, 100)
self.assertEqual(h, 100)
# Verify cropped size
cropped_img = Image.open(BytesIO(base64.b64decode(cropped_b64)))
self.assertEqual(cropped_img.size, (100, 80))
def test_map_coordinates_back(self):
ocr_results = [
{
"text": "test",
"position": [{"x": 10, "y": 10}, {"x": 20, "y": 20}],
}
]
mapped = _map_coordinates_back(ocr_results, status_bar_height=100)
self.assertEqual(mapped[0]["position"][0]["y"], 110)
self.assertEqual(mapped[0]["position"][1]["y"], 120)
def test_fuse_ocr_with_xml(self):
xml = [
{
"class": "android.widget.Button",
"bounds": "[10,100][200,150]",
"text": "Submit",
},
{
"class": "android.widget.TextView",
"bounds": "[10,200][200,250]",
"text": "",
},
]
ocr = [
# Should be discarded (similar to Submit)
{
"text": "Submit",
"position": [{"x": 15, "y": 105}, {"x": 95, "y": 145}],
},
# Multiple OCRs matching the same node (TextView at [10,200])
# They are on the same line, should be merged in order
{
"text": "Result",
"position": [{"x": 15, "y": 205}, {"x": 95, "y": 245}],
},
{
"text": "Found",
"position": [{"x": 105, "y": 205}, {"x": 185, "y": 245}],
},
# Unmatched OCRs that are close to each other (should be merged into one virtual node)
{
"text": "Floating",
"position": [{"x": 15, "y": 305}, {"x": 95, "y": 345}],
},
{
"text": "Text",
"position": [{"x": 105, "y": 305}, {"x": 185, "y": 345}],
},
# Unmatched OCR far away (should be separate virtual node)
{
"text": "Far",
"position": [{"x": 15, "y": 405}, {"x": 95, "y": 445}],
},
]
fused = fuse_ocr_with_xml(xml, ocr)
# Check first node (Submit) - should NOT have ocr_text
self.assertNotIn("ocr_text", fused[0])
# Check second node (Result Found) - should have aggregated ocr_elements
self.assertEqual(fused[1]["ocr_elements"][0]["text"], "Result Found")
self.assertIn("[OCR]", fused[1]["class"])
# We expect 0 virtual nodes now as per new strategy.
# Total nodes should be just the original 2 XML nodes.
self.assertEqual(len(fused), 2)
def test_banded_overlap_matching(self):
# Screen is 1080x2400
# XML has a giant container and a small correct button.
xml = [
# Giant container (FrameLayout, covers a lot of screen)
{
"class": "android.widget.FrameLayout",
"bounds": "[0,80][1080,1000]",
"text": "",
},
# Small correct button
{
"class": "android.widget.Button",
"bounds": "[185,84][512,114]",
"text": "",
},
]
# OCR Search is slightly shifted to the left of the Button.
# Button: [185,84][512,114] -> area 327 * 30 = 9810
# OCR: [174,89][326,109] -> area 152 * 20 = 3040
# Overlap with Button:
# Intersection: [185,89][326,109] -> area 141 * 20 = 2820
# Ratio: 2820 / 3040 = 0.927 (Band 1)
# Overlap with FrameLayout:
# OCR is fully inside FrameLayout [0,80][1080,1000]
# Ratio: 1.0 (Band 1)
# Both are in Band 1 (>=0.9).
# The Button has MUCH smaller area than FrameLayout (9810 < 993600).
# So Button should win despite having lower overlap (0.927 < 1.0).
ocr = [
{
"text": "Search",
"position": [
{"x": 174, "y": 89},
{"x": 326, "y": 89},
{"x": 326, "y": 109},
{"x": 174, "y": 109},
],
}
]
fused = fuse_ocr_with_xml(xml, ocr)
# Check that OCR text "Search" is mounted on the Button node (index 1)
self.assertIn("ocr_elements", fused[1])
self.assertEqual(fused[1]["ocr_elements"][0]["text"], "Search")
# And NOT on the FrameLayout node (index 0)
self.assertNotIn("ocr_elements", fused[0])
def test_giant_node_exclusion(self):
# Screen is 1080x2400. Area = 2,592,000.
# 10% area = 259,200.
# FrameLayout is 1080x1000 -> area 1,080,000 (>10%).
# It should be excluded from matching entirely.
# Since there is no other node, the OCR should be discarded.
xml = [
{
"class": "android.widget.FrameLayout",
"bounds": "[0,0][1080,1000]",
"text": "",
},
]
ocr = [
# Some floating developer text "dX : 0.0"
{
"text": "dX : 0.0",
"position": [
{"x": 144, "y": 55},
{"x": 215, "y": 55},
{"x": 215, "y": 63},
{"x": 144, "y": 63},
],
}
]
fused = fuse_ocr_with_xml(xml, ocr)
# The giant FrameLayout should NOT have ocr_elements
self.assertNotIn("ocr_elements", fused[0])
# The class should NOT be updated with [OCR]
self.assertNotIn("[OCR]", fused[0].get("class", ""))
if __name__ == "__main__":
unittest.main()
+650
View File
@@ -0,0 +1,650 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import hashlib
from artemis.utils.task_tree import (
build_plan_and_history,
get_active_subgoal_hashes,
get_recent_subgoal_hashes,
)
def test_build_plan_and_history_separated_layout():
plan = """- [ ] Open Settings app
- [ ] Navigate to System settings"""
# No steps executed
output = build_plan_and_history(plan, [], "default")
assert plan in output
assert "--- Execution History ---" in output
assert "No steps executed yet." in output
# With steps
steps = [
{
"step_id": "step_1",
"step_number": 1,
"relative_time": "2.5s",
"summary": "Launched settings app",
"operator_raw_thinking": "Thought 1",
"action_taken": [{"action": "launch_app"}],
"last_execution_result": {"status": "success"},
}
]
output = build_plan_and_history(plan, steps, "default", last_n_detailed=1)
assert plan in output
assert "--- Execution History ---" in output
assert "- **Step 1 (Most Recent Step, Start: 2.5s)**" in output
assert " - Thought 1" in output
assert " * [Planned Action]: Launched app 'None'" in output
def test_build_plan_and_history_granularities():
plan = """- [x] Open Settings app
- [/] Navigate to System settings
- [ ] Navigate to Languages & input"""
subgoal_hash_1 = hashlib.md5(b"Open Settings app").hexdigest()
subgoal_hash_2 = hashlib.md5(b"Navigate to System settings").hexdigest()
steps = [
{
"step_id": "step_1",
"step_number": 1,
"relative_time": "2.5s",
"summary": "Launched settings",
"operator_raw_thinking": "Thought 1",
"action_taken": [{"action": "launch_app"}],
"last_execution_result": {"status": "success"},
"extra_metadata": {"subgoal_hash": subgoal_hash_1},
},
{
"step_id": "step_2",
"step_number": 2,
"relative_time": "8.0s",
"summary": "Swiped down",
"operator_raw_thinking": "Thought 2",
"action_taken": [{"action": "swipe"}],
"last_execution_result": {"status": "success"},
"extra_metadata": {"subgoal_hash": subgoal_hash_2},
},
{
"step_id": "step_3",
"step_number": 3,
"relative_time": "15.0s",
"summary": "Clicked System settings",
"operator_raw_thinking": "Thought 3",
"action_taken": [{"action": "click"}],
"last_execution_result": {"status": "success"},
"extra_metadata": {"subgoal_hash": subgoal_hash_2},
},
]
# Test last_n_detailed = 0 (high-level analysis context, all summarized)
output_0 = build_plan_and_history(plan, steps, subgoal_hash_2, last_n_detailed=0)
assert "- *Step 1 (Start: 2.5s): Launched settings*" in output_0
assert "- *Step 2 (Start: 8.0s): Swiped down*" in output_0
assert "- *Step 3 (Start: 15.0s): Clicked System settings*" in output_0
assert "*Thought*: Thought 3" not in output_0
# Test last_n_detailed = 1 (normal execution context, only last detailed)
output_1 = build_plan_and_history(plan, steps, subgoal_hash_2, last_n_detailed=1)
assert "- *Step 1 (Start: 2.5s): Launched settings*" in output_1
assert "- *Step 2 (Start: 8.0s): Swiped down*" in output_1
assert "- **Step 3 (Most Recent Step, Start: 15.0s)**" in output_1
assert " - Thought 3" in output_1
# Test last_n_detailed = 3 (committee context, last 3 detailed)
output_3 = build_plan_and_history(plan, steps, subgoal_hash_2, last_n_detailed=3)
assert "- **Step 1 (Start: 2.5s)**" in output_3
assert " - Thought 1" in output_3
assert "- **Step 2 (Start: 8.0s)**" in output_3
assert " - Thought 2" in output_3
assert "- **Step 3 (Most Recent Step, Start: 15.0s)**" in output_3
assert " - Thought 3" in output_3
# Test keep_subgoal_hashes (milestone filtering for checker/diagnoser)
# Only keep steps for subgoal_hash_2
output_filter = build_plan_and_history(
plan, steps, subgoal_hash_2, keep_subgoal_hashes={subgoal_hash_2}
)
assert "Launched settings" not in output_filter # Step 1 is filtered out!
assert "Swiped down" in output_filter # Step 2 belongs to subgoal_hash_2, kept!
assert " - Thought 3" in output_filter # Step 3 is the last step, kept detailed!
def test_get_active_subgoal_hashes_fallback():
# Scenario 1: Normal Active Subgoal (Level 1 [/] active)
plan_normal = """- [x] Open Settings app
- [/] Navigate to System settings
- [ ] Navigate to Languages & input"""
h_settings = hashlib.md5(b"Open Settings app").hexdigest()
h_system = hashlib.md5(b"Navigate to System settings").hexdigest()
parent, sub = get_active_subgoal_hashes(plan_normal)
assert parent == h_system
assert sub is None
# Scenario 2: Indented Sub-subgoal Active (Level 2 [/] active, consolidated to parent)
plan_indented = """- [x] Open Settings app
- [/] Navigate to System settings
- [/] Scroll down to find System settings
- [ ] Navigate to Languages & input"""
parent, sub = get_active_subgoal_hashes(plan_indented)
assert parent == h_system
assert sub is None
# Scenario 3: Safe Fallback (All pending "[ ]", no active subgoal)
plan_fallback = """- [ ] Open Settings app
- [ ] Navigate to System settings
- [ ] Navigate to Languages & input"""
parent, sub = get_active_subgoal_hashes(plan_fallback)
assert parent == h_settings # Successfully fell back to the first subgoal's hash!
assert sub is None
# Scenario 4: All Completed (All "[x]", no active subgoal)
plan_all_done = """- [x] Open Settings app
- [x] Navigate to System settings
- [x] Navigate to Languages & input"""
parent, sub = get_active_subgoal_hashes(plan_all_done)
assert parent == "default"
assert sub is None
def test_build_plan_and_history_sliding_window():
plan = """- [x] Subgoal 1
- [x] Subgoal 2
- [/] Subgoal 3"""
subgoal_hash_1 = hashlib.md5(b"Subgoal 1").hexdigest()
subgoal_hash_2 = hashlib.md5(b"Subgoal 2").hexdigest()
subgoal_hash_3 = hashlib.md5(b"Subgoal 3").hexdigest()
steps = [
{
"step_id": "step_1",
"step_number": 1,
"summary": "Step 1 summary",
"extra_metadata": {"subgoal_hash": subgoal_hash_1},
},
{
"step_id": "step_2",
"step_number": 2,
"summary": "Step 2 summary",
"extra_metadata": {"subgoal_hash": subgoal_hash_1},
},
{
"step_id": "step_3",
"step_number": 3,
"summary": "Step 3 summary",
"extra_metadata": {"subgoal_hash": subgoal_hash_2},
},
{
"step_id": "step_4",
"step_number": 4,
"summary": "Step 4 summary",
"extra_metadata": {"subgoal_hash": subgoal_hash_3},
},
{
"step_id": "step_5",
"step_number": 5,
"summary": "Step 5 summary",
"extra_metadata": {"subgoal_hash": subgoal_hash_3},
},
{
"step_id": "step_6",
"step_number": 6,
"summary": "Step 6 summary",
"extra_metadata": {"subgoal_hash": subgoal_hash_3},
},
{
"step_id": "step_7",
"step_number": 7,
"summary": "Step 7 summary",
"extra_metadata": {"subgoal_hash": subgoal_hash_3},
},
]
# Case 1: min_summaries = 2
# subgoal 3 has 4 steps (4, 5, 6, 7). This already exceeds min_summaries=2.
# So older steps (1, 2, 3) from completed subgoals should be compressed/hidden!
output_c1 = build_plan_and_history(
plan, steps, subgoal_hash_3, min_summaries=2, last_n_detailed=0
)
assert "Step 1 summary" not in output_c1
assert "Step 2 summary" not in output_c1
assert "Step 3 summary" not in output_c1
assert "Step 4 summary" in output_c1
assert "Step 5 summary" in output_c1
assert "Step 6 summary" in output_c1
assert "Step 7 summary" in output_c1
# Case 2: min_summaries = 5
# subgoal 3 has 4 steps (4, 5, 6, 7), which translates to 3 non-last steps.
# To reach min_summaries=5 non-last steps, the sliding window slides back to keep Step 3 and Step 2 visible!
# But Step 1 should still be compressed/hidden.
output_c2 = build_plan_and_history(
plan, steps, subgoal_hash_3, min_summaries=5, last_n_detailed=0
)
assert "Step 1 summary" not in output_c2
assert "Step 2 summary" in output_c2 # Kept visible by sliding window!
assert "Step 3 summary" in output_c2 # Kept visible by sliding window!
assert "Step 4 summary" in output_c2
assert "Step 5 summary" in output_c2
assert "Step 6 summary" in output_c2
assert "Step 7 summary" in output_c2
def test_build_plan_and_history_clean_success_result():
plan = "- [ ] Subgoal 1"
steps = [
{
"step_id": "step_1",
"step_number": 1,
"relative_time": "2.5s",
"summary": "Swiped screen",
"action_taken": [{"action": "swipe"}],
"last_execution_result": {
"executed_actions": [{"action": "swipe"}],
"status": "success",
},
}
]
output = build_plan_and_history(plan, steps, "default", last_n_detailed=1)
# The output should contain "status": "success" but NOT the redundant "executed_actions"
# Success results are stripped under Factual Realism
assert "* [Validator Execution Result]" not in output
assert "executed_actions" not in output
def test_build_plan_and_history_with_tool_calls():
plan = "- [ ] Subgoal 1"
steps = [
{
"step_id": "step_1",
"step_number": 1,
"relative_time": "2.5s",
"summary": "Read note and swiped",
"action_taken": [{"action": "swipe"}],
"last_execution_result": {"status": "success"},
"tool_calls": [
{
"name": "read_note",
"payload": {
"args": {"key": "other_note"},
"result": "- [ ] Milestone",
},
"status": "success",
}
],
}
]
output = build_plan_and_history(plan, steps, "default", last_n_detailed=1)
assert "* [Operator Decision Loop]:" in output
assert '`read_note({"key": "other_note"})` -> - [ ] Milestone' in output
def test_build_plan_and_history_with_interleaved_events():
plan = "- [ ] Subgoal 1"
steps = [
{
"step_id": "step_1",
"step_number": 1,
"relative_time": "2.5s",
"summary": "Read note and thought",
"action_taken": [{"action": "swipe"}],
"last_execution_result": {"status": "success"},
"interleaved_events": [
{"type": "thought", "content": "I want to search for files."},
{
"type": "tool_call",
"name": "list_notes",
"args": {"dir": "/notes"},
"result": ["notes.txt"],
},
{"type": "thought", "content": "I see notes.txt, let's read it."},
{
"type": "tool_call",
"name": "read_note",
"args": {"file": "notes.txt"},
"result": "Milestone plan",
},
],
}
]
output = build_plan_and_history(plan, steps, "default", last_n_detailed=1)
# It should render interleaved thoughts and tool calls in natural order
assert " - I want to search for files." in output
assert ' - [Tool Call]: `list_notes({"dir": "/notes"})` -> [\'notes.txt\']' in output
assert " - I see notes.txt, let's read it." in output
assert ' - [Tool Call]: `read_note({"file": "notes.txt"})` -> Milestone plan' in output
# It should use the new section header
assert "* [Operator Decision Loop]:" in output
def test_build_plan_and_history_concatenates_thoughts():
plan = "- [ ] Subgoal 1"
steps = [
{
"step_id": "step_1",
"step_number": 1,
"relative_time": "2.5s",
"summary": "Step with multiple thoughts",
"action_taken": [{"action": "click", "thought": "Action thought"}],
"operator_raw_thinking": "Raw thought",
"last_execution_result": {"status": "success"},
"interleaved_events": [
{"type": "thought", "content": "Event thought"},
],
}
]
output = build_plan_and_history(plan, steps, "default", last_n_detailed=1)
# Check that all thoughts are concatenated and visible in the output
assert " - Action thought" in output
assert " - Event thought" in output
assert " - Raw thought" in output
def test_build_plan_and_history_chronological_last_step():
plan = "- [ ] Subgoal 1"
steps = [
{
"step_id": "step_1",
"step_number": 1,
"relative_time": "2.5s",
"summary": "Completed search",
"action_taken": [{"action": "click", "target_text": "Search"}],
"last_execution_result": {"status": "success"},
"interleaved_events": [
{"type": "thought", "content": "I should find search icon."},
{
"type": "tool_call",
"name": "ask_explorer",
"args": {"query": "search icon"},
"result": {"center": [500, 600]},
},
{
"type": "native_thought",
"content": "Clicking coordinates [500, 600] now.",
},
],
}
]
output = build_plan_and_history(
plan, steps, "default", last_n_detailed=1, chronological_last_step=True
)
assert plan in output
assert "--- Execution History ---" in output
assert "- **Step 1 (Most Recent Step, Start: 2.5s)**" in output
assert "* [Operator Decision Loop]:" in output
assert " - I should find search icon." in output
assert (
'- [Tool Call]: `ask_explorer({"query": "search icon"})` -> {"center":'
" [500, 600]}" in output
)
assert " - Clicking coordinates [500, 600] now." in output
assert "* [Planned Action]: Tapped 'Search' at None" in output
assert "* [Validator Execution Result]" not in output
def test_build_plan_and_history_safety_net_and_failure_analyzer():
plan = "- [ ] Subgoal 1"
steps = [
{
"step_id": "step_1",
"step_number": 1,
"relative_time": "5.0s",
"summary": "Repaired click search",
"action_taken": [{"action": "click", "target_text": "Search"}],
"last_execution_result": {"status": "success"},
"interleaved_events": [
{"type": "thought", "content": "Let's click search."},
# Safety net failure
{
"type": "tool_call",
"name": "safety_net_validation",
"args": {"target": "Search"},
"result": [
False,
"TARGET_DISAPPEARED",
"Target button is not visible",
],
},
# Failure Analyzer intervention
{
"type": "failure_analyzer_thought",
"content": ("The search button is covered by keyboard, need to go BACK first."),
},
{
"type": "tool_call",
"name": "_exec_press_key",
"args": {"key": "BACK", "state": "<State>"},
"result": ["Pressed key BACK", "<Bytes>", "/path/to/shot.jpg"],
},
{
"type": "failure_analyzer_thought",
"content": "Keyboard is closed, now re-executing search tap.",
},
{
"type": "tool_call",
"name": "_exec_click",
"args": {"target": [500, 600]},
"result": ["Tapped search button successfully", "<Bytes>"],
},
{
"type": "tool_call",
"name": "report_failure_analysis",
"args": {
"status": "fixed",
"analysis": "Closed keyboard and tapped search.",
},
"result": "Success",
},
],
}
]
output = build_plan_and_history(
plan, steps, "default", last_n_detailed=1, chronological_last_step=True
)
assert "- **Step 1 (Most Recent Step, Start: 5.0s)**" in output
assert "* [Operator Decision Loop]:" in output
assert " - Let's click search." in output
assert (
"* [Planned Action]: Tapped 'Search' at None (Intercepted by"
" Pre-Execution Safety Net)" in output
)
assert "* [Pre-Execution Safety Net]:" in output
assert "- [Safety Net Check]: (Action not executed: Target button is not visible)" in output
assert "* [Failure Analyzer Recovery Loop]:" in output
assert " - [Tool Call]: Pressed key 'BACK'" in output
assert " - [Tool Call]: Tapped element at [500, 600]" in output
def test_build_plan_and_history_repaired_result():
plan = "- [ ] Subgoal 1"
steps = [
{
"step_id": "step_1",
"step_number": 1,
"relative_time": "2.5s",
"summary": "Tapped button with repair",
"action_taken": [{"action": "click"}],
"last_execution_result": {
"execution": [
{
"action": "click",
"attempts": ["pre-validation failed"],
"repair": ("Tapped center to wake controls and then tapped target"),
}
],
"status": "success",
"repair_status": "fixed",
},
}
]
output = build_plan_and_history(plan, steps, "default", last_n_detailed=1)
assert " * [Validator Execution Result]: Action executed." not in output
assert " * [Result]: Repaired: Tapped center to wake controls and then tapped target" in output
def test_get_recent_subgoal_hashes_robust_milestone_exclusion(tmp_path):
# Scenario:
# 1. Milestone 1 is completed (hash_completed_1).
# 2. We had a failed attempt 'hash_failed_old' during Milestone 1.
# 3. Milestone 2 is completed (hash_completed_2).
# 4. We are now working on Milestone C (hash_active_c).
# 5. Milestone C had two previous failed renamed versions: Failed Attempt A and Failed Attempt B.
# 6. subgoal_hash_chain.json maps: A -> B -> C.
base_dir = tmp_path
notes_dir = base_dir / "notes"
notes_dir.mkdir(parents=True, exist_ok=True)
chain_path = notes_dir / "subgoal_hash_chain.json"
hash_completed_1 = hashlib.md5(b"Completed Milestone 1").hexdigest()
hash_completed_2 = hashlib.md5(b"Completed Milestone 2").hexdigest()
hash_active_c = hashlib.md5(b"Active Milestone C").hexdigest()
hash_failed_old = hashlib.md5(b"Failed Attempt Old").hexdigest()
hash_failed_a = hashlib.md5(b"Failed Attempt A").hexdigest()
hash_failed_b = hashlib.md5(b"Failed Attempt B").hexdigest()
# Write the hash chain representing: A -> B -> C
import json
chain_data = {hash_failed_a: hash_failed_b, hash_failed_b: hash_active_c}
chain_path.write_text(json.dumps(chain_data), encoding="utf-8")
steps = [
# Steps from Completed Milestone 1
{"step_id": "s1", "extra_metadata": {"subgoal_hash": hash_completed_1}},
# Old failed attempt during Milestone 1
{"step_id": "s2", "extra_metadata": {"subgoal_hash": hash_failed_old}},
# Steps from Completed Milestone 2 (Most recent completed milestone)
{"step_id": "s3", "extra_metadata": {"subgoal_hash": hash_completed_2}},
{"step_id": "s4", "extra_metadata": {"subgoal_hash": hash_completed_2}},
# Steps from Failed Attempt A (Part of the active task slot's past renames)
{"step_id": "s5", "extra_metadata": {"subgoal_hash": hash_failed_a}},
{"step_id": "s6", "extra_metadata": {"subgoal_hash": hash_failed_a}},
# Steps from Failed Attempt B (Part of the active task slot's past renames)
{"step_id": "s7", "extra_metadata": {"subgoal_hash": hash_failed_b}},
{"step_id": "s8", "extra_metadata": {"subgoal_hash": hash_failed_b}},
# Steps from Active Milestone C
{"step_id": "s9", "extra_metadata": {"subgoal_hash": hash_active_c}},
]
# Test raw transitive alias resolver
from artemis.utils.task_tree import get_all_subgoal_aliases
aliases = get_all_subgoal_aliases(hash_active_c, base_dir)
assert aliases == {hash_active_c, hash_failed_b, hash_failed_a}
keep_hashes = get_recent_subgoal_hashes(steps, hash_active_c, base_dir)
# Should keep:
# 1. The current active subgoal (hash_active_c)
assert hash_active_c in keep_hashes
# 2. The failed/renamed subgoals during active slot (hash_failed_b, hash_failed_a) - resolved transitively via chain
assert hash_failed_b in keep_hashes
assert hash_failed_a in keep_hashes
# 3. The single most recent completed subgoal (hash_completed_2) for continuity context
assert hash_completed_2 in keep_hashes
# Should prune:
# 4. Older completed subgoals (hash_completed_1)
assert hash_completed_1 not in keep_hashes
# 5. Older failed subgoals from previously completed milestones (hash_failed_old)
assert hash_failed_old not in keep_hashes
def test_build_plan_and_history_for_failure_analyzer():
plan_with_subgoals = """- [/] Goal 1
- [x] Subtask 1.1
- [/] Subtask 1.2"""
steps = [
{
"step_id": "step_1",
"step_number": 1,
"relative_time": "12.3s",
"summary": "Tapped full screen",
"action_taken": [{"action": "click", "target_text": "Fullscreen"}],
"operator_raw_thinking": "Let's tap it",
"operator_native_thinking": "Thinking to tap",
"last_execution_result": {"status": "failed"},
"interleaved_events": [
{"type": "thought", "content": "I should find fullscreen icon."},
{
"type": "tool_call",
"name": "safety_net_validation",
"args": {"target": "Fullscreen"},
"result": [
False,
"TARGET_DISAPPEARED",
"Target button is not visible",
],
},
],
}
]
# Test with subgoals and for_failure_analyzer=True
output_analyzer_with_plan = build_plan_and_history(
plan_with_subgoals,
steps,
"default",
last_n_detailed=1,
for_failure_analyzer=True,
)
assert "--- Task Plan ---" in output_analyzer_with_plan
assert (
"*(Note: Provided for context only. Do not execute the remaining"
" plan.)*" in output_analyzer_with_plan
)
assert "--- Execution History ---" in output_analyzer_with_plan
assert (
"- **Step 1 (Most Recent Step (Failed to execute, this is the step you"
" need to focus on and repair), Start: 12.3s)**" in output_analyzer_with_plan
)
assert "[Operator Decision Loop]" in output_analyzer_with_plan
assert "I should find fullscreen icon." in output_analyzer_with_plan
assert (
"* [Planned Action]: Tapped 'Fullscreen' at None (Intercepted by"
" Pre-Execution Safety Net)" in output_analyzer_with_plan
)
# Test with empty plan and for_failure_analyzer=True (should completely omit plan block)
output_analyzer_empty_plan = build_plan_and_history(
"", steps, "default", last_n_detailed=1, for_failure_analyzer=True
)
assert "--- Task Plan ---" not in output_analyzer_empty_plan
assert (
"*(Note: Provided for context only. Do not execute the remaining"
" plan.)*" not in output_analyzer_empty_plan
)
assert "--- Execution History ---" in output_analyzer_empty_plan
+349
View File
@@ -0,0 +1,349 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from artemis.utils.ui_filter import (
_clip_bounds,
_is_semantic_empty,
filter_node,
filter_ui_hierarchy,
)
def test_is_semantic_empty():
# Interactive or textual nodes are NOT empty
assert not _is_semantic_empty({"text": "Hello"})
assert not _is_semantic_empty({"content-desc": "Button"})
assert not _is_semantic_empty({"clickable": True})
assert not _is_semantic_empty({"clickable": "true"})
assert not _is_semantic_empty({"focusable": True})
assert not _is_semantic_empty({"className": "android.widget.ImageView"})
# Empty containers ARE semantic empty
assert _is_semantic_empty(
{
"text": "",
"clickable": False,
"className": "android.widget.FrameLayout",
}
)
assert _is_semantic_empty({})
def test_clip_bounds():
node = {"bounds": "[10,-10][1200,100]"}
bounds = {"left": 10, "top": -10, "right": 1200, "bottom": 100}
ancestor_bounds = {"left": 0, "top": 0, "right": 1080, "bottom": 2400}
clipped = _clip_bounds(node, bounds, ancestor_bounds)
assert clipped["left"] == 10
assert clipped["top"] == 0
assert clipped["right"] == 1080
assert clipped["bottom"] == 100
assert node["bounds"] == "[10,0][1080,100]"
assert "is_clipped" not in node
def test_clip_bounds_fully_clipped():
node = {"bounds": "[1200,2500][1300,2600]"}
bounds = {"left": 1200, "top": 2500, "right": 1300, "bottom": 2600}
ancestor_bounds = {"left": 0, "top": 0, "right": 1080, "bottom": 2400}
clipped = _clip_bounds(node, bounds, ancestor_bounds)
assert clipped["left"] == 1080
assert clipped["top"] == 2400
assert clipped["right"] == 1080
assert clipped["bottom"] == 2400
assert node["is_clipped"] is True
assert node["bounds"] == "[1080,2400][1080,2400]"
def test_semantic_hoisting():
hierarchy = [
{
"bounds": "[0,0][100,100]",
"text": "",
"clickable": False,
"children": [
{
"bounds": "[10,10][90,90]",
"text": "Child Button",
"clickable": True,
}
],
}
]
# Mode: hoist
res_hoist = filter_ui_hierarchy(hierarchy, semantic_pruning="hoist")
assert len(res_hoist) == 1
assert res_hoist[0]["text"] == "Child Button"
# Mode: safe
res_safe = filter_ui_hierarchy(hierarchy, semantic_pruning="safe")
assert len(res_safe) == 1
assert res_safe[0]["text"] == ""
assert len(res_safe[0]["children"]) == 1
def test_dynamic_min_size():
# Short side is 1000. 0.5% of 1000 is 5px.
# Element size is 4x4 -> filtered out.
node = {"bounds": "[0,0][4,4]", "text": "Tiny", "clickable": True}
res = filter_node(node, 1000, 2000, min_element_size=2, use_dynamic_min_size=True)
assert res is None
# Element size is 6x6 -> kept.
node = {"bounds": "[0,0][6,6]", "text": "Good", "clickable": True}
res = filter_node(node, 1000, 2000, min_element_size=2, use_dynamic_min_size=True)
assert res is not None
def test_ancestor_bounds_propagation():
hierarchy = {
"bounds": "[0,0][100,100]",
"text": "Parent",
"clickable": True,
"children": [{"bounds": "[50,50][150,150]", "text": "Child", "clickable": True}],
}
# The child is at [50,50][150,150], but parent is [0,0][100,100].
# So child should be clipped to [50,50][100,100].
res = filter_node(hierarchy, 1000, 1000, clip_bounds=True)
assert res is not None
assert res["children"][0]["bounds"] == "[50,50][100,100]"
def test_sibling_merging():
hierarchy = {
"clickable": True,
"bounds": "[0,0][200,200]",
"children": [
{
"className": "android.widget.ImageView",
"bounds": "[10,10][50,50]",
"clickable": False,
"focusable": False,
},
{
"className": "android.widget.TextView",
"text": "Settings",
"bounds": "[60,10][150,50]",
"clickable": False,
"focusable": False,
},
],
}
# The parent is clickable, has 1 text child and 1 image child without text.
# They should be merged!
res = filter_node(hierarchy, 1000, 1000)
assert res is not None
assert "children" in res
assert len(res["children"]) == 1
assert res["children"][0]["text"] == "Settings"
assert res["children"][0]["content-desc"] == "[Icon]"
assert res["children"][0]["bounds"] == "[10,10][150,50]"
def test_sibling_merging_not_triggered_if_child_clickable():
hierarchy = {
"clickable": True,
"bounds": "[0,0][200,200]",
"children": [
{
"className": "android.widget.ImageView",
"bounds": "[10,10][50,50]",
"clickable": True, # Child is clickable
"focusable": False,
},
{
"className": "android.widget.TextView",
"text": "Settings",
"bounds": "[60,10][150,50]",
"clickable": False,
"focusable": False,
},
],
}
res = filter_node(hierarchy, 1000, 1000)
assert res is not None
assert "children" in res
assert len(res["children"]) == 2 # No merge
def test_sibling_merging_not_triggered_if_form_control():
hierarchy = {
"clickable": True,
"bounds": "[0,0][200,200]",
"children": [
{
"className": "android.widget.CheckBox", # Form control
"bounds": "[10,10][50,50]",
"clickable": True, # Make it clickable so it's not pruned
"focusable": False,
},
{
"className": "android.widget.TextView",
"text": "Enable",
"bounds": "[60,10][150,50]",
"clickable": False,
"focusable": False,
},
],
}
res = filter_node(hierarchy, 1000, 1000)
assert res is not None
assert "children" in res
assert len(res["children"]) == 2 # No merge
def test_parent_child_redundancy():
# Case 1: Redundant TextView removed
hierarchy = {
"text": "Confirm",
"clickable": True,
"bounds": "[0,0][200,200]",
"children": [
{
"className": "android.widget.TextView",
"text": "Confirm",
"bounds": "[10,10][190,190]",
"clickable": False,
"focusable": False,
}
],
}
res = filter_node(hierarchy, 1000, 1000)
assert res is not None
assert "children" in res
assert len(res["children"]) == 0 # Child removed
# Case 2: Redundant CheckBox text cleared
hierarchy2 = {
"text": "Confirm",
"clickable": True,
"bounds": "[0,0][200,200]",
"children": [
{
"className": "android.widget.CheckBox",
"text": "Confirm",
"bounds": "[10,10][190,190]",
"clickable": False,
"focusable": False,
}
],
}
res2 = filter_node(hierarchy2, 1000, 1000)
assert res2 is not None
assert "children" in res2
assert len(res2["children"]) == 1
assert res2["children"][0]["text"] == ""
# Case 3: Redundant interactive node text cleared
hierarchy3 = {
"text": "Confirm",
"clickable": True,
"bounds": "[0,0][200,200]",
"children": [
{
"className": "android.widget.TextView",
"text": "Confirm",
"bounds": "[10,10][190,190]",
"clickable": True,
"focusable": False,
}
],
}
res3 = filter_node(hierarchy3, 1000, 1000)
assert res3 is not None
assert "children" in res3
assert len(res3["children"]) == 1
assert res3["children"][0]["text"] == ""
# Case 4: No match
hierarchy4 = {
"text": "Confirm",
"clickable": True,
"bounds": "[0,0][200,200]",
"children": [
{
"className": "android.widget.TextView",
"text": "Cancel",
"bounds": "[10,10][190,190]",
"clickable": False,
"focusable": False,
}
],
}
res4 = filter_node(hierarchy4, 1000, 1000)
assert res4 is not None
assert "children" in res4
assert len(res4["children"]) == 1
assert res4["children"][0]["text"] == "Cancel"
def test_fixed_system_bars_clamping_and_discarding():
# Simulate a full-width fixed bottom navigation bar at [0, 2250][1080, 2400]
# and two candidates: one slightly occluded (should be safely clamped) and one mostly covered (should be discarded).
hierarchy = [
{
"resource-id": "com.google.android.youtube:id/bottom_navigation",
"bounds": "[0,2250][1080,2400]",
"scrollable": False,
"clickable": False,
},
{
"text": "Partially occluded video item",
"bounds": "[0,2100][1080,2300]",
"scrollable": False,
"clickable": True,
},
{
"text": "Almost entirely covered sliver item",
"bounds": "[0,2245][1080,2300]",
"scrollable": False,
"clickable": True,
},
]
filtered = filter_ui_hierarchy(
hierarchy, screen_width=1080, screen_height=2400, min_element_size=20
)
# The bottom_navigation bar itself is kept (or filtered depending on semantics, here kept if not empty / checked).
# Specifically check the clamped item and discarded item:
texts = [el.get("text") for el in filtered]
assert "Partially occluded video item" in texts
assert "Almost entirely covered sliver item" not in texts
clamped_item = next(el for el in filtered if el.get("text") == "Partially occluded video item")
# Its original bottom was 2300, clamped safely to the bar's top at 2250
assert clamped_item["bounds"] == "[0,2100][1080,2250]"
def test_mutual_occlusion_warning_injection():
from artemis.utils.visualization import format_minimal_list_with_elements
# Simulate two overlapping non-ancestor interactive items with >= 50% overlap
fused_xml = [
{"text": "Search Video Result", "bounds": "[50,1800][1000,2000]"},
{"text": "Create Floating Button", "bounds": "[750,1850][1000,2000]"},
]
minimal_list, elements, labels = format_minimal_list_with_elements(
fused_xml, width=1080, height=2400
)
assert "WARNING: may overlap with [2], possible occlusion" in minimal_list
assert "WARNING: may overlap with [1], possible occlusion" in minimal_list
+243
View File
@@ -0,0 +1,243 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import patch
from artemis.utils.ui_hierarchy import (
ElementBounds,
Point,
find_element_by_resource_id,
get_bounds_for_element,
get_element_text,
is_element_focused,
text_input_is_empty,
)
def test_text_input_is_empty():
assert text_input_is_empty(text=None, hint_text=None)
assert text_input_is_empty(text="", hint_text=None)
assert text_input_is_empty(text="", hint_text="")
assert text_input_is_empty(text="text", hint_text="text")
assert not text_input_is_empty(text="text", hint_text=None)
assert not text_input_is_empty(text="text", hint_text="")
def test_find_element_by_resource_id():
ui_hierarchy = [
{
"resourceId": "com.example:id/button1",
"text": "Button 1",
"children": [],
},
{
"resourceId": "com.example:id/container",
"children": [
{
"resourceId": "com.example:id/nested_button",
"text": "Nested Button",
"children": [],
}
],
},
]
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button1")
assert result is not None
assert result["resourceId"] == "com.example:id/button1"
assert result["text"] == "Button 1"
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/nested_button")
assert result is not None
assert result["resourceId"] == "com.example:id/nested_button"
assert result["text"] == "Nested Button"
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/nonexistent")
assert result is None
result = find_element_by_resource_id([], "com.example:id/button1")
assert result is None
def test_find_element_by_resource_id_with_index():
ui_hierarchy = [
{
"resourceId": "com.example:id/button",
"text": "Button 1",
"children": [],
},
{
"resourceId": "com.example:id/button",
"text": "Button 2",
"children": [],
},
{
"resourceId": "com.example:id/container",
"children": [
{
"resourceId": "com.example:id/button",
"text": "Button 3",
"children": [],
}
],
},
]
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button", index=0)
assert result is not None
assert result["text"] == "Button 1"
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button", index=1)
assert result is not None
assert result["text"] == "Button 2"
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button", index=2)
assert result is not None
assert result["text"] == "Button 3"
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button", index=3)
assert result is None
def test_find_element_by_resource_id_rich_hierarchy():
rich_hierarchy = [
{
"attributes": {"resource-id": "com.example:id/button1"},
"children": [],
},
{
"attributes": {"resource-id": "com.example:id/container"},
"children": [
{
"attributes": {"resource-id": "com.example:id/nested_button"},
"children": [],
}
],
},
]
result = find_element_by_resource_id(
rich_hierarchy, "com.example:id/button1", is_rich_hierarchy=True
)
assert result is not None
assert result["resource-id"] == "com.example:id/button1"
result = find_element_by_resource_id(
rich_hierarchy, "com.example:id/nested_button", is_rich_hierarchy=True
)
assert result is not None
assert result["resource-id"] == "com.example:id/nested_button"
result = find_element_by_resource_id(
rich_hierarchy, "com.example:id/nonexistent", is_rich_hierarchy=True
)
assert result is None
def test_is_element_focused():
focused_element = {"focused": "true"}
assert is_element_focused(focused_element)
non_focused_element = {"focused": "false"}
assert not is_element_focused(non_focused_element)
no_focused_element = {"text": "some text"}
assert not is_element_focused(no_focused_element)
none_focused_element = {"focused": None}
assert not is_element_focused(none_focused_element)
def test_get_element_text():
element = {"text": "Button Text", "hintText": "Hint Text"}
assert get_element_text(element) == "Button Text"
assert get_element_text(element, hint_text=False) == "Button Text"
assert get_element_text(element, hint_text=True) == "Hint Text"
element_no_text = {"hintText": "Hint Text"}
assert get_element_text(element_no_text) is None
assert get_element_text(element_no_text, hint_text=True) == "Hint Text"
element_no_hint = {"text": "Button Text"}
assert get_element_text(element_no_hint) == "Button Text"
assert get_element_text(element_no_hint, hint_text=True) is None
empty_element = {}
assert get_element_text(empty_element) is None
assert get_element_text(empty_element, hint_text=True) is None
def test_get_bounds_for_element():
element_with_bounds = {"bounds": {"x": 10, "y": 20, "width": 100, "height": 50}}
bounds = get_bounds_for_element(element_with_bounds)
assert bounds is not None
assert isinstance(bounds, ElementBounds)
assert bounds.x == 10
assert bounds.y == 20
assert bounds.width == 100
assert bounds.height == 50
element_no_bounds = {"text": "Button"}
bounds = get_bounds_for_element(element_no_bounds)
assert bounds is None
# Suppress logger output for the invalid bounds test case
with patch("artemis.utils.ui_hierarchy.logger.error"):
element_invalid_bounds = {
"bounds": {
"x": "invalid", # Should be int
"y": 20,
"width": 100,
"height": 50,
}
}
bounds = get_bounds_for_element(element_invalid_bounds)
assert bounds is None
def test_element_bounds():
bounds = ElementBounds(x=10, y=20, width=100, height=50)
center = bounds.get_center()
assert isinstance(center, Point)
assert center.x == 60
assert center.y == 45
center_point = bounds.get_relative_point(0.5, 0.5)
assert isinstance(center_point, Point)
assert center_point.x == 60
assert center_point.y == 45
top_left = bounds.get_relative_point(0.0, 0.0)
assert top_left.x == 10
assert top_left.y == 20
bottom_right = bounds.get_relative_point(1.0, 1.0)
assert bottom_right.x == 110
assert bottom_right.y == 70
custom_point = bounds.get_relative_point(0.95, 0.95)
assert custom_point.x == 105
assert custom_point.y == 67
if __name__ == "__main__":
test_text_input_is_empty()
test_find_element_by_resource_id()
test_find_element_by_resource_id_with_index()
test_find_element_by_resource_id_rich_hierarchy()
test_is_element_focused()
test_get_element_text()
test_get_bounds_for_element()
test_element_bounds()
print("All tests passed")
+77
View File
@@ -0,0 +1,77 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from artemis.utils.verification import (
append_verification_chat,
get_verification_chat_path,
get_verification_chat_rounds,
read_verification_chat,
)
def test_get_verification_chat_path(tmp_path):
subgoal_hash = "test_hash"
expected_path = tmp_path / "notes" / f"verification_chat_{subgoal_hash}.json"
path = get_verification_chat_path(tmp_path, subgoal_hash)
assert path == expected_path
def test_read_verification_chat_not_exists(tmp_path):
path = tmp_path / "non_existent.json"
turns = read_verification_chat(path)
assert turns == []
def test_read_verification_chat_exists(tmp_path):
path = tmp_path / "chat.json"
expected_turns = [{"role": "operator", "round": 1, "content": "hello"}]
path.write_text(json.dumps(expected_turns), encoding="utf-8")
turns = read_verification_chat(path)
assert turns == expected_turns
def test_get_verification_chat_rounds():
turns = [
{"role": "operator", "round": 1, "content": "op1"},
{"role": "checker", "round": 1, "content": "chk1"},
{"role": "operator", "round": 2, "content": "op2"},
]
max_op, max_chk = get_verification_chat_rounds(turns)
assert max_op == 2
assert max_chk == 1
def test_get_verification_chat_rounds_empty():
max_op, max_chk = get_verification_chat_rounds([])
assert max_op == 0
assert max_chk == 0
def test_append_verification_chat(tmp_path):
path = tmp_path / "chat.json"
# Append to empty/non-existent
success = append_verification_chat(path, "operator", "hello", 1)
assert success
assert path.exists()
turns = json.loads(path.read_text(encoding="utf-8"))
assert turns == [{"role": "operator", "round": 1, "content": "hello"}]
# Append again
success = append_verification_chat(path, "checker", "hi", 2)
assert success
turns = json.loads(path.read_text(encoding="utf-8"))
assert len(turns) == 2
assert turns[1] == {"role": "checker", "round": 2, "content": "hi"}