test: add comprehensive unit, integration, and e2e test suites
Introduce complete test coverage for the ARTEMIS platform across multiple layers: Unit tests: Add tests for agent nodes (Planner, Operator, Validator, Diagnoser, FlashRunner, Explorer), tools, utils (OCR/XML fusion, UI hierarchy), and data engine. Integration tests: Add live tool verification, reactive runner, IME, and standalone workflow tests. E2E tests: Add end-to-end scenario tests for media playback and failure recovery flows. Fixtures & mocks: Add mock inputs (audio, video, screenshot) and test utilities.
This commit is contained in:
1 parent
c1803fcb86
commit
5d2283deaa
115 files changed
+22813
No files matched your search
@@ -0,0 +1,98 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.agents.checker.checker import CheckerResult, run_async_check
|
||||
from artemis.context import ArtemisContext
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_async_check(tmp_path):
|
||||
notes_dir = tmp_path / "notes"
|
||||
notes_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Put a dummy task plan file
|
||||
(notes_dir / "task_plan.md").write_text("Plan")
|
||||
|
||||
# Put a dummy verification_chat file
|
||||
import hashlib
|
||||
|
||||
subgoal_text = "Open WhatsApp"
|
||||
s_hash = hashlib.md5(subgoal_text.encode("utf-8")).hexdigest()
|
||||
(notes_dir / f"verification_chat_{s_hash}.json").write_text("[]")
|
||||
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = MagicMock()
|
||||
mock_ctx.data_engine.base_dir = str(tmp_path)
|
||||
mock_ctx.data_engine.get_agent_friendly_steps.return_value = []
|
||||
mock_ctx.execution_setup = None
|
||||
|
||||
# Mock controller for screen data
|
||||
mock_controller = MagicMock()
|
||||
mock_controller.take_screenshot = AsyncMock(return_value="dummy_screenshot_base64")
|
||||
|
||||
from artemis.controllers.device_controller import ScreenDataResponse
|
||||
|
||||
mock_screen_data = ScreenDataResponse(
|
||||
base64="dummy_screenshot_base64",
|
||||
elements=[],
|
||||
width=1080,
|
||||
height=2400,
|
||||
platform="android",
|
||||
)
|
||||
mock_controller.get_screen_data = AsyncMock(return_value=mock_screen_data)
|
||||
|
||||
# Mock LLM response with with_structured_output
|
||||
mock_llm = MagicMock()
|
||||
mock_response = MagicMock()
|
||||
mock_response.content = "Task completed successfully."
|
||||
mock_response.tool_calls = []
|
||||
|
||||
async def mock_astream(*args, **kwargs):
|
||||
yield mock_response
|
||||
|
||||
mock_llm.astream.side_effect = mock_astream
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
|
||||
mock_structured_llm = MagicMock()
|
||||
mock_structured_llm.ainvoke = AsyncMock(
|
||||
return_value=CheckerResult(success=True, reason="Task completed successfully.")
|
||||
)
|
||||
mock_llm.with_structured_output.return_value = mock_structured_llm
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.checker.checker.UnifiedMobileController",
|
||||
return_value=mock_controller,
|
||||
),
|
||||
patch("artemis.agents.checker.checker.get_llm", return_value=mock_llm),
|
||||
patch(
|
||||
"artemis.utils.task_tree.build_plan_and_history",
|
||||
return_value="Task tree",
|
||||
),
|
||||
):
|
||||
result = await run_async_check(
|
||||
mock_ctx,
|
||||
subgoal_text=subgoal_text,
|
||||
subgoal_hash=s_hash,
|
||||
raw_perception_data={"screenshot_b64": "dummy", "width": 1080, "height": 2400},
|
||||
latest_ui_hierarchy=[],
|
||||
)
|
||||
|
||||
assert result["status"] == "success"
|
||||
assert result["reason"] == "Task completed successfully."
|
||||
assert mock_llm.astream.called
|
||||
assert mock_llm.with_structured_output.called
|
||||
@@ -0,0 +1,308 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
from langchain_core.messages import AIMessage, ToolMessage
|
||||
from langgraph.types import Command
|
||||
from artemis.agents.diagnoser.diagnoser import Diagnoser
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.state import State
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context():
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
ctx.llm_config = Mock()
|
||||
ctx.data_engine = Mock()
|
||||
ctx.data_engine.base_dir = "/tmp"
|
||||
ctx.data_engine.get_agent_friendly_steps.return_value = []
|
||||
|
||||
# Mock execution_setup
|
||||
mock_setup = Mock()
|
||||
mock_setup.video_recording_tools_enabled = True
|
||||
ctx.execution_setup = mock_setup
|
||||
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_state():
|
||||
state = Mock(spec=State)
|
||||
state.initial_goal = "Test goal"
|
||||
state.latest_screenshot = None
|
||||
state.latest_ui_hierarchy = []
|
||||
return state
|
||||
|
||||
|
||||
@patch("artemis.agents.diagnoser.diagnoser.get_llm")
|
||||
@patch("artemis.agents.diagnoser.diagnoser.get_video_analyzer_tool")
|
||||
@patch("artemis.agents.diagnoser.diagnoser.get_analyze_logs_tool")
|
||||
@pytest.mark.asyncio
|
||||
async def test_diagnostic_agent_simple_call(
|
||||
mock_get_log_tool,
|
||||
mock_get_video_tool,
|
||||
mock_get_llm,
|
||||
mock_context,
|
||||
mock_state,
|
||||
):
|
||||
"""Test that DiagnosticAgent calls LLM and returns result."""
|
||||
mock_llm = Mock()
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
|
||||
mock_response = AIMessage(content="I diagnosed the issue: it is a network error.")
|
||||
|
||||
async def mock_astream(*args, **kwargs):
|
||||
yield mock_response
|
||||
|
||||
mock_llm.astream.side_effect = mock_astream
|
||||
mock_get_llm.return_value = mock_llm
|
||||
|
||||
# Mock tools
|
||||
mock_video_tool = Mock()
|
||||
mock_video_tool.name = "video_analyzer"
|
||||
mock_get_video_tool.return_value = mock_video_tool
|
||||
|
||||
mock_log_tool = Mock()
|
||||
mock_log_tool.name = "analyze_logs"
|
||||
mock_get_log_tool.return_value = mock_log_tool
|
||||
|
||||
agent = Diagnoser(mock_context)
|
||||
result = await agent.run("Diagnose why the app crashed", mock_state)
|
||||
|
||||
assert result == "I diagnosed the issue: it is a network error."
|
||||
|
||||
|
||||
@patch("artemis.agents.diagnoser.diagnoser.get_llm")
|
||||
@patch("artemis.agents.diagnoser.diagnoser.get_video_analyzer_tool")
|
||||
@patch("artemis.agents.diagnoser.diagnoser.get_analyze_logs_tool")
|
||||
@pytest.mark.asyncio
|
||||
async def test_diagnostic_agent_tool_calls(
|
||||
mock_get_log_tool,
|
||||
mock_get_video_tool,
|
||||
mock_get_llm,
|
||||
mock_context,
|
||||
mock_state,
|
||||
):
|
||||
"""Test that DiagnosticAgent handles tool calls."""
|
||||
mock_llm = Mock()
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
|
||||
# Simulate response with tool calls
|
||||
mock_response = AIMessage(
|
||||
content="I need to check logs and video.",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "analyze_logs",
|
||||
"args": {"specific_query": "Check for crashes"},
|
||||
"id": "call_1",
|
||||
},
|
||||
{
|
||||
"name": "video_analyzer",
|
||||
"args": {
|
||||
"time_description": "around 10s",
|
||||
"purpose": "Check UI",
|
||||
},
|
||||
"id": "call_2",
|
||||
},
|
||||
],
|
||||
)
|
||||
# Second call returns final answer
|
||||
mock_final_response = AIMessage(content="Diagnosis complete.")
|
||||
|
||||
responses = [mock_response, mock_final_response]
|
||||
call_count = 0
|
||||
|
||||
async def mock_astream(*args, **kwargs):
|
||||
nonlocal call_count
|
||||
if call_count < len(responses):
|
||||
yield responses[call_count]
|
||||
call_count += 1
|
||||
|
||||
mock_llm.astream.side_effect = mock_astream
|
||||
mock_get_llm.return_value = mock_llm
|
||||
|
||||
# Mock tools
|
||||
mock_log_tool = AsyncMock()
|
||||
mock_log_tool.name = "analyze_logs"
|
||||
mock_log_tool.args = {"specific_query": None, "state": None}
|
||||
mock_log_tool.ainvoke.return_value = "Logs show no crash."
|
||||
mock_get_log_tool.return_value = mock_log_tool
|
||||
|
||||
mock_video_tool = AsyncMock()
|
||||
mock_video_tool.name = "video_analyzer"
|
||||
mock_video_tool.args = {
|
||||
"time_description": None,
|
||||
"purpose": None,
|
||||
"state": None,
|
||||
}
|
||||
mock_command = Command(
|
||||
update={
|
||||
VALIDATOR_MESSAGES_KEY: [
|
||||
ToolMessage(content="Video shows UI loading.", tool_call_id="call_2")
|
||||
]
|
||||
}
|
||||
)
|
||||
mock_video_tool.ainvoke.return_value = mock_command
|
||||
mock_get_video_tool.return_value = mock_video_tool
|
||||
|
||||
agent = Diagnoser(mock_context)
|
||||
result = await agent.run("Diagnose issue", mock_state)
|
||||
|
||||
assert result == "Diagnosis complete."
|
||||
mock_log_tool.ainvoke.assert_called_once_with(
|
||||
{"specific_query": "Check for crashes", "state": mock_state}
|
||||
)
|
||||
mock_video_tool.ainvoke.assert_called_once()
|
||||
|
||||
|
||||
@patch("artemis.agents.diagnoser.diagnoser.get_llm")
|
||||
@patch("artemis.agents.diagnoser.diagnoser.get_video_analyzer_tool")
|
||||
@patch("artemis.agents.diagnoser.diagnoser.get_analyze_logs_tool")
|
||||
@pytest.mark.asyncio
|
||||
async def test_diagnostic_agent_no_video_tool_when_disabled(
|
||||
mock_get_log_tool,
|
||||
mock_get_video_tool,
|
||||
mock_get_llm,
|
||||
mock_context,
|
||||
mock_state,
|
||||
):
|
||||
"""Test that Diagnoser does not bind video tool when disabled."""
|
||||
# Set video recording to False
|
||||
mock_context.execution_setup.video_recording_tools_enabled = False
|
||||
|
||||
mock_llm = Mock()
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
|
||||
mock_response = AIMessage(content="Diagnosis complete.")
|
||||
|
||||
async def mock_astream(*args, **kwargs):
|
||||
yield mock_response
|
||||
|
||||
mock_llm.astream.side_effect = mock_astream
|
||||
mock_get_llm.return_value = mock_llm
|
||||
|
||||
# Mock tools
|
||||
mock_video_tool = Mock()
|
||||
mock_video_tool.name = "video_analyzer"
|
||||
mock_get_video_tool.return_value = mock_video_tool
|
||||
|
||||
mock_log_tool = Mock()
|
||||
mock_log_tool.name = "analyze_logs"
|
||||
mock_get_log_tool.return_value = mock_log_tool
|
||||
|
||||
agent = Diagnoser(mock_context)
|
||||
await agent.run("Diagnose issue", mock_state)
|
||||
|
||||
# Verify bind_tools was called
|
||||
mock_llm.bind_tools.assert_called_once()
|
||||
bound_tools = mock_llm.bind_tools.call_args[1]["tools"]
|
||||
|
||||
# Verify video_analyzer is NOT in bound tools
|
||||
tool_names = [t.name for t in bound_tools if hasattr(t, "name")]
|
||||
assert "video_analyzer" not in tool_names
|
||||
# Verify log tool IS in bound tools (should still be there)
|
||||
assert "analyze_logs" in tool_names
|
||||
|
||||
|
||||
@patch("artemis.agents.diagnoser.diagnoser.get_llm")
|
||||
@patch("artemis.tools.video_tool.VideoAnalyzer")
|
||||
@pytest.mark.asyncio
|
||||
async def test_diagnoser_background_job_lifecycle(
|
||||
mock_video_analyzer_class, mock_get_llm, mock_state
|
||||
):
|
||||
# Setup context
|
||||
ctx = ArtemisContext.model_construct(
|
||||
device=Mock(),
|
||||
llm_config=Mock(),
|
||||
execution_setup=Mock(video_recording_tools_enabled=True),
|
||||
)
|
||||
ctx.data_engine = None
|
||||
|
||||
# The Mock VideoAnalyzer agent running inside the background job
|
||||
mock_agent_instance = AsyncMock()
|
||||
|
||||
async def mock_agent_run(time_desc, purpose):
|
||||
return "Video shows step successfully completed.", "success"
|
||||
|
||||
mock_agent_instance.run.side_effect = mock_agent_run
|
||||
mock_video_analyzer_class.return_value = mock_agent_instance
|
||||
|
||||
# Prepare LLM responses
|
||||
mock_llm = Mock()
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
|
||||
# Turn 1: LLM requests tool call
|
||||
mock_response_1 = AIMessage(
|
||||
content="Checking the video first.",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "video_analyzer",
|
||||
"args": {
|
||||
"time_description": "from 0s to 5s",
|
||||
"purpose": "check UI",
|
||||
},
|
||||
"id": "call_video_1",
|
||||
}
|
||||
],
|
||||
)
|
||||
# Turn 2: LLM gets the initial job start message, continues, then background job finishes, and result is injected.
|
||||
mock_response_2 = AIMessage(content="Final complete diagnosis.")
|
||||
|
||||
responses = [mock_response_1, mock_response_2]
|
||||
call_count = 0
|
||||
|
||||
async def mock_astream(*args, **kwargs):
|
||||
nonlocal call_count
|
||||
if call_count < len(responses):
|
||||
yield responses[call_count]
|
||||
call_count += 1
|
||||
|
||||
mock_llm.astream.side_effect = mock_astream
|
||||
mock_get_llm.return_value = mock_llm
|
||||
|
||||
# Mock asanitize_update on mock_state
|
||||
async def mock_asanitize_update(ctx, update, agent):
|
||||
return update
|
||||
|
||||
mock_state.asanitize_update = mock_asanitize_update
|
||||
|
||||
# Instantiate the Diagnoser
|
||||
agent = Diagnoser(ctx)
|
||||
|
||||
# We want to use the actual video analyzer tool so it triggers the background task
|
||||
from artemis.tools.video_tool import get_video_analyzer_tool
|
||||
|
||||
real_video_tool = get_video_analyzer_tool(ctx, role="diagnoser")
|
||||
|
||||
with patch(
|
||||
"artemis.tools.log_tool.get_analyze_logs_tool",
|
||||
return_value=Mock(name="analyze_logs"),
|
||||
):
|
||||
with patch(
|
||||
"artemis.tools.video_tool.get_video_analyzer_tool",
|
||||
return_value=real_video_tool,
|
||||
):
|
||||
result = await agent.run("Diagnose video issue", mock_state)
|
||||
|
||||
assert result == "Final complete diagnosis."
|
||||
# Verify job registry
|
||||
assert len(ctx.background_jobs) == 1
|
||||
job_id = list(ctx.background_jobs.keys())[0]
|
||||
job = ctx.background_jobs[job_id]
|
||||
assert job["tool_call_id"] == "call_video_1"
|
||||
assert job["status"] == "completed"
|
||||
assert job["result"] == "Video shows step successfully completed."
|
||||
assert job["consumed"] is True
|
||||
@@ -0,0 +1,343 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
from adbutils import AdbClient
|
||||
from langchain_core.messages import AIMessage, ToolMessage
|
||||
from artemis.agents.diagnoser.diagnoser import Diagnoser
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.state import State
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def ensure_device_connected():
|
||||
try:
|
||||
client = AdbClient()
|
||||
if not client.device_list():
|
||||
pytest.skip("No active Android device/emulator found. Skipping diagnostic tool tests.")
|
||||
except Exception as e:
|
||||
pytest.skip(f"ADB Client connection failed: {e}. Skipping tests.")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context():
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
ctx.llm_config = Mock()
|
||||
ctx.data_engine = Mock()
|
||||
ctx.data_engine.base_dir = "/tmp"
|
||||
ctx.data_engine.get_agent_friendly_steps.return_value = []
|
||||
|
||||
# Mock execution_setup
|
||||
mock_setup = Mock()
|
||||
mock_setup.video_recording_tools_enabled = True
|
||||
ctx.execution_setup = mock_setup
|
||||
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_state():
|
||||
state = Mock(spec=State)
|
||||
state.initial_goal = "Test goal"
|
||||
state.latest_screenshot = None
|
||||
state.latest_ui_hierarchy = []
|
||||
return state
|
||||
|
||||
|
||||
@patch("artemis.agents.diagnoser.diagnoser.get_llm")
|
||||
@patch("artemis.tools.video_tool.get_video_analyzer_tool")
|
||||
@patch("artemis.tools.log_tool.get_analyze_logs_tool")
|
||||
@patch("artemis.agents.diagnoser.diagnoser.get_ui_hierarchy_tool")
|
||||
@pytest.mark.asyncio
|
||||
async def test_maestro_uninstall_timeout(
|
||||
mock_get_ui_tool,
|
||||
mock_get_log_tool,
|
||||
mock_get_video_tool,
|
||||
mock_get_llm,
|
||||
mock_context,
|
||||
mock_state,
|
||||
):
|
||||
"""Test Maestro Uninstall Timeout / Blocker Failure."""
|
||||
# Mock LLM calls
|
||||
mock_llm = Mock()
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
|
||||
mock_response_1 = AIMessage(
|
||||
content="I need to check the UI hierarchy.",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "get_ui_hierarchy",
|
||||
"args": {},
|
||||
"id": "call_hierarchy",
|
||||
}
|
||||
],
|
||||
)
|
||||
mock_response_2 = AIMessage(
|
||||
content=(
|
||||
"Diagnoser identifies the blocker failure and recommends restarting"
|
||||
" ADB/uninstalling conflicting tools manually."
|
||||
)
|
||||
)
|
||||
|
||||
responses = [mock_response_1, mock_response_2]
|
||||
call_count = 0
|
||||
|
||||
async def mock_astream(*args, **kwargs):
|
||||
nonlocal call_count
|
||||
if call_count < len(responses):
|
||||
yield responses[call_count]
|
||||
call_count += 1
|
||||
|
||||
mock_llm.astream.side_effect = mock_astream
|
||||
mock_get_llm.return_value = mock_llm
|
||||
|
||||
# Mock tool
|
||||
mock_ui_tool = AsyncMock()
|
||||
mock_ui_tool.name = "get_ui_hierarchy"
|
||||
mock_ui_tool.args = {}
|
||||
mock_ui_tool.ainvoke.return_value = "Error: pm list packages timed out"
|
||||
mock_get_ui_tool.return_value = mock_ui_tool
|
||||
|
||||
# Run agent
|
||||
agent = Diagnoser(mock_context)
|
||||
result = await agent.run("Diagnose why the app crashed", mock_state)
|
||||
|
||||
assert "recommends restarting ADB" in result
|
||||
assert "uninstalling conflicting tools" in result
|
||||
|
||||
# Assert get_ui_hierarchy was called
|
||||
mock_ui_tool.ainvoke.assert_called_once()
|
||||
|
||||
# Assert that the ToolMessage was sent back to LLM
|
||||
assert len(mock_llm.astream.call_args_list) == 2
|
||||
sent_messages = mock_llm.astream.call_args_list[1][0][0]
|
||||
tool_message = sent_messages[-2]
|
||||
assert isinstance(tool_message, ToolMessage)
|
||||
assert tool_message.tool_call_id == "call_hierarchy"
|
||||
assert tool_message.content == "Error: pm list packages timed out"
|
||||
assert tool_message.status == "error"
|
||||
|
||||
|
||||
@patch("artemis.agents.diagnoser.diagnoser.get_llm")
|
||||
@patch("artemis.tools.video_tool.get_video_analyzer_tool")
|
||||
@patch("artemis.tools.log_tool.get_analyze_logs_tool")
|
||||
@patch("artemis.agents.diagnoser.diagnoser.get_ui_hierarchy_tool")
|
||||
@pytest.mark.asyncio
|
||||
async def test_uiautomator2_connection_lost(
|
||||
mock_get_ui_tool,
|
||||
mock_get_log_tool,
|
||||
mock_get_video_tool,
|
||||
mock_get_llm,
|
||||
mock_context,
|
||||
mock_state,
|
||||
):
|
||||
"""Test UIAutomator2 Daemon Connection Lost."""
|
||||
# Mock LLM
|
||||
mock_llm = Mock()
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
|
||||
mock_response_1 = AIMessage(
|
||||
content="Let me check the UI hierarchy.",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "get_ui_hierarchy",
|
||||
"args": {},
|
||||
"id": "call_hierarchy_lost",
|
||||
}
|
||||
],
|
||||
)
|
||||
mock_response_2 = AIMessage(content="Diagnoser reports connection loss with UIAutomator2.")
|
||||
|
||||
responses = [mock_response_1, mock_response_2]
|
||||
call_count = 0
|
||||
|
||||
async def mock_astream(*args, **kwargs):
|
||||
nonlocal call_count
|
||||
if call_count < len(responses):
|
||||
yield responses[call_count]
|
||||
call_count += 1
|
||||
|
||||
mock_llm.astream.side_effect = mock_astream
|
||||
mock_get_llm.return_value = mock_llm
|
||||
|
||||
# Mock tool to throw ConnectionError
|
||||
mock_ui_tool = AsyncMock()
|
||||
mock_ui_tool.name = "get_ui_hierarchy"
|
||||
mock_ui_tool.args = {}
|
||||
mock_ui_tool.ainvoke.side_effect = ConnectionError(
|
||||
"UIAutomator2 connection lost, reconnecting..."
|
||||
)
|
||||
mock_get_ui_tool.return_value = mock_ui_tool
|
||||
|
||||
# Run agent
|
||||
agent = Diagnoser(mock_context)
|
||||
result = await agent.run("Diagnose crash", mock_state)
|
||||
|
||||
assert "reports connection loss" in result
|
||||
|
||||
# Assert that tool error is wrapped and returned in the ToolMessage
|
||||
assert len(mock_llm.astream.call_args_list) == 2
|
||||
sent_messages = mock_llm.astream.call_args_list[1][0][0]
|
||||
tool_message = sent_messages[-2]
|
||||
assert isinstance(tool_message, ToolMessage)
|
||||
assert tool_message.tool_call_id == "call_hierarchy_lost"
|
||||
assert "UIAutomator2 connection lost, reconnecting..." in tool_message.content
|
||||
assert tool_message.status == "error"
|
||||
|
||||
# Verify that the trace span recorded the failure
|
||||
record_trace_calls = mock_context.data_engine.record_trace.call_args_list
|
||||
tool_trace_calls = [c for c in record_trace_calls if c[1].get("name") == "get_ui_hierarchy"]
|
||||
failed_trace = next((c for c in tool_trace_calls if c[1].get("status") == "failed"), None)
|
||||
assert failed_trace is not None
|
||||
assert "UIAutomator2 connection lost, reconnecting..." in failed_trace[1]["payload"]["error"]
|
||||
|
||||
|
||||
@patch("artemis.agents.diagnoser.diagnoser.get_llm")
|
||||
@patch("artemis.tools.video_tool.get_video_analyzer_tool")
|
||||
@patch("artemis.tools.log_tool.get_analyze_logs_tool")
|
||||
@patch("artemis.agents.diagnoser.diagnoser.get_ui_hierarchy_tool")
|
||||
@pytest.mark.asyncio
|
||||
async def test_ui_hierarchy_dump_stall(
|
||||
mock_get_ui_tool,
|
||||
mock_get_log_tool,
|
||||
mock_get_video_tool,
|
||||
mock_get_llm,
|
||||
mock_context,
|
||||
mock_state,
|
||||
):
|
||||
"""Test UI Hierarchy Dump Stall (UI Thread Frozen)."""
|
||||
# Mock LLM
|
||||
mock_llm = Mock()
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
|
||||
mock_response_1 = AIMessage(
|
||||
content="Let me fetch the UI hierarchy to see what's on screen.",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "get_ui_hierarchy",
|
||||
"args": {},
|
||||
"id": "call_hierarchy_stall",
|
||||
}
|
||||
],
|
||||
)
|
||||
mock_response_2 = AIMessage(
|
||||
content=(
|
||||
"Diagnoser identifies frozen UI and suggests rendering-safe"
|
||||
" recovery actions (like ADB wake commands or a wait/retry cycle)."
|
||||
)
|
||||
)
|
||||
|
||||
responses = [mock_response_1, mock_response_2]
|
||||
call_count = 0
|
||||
|
||||
async def mock_astream(*args, **kwargs):
|
||||
nonlocal call_count
|
||||
if call_count < len(responses):
|
||||
yield responses[call_count]
|
||||
call_count += 1
|
||||
|
||||
mock_llm.astream.side_effect = mock_astream
|
||||
mock_get_llm.return_value = mock_llm
|
||||
|
||||
# Mock tool
|
||||
mock_ui_tool = AsyncMock()
|
||||
mock_ui_tool.name = "get_ui_hierarchy"
|
||||
mock_ui_tool.args = {}
|
||||
mock_ui_tool.ainvoke.return_value = (
|
||||
"Error retrieving UI hierarchy: dump_hierarchy timed out after 3 seconds"
|
||||
)
|
||||
mock_get_ui_tool.return_value = mock_ui_tool
|
||||
|
||||
# Run agent
|
||||
agent = Diagnoser(mock_context)
|
||||
result = await agent.run("Diagnose stall", mock_state)
|
||||
|
||||
assert "rendering-safe recovery actions" in result
|
||||
|
||||
# Assert ToolMessage was passed back with correct error details
|
||||
assert len(mock_llm.astream.call_args_list) == 2
|
||||
sent_messages = mock_llm.astream.call_args_list[1][0][0]
|
||||
tool_message = sent_messages[-2]
|
||||
assert isinstance(tool_message, ToolMessage)
|
||||
assert tool_message.tool_call_id == "call_hierarchy_stall"
|
||||
assert "dump_hierarchy timed out after 3 seconds" in tool_message.content
|
||||
assert tool_message.status == "error"
|
||||
|
||||
|
||||
@patch("artemis.agents.diagnoser.diagnoser.get_llm")
|
||||
@patch("artemis.tools.video_tool.get_video_analyzer_tool")
|
||||
@patch("artemis.tools.log_tool.get_analyze_logs_tool")
|
||||
@patch("artemis.agents.diagnoser.diagnoser.get_ui_hierarchy_tool")
|
||||
@pytest.mark.asyncio
|
||||
async def test_device_offline(
|
||||
mock_get_ui_tool,
|
||||
mock_get_log_tool,
|
||||
mock_get_video_tool,
|
||||
mock_get_llm,
|
||||
mock_context,
|
||||
mock_state,
|
||||
):
|
||||
"""Test Device Offline (ADB Transport Lost)."""
|
||||
# Mock LLM
|
||||
mock_llm = Mock()
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
|
||||
mock_response_1 = AIMessage(
|
||||
content="Let's grab the UI hierarchy.",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "get_ui_hierarchy",
|
||||
"args": {},
|
||||
"id": "call_hierarchy_offline",
|
||||
}
|
||||
],
|
||||
)
|
||||
mock_response_2 = AIMessage(
|
||||
content=("The device is offline. Diagnoser flags the offline device state.")
|
||||
)
|
||||
|
||||
responses = [mock_response_1, mock_response_2]
|
||||
call_count = 0
|
||||
|
||||
async def mock_astream(*args, **kwargs):
|
||||
nonlocal call_count
|
||||
if call_count < len(responses):
|
||||
yield responses[call_count]
|
||||
call_count += 1
|
||||
|
||||
mock_llm.astream.side_effect = mock_astream
|
||||
mock_get_llm.return_value = mock_llm
|
||||
|
||||
# Mock tool
|
||||
mock_ui_tool = AsyncMock()
|
||||
mock_ui_tool.name = "get_ui_hierarchy"
|
||||
mock_ui_tool.args = {}
|
||||
mock_ui_tool.ainvoke.side_effect = RuntimeError("device offline")
|
||||
mock_get_ui_tool.return_value = mock_ui_tool
|
||||
|
||||
# Run agent
|
||||
agent = Diagnoser(mock_context)
|
||||
result = await agent.run("Diagnose offline", mock_state)
|
||||
|
||||
assert "flags the offline device state" in result
|
||||
|
||||
# Assert ToolMessage was passed back with correct error details
|
||||
assert len(mock_llm.astream.call_args_list) == 2
|
||||
sent_messages = mock_llm.astream.call_args_list[1][0][0]
|
||||
tool_message = sent_messages[-2]
|
||||
assert isinstance(tool_message, ToolMessage)
|
||||
assert tool_message.tool_call_id == "call_hierarchy_offline"
|
||||
assert "device offline" in tool_message.content
|
||||
assert tool_message.status == "error"
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,127 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for Universal Explorer and Multi-Model Grounding Engine."""
|
||||
|
||||
import json
|
||||
from unittest.mock import AsyncMock, MagicMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import AIMessage
|
||||
|
||||
from artemis.agents.explorer.explorer import (
|
||||
UNIVERSAL_EXPLORER_TOOLS,
|
||||
Explorer,
|
||||
)
|
||||
from artemis.config import settings
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.state import State
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context(tmp_path):
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
ctx.llm_config = Mock()
|
||||
mock_llm_cfg = Mock()
|
||||
mock_llm_cfg.model = "claude-3-7-sonnet"
|
||||
mock_llm_cfg.temperature = 0.1
|
||||
ctx.llm_config.explorer = mock_llm_cfg
|
||||
|
||||
ctx.device = Mock()
|
||||
ctx.device.device_width = 1080
|
||||
ctx.device.device_height = 2400
|
||||
|
||||
ctx.agent_config = Mock()
|
||||
ctx.agent_config.blacklisted_tools = {}
|
||||
|
||||
ctx.data_engine = Mock()
|
||||
ctx.data_engine.base_dir = tmp_path
|
||||
return ctx
|
||||
|
||||
|
||||
def test_explorer_universal_engine_detection(mock_context):
|
||||
"""Test that Explorer routes to Universal Engine when non-Gemini model is configured."""
|
||||
with patch.object(settings, "GOOGLE_API_KEY", None):
|
||||
agent = Explorer(mock_context)
|
||||
assert agent.use_native_gemini is False
|
||||
assert agent.model_name == "claude-3-7-sonnet"
|
||||
|
||||
|
||||
def test_universal_explorer_tool_schemas():
|
||||
"""Verify schema integrity of Universal Explorer tool declarations."""
|
||||
assert len(UNIVERSAL_EXPLORER_TOOLS) == 6
|
||||
names = [t["function"]["name"] for t in UNIVERSAL_EXPLORER_TOOLS]
|
||||
assert "ask_perception_tool" in names
|
||||
assert "detect_objects" in names
|
||||
assert "get_ocr_list" in names
|
||||
assert "ask_image_processor" in names
|
||||
assert "inspect_region" in names
|
||||
assert "submit_answer" in names
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_universal_explorer_success(mock_context, tmp_path):
|
||||
"""Test universal Explorer tool invocation and submit_answer coordinate parsing."""
|
||||
agent = Explorer(mock_context)
|
||||
agent.use_native_gemini = False
|
||||
|
||||
fake_screenshot = tmp_path / "screenshot.jpg"
|
||||
fake_screenshot.write_bytes(b"fake_image_bytes")
|
||||
|
||||
# Mock tool call response for submit_answer
|
||||
msg_response = AIMessage(
|
||||
content="Submitting grounded elements.",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "submit_answer",
|
||||
"args": {
|
||||
"candidates": [
|
||||
{
|
||||
"label": "D1",
|
||||
"coords": [500, 600],
|
||||
"description": "Settings button",
|
||||
}
|
||||
],
|
||||
"fallback_message": "",
|
||||
},
|
||||
"id": "exp_call_1",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
mock_llm = MagicMock()
|
||||
mock_bound = MagicMock()
|
||||
mock_bound.ainvoke = AsyncMock(return_value=msg_response)
|
||||
mock_llm.bind_tools.return_value = mock_bound
|
||||
|
||||
with patch(
|
||||
"artemis.agents.explorer.explorer.get_llm",
|
||||
return_value=mock_llm,
|
||||
):
|
||||
mock_state = Mock(spec=State)
|
||||
res_str = await agent._run_universal(
|
||||
query="find settings icon",
|
||||
context_feedback="",
|
||||
screenshot_path=str(fake_screenshot),
|
||||
state=mock_state,
|
||||
minimal_list="",
|
||||
version="pro",
|
||||
prompt_template="You are an explorer.",
|
||||
max_iterations=3,
|
||||
)
|
||||
|
||||
res = json.loads(res_str)
|
||||
assert len(res["candidates"]) == 1
|
||||
assert res["candidates"][0]["label"] == "D1"
|
||||
assert res["candidates"][0]["coords"] == [500, 600]
|
||||
@@ -0,0 +1,213 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for Universal Failure Analyzer."""
|
||||
|
||||
import base64
|
||||
from unittest.mock import AsyncMock, MagicMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import AIMessage
|
||||
|
||||
from artemis.agents.validator.failure_analyzer import (
|
||||
FailureAnalyzer,
|
||||
PixelTargetDisappearedStrategy,
|
||||
TargetDisappearedStrategy,
|
||||
ValidationErrorCategory,
|
||||
VALIDATOR_TOOLS_DECLARATION,
|
||||
)
|
||||
from artemis.context import ArtemisContext
|
||||
|
||||
|
||||
class DummyState:
|
||||
def __init__(self, latest_screenshot=None):
|
||||
self.latest_screenshot = latest_screenshot
|
||||
self.indexed_points = []
|
||||
self.indexed_elements = []
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context(tmp_path):
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
ctx.llm_config = Mock()
|
||||
mock_llm_cfg = Mock()
|
||||
mock_llm_cfg.model = "gemini-2.5-pro"
|
||||
mock_llm_cfg.temperature = 0.1
|
||||
ctx.llm_config.get_agent.return_value = mock_llm_cfg
|
||||
|
||||
ctx.device = Mock()
|
||||
ctx.device.device_width = 1080
|
||||
ctx.device.device_height = 2400
|
||||
|
||||
ctx.data_engine = Mock()
|
||||
ctx.data_engine.base_dir = tmp_path
|
||||
ctx.data_engine.get_agent_friendly_steps.return_value = []
|
||||
ctx.adb_client = None
|
||||
ctx.driver = Mock()
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failure_analyzer_cannot_fix(mock_context, tmp_path):
|
||||
"""Test that FailureAnalyzer natively executes and handles 'cannot_fix' report."""
|
||||
screenshot_file = tmp_path / "shot.jpg"
|
||||
screenshot_file.write_bytes(b"dummy_bytes")
|
||||
|
||||
state = DummyState(latest_screenshot=str(screenshot_file))
|
||||
failed_action = {"action": "click", "coordinates": [100, 200]}
|
||||
error_msg = "Element not found"
|
||||
|
||||
analyzer = FailureAnalyzer(mock_context)
|
||||
|
||||
mock_llm = MagicMock()
|
||||
mock_response = AIMessage(
|
||||
content="I see the error.",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "report_failure_analysis",
|
||||
"args": {
|
||||
"status": "cannot_fix",
|
||||
"analysis": "The element was definitely not there.",
|
||||
},
|
||||
"id": "call_1",
|
||||
}
|
||||
],
|
||||
)
|
||||
mock_bound = MagicMock()
|
||||
mock_bound.ainvoke = AsyncMock(return_value=mock_response)
|
||||
mock_llm.bind_tools.return_value = mock_bound
|
||||
|
||||
with (
|
||||
patch("artemis.agents.validator.failure_analyzer.get_llm", return_value=mock_llm),
|
||||
patch(
|
||||
"artemis.agents.validator.failure_analyzer.UnifiedMobileController"
|
||||
) as mock_controller_cls,
|
||||
):
|
||||
mock_controller = Mock()
|
||||
mock_controller_cls.return_value = mock_controller
|
||||
mock_device_data = Mock()
|
||||
mock_device_data.base64 = base64.b64encode(b"mocked_screenshot").decode()
|
||||
mock_device_data.elements = []
|
||||
mock_device_data.width = 1080
|
||||
mock_device_data.height = 2400
|
||||
mock_controller.get_screen_data = AsyncMock(return_value=mock_device_data)
|
||||
mock_controller.get_ui_elements = AsyncMock(return_value=mock_device_data.elements)
|
||||
|
||||
result = await analyzer.analyze(
|
||||
state,
|
||||
failed_action,
|
||||
error_msg,
|
||||
pre_screenshot=base64.b64encode(b"dummy").decode(),
|
||||
post_screenshot=base64.b64encode(b"dummy").decode(),
|
||||
)
|
||||
|
||||
assert result["status"] == "cannot_fix"
|
||||
assert "not there" in result["analysis"]
|
||||
assert result["new_remaining_actions"] == []
|
||||
|
||||
|
||||
def test_report_failure_analysis_tool_declaration():
|
||||
"""Test that report_failure_analysis tool declaration is correct and matches."""
|
||||
report_tool = next(
|
||||
(t for t in VALIDATOR_TOOLS_DECLARATION if t.name == "report_failure_analysis"),
|
||||
None,
|
||||
)
|
||||
assert report_tool is not None
|
||||
props = report_tool.parameters.get("properties", {})
|
||||
assert "analysis" in props
|
||||
assert "single paragraph" in props["analysis"]["description"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failure_analyzer_strategy_routing(mock_context):
|
||||
"""Test that FailureAnalyzer selects appropriate strategies based on error category."""
|
||||
analyzer = FailureAnalyzer(mock_context)
|
||||
|
||||
strat_disappeared = analyzer._select_strategy(ValidationErrorCategory.TARGET_DISAPPEARED)
|
||||
assert isinstance(strat_disappeared, TargetDisappearedStrategy)
|
||||
assert "target_disappeared" in strat_disappeared.get_prompt_template_name()
|
||||
|
||||
strat_pixel = analyzer._select_strategy(ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED)
|
||||
assert isinstance(strat_pixel, PixelTargetDisappearedStrategy)
|
||||
assert "pixel_target_disappeared" in strat_pixel.get_prompt_template_name()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failure_analyzer_click_sequence(mock_context, tmp_path):
|
||||
"""Test click_sequence execution in FailureAnalyzer."""
|
||||
screenshot_file = tmp_path / "shot.jpg"
|
||||
screenshot_file.write_bytes(b"dummy_bytes")
|
||||
|
||||
state = DummyState(latest_screenshot=str(screenshot_file))
|
||||
analyzer = FailureAnalyzer(mock_context)
|
||||
|
||||
# Turn 1: Call click_sequence tool with direct coordinates
|
||||
call_step_1 = AIMessage(
|
||||
content="Let me click in sequence.",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "click_sequence",
|
||||
"args": {"sequence": [[100, 200], [300, 400]]},
|
||||
"id": "call_seq_1",
|
||||
}
|
||||
],
|
||||
)
|
||||
# Turn 2: Report failure fixed
|
||||
call_step_2 = AIMessage(
|
||||
content="Repaired successfully.",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "report_failure_analysis",
|
||||
"args": {
|
||||
"status": "fixed",
|
||||
"analysis": "Sequence clicked successfully.",
|
||||
},
|
||||
"id": "call_report_2",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
mock_llm = MagicMock()
|
||||
mock_bound = MagicMock()
|
||||
mock_bound.ainvoke = AsyncMock(side_effect=[call_step_1, call_step_2])
|
||||
mock_llm.bind_tools.return_value = mock_bound
|
||||
|
||||
with (
|
||||
patch("artemis.agents.validator.failure_analyzer.get_llm", return_value=mock_llm),
|
||||
patch(
|
||||
"artemis.agents.validator.failure_analyzer.UnifiedMobileController"
|
||||
) as mock_controller_cls,
|
||||
):
|
||||
mock_controller = Mock()
|
||||
mock_controller_cls.return_value = mock_controller
|
||||
mock_device_data = Mock()
|
||||
mock_device_data.base64 = base64.b64encode(b"mocked_screenshot").decode()
|
||||
mock_device_data.elements = []
|
||||
mock_device_data.width = 1080
|
||||
mock_device_data.height = 2400
|
||||
mock_controller.get_screen_data = AsyncMock(return_value=mock_device_data)
|
||||
mock_controller.get_ui_elements = AsyncMock(return_value=mock_device_data.elements)
|
||||
mock_controller.tap_at = AsyncMock(return_value=Mock(error=None))
|
||||
|
||||
result = await analyzer.analyze(
|
||||
state,
|
||||
{"action": "click", "coordinates": [100, 200]},
|
||||
"Element not found",
|
||||
pre_screenshot=base64.b64encode(b"dummy").decode(),
|
||||
post_screenshot=base64.b64encode(b"dummy").decode(),
|
||||
error_category=ValidationErrorCategory.TARGET_DISAPPEARED,
|
||||
)
|
||||
|
||||
assert result["status"] == "fixed"
|
||||
assert mock_controller.tap_at.call_count == 2
|
||||
@@ -0,0 +1,460 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import base64
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
from artemis.agents.validator.failure_analyzer import (
|
||||
_exec_click,
|
||||
_exec_click_sequence,
|
||||
_exec_input_text,
|
||||
_exec_long_press,
|
||||
_exec_manage_app,
|
||||
_exec_press_key,
|
||||
_exec_swipe,
|
||||
_exec_wait_for_delay,
|
||||
)
|
||||
from artemis.context import ArtemisContext
|
||||
import pytest
|
||||
|
||||
|
||||
class DummyState:
|
||||
def __init__(self, latest_screenshot=None):
|
||||
self.latest_screenshot = latest_screenshot
|
||||
self.indexed_points = []
|
||||
self.indexed_elements = []
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context(tmp_path):
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
ctx.device = Mock()
|
||||
ctx.device.device_width = 1080
|
||||
ctx.device.device_height = 2400
|
||||
ctx.data_engine = Mock()
|
||||
ctx.data_engine.base_dir = tmp_path
|
||||
ctx.data_engine.get_or_create_image.return_value = "dummy_image_name"
|
||||
ctx.data_engine.get_image_path.return_value = tmp_path / "dummy_image_name.jpg"
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_controller():
|
||||
controller = Mock()
|
||||
mock_device_data = Mock()
|
||||
mock_device_data.base64 = base64.b64encode(b"mocked_screenshot").decode()
|
||||
mock_device_data.elements = []
|
||||
mock_device_data.width = 1080
|
||||
mock_device_data.height = 2400
|
||||
controller.get_screen_data = AsyncMock(return_value=mock_device_data)
|
||||
controller.get_ui_elements = AsyncMock(return_value=mock_device_data.elements)
|
||||
controller.tap_at = AsyncMock(return_value=Mock(error=None))
|
||||
controller.type_text = AsyncMock(return_value=True)
|
||||
controller.erase_text = AsyncMock(return_value=True)
|
||||
controller.swipe_coords = AsyncMock(return_value=None)
|
||||
controller.press_enter = AsyncMock(return_value=True)
|
||||
controller.go_back = AsyncMock(return_value=True)
|
||||
controller.go_home = AsyncMock(return_value=True)
|
||||
controller.press_key = AsyncMock(return_value=True)
|
||||
controller.terminate_app = AsyncMock(return_value=True)
|
||||
return controller
|
||||
|
||||
|
||||
# ==========================================
|
||||
# SLIDE 1: click
|
||||
# ==========================================
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_click_happy_path(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
outcome, img_bytes, shot_path, xml_list = await _exec_click(
|
||||
target=[500, 600],
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
times=2,
|
||||
delay_ms=150,
|
||||
)
|
||||
assert "Clicked at [500, 600]" in outcome
|
||||
assert mock_controller.tap_at.call_count == 1
|
||||
mock_controller.tap_at.assert_any_call(540, 1440, times=2, delay_ms=150)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_click_input_validation(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
# Coordinates holding non-integer values
|
||||
outcome, _, _, _ = await _exec_click(
|
||||
target=["invalid", "invalid"],
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Error during click:" in outcome
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_click_edge_cases(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
# Underflow/overflow coordinates should be clamped to valid boundary
|
||||
outcome, _, _, _ = await _exec_click(
|
||||
target=[-10, 1050],
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Clicked at [-10, 1050]" in outcome
|
||||
mock_controller.tap_at.assert_called_with(0, 2399, times=1, delay_ms=100)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_click_graceful_failure(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
mock_controller.tap_at.return_value = Mock(error="Click failed connection lost")
|
||||
outcome, _, _, _ = await _exec_click(
|
||||
target=[500, 600],
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Error executing click: Click failed connection lost" in outcome
|
||||
|
||||
|
||||
# ==========================================
|
||||
# SLIDE 2: click_sequence
|
||||
# ==========================================
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_click_sequence_happy_path(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
state.indexed_points = [[540, 1200]] # Mapped from index 1 (1-indexed)
|
||||
outcome, img_bytes, shot_path, xml_list = await _exec_click_sequence(
|
||||
sequence=[1, [300, 400]],
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Sequence clicked successfully" in outcome
|
||||
assert mock_controller.tap_at.call_count == 2
|
||||
mock_controller.tap_at.assert_any_call(540, 1200)
|
||||
mock_controller.tap_at.assert_any_call(324, 960)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_click_sequence_input_validation(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
state.indexed_points = [[540, 1200], [270, 600]]
|
||||
|
||||
# Subsequent elements can also be integers (mixed array of indexes and coordinates)
|
||||
outcome, _, _, _ = await _exec_click_sequence(
|
||||
sequence=[1, 2, [300, 400]],
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Sequence clicked successfully" in outcome
|
||||
assert mock_controller.tap_at.call_count == 3
|
||||
mock_controller.tap_at.assert_any_call(540, 1200)
|
||||
mock_controller.tap_at.assert_any_call(270, 600)
|
||||
mock_controller.tap_at.assert_any_call(324, 960)
|
||||
|
||||
# Reset indexed_points since _exec_click_sequence overwrites them
|
||||
# via _capture_screenshot_and_parse_ui
|
||||
state.indexed_points = [[540, 1200], [270, 600]]
|
||||
|
||||
# Index out of bounds
|
||||
outcome, _, _, _ = await _exec_click_sequence(
|
||||
sequence=[5, [300, 400]],
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Error during click sequence: Index 5 is out of range" in outcome
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_click_sequence_edge_cases(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
# Coordinates in sequence outside range should be clamped
|
||||
outcome, _, _, _ = await _exec_click_sequence(
|
||||
sequence=[[0, 0], [1200, -50]],
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Sequence clicked successfully" in outcome
|
||||
mock_controller.tap_at.assert_any_call(0, 0)
|
||||
mock_controller.tap_at.assert_any_call(1079, 0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_click_sequence_graceful_failure(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
mock_controller.tap_at.side_effect = [
|
||||
Mock(error=None),
|
||||
Mock(error="Connection error on step 2"),
|
||||
]
|
||||
outcome, _, _, _ = await _exec_click_sequence(
|
||||
sequence=[[100, 200], [300, 400]],
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Error executing click at" in outcome
|
||||
assert "Connection error on step 2" in outcome
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_click_sequence_stringified_formats(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
state.indexed_points = [[540, 1200]]
|
||||
# Test stringified coordinate formats commonly generated by LLMs:
|
||||
# 1. string representation of lists: ["[500, 280]", "[884, 362]"]
|
||||
outcome, _, _, _ = await _exec_click_sequence(
|
||||
sequence=["[500, 280]", "[884, 362]"],
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Sequence clicked successfully" in outcome
|
||||
mock_controller.tap_at.assert_any_call(540, 672)
|
||||
mock_controller.tap_at.assert_any_call(954, 868)
|
||||
|
||||
# 2. comma-separated strings: ["500,280", "884,362"]
|
||||
outcome, _, _, _ = await _exec_click_sequence(
|
||||
sequence=["500,280", "884,362"],
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Sequence clicked successfully" in outcome
|
||||
|
||||
# 3. whole sequence as a string
|
||||
outcome, _, _, _ = await _exec_click_sequence(
|
||||
sequence="[[500, 280], [884, 362]]",
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Sequence clicked successfully" in outcome
|
||||
|
||||
|
||||
# ==========================================
|
||||
# SLIDE 3: long_press
|
||||
# ==========================================
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_long_press_happy_path(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
outcome, img_bytes, shot_path, xml_list = await _exec_long_press(
|
||||
target=[500, 500],
|
||||
duration=1500,
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Long pressed at [500, 500]" in outcome
|
||||
mock_controller.tap_at.assert_called_once_with(
|
||||
540, 1200, long_press=True, long_press_duration=1500
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_long_press_input_validation(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
# Invalid target type
|
||||
outcome, _, _, _ = await _exec_long_press(
|
||||
target="not-a-list",
|
||||
duration=1500,
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Error during long press:" in outcome
|
||||
|
||||
|
||||
# ==========================================
|
||||
# SLIDE 4: input_text
|
||||
# ==========================================
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_input_text_happy_path(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
outcome, img_bytes, shot_path, xml_list = await _exec_input_text(
|
||||
text="hello",
|
||||
target=[500, 500],
|
||||
clear_exist=True,
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Executed typing 'hello'." in outcome
|
||||
mock_controller.tap_at.assert_called_once_with(540, 1200)
|
||||
mock_controller.erase_text.assert_called_once()
|
||||
mock_controller.type_text.assert_called_once_with("hello")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_input_text_input_validation(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
outcome, _, _, _ = await _exec_input_text(
|
||||
text="hello",
|
||||
target="invalid",
|
||||
clear_exist=True,
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Error during input text:" in outcome
|
||||
|
||||
|
||||
# ==========================================
|
||||
# SLIDE 5: swipe
|
||||
# ==========================================
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swipe_happy_path_direction(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
outcome, img_bytes, shot_path, xml_list = await _exec_swipe(
|
||||
action="up",
|
||||
duration=400,
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Swipe completed successfully" in outcome
|
||||
mock_controller.swipe_coords.assert_called_once_with(648, 1920, 648, 480, 400)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swipe_happy_path_coords(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
outcome, img_bytes, shot_path, xml_list = await _exec_swipe(
|
||||
action=[100, 900, 100, 100],
|
||||
duration=400,
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Swipe completed successfully" in outcome
|
||||
mock_controller.swipe_coords.assert_called_once_with(108, 2160, 108, 240, 400)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swipe_input_validation(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
outcome, _, _, _ = await _exec_swipe(
|
||||
action="invalid_direction",
|
||||
duration=400,
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Error during swipe: Invalid direction:" in outcome
|
||||
|
||||
|
||||
# ==========================================
|
||||
# SLIDE 6: press_key
|
||||
# ==========================================
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_press_key_happy_path(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
outcome, img_bytes, shot_path, xml_list = await _exec_press_key(
|
||||
key="BACK",
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Executed key press 'BACK'." in outcome
|
||||
mock_controller.go_back.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_press_key_input_validation(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
outcome, _, _, _ = await _exec_press_key(
|
||||
key="INVALID_KEY",
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
# The current execution doesn't raise error on unknown keys, just logs error
|
||||
assert "Error executing key press 'INVALID_KEY'." in outcome
|
||||
|
||||
|
||||
# ==========================================
|
||||
# SLIDE 7: manage_app
|
||||
# ==========================================
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manage_app_happy_path(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
with (
|
||||
patch("artemis.agents.validator.tool_declarations.find_package") as mock_find_package,
|
||||
patch("artemis.agents.validator.tool_declarations.launch_app_with_retries") as mock_launch,
|
||||
):
|
||||
mock_find_package.return_value = "com.google.android.youtube"
|
||||
mock_launch.return_value = (True, "")
|
||||
|
||||
outcome, img_bytes, shot_path, xml_list = await _exec_manage_app(
|
||||
action="launch",
|
||||
app_name="YouTube",
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Launched app 'YouTube' (com.google.android.youtube) successfully." in outcome
|
||||
mock_find_package.assert_called_once_with(mock_context, "YouTube", use_fallback=False)
|
||||
mock_launch.assert_called_once_with(mock_context, "com.google.android.youtube")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manage_app_input_validation(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
with patch("artemis.agents.validator.tool_declarations.find_package") as mock_find_package:
|
||||
mock_find_package.return_value = None
|
||||
|
||||
outcome, _, _, _ = await _exec_manage_app(
|
||||
action="launch",
|
||||
app_name="NonExistentApp",
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Error finding package for app: NonExistentApp" in outcome
|
||||
|
||||
|
||||
# ==========================================
|
||||
# SLIDE 8: wait_for_delay
|
||||
# ==========================================
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wait_for_delay_happy_path(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
outcome, img_bytes, shot_path, xml_list = await _exec_wait_for_delay(
|
||||
time_in_ms=100,
|
||||
state=state,
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Waited for 100ms successfully." in outcome
|
||||
@@ -0,0 +1,113 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for Universal FlashRunner."""
|
||||
|
||||
from unittest.mock import Mock, patch
|
||||
import pytest
|
||||
from langchain_core.messages import HumanMessage, ToolMessage
|
||||
|
||||
from artemis.agents.flash.runner import FlashRunner
|
||||
from artemis.context import ArtemisContext
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context():
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
ctx.llm_config = Mock()
|
||||
mock_llm_cfg = Mock()
|
||||
mock_llm_cfg.model = "gemini-2.5-flash"
|
||||
mock_llm_cfg.temperature = 0.1
|
||||
ctx.llm_config.get_agent.return_value = mock_llm_cfg
|
||||
|
||||
ctx.device = Mock()
|
||||
ctx.device.device_width = 1080
|
||||
ctx.device.device_height = 2400
|
||||
ctx.data_engine = None
|
||||
ctx.adb_client = None
|
||||
ctx.driver = Mock()
|
||||
return ctx
|
||||
|
||||
|
||||
def test_flash_runner_tools_initialization(mock_context):
|
||||
"""Verify FlashRunner gathers correct active tools."""
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = FlashRunner(mock_context, goal="Test Open Settings")
|
||||
tools = runner._get_tools()
|
||||
|
||||
tool_names = [t.name for t in tools]
|
||||
assert "click" in tool_names
|
||||
assert "click_sequence" in tool_names
|
||||
assert "swipe" in tool_names
|
||||
assert "input_text" in tool_names
|
||||
assert "ask_explorer" in tool_names
|
||||
assert "report_task_status" in tool_names
|
||||
# Validator's report_failure_analysis should NOT be in FlashRunner
|
||||
assert "report_failure_analysis" not in tool_names
|
||||
|
||||
|
||||
def test_flash_runner_screenshot_pruning(mock_context):
|
||||
"""Verify that earlier screenshot blocks are pruned, keeping only the latest."""
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = FlashRunner(mock_context, goal="Test Pruning")
|
||||
|
||||
messages = [
|
||||
HumanMessage(
|
||||
content=[
|
||||
{"type": "text", "text": "Turn 1 goal"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "data:image/jpeg;base64,SCREENSHOT_1"},
|
||||
},
|
||||
]
|
||||
),
|
||||
ToolMessage(
|
||||
tool_call_id="tc1",
|
||||
name="click",
|
||||
content=[
|
||||
{"type": "text", "text": "Clicked button"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "data:image/jpeg;base64,SCREENSHOT_2"},
|
||||
},
|
||||
],
|
||||
),
|
||||
ToolMessage(
|
||||
tool_call_id="tc2",
|
||||
name="swipe",
|
||||
content=[
|
||||
{"type": "text", "text": "Swiped up"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "data:image/jpeg;base64,SCREENSHOT_3"},
|
||||
},
|
||||
],
|
||||
),
|
||||
]
|
||||
|
||||
runner._prune_intermediate_screenshots(messages)
|
||||
|
||||
# Check Turn 1: image should be removed
|
||||
assert len(messages[0].content) == 1
|
||||
assert messages[0].content[0]["type"] == "text"
|
||||
|
||||
# Check Turn 2 (Tool 1): image should be removed
|
||||
assert len(messages[1].content) == 1
|
||||
assert messages[1].content[0]["type"] == "text"
|
||||
|
||||
# Check Turn 3 (Tool 2 - Latest): image MUST be preserved
|
||||
assert len(messages[2].content) == 2
|
||||
assert messages[2].content[0]["type"] == "text"
|
||||
assert messages[2].content[1]["type"] == "image_url"
|
||||
assert messages[2].content[1]["image_url"]["url"] == "data:image/jpeg;base64,SCREENSHOT_3"
|
||||
@@ -0,0 +1,289 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for VisualStepSummarizer and Context Compressor in Flash profile."""
|
||||
|
||||
import asyncio
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
import pytest
|
||||
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
|
||||
|
||||
from artemis.agents.flash.context_compressor import compress_flash_messages
|
||||
from artemis.agents.flash.summarizer import VisualStepSummarizer
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.sdk.builders import Builders
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context():
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
ctx.data_engine = None
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_summarizer_dispatch_and_caching(mock_context):
|
||||
"""Verify non-blocking dispatch and summary generation."""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
|
||||
mock_llm_response = AIMessage(
|
||||
content="[Step 1 State: Tapped 'Login' button. Home view opened.]"
|
||||
)
|
||||
summarizer._llm = Mock()
|
||||
summarizer._llm.ainvoke = AsyncMock(return_value=mock_llm_response)
|
||||
|
||||
summarizer.dispatch(
|
||||
step_number=1,
|
||||
action_name="click",
|
||||
action_args={"target": [500, 600]},
|
||||
pre_img_bytes=b"fake_pre",
|
||||
post_img_bytes=b"fake_post",
|
||||
exec_outcome="Clicked successfully",
|
||||
)
|
||||
|
||||
# Await background task flush
|
||||
await summarizer.flush()
|
||||
|
||||
assert summarizer.has_summary(1)
|
||||
summary = summarizer.get_summary(1)
|
||||
assert "[Step 1 State: Tapped 'Login' button. Home view opened.]" in summary
|
||||
|
||||
# Verify memory optimization: image buffers are released
|
||||
assert summarizer._step_inputs[1]["pre_img_bytes"] is None
|
||||
assert summarizer._step_inputs[1]["post_img_bytes"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_summarizer_retrigger_stalled_steps(mock_context):
|
||||
"""Verify that earlier stalled steps are re-triggered when subsequent steps arrive."""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
attempt_count = 0
|
||||
|
||||
async def mock_ainvoke(messages):
|
||||
nonlocal attempt_count
|
||||
attempt_count += 1
|
||||
# Fail the first attempt (Step 1 first call)
|
||||
if attempt_count == 1:
|
||||
raise asyncio.CancelledError("Simulated timeout")
|
||||
# Check messages for step number
|
||||
msg_str = str(messages)
|
||||
if "Step 1" in msg_str:
|
||||
return AIMessage(content="[Step 1 State: Recovered Step 1 summary]")
|
||||
return AIMessage(content="[Step 2 State: Step 2 summary]")
|
||||
|
||||
summarizer._llm = Mock()
|
||||
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
|
||||
|
||||
# Dispatch step 1 (fails on first attempt)
|
||||
summarizer.dispatch(
|
||||
step_number=1,
|
||||
action_name="click",
|
||||
action_args={"target": [100, 100]},
|
||||
pre_img_bytes=b"pre1",
|
||||
post_img_bytes=b"post1",
|
||||
exec_outcome="Outcome 1",
|
||||
)
|
||||
await summarizer.flush()
|
||||
assert not summarizer.has_summary(1)
|
||||
|
||||
# Dispatch step 2 (should detect stalled Step 1 and re-trigger)
|
||||
summarizer.dispatch(
|
||||
step_number=2,
|
||||
action_name="swipe",
|
||||
action_args={"action": "up"},
|
||||
pre_img_bytes=b"pre2",
|
||||
post_img_bytes=b"post2",
|
||||
exec_outcome="Outcome 2",
|
||||
)
|
||||
await summarizer.flush()
|
||||
|
||||
assert summarizer.has_summary(1)
|
||||
assert summarizer.has_summary(2)
|
||||
assert "Recovered Step 1" in summarizer.get_summary(1)
|
||||
|
||||
|
||||
def test_context_compressor_with_ready_summaries(mock_context):
|
||||
"""Verify that ready summaries replace past image blocks and prune outdated XML."""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
summarizer._summaries[1] = "[Step 1 State: Tapped Search bar. Keyboard appeared.]"
|
||||
summarizer._summaries[2] = "[Step 2 State: Typed 'flights'. Suggestions displayed.]"
|
||||
|
||||
messages = [
|
||||
HumanMessage(
|
||||
content=[
|
||||
{"type": "text", "text": "Task: Search flights"},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_1"}},
|
||||
{"type": "text", "text": "--- UI Element List ---\nOld Tree 1"},
|
||||
]
|
||||
),
|
||||
ToolMessage(
|
||||
tool_call_id="tc1",
|
||||
name="click",
|
||||
content=[
|
||||
{"type": "text", "text": "Action 'click' completed."},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_2"}},
|
||||
{"type": "text", "text": "--- UI Element List ---\nOld Tree 2"},
|
||||
],
|
||||
),
|
||||
ToolMessage(
|
||||
tool_call_id="tc2",
|
||||
name="input_text",
|
||||
content=[
|
||||
{"type": "text", "text": "Action 'input_text' completed."},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_3"}},
|
||||
{"type": "text", "text": "--- UI Element List ---\nLive Latest Tree 3"},
|
||||
],
|
||||
),
|
||||
]
|
||||
|
||||
compress_flash_messages(messages, summarizer=summarizer, prune_history_xml=True)
|
||||
|
||||
# Initial HumanMessage: Historical initial image pruned, task text preserved
|
||||
assert len(messages[0].content) == 1
|
||||
assert messages[0].content[0]["text"] == "Task: Search flights"
|
||||
|
||||
# Step 1 (ToolMessage 1): Image replaced with Step 1 summary, old XML pruned
|
||||
assert len(messages[1].content) == 2
|
||||
assert messages[1].content[0]["text"] == "Action 'click' completed."
|
||||
assert messages[1].content[1]["text"] == "[Step 1 State: Tapped Search bar. Keyboard appeared.]"
|
||||
|
||||
# Step 2 (ToolMessage 2 - Latest live screen): Image and latest live XML MUST be preserved
|
||||
assert len(messages[2].content) == 3
|
||||
assert messages[2].content[0]["text"] == "Action 'input_text' completed."
|
||||
assert messages[2].content[1]["type"] == "image_url"
|
||||
assert "Live Latest Tree 3" in messages[2].content[2]["text"]
|
||||
|
||||
|
||||
def test_context_compressor_graceful_fallback(mock_context):
|
||||
"""Verify that when a summary is not ready yet, it cleanly falls back to standard pruned state."""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
# Step 1 has no summary in cache
|
||||
|
||||
messages = [
|
||||
HumanMessage(
|
||||
content=[
|
||||
{"type": "text", "text": "Task: Search"},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_1"}},
|
||||
]
|
||||
),
|
||||
ToolMessage(
|
||||
tool_call_id="tc1",
|
||||
name="click",
|
||||
content=[
|
||||
{"type": "text", "text": "Action 'click' completed."},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_2"}},
|
||||
],
|
||||
),
|
||||
]
|
||||
|
||||
compress_flash_messages(messages, summarizer=summarizer, prune_history_xml=True)
|
||||
|
||||
# Turn 1 should simply have image omitted (standard fallback)
|
||||
assert len(messages[0].content) == 1
|
||||
assert messages[0].content[0]["text"] == "Task: Search"
|
||||
|
||||
# Turn 2 (Latest) preserves image
|
||||
assert len(messages[1].content) == 2
|
||||
assert messages[1].content[1]["type"] == "image_url"
|
||||
|
||||
|
||||
def test_flash_config_and_builder():
|
||||
"""Verify Flash profile configuration model and SDK builder fluent API."""
|
||||
cfg = Builders.AgentConfig.with_flash_config(
|
||||
max_turns=25,
|
||||
explorer_mode="flash",
|
||||
step_summarizer=True,
|
||||
step_summarizer_model="gemini-2.5-flash-lite",
|
||||
prune_history_xml=True,
|
||||
).build()
|
||||
|
||||
assert cfg.flash.max_turns == 25
|
||||
assert cfg.flash.explorer_mode == "flash"
|
||||
assert cfg.flash.step_summarizer.enabled is True
|
||||
assert cfg.flash.step_summarizer.model == "gemini-2.5-flash-lite"
|
||||
assert cfg.flash.step_summarizer.prune_history_xml is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_data_engine_integration_with_step_summarizer(mock_context):
|
||||
"""Verify that generated summaries sync into DataEngine and are visible to get_agent_friendly_steps."""
|
||||
mock_de = Mock()
|
||||
mock_de.update_step_summary = Mock()
|
||||
mock_context.data_engine = mock_de
|
||||
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
summarizer._llm = Mock()
|
||||
summarizer._llm.ainvoke = AsyncMock(
|
||||
return_value=AIMessage(content="[Step 1 State: Input text confirmed.]")
|
||||
)
|
||||
|
||||
summarizer.dispatch(
|
||||
step_number=1,
|
||||
action_name="input_text",
|
||||
action_args={"text": "hello"},
|
||||
pre_img_bytes=b"pre",
|
||||
post_img_bytes=b"post",
|
||||
exec_outcome="Executed typing",
|
||||
)
|
||||
await summarizer.flush()
|
||||
|
||||
mock_de.update_step_summary.assert_called_once_with(1, "[Step 1 State: Input text confirmed.]")
|
||||
|
||||
|
||||
def test_draw_action_overlay_rendering():
|
||||
"""Verify that draw_action_overlay_on_image correctly renders markers on images."""
|
||||
import io
|
||||
from PIL import Image
|
||||
from artemis.utils.visualization import draw_action_overlay_on_image
|
||||
|
||||
# Create a blank RGB image in memory
|
||||
img = Image.new("RGB", (1080, 2400), color="white")
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, format="JPEG")
|
||||
raw_bytes = buf.getvalue()
|
||||
|
||||
# 1. Click overlay
|
||||
annotated_click = draw_action_overlay_on_image(
|
||||
image_bytes=raw_bytes,
|
||||
action_name="click",
|
||||
action_args={"target": [540, 1200]},
|
||||
)
|
||||
assert len(annotated_click) > 0
|
||||
assert annotated_click != raw_bytes
|
||||
|
||||
# 2. Swipe overlay
|
||||
annotated_swipe = draw_action_overlay_on_image(
|
||||
image_bytes=raw_bytes,
|
||||
action_name="swipe",
|
||||
action_args={"start": [500, 1800], "end": [500, 600]},
|
||||
)
|
||||
assert len(annotated_swipe) > 0
|
||||
assert annotated_swipe != raw_bytes
|
||||
|
||||
# 3. Click Sequence overlay
|
||||
annotated_seq = draw_action_overlay_on_image(
|
||||
image_bytes=raw_bytes,
|
||||
action_name="click_sequence",
|
||||
action_args={"sequence": [[200, 300], [400, 500], [600, 700]]},
|
||||
)
|
||||
assert len(annotated_seq) > 0
|
||||
assert annotated_seq != raw_bytes
|
||||
|
||||
# 4. Fail-safe on corrupt bytes
|
||||
corrupt_result = draw_action_overlay_on_image(
|
||||
image_bytes=b"invalid_bytes",
|
||||
action_name="click",
|
||||
action_args={"target": [100, 100]},
|
||||
)
|
||||
assert corrupt_result == b"invalid_bytes"
|
||||
@@ -0,0 +1,377 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from langchain_core.messages import ToolMessage
|
||||
from artemis.agents.history_analyzer.history_analyzer import HistoryAnalyzer
|
||||
from artemis.context import ArtemisContext
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_history_analyzer_no_history():
|
||||
# Mock context with a data engine that has no steps
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = MagicMock()
|
||||
mock_ctx.data_engine.get_agent_friendly_steps.return_value = []
|
||||
|
||||
analyzer = HistoryAnalyzer(mock_ctx)
|
||||
result = await analyzer.run("What happened?")
|
||||
assert result == "No history recorded for this session yet."
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_history_analyzer_simple_query_no_tool_call():
|
||||
# Mock context and data engine
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = MagicMock()
|
||||
mock_ctx.data_engine.base_dir = "/tmp/fake_traces"
|
||||
|
||||
steps = [
|
||||
{
|
||||
"step_number": 1,
|
||||
"relative_time": "1.2s",
|
||||
"summary": "Opened the settings app",
|
||||
},
|
||||
{
|
||||
"step_number": 2,
|
||||
"relative_time": "4.5s",
|
||||
"summary": "Toggled the wifi switch",
|
||||
},
|
||||
]
|
||||
mock_ctx.data_engine.get_agent_friendly_steps.return_value = steps
|
||||
|
||||
# Mock LLM to return a direct text response (no tool calls)
|
||||
mock_response = MagicMock()
|
||||
mock_response.content = "First you opened settings, then you toggled wifi."
|
||||
mock_response.tool_calls = []
|
||||
|
||||
mock_llm = MagicMock()
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
|
||||
async def mock_astream(*args, **kwargs):
|
||||
yield mock_response
|
||||
|
||||
mock_llm.astream.side_effect = mock_astream
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.history_analyzer.history_analyzer.get_llm",
|
||||
return_value=mock_llm,
|
||||
),
|
||||
patch("pathlib.Path.exists", return_value=False), # Force system prompt fallback
|
||||
):
|
||||
analyzer = HistoryAnalyzer(mock_ctx)
|
||||
result = await analyzer.run("Summarize the session")
|
||||
|
||||
assert result == "First you opened settings, then you toggled wifi."
|
||||
assert mock_llm.astream.called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_history_analyzer_detailed_query_with_tool_call():
|
||||
# Mock context and data engine with full details
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = MagicMock()
|
||||
mock_ctx.data_engine.base_dir = "/tmp/fake_traces"
|
||||
|
||||
steps = [
|
||||
{
|
||||
"step_number": 1,
|
||||
"relative_time": "1.2s",
|
||||
"summary": "Opened the settings app",
|
||||
"action_taken": {"action": "tap", "coordinates": [500, 600]},
|
||||
"operator_raw_thinking": "Need to open settings to configure wifi",
|
||||
"last_execution_result": {"status": "success"},
|
||||
},
|
||||
{
|
||||
"step_number": 2,
|
||||
"relative_time": "4.5s",
|
||||
"summary": "Toggled the wifi switch",
|
||||
"action_taken": {"action": "tap", "coordinates": [100, 200]},
|
||||
"operator_raw_thinking": "Toggle the switch to turn wifi on",
|
||||
"last_execution_result": {"status": "success"},
|
||||
},
|
||||
]
|
||||
mock_ctx.data_engine.get_agent_friendly_steps.return_value = steps
|
||||
|
||||
# Turn 1 LLM response: Tool call to get_step_details for step 2
|
||||
mock_response_turn_1 = MagicMock()
|
||||
mock_response_turn_1.content = ""
|
||||
mock_tool_call = {
|
||||
"name": "get_step_details",
|
||||
"args": {"start_step": 2, "end_step": 2},
|
||||
"id": "call_123456",
|
||||
}
|
||||
mock_response_turn_1.tool_calls = [mock_tool_call]
|
||||
|
||||
# Turn 2 LLM response: Final answer using the details
|
||||
mock_response_turn_2 = MagicMock()
|
||||
mock_response_turn_2.content = (
|
||||
"In step 2, the operator was thinking: 'Toggle the switch to turn wifi on'."
|
||||
)
|
||||
mock_response_turn_2.tool_calls = []
|
||||
|
||||
mock_llm = MagicMock()
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
responses = [mock_response_turn_1, mock_response_turn_2]
|
||||
call_count = 0
|
||||
|
||||
async def mock_astream(*args, **kwargs):
|
||||
nonlocal call_count
|
||||
if call_count < len(responses):
|
||||
yield responses[call_count]
|
||||
call_count += 1
|
||||
|
||||
mock_llm.astream.side_effect = mock_astream
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.history_analyzer.history_analyzer.get_llm",
|
||||
return_value=mock_llm,
|
||||
),
|
||||
patch("pathlib.Path.exists", return_value=False), # Force fallback prompt
|
||||
):
|
||||
analyzer = HistoryAnalyzer(mock_ctx)
|
||||
result = await analyzer.run("What was the operator thinking in step 2?")
|
||||
|
||||
assert (
|
||||
result == "In step 2, the operator was thinking: 'Toggle the switch to turn wifi on'."
|
||||
)
|
||||
assert mock_llm.astream.call_count == 2
|
||||
|
||||
# Verify the tool was invoked with correct parameters internally
|
||||
tool_msg = mock_llm.astream.call_args_list[1][0][0][
|
||||
-2
|
||||
] # ToolMessage is second to last after final response is appended
|
||||
assert isinstance(tool_msg, ToolMessage)
|
||||
assert tool_msg.tool_call_id == "call_123456"
|
||||
|
||||
# Parse the JSON in the ToolMessage content
|
||||
tool_result = json.loads(tool_msg.content)
|
||||
assert len(tool_result) == 1
|
||||
assert tool_result[0]["step_number"] == 2
|
||||
assert tool_result[0]["operator_raw_thinking"] == "Toggle the switch to turn wifi on"
|
||||
assert tool_result[0]["action_taken"] == {
|
||||
"action": "tap",
|
||||
"coordinates": [100, 200],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_history_analyzer_read_note_tool_call():
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = MagicMock()
|
||||
mock_ctx.data_engine.base_dir = "/tmp/fake_traces"
|
||||
|
||||
steps = [{"step_number": 1, "relative_time": "1.0s", "summary": "Initiated"}]
|
||||
mock_ctx.data_engine.get_agent_friendly_steps.return_value = steps
|
||||
|
||||
# Turn 1 LLM response: Tool call to list_notes
|
||||
mock_response_turn_1 = MagicMock()
|
||||
mock_response_turn_1.content = ""
|
||||
mock_tool_call_list = {"name": "list_notes", "args": {}, "id": "call_list"}
|
||||
mock_response_turn_1.tool_calls = [mock_tool_call_list]
|
||||
|
||||
# Turn 2 LLM response: Tool call to read_note for tactical_plan
|
||||
mock_response_turn_2 = MagicMock()
|
||||
mock_response_turn_2.content = ""
|
||||
mock_tool_call_read = {
|
||||
"name": "read_note",
|
||||
"args": {"key": "tactical_plan"},
|
||||
"id": "call_read",
|
||||
}
|
||||
mock_response_turn_2.tool_calls = [mock_tool_call_read]
|
||||
|
||||
# Turn 3 LLM response: Final answer
|
||||
mock_response_turn_3 = MagicMock()
|
||||
mock_response_turn_3.content = "The plan was to toggle wifi."
|
||||
mock_response_turn_3.tool_calls = []
|
||||
|
||||
mock_llm = MagicMock()
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
responses = [
|
||||
mock_response_turn_1,
|
||||
mock_response_turn_2,
|
||||
mock_response_turn_3,
|
||||
]
|
||||
call_count = 0
|
||||
|
||||
async def mock_astream(*args, **kwargs):
|
||||
nonlocal call_count
|
||||
if call_count < len(responses):
|
||||
yield responses[call_count]
|
||||
call_count += 1
|
||||
|
||||
mock_llm.astream.side_effect = mock_astream
|
||||
|
||||
# Mock the file system read using patch
|
||||
fake_plan_content = "Goal: Toggle wifi"
|
||||
|
||||
def mock_exists(path):
|
||||
# Let Path("/tmp/fake_traces/notes").exists() and notes/tactical_plan.md exist
|
||||
return True
|
||||
|
||||
def mock_read_text(self, encoding="utf-8"):
|
||||
return fake_plan_content
|
||||
|
||||
def mock_glob(self, pattern):
|
||||
return [Path("/tmp/fake_traces/notes/tactical_plan.md")]
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.history_analyzer.history_analyzer.get_llm",
|
||||
return_value=mock_llm,
|
||||
),
|
||||
patch("pathlib.Path.exists", mock_exists),
|
||||
patch("pathlib.Path.read_text", mock_read_text),
|
||||
patch("pathlib.Path.glob", mock_glob),
|
||||
):
|
||||
analyzer = HistoryAnalyzer(mock_ctx)
|
||||
result = await analyzer.run("What is the saved tactical plan?")
|
||||
|
||||
assert result == "The plan was to toggle wifi."
|
||||
assert mock_llm.astream.call_count == 3
|
||||
|
||||
# Verify list_notes response
|
||||
list_msg = mock_llm.astream.call_args_list[1][0][0][
|
||||
3
|
||||
] # Index 3 contains the ToolMessage for list_notes
|
||||
assert isinstance(list_msg, ToolMessage)
|
||||
assert list_msg.tool_call_id == "call_list"
|
||||
assert list_msg.content == "Here are all the notes:\n- tactical_plan (1 lines)"
|
||||
|
||||
# Verify read_note response
|
||||
read_msg = mock_llm.astream.call_args_list[2][0][0][
|
||||
5
|
||||
] # Index 5 contains the ToolMessage for read_note
|
||||
assert isinstance(read_msg, ToolMessage)
|
||||
assert read_msg.tool_call_id == "call_read"
|
||||
assert (
|
||||
read_msg.content == "Successfully read note 'tactical_plan'. 'tactical_plan' note"
|
||||
f" content:\n{fake_plan_content}"
|
||||
)
|
||||
|
||||
|
||||
def test_history_analyzer_robust_tools_behavior():
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = MagicMock()
|
||||
mock_ctx.data_engine.base_dir = "/tmp/fake_traces"
|
||||
|
||||
steps = [
|
||||
{
|
||||
"step_number": 1,
|
||||
"relative_time": "1.0s",
|
||||
"summary": "Action 1",
|
||||
"action_taken": {"action": "tap"},
|
||||
}
|
||||
]
|
||||
|
||||
analyzer = HistoryAnalyzer(mock_ctx)
|
||||
|
||||
# 1. Test get_step_details with string inputs
|
||||
details_tool = analyzer._get_step_details_tool(steps)
|
||||
result_details = details_tool.invoke({"start_step": "1", "end_step": "1"})
|
||||
parsed_result = json.loads(result_details)
|
||||
assert len(parsed_result) == 1
|
||||
assert parsed_result[0]["step_number"] == 1
|
||||
|
||||
# 2. Test read_note with .md suffix
|
||||
from artemis.tools.scratchpad import get_read_note_tool_pure
|
||||
|
||||
read_tool = get_read_note_tool_pure(analyzer.ctx)
|
||||
|
||||
def mock_exists(path):
|
||||
return "tactical_plan.md" in str(path)
|
||||
|
||||
with (
|
||||
patch("pathlib.Path.exists", mock_exists),
|
||||
patch("pathlib.Path.read_text", return_value="Tactical plan content"),
|
||||
):
|
||||
result_read = read_tool.invoke({"key": "tactical_plan.md"})
|
||||
assert (
|
||||
result_read == "Successfully read note 'tactical_plan.md'. 'tactical_plan.md'"
|
||||
" note content:\nTactical plan content"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_history_analyzer_integration_with_task_tree():
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = MagicMock()
|
||||
mock_ctx.data_engine.base_dir = "/tmp/fake_traces"
|
||||
|
||||
steps = [
|
||||
{
|
||||
"step_id": "step_1",
|
||||
"step_number": 1,
|
||||
"relative_time": "1.0s",
|
||||
"summary": "Step 1",
|
||||
},
|
||||
]
|
||||
mock_ctx.data_engine.get_agent_friendly_steps.return_value = steps
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.content = "Answer"
|
||||
mock_response.tool_calls = []
|
||||
|
||||
mock_llm = MagicMock()
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
|
||||
async def mock_astream(*args, **kwargs):
|
||||
yield mock_response
|
||||
|
||||
mock_llm.astream.side_effect = mock_astream
|
||||
|
||||
fake_plan = """- [/] Active Subgoal
|
||||
- [ ] Pending Subgoal"""
|
||||
|
||||
mock_path = MagicMock()
|
||||
mock_path.exists.return_value = True
|
||||
mock_path.read_text.return_value = fake_plan
|
||||
|
||||
# Mock build_plan_and_history
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.history_analyzer.history_analyzer.get_llm",
|
||||
return_value=mock_llm,
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.history_analyzer.history_analyzer.get_note_file_path",
|
||||
return_value=mock_path,
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.history_analyzer.history_analyzer.build_plan_and_history"
|
||||
) as mock_build,
|
||||
):
|
||||
mock_build.return_value = "Mocked operation history"
|
||||
analyzer = HistoryAnalyzer(mock_ctx)
|
||||
await analyzer.run("query")
|
||||
|
||||
# Verify build_plan_and_history call args
|
||||
mock_build.assert_called_once()
|
||||
args, kwargs = mock_build.call_args
|
||||
assert args[0] == fake_plan
|
||||
assert args[1] == steps
|
||||
|
||||
import hashlib
|
||||
|
||||
expected_hash = hashlib.md5(b"Active Subgoal").hexdigest()
|
||||
assert args[2] == expected_hash
|
||||
|
||||
assert kwargs.get("last_n_detailed") == 1
|
||||
assert kwargs.get("min_summaries") == len(steps)
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,508 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
from pydantic import BaseModel
|
||||
import pytest
|
||||
|
||||
from artemis.agents.outputter.outputter import outputter
|
||||
from artemis.config import LLM, OutputConfig # noqa: E402
|
||||
from artemis.context import ArtemisContext # noqa: E402
|
||||
from artemis.utils.logger import get_logger # noqa: E402
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class _CyFunctionDetectorMeta(type):
|
||||
def __instancecheck__(self, instance):
|
||||
name = type(instance).__name__
|
||||
return (
|
||||
name
|
||||
in (
|
||||
"cyfunction",
|
||||
"cython_function_or_method",
|
||||
"builtin_function_or_method",
|
||||
)
|
||||
or "cyfunction" in name.lower()
|
||||
)
|
||||
|
||||
|
||||
class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta):
|
||||
pass
|
||||
|
||||
|
||||
class MockPydanticSchema(BaseModel):
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
color: str
|
||||
price: float
|
||||
currency_symbol: str
|
||||
website_url: str
|
||||
|
||||
|
||||
mock_dict = {
|
||||
"color": "green",
|
||||
"price": 20,
|
||||
"currency_symbol": "$",
|
||||
"website_url": "http://superwebsite.fr",
|
||||
}
|
||||
|
||||
|
||||
class DummyState:
|
||||
def __init__(self, messages, initial_goal, operator_raw_data=None):
|
||||
self.messages = messages
|
||||
self.initial_goal = initial_goal
|
||||
self.operator_raw_data = operator_raw_data
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context():
|
||||
"""Create a properly mocked context with all required fields."""
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
ctx.llm_config = {
|
||||
"planner": LLM(provider="openai", model="gpt-5-nano"),
|
||||
"operator": LLM(provider="openai", model="gpt-5-nano"),
|
||||
"validator": LLM(provider="openai", model="gpt-5-nano"),
|
||||
}
|
||||
ctx.device = Mock()
|
||||
ctx.data_engine = None
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_state():
|
||||
"""Create a mock state with test data."""
|
||||
return DummyState(
|
||||
messages=[],
|
||||
initial_goal="Find a green product on my website",
|
||||
)
|
||||
|
||||
|
||||
def setup_mock_llm(mock_get_llm, react_response_content="Paris", structured_response=None):
|
||||
mock_llm = Mock()
|
||||
mock_llm_fallback = Mock()
|
||||
|
||||
# Mock bind_tools
|
||||
mock_llm_with_tools = Mock()
|
||||
mock_llm_with_tools.ainvoke = AsyncMock()
|
||||
mock_react_response = Mock()
|
||||
mock_react_response.content = react_response_content
|
||||
mock_react_response.tool_calls = []
|
||||
mock_llm_with_tools.ainvoke.return_value = mock_react_response
|
||||
|
||||
mock_llm.bind_tools.return_value = mock_llm_with_tools
|
||||
mock_llm_fallback.bind_tools.return_value = mock_llm_with_tools
|
||||
|
||||
# Mock with_structured_output
|
||||
mock_structured_llm = Mock()
|
||||
mock_structured_llm.ainvoke = AsyncMock()
|
||||
if structured_response is not None:
|
||||
mock_structured_llm.ainvoke.return_value = structured_response
|
||||
mock_llm.with_structured_output.return_value = mock_structured_llm
|
||||
mock_llm_fallback.with_structured_output.return_value = mock_structured_llm
|
||||
|
||||
def get_llm_side_effect(ctx, name, is_utils=False, use_fallback=False):
|
||||
if use_fallback:
|
||||
return mock_llm_fallback
|
||||
return mock_llm
|
||||
|
||||
mock_get_llm.side_effect = get_llm_side_effect
|
||||
|
||||
return mock_llm, mock_llm_with_tools, mock_structured_llm
|
||||
|
||||
|
||||
@patch("artemis.agents.outputter.outputter.get_llm")
|
||||
@pytest.mark.asyncio
|
||||
async def test_outputter_with_pydantic_model(mock_get_llm, mock_context, mock_state):
|
||||
"""Test outputter with Pydantic model output."""
|
||||
expected_structured = MockPydanticSchema(
|
||||
color="green",
|
||||
price=20,
|
||||
currency_symbol="$",
|
||||
website_url="http://superwebsite.fr",
|
||||
)
|
||||
setup_mock_llm(
|
||||
mock_get_llm,
|
||||
react_response_content="Raw details about green product",
|
||||
structured_response=expected_structured,
|
||||
)
|
||||
|
||||
config = OutputConfig(
|
||||
structured_output=MockPydanticSchema,
|
||||
output_description=None,
|
||||
)
|
||||
|
||||
result = await outputter(ctx=mock_context, output_config=config, graph_output=mock_state)
|
||||
|
||||
assert isinstance(result, dict)
|
||||
assert result.get("color") == "green"
|
||||
|
||||
|
||||
@patch("artemis.agents.outputter.outputter.get_llm")
|
||||
@pytest.mark.asyncio
|
||||
async def test_outputter_with_dict(mock_get_llm, mock_context, mock_state):
|
||||
"""Test outputter with dictionary output."""
|
||||
expected_dict = {
|
||||
"color": "green",
|
||||
"price": 20,
|
||||
"currency_symbol": "$",
|
||||
"website_url": "http://superwebsite.fr",
|
||||
}
|
||||
setup_mock_llm(
|
||||
mock_get_llm,
|
||||
react_response_content="Raw details",
|
||||
structured_response=expected_dict,
|
||||
)
|
||||
|
||||
config = OutputConfig(
|
||||
structured_output=mock_dict,
|
||||
output_description=None,
|
||||
)
|
||||
|
||||
result = await outputter(ctx=mock_context, output_config=config, graph_output=mock_state)
|
||||
|
||||
assert isinstance(result, dict)
|
||||
assert result.get("color") == "green"
|
||||
assert result.get("price") == 20
|
||||
assert result.get("currency_symbol") == "$"
|
||||
assert result.get("website_url") == "http://superwebsite.fr"
|
||||
|
||||
|
||||
@patch("artemis.agents.outputter.outputter.get_llm")
|
||||
@pytest.mark.asyncio
|
||||
async def test_outputter_with_natural_language_output(mock_get_llm, mock_context, mock_state):
|
||||
"""Test outputter with natural language description output (returns JSON string)."""
|
||||
expected_json = (
|
||||
'{"color": "green", "price": 20, "currency_symbol": "$", "website_url":'
|
||||
' "http://superwebsite.fr"}'
|
||||
)
|
||||
setup_mock_llm(mock_get_llm, react_response_content=expected_json)
|
||||
|
||||
config = OutputConfig(
|
||||
structured_output=None,
|
||||
output_description=(
|
||||
"A JSON object with a color, a price, a currency_symbol and a website_url key"
|
||||
),
|
||||
)
|
||||
|
||||
result = await outputter(ctx=mock_context, output_config=config, graph_output=mock_state)
|
||||
|
||||
assert isinstance(result, dict)
|
||||
assert result.get("color") == "green"
|
||||
assert result.get("price") == 20
|
||||
assert result.get("currency_symbol") == "$"
|
||||
assert result.get("website_url") == "http://superwebsite.fr"
|
||||
|
||||
|
||||
@patch("artemis.agents.outputter.outputter.get_llm")
|
||||
@pytest.mark.asyncio
|
||||
async def test_outputter_with_history_and_image(mock_get_llm, mock_context):
|
||||
"""Test outputter with explicit history and image."""
|
||||
expected_json = '{"answer": "Paris"}'
|
||||
_, mock_llm_with_tools, _ = setup_mock_llm(mock_get_llm, react_response_content=expected_json)
|
||||
|
||||
config = OutputConfig(
|
||||
structured_output=None,
|
||||
output_description="The capital of France",
|
||||
)
|
||||
|
||||
class StateWithRawData:
|
||||
def __init__(self, messages, initial_goal, operator_raw_data):
|
||||
self.messages = messages
|
||||
self.initial_goal = initial_goal
|
||||
self.operator_raw_data = operator_raw_data
|
||||
|
||||
state = StateWithRawData(
|
||||
messages=[],
|
||||
initial_goal="What is the capital of France?",
|
||||
operator_raw_data={"screenshot_b64": "mock_b64_data"},
|
||||
)
|
||||
|
||||
await outputter(
|
||||
ctx=mock_context,
|
||||
output_config=config,
|
||||
graph_output=state,
|
||||
plan_and_history="Plan: Find capital\nStep 1: Opened browser",
|
||||
)
|
||||
|
||||
call_args = mock_llm_with_tools.ainvoke.call_args
|
||||
assert call_args is not None
|
||||
messages = call_args[0][0]
|
||||
|
||||
assert (
|
||||
"Your sole objective is to verify whether the user's initial goal was"
|
||||
" achieved" in messages[0].content
|
||||
)
|
||||
|
||||
human_content = messages[1].content
|
||||
assert isinstance(human_content, list)
|
||||
assert human_content[0]["type"] == "text"
|
||||
assert "Step 1: Opened browser" in human_content[0]["text"]
|
||||
assert "Plan: Find capital" in human_content[0]["text"]
|
||||
assert human_content[1]["type"] == "image_url"
|
||||
assert human_content[1]["image_url"]["url"] == "data:image/jpeg;base64,mock_b64_data"
|
||||
|
||||
|
||||
@patch("artemis.agents.outputter.outputter.get_llm")
|
||||
@pytest.mark.asyncio
|
||||
async def test_outputter_builds_concise_history(mock_get_llm):
|
||||
"""Test that the outputter dynamically builds clean Summary ➡️ Action history."""
|
||||
expected_json = '{"answer": "Success"}'
|
||||
_, mock_llm_with_tools, _ = setup_mock_llm(mock_get_llm, react_response_content=expected_json)
|
||||
|
||||
config = OutputConfig(
|
||||
structured_output=None,
|
||||
output_description="Test output",
|
||||
)
|
||||
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
ctx.llm_config = {
|
||||
"planner": LLM(provider="openai", model="gpt-5-nano"),
|
||||
"operator": LLM(provider="openai", model="gpt-5-nano"),
|
||||
"validator": LLM(provider="openai", model="gpt-5-nano"),
|
||||
}
|
||||
ctx.device = Mock()
|
||||
|
||||
mock_data_engine = Mock()
|
||||
steps = [
|
||||
{
|
||||
"step_id": "step_1",
|
||||
"step_number": 1,
|
||||
"relative_time": "2.5s",
|
||||
"summary": "Tapped search button",
|
||||
"action_taken": [{"action": "click", "target_text": "Search"}],
|
||||
"last_execution_result": {"status": "success"},
|
||||
}
|
||||
]
|
||||
mock_data_engine.get_agent_friendly_steps.return_value = steps
|
||||
mock_data_engine.base_dir = "/tmp/mock_session"
|
||||
ctx.data_engine = mock_data_engine
|
||||
|
||||
class StateWithRawData:
|
||||
def __init__(self, messages, initial_goal, operator_raw_data):
|
||||
self.messages = messages
|
||||
self.initial_goal = initial_goal
|
||||
self.operator_raw_data = operator_raw_data
|
||||
|
||||
state = StateWithRawData(
|
||||
messages=[],
|
||||
initial_goal="Search for something",
|
||||
operator_raw_data=None,
|
||||
)
|
||||
|
||||
# Test Case 1: Fallback path (no plan file)
|
||||
from pathlib import Path
|
||||
|
||||
original_exists = Path.exists
|
||||
|
||||
def mock_exists_fallback(self, *args, **kwargs):
|
||||
if self.name == "task_plan.md":
|
||||
return False
|
||||
return original_exists(self, *args, **kwargs)
|
||||
|
||||
with patch("pathlib.Path.exists", mock_exists_fallback):
|
||||
await outputter(
|
||||
ctx=ctx,
|
||||
output_config=config,
|
||||
graph_output=state,
|
||||
)
|
||||
|
||||
call_args = mock_llm_with_tools.ainvoke.call_args
|
||||
assert call_args is not None
|
||||
messages = call_args[0][0]
|
||||
human_content_fallback = messages[1].content[0]["text"]
|
||||
|
||||
assert "--- Execution History ---" in human_content_fallback
|
||||
assert "- *Step 1 (Start: 2.5s): Tapped search button*" in human_content_fallback
|
||||
assert "*Action*: Tapped 'Search' at None" in human_content_fallback
|
||||
|
||||
# Reset mock call history
|
||||
mock_llm_with_tools.ainvoke.reset_mock()
|
||||
|
||||
# Test Case 2: Standard path (plan file exists)
|
||||
def mock_exists_standard(self, *args, **kwargs):
|
||||
if self.name == "task_plan.md":
|
||||
return True
|
||||
return original_exists(self, *args, **kwargs)
|
||||
|
||||
original_read_text = Path.read_text
|
||||
|
||||
def mock_read_text(self, *args, **kwargs):
|
||||
if self.name == "task_plan.md":
|
||||
return "- [ ] Goal 1\n - [/] Subgoal 1.1"
|
||||
return original_read_text(self, *args, **kwargs)
|
||||
|
||||
with (
|
||||
patch("pathlib.Path.exists", mock_exists_standard),
|
||||
patch("pathlib.Path.read_text", mock_read_text),
|
||||
):
|
||||
await outputter(
|
||||
ctx=ctx,
|
||||
output_config=config,
|
||||
graph_output=state,
|
||||
)
|
||||
|
||||
call_args = mock_llm_with_tools.ainvoke.call_args
|
||||
assert call_args is not None
|
||||
messages = call_args[0][0]
|
||||
human_content_standard = messages[1].content[0]["text"]
|
||||
|
||||
assert "--- Execution History ---" in human_content_standard
|
||||
assert "- *Step 1 (Start: 2.5s): Tapped search button*" in human_content_standard
|
||||
assert "*Action*: Tapped 'Search' at None" in human_content_standard
|
||||
|
||||
|
||||
@patch("artemis.tools.video_tool.VideoAnalyzer")
|
||||
@patch("artemis.agents.outputter.outputter.get_llm")
|
||||
@pytest.mark.asyncio
|
||||
async def test_outputter_executes_video_analyzer_tool(
|
||||
mock_get_llm, mock_video_analyzer, mock_context, mock_state
|
||||
):
|
||||
"""Test that the outputter ReAct loop can successfully invoke the video_analyzer tool."""
|
||||
# Setup mock VideoAnalyzer
|
||||
mock_agent_instance = Mock()
|
||||
mock_agent_instance.run = AsyncMock(
|
||||
return_value=("The user played a video for 5 seconds.", "success")
|
||||
)
|
||||
mock_video_analyzer.return_value = mock_agent_instance
|
||||
|
||||
# Setup mock LLM with two turns
|
||||
mock_llm = Mock()
|
||||
mock_llm_fallback = Mock()
|
||||
|
||||
mock_llm_with_tools = Mock()
|
||||
mock_llm_with_tools.ainvoke = AsyncMock()
|
||||
|
||||
# Turn 1: LLM calls video_analyzer tool
|
||||
tool_call = {
|
||||
"name": "video_analyzer",
|
||||
"args": {
|
||||
"time_description": "from 5s to 10s",
|
||||
"purpose": "verify video played",
|
||||
},
|
||||
"id": "call_123",
|
||||
}
|
||||
msg_turn1 = Mock()
|
||||
msg_turn1.content = ""
|
||||
msg_turn1.tool_calls = [tool_call]
|
||||
|
||||
# Turn 2: LLM returns final answer
|
||||
msg_turn2 = Mock()
|
||||
msg_turn2.content = "Video played successfully."
|
||||
msg_turn2.tool_calls = []
|
||||
|
||||
mock_llm_with_tools.ainvoke.side_effect = [msg_turn1, msg_turn2]
|
||||
|
||||
mock_llm.bind_tools.return_value = mock_llm_with_tools
|
||||
mock_llm_fallback.bind_tools.return_value = mock_llm_with_tools
|
||||
|
||||
def get_llm_side_effect(ctx, name, is_utils=False, use_fallback=False):
|
||||
if use_fallback:
|
||||
return mock_llm_fallback
|
||||
return mock_llm
|
||||
|
||||
mock_get_llm.side_effect = get_llm_side_effect
|
||||
|
||||
config = OutputConfig(
|
||||
structured_output=None,
|
||||
output_description="Verify video",
|
||||
)
|
||||
|
||||
result = await outputter(ctx=mock_context, output_config=config, graph_output=mock_state)
|
||||
|
||||
# Assert VideoAnalyzer was called correctly
|
||||
mock_video_analyzer.assert_called_once_with(mock_context)
|
||||
mock_agent_instance.run.assert_called_once_with("from 5s to 10s", "verify video played")
|
||||
|
||||
# Assert LLM's ainvoke was called twice
|
||||
assert mock_llm_with_tools.ainvoke.call_count == 2
|
||||
|
||||
# Assert tool message was appended (messages list is mutated in place,
|
||||
# so it eventually has 5 elements)
|
||||
sent_messages = mock_llm_with_tools.ainvoke.call_args_list[1][0][0]
|
||||
assert len(sent_messages) == 5
|
||||
|
||||
# Verify the sequence of messages
|
||||
assert sent_messages[2] == msg_turn1
|
||||
|
||||
tool_message = sent_messages[3]
|
||||
from langchain_core.messages import ToolMessage
|
||||
|
||||
assert isinstance(tool_message, ToolMessage)
|
||||
assert tool_message.content == "The user played a video for 5 seconds."
|
||||
assert tool_message.status == "success"
|
||||
|
||||
assert sent_messages[4] == msg_turn2
|
||||
|
||||
# Assert final result
|
||||
assert result == "Video played successfully."
|
||||
|
||||
|
||||
@patch("artemis.agents.outputter.outputter.get_llm")
|
||||
@pytest.mark.asyncio
|
||||
async def test_outputter_executes_save_note_tool(mock_get_llm, mock_context, mock_state):
|
||||
"""Test that the outputter ReAct loop can successfully invoke the save_note tool."""
|
||||
# Setup mock LLM with two turns
|
||||
mock_llm = Mock()
|
||||
mock_llm_fallback = Mock()
|
||||
|
||||
mock_llm_with_tools = Mock()
|
||||
mock_llm_with_tools.ainvoke = AsyncMock()
|
||||
|
||||
# Turn 1: LLM calls save_note tool
|
||||
tool_call = {
|
||||
"name": "save_note",
|
||||
"args": {"key": "verification_code", "content": "123456"},
|
||||
"id": "call_save_123",
|
||||
}
|
||||
msg_turn1 = Mock()
|
||||
msg_turn1.content = ""
|
||||
msg_turn1.tool_calls = [tool_call]
|
||||
|
||||
# Turn 2: LLM returns final answer
|
||||
msg_turn2 = Mock()
|
||||
msg_turn2.content = "Verification code is 123456."
|
||||
msg_turn2.tool_calls = []
|
||||
|
||||
mock_llm_with_tools.ainvoke.side_effect = [msg_turn1, msg_turn2]
|
||||
|
||||
mock_llm.bind_tools.return_value = mock_llm_with_tools
|
||||
mock_llm_fallback.bind_tools.return_value = mock_llm_with_tools
|
||||
|
||||
def get_llm_side_effect(ctx, name, is_utils=False, use_fallback=False):
|
||||
if use_fallback:
|
||||
return mock_llm_fallback
|
||||
return mock_llm
|
||||
|
||||
mock_get_llm.side_effect = get_llm_side_effect
|
||||
|
||||
config = OutputConfig(
|
||||
structured_output=None,
|
||||
output_description="Save verification code to note",
|
||||
)
|
||||
|
||||
# Setup mock data_engine on context to support saving notes
|
||||
mock_data_engine = Mock()
|
||||
mock_data_engine.base_dir = "/tmp/mock_session"
|
||||
mock_context.data_engine = mock_data_engine
|
||||
|
||||
# Patch save_note_content utility to avoid writing to disk
|
||||
with patch("artemis.tools.scratchpad.save_note_content") as mock_save_content:
|
||||
result = await outputter(ctx=mock_context, output_config=config, graph_output=mock_state)
|
||||
mock_save_content.assert_called_once_with(
|
||||
"/tmp/mock_session", "verification_code", "123456"
|
||||
)
|
||||
|
||||
# Assert final result
|
||||
assert result == "Verification code is 123456."
|
||||
@@ -0,0 +1,102 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import json
|
||||
from unittest.mock import MagicMock, mock_open, patch
|
||||
|
||||
from artemis.agents.outputter.tools import (
|
||||
get_search_history_tool,
|
||||
get_step_details_tool,
|
||||
get_step_screenshot_tool,
|
||||
)
|
||||
from artemis.context import ArtemisContext
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context():
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.data_engine = MagicMock()
|
||||
ctx.data_engine.global_base_dir = "/tmp/fake_traces"
|
||||
return ctx
|
||||
|
||||
|
||||
def test_get_step_details_tool():
|
||||
steps = [
|
||||
{
|
||||
"step_number": 1,
|
||||
"relative_time": "1s",
|
||||
"summary": "Step 1",
|
||||
"action_taken": "tap",
|
||||
},
|
||||
{
|
||||
"step_number": 2,
|
||||
"relative_time": "2s",
|
||||
"summary": "Step 2",
|
||||
"action_taken": "swipe",
|
||||
},
|
||||
]
|
||||
tool = get_step_details_tool(steps)
|
||||
|
||||
# Test valid range
|
||||
result = tool.invoke({"start_step": 1, "end_step": 1})
|
||||
parsed = json.loads(result)
|
||||
assert len(parsed) == 1
|
||||
assert parsed[0]["step_number"] == 1
|
||||
assert parsed[0]["action_taken"] == "tap"
|
||||
|
||||
# Test invalid range
|
||||
result_invalid = tool.invoke({"start_step": 3, "end_step": 4})
|
||||
assert "No steps found" in result_invalid
|
||||
|
||||
|
||||
def test_get_step_screenshot_tool(mock_context):
|
||||
steps = [
|
||||
{"step_number": 1, "pre_image_name": "img1", "post_image_name": "img2"},
|
||||
]
|
||||
tool = get_step_screenshot_tool(mock_context, steps)
|
||||
|
||||
# Mock open and exists
|
||||
with (
|
||||
patch("pathlib.Path.exists", return_value=True),
|
||||
patch("builtins.open", mock_open(read_data=b"fake_image_bytes")),
|
||||
):
|
||||
result = tool.invoke({"step_number": 1})
|
||||
assert isinstance(result, list)
|
||||
assert result[0]["type"] == "text"
|
||||
assert result[1]["type"] == "image_url"
|
||||
assert "data:image/jpeg;base64" in result[1]["image_url"]["url"]
|
||||
|
||||
|
||||
def test_search_history_tool(mock_context):
|
||||
tool = get_search_history_tool(mock_context)
|
||||
|
||||
# Mock steps returned by storage
|
||||
mock_step_1 = MagicMock()
|
||||
mock_step_1.step_number = 1
|
||||
mock_step_1.pre_image_name = "img1"
|
||||
mock_step_1.post_image_name = None
|
||||
|
||||
mock_context.data_engine.storage.get_steps.return_value = [mock_step_1]
|
||||
|
||||
# Mock search_ui_by_hash
|
||||
mock_context.data_engine.storage.search_ui_by_hash.return_value = [
|
||||
{"type": "ocr", "matched_text": "verification code: 1234", "score": 0.9}
|
||||
]
|
||||
|
||||
result = tool.invoke({"query": "verification"})
|
||||
parsed = json.loads(result)
|
||||
assert len(parsed) == 1
|
||||
assert parsed[0]["step_number"] == 1
|
||||
assert parsed[0]["matches"][0]["matched_text"] == "verification code: 1234"
|
||||
@@ -0,0 +1,89 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from artemis.agents.planner.planner import PlannerNode
|
||||
from artemis.context import DevicePlatform, ArtemisContext
|
||||
import pytest
|
||||
|
||||
|
||||
class DummyState:
|
||||
def __init__(
|
||||
self,
|
||||
initial_goal,
|
||||
needs_replan=False,
|
||||
operator_raw_data=None,
|
||||
operator_replan_reason=None,
|
||||
):
|
||||
self.initial_goal = initial_goal
|
||||
self.needs_replan = needs_replan
|
||||
self.operator_raw_data = operator_raw_data
|
||||
self.operator_replan_reason = operator_replan_reason
|
||||
|
||||
async def asanitize_update(self, ctx, update, agent):
|
||||
return update
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context():
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
ctx.device = Mock()
|
||||
ctx.device.mobile_platform = DevicePlatform.ANDROID
|
||||
ctx.adb_client = Mock()
|
||||
ctx.ui_adb_client = Mock()
|
||||
mock_screen_data = Mock()
|
||||
mock_screen_data.base64 = "YmFzZTY0"
|
||||
mock_screen_data.elements = []
|
||||
mock_screen_data.width = 1080
|
||||
mock_screen_data.height = 2400
|
||||
ctx.ui_adb_client.get_screen_data.return_value = mock_screen_data
|
||||
ctx.data_engine = None
|
||||
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_planner_initial_plan(mock_context):
|
||||
state = DummyState(
|
||||
initial_goal="Test Goal",
|
||||
operator_raw_data={"screenshot_b64": "dummy_base64_string"},
|
||||
)
|
||||
node = PlannerNode(mock_context)
|
||||
|
||||
mock_llm = Mock()
|
||||
mock_response = Mock()
|
||||
mock_response.content = "- [ ] Subgoal 1"
|
||||
mock_response.tool_calls = []
|
||||
|
||||
async def mock_astream(*args, **kwargs):
|
||||
yield mock_response
|
||||
|
||||
mock_llm.astream.side_effect = mock_astream
|
||||
mock_llm.bind_tools = Mock(return_value=mock_llm)
|
||||
|
||||
with patch("artemis.agents.planner.planner.get_llm", return_value=mock_llm):
|
||||
await node(state)
|
||||
|
||||
assert mock_llm.astream.called
|
||||
args, kwargs = mock_llm.astream.call_args
|
||||
messages = args[0]
|
||||
|
||||
system_msg = messages[0].content
|
||||
assert "## Role" in system_msg
|
||||
|
||||
human_msg_content = messages[1].content
|
||||
assert isinstance(human_msg_content, list)
|
||||
assert human_msg_content[0]["type"] == "text"
|
||||
assert "Goal: Test Goal" in human_msg_content[0]["text"]
|
||||
@@ -0,0 +1,112 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import asyncio
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.agents.operator.operator import OperatorNode
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.graph import execution_check_node
|
||||
from artemis.graph.state import State
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_snapshot_rollback_flow(tmp_path):
|
||||
# Setup dummy directories
|
||||
base_dir = tmp_path / "trace"
|
||||
notes_dir = base_dir / "notes"
|
||||
notes_dir.mkdir(parents=True)
|
||||
|
||||
task_plan_path = notes_dir / "task_plan.md"
|
||||
task_plan_path.write_text("Initial Plan", encoding="utf-8")
|
||||
|
||||
# Mock Context
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = MagicMock()
|
||||
mock_ctx.data_engine.base_dir = base_dir
|
||||
mock_ctx.execution_setup = None
|
||||
|
||||
# Mock State
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.subagent_calls = []
|
||||
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
|
||||
mock_state.initial_goal = "Test goal"
|
||||
mock_state.short_term_memory = ""
|
||||
mock_state.operator_raw_data = {
|
||||
"screenshot_b64": "YmFzZTY0",
|
||||
"xml_hierarchy": "<hierarchy></hierarchy>",
|
||||
"ocr_results": [],
|
||||
}
|
||||
mock_state.current_step_id = "test-step-id"
|
||||
|
||||
# Mock Controller and LLM to let Operator run minimally
|
||||
mock_controller = MagicMock()
|
||||
mock_controller.take_screenshot = AsyncMock(return_value="YmFzZTY0")
|
||||
mock_device_data = MagicMock()
|
||||
mock_device_data.elements = []
|
||||
mock_controller.get_screen_data = AsyncMock(return_value=mock_device_data)
|
||||
|
||||
mock_llm = MagicMock()
|
||||
mock_response = MagicMock()
|
||||
mock_response.tool_calls = []
|
||||
|
||||
async def mock_astream(*args, **kwargs):
|
||||
yield mock_response
|
||||
|
||||
mock_llm.astream.side_effect = mock_astream
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
|
||||
# 1. Run Operator to trigger snapshot
|
||||
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
|
||||
node = OperatorNode(mock_ctx)
|
||||
await node(mock_state)
|
||||
|
||||
# Verify snapshot was created
|
||||
snapshot_dir = notes_dir.with_name("notes_snapshot")
|
||||
assert snapshot_dir.exists()
|
||||
assert (snapshot_dir / "task_plan.md").read_text(encoding="utf-8") == "Initial Plan"
|
||||
assert mock_ctx.task_plan_snapshot == snapshot_dir
|
||||
|
||||
# 2. Simulate Operator optimistic modification
|
||||
task_plan_path.write_text("Modified Plan", encoding="utf-8")
|
||||
|
||||
# 3. Run execution_check_node with failure to trigger rollback
|
||||
future = asyncio.Future()
|
||||
future.set_result({"status": "failed"})
|
||||
mock_ctx.checker_task = future
|
||||
|
||||
await execution_check_node(mock_state, mock_ctx)
|
||||
|
||||
# Verify rollback restored the file
|
||||
assert task_plan_path.read_text(encoding="utf-8") == "Initial Plan"
|
||||
|
||||
# 4. Run Operator again to create new snapshot (simulate next attempt)
|
||||
# First modify it again to simulate operator working on restored plan
|
||||
task_plan_path.write_text("Modified Plan 2", encoding="utf-8")
|
||||
|
||||
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
|
||||
await node(mock_state)
|
||||
|
||||
assert (snapshot_dir / "task_plan.md").read_text(encoding="utf-8") == "Modified Plan 2"
|
||||
|
||||
# 5. Run execution_check_node with success to trigger commit (cleanup)
|
||||
future_success = asyncio.Future()
|
||||
future_success.set_result({"status": "success"})
|
||||
mock_ctx.checker_task = future_success
|
||||
|
||||
await execution_check_node(mock_state, mock_ctx)
|
||||
|
||||
# Verify snapshot was deleted
|
||||
assert not snapshot_dir.exists()
|
||||
@@ -0,0 +1,216 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import json
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
from artemis.agents.summarizer.summarizer import SummarizerNode
|
||||
from artemis.context import ArtemisContext
|
||||
import pytest
|
||||
|
||||
|
||||
class DummyState:
|
||||
def __init__(
|
||||
self,
|
||||
structured_decisions=None,
|
||||
operator_raw_thinking=None,
|
||||
operator_native_thinking=None,
|
||||
last_execution_result=None,
|
||||
current_step_id=None,
|
||||
):
|
||||
self.structured_decisions = structured_decisions
|
||||
self.operator_raw_thinking = operator_raw_thinking
|
||||
self.operator_native_thinking = operator_native_thinking
|
||||
self.last_execution_result = last_execution_result
|
||||
self.current_step_id = current_step_id
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context():
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
ctx.device = None
|
||||
ctx.data_engine = Mock()
|
||||
ctx.background_tasks = []
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_summarizer_success(mock_context):
|
||||
"""Test that SummarizerNode successfully generates summary from raw thinking and execution result."""
|
||||
decisions = json.dumps([{"action": "tap", "coordinates": [500, 600]}])
|
||||
thinking = "The operator clicked the search button."
|
||||
result = {
|
||||
"status": "success",
|
||||
"executed_actions": [{"action": "tap", "coordinates": [500, 600]}],
|
||||
}
|
||||
|
||||
state = DummyState(
|
||||
structured_decisions=decisions,
|
||||
operator_raw_thinking=thinking,
|
||||
last_execution_result=result,
|
||||
current_step_id="12345678-1234-5678-1234-567812345678",
|
||||
)
|
||||
|
||||
# Mock LLM response
|
||||
mock_response = Mock()
|
||||
mock_response.content = "Step completed successfully."
|
||||
mock_llm = MagicMock()
|
||||
|
||||
async def mock_astream(*args, **kwargs):
|
||||
yield mock_response
|
||||
|
||||
mock_llm.astream.side_effect = mock_astream
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.summarizer.summarizer.get_llm", return_value=mock_llm
|
||||
) as mock_get_llm,
|
||||
patch("asyncio.create_task") as mock_create_task,
|
||||
):
|
||||
# Run the node call
|
||||
node = SummarizerNode(mock_context)
|
||||
await node(state)
|
||||
|
||||
# Verify task was registered in context
|
||||
assert len(mock_context.background_tasks) == 1
|
||||
assert mock_context.background_tasks[0] == mock_create_task.return_value
|
||||
|
||||
# Verify background task was spawned
|
||||
assert mock_create_task.called
|
||||
background_coroutine = mock_create_task.call_args[0][0]
|
||||
|
||||
# Await the background task synchronously for test verification
|
||||
await background_coroutine
|
||||
|
||||
# Verify LLM was invoked with proper structured messages
|
||||
assert mock_llm.astream.called
|
||||
invoked_messages = mock_llm.astream.call_args[0][0]
|
||||
|
||||
system_msg = invoked_messages[0].content
|
||||
human_msg = invoked_messages[1].content
|
||||
|
||||
assert "Summarizer Agent" in system_msg
|
||||
assert "Decisions made" in human_msg
|
||||
assert "The operator clicked the search button." in human_msg
|
||||
|
||||
# Verify Data Engine updated the step summary
|
||||
from uuid import UUID
|
||||
|
||||
mock_context.data_engine.update_step_summary.assert_called_once_with(
|
||||
UUID("12345678-1234-5678-1234-567812345678"),
|
||||
"Step completed successfully.",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_summarizer_with_native_thinking(mock_context):
|
||||
"""Test that SummarizerNode successfully generates summary from both explicit and native thoughts."""
|
||||
decisions = json.dumps([{"action": "tap", "coordinates": [500, 600]}])
|
||||
raw_thinking = "Explicit monologue."
|
||||
native_thinking = "Silent core reasoning."
|
||||
result = {"status": "success"}
|
||||
|
||||
state = DummyState(
|
||||
structured_decisions=decisions,
|
||||
operator_raw_thinking=raw_thinking,
|
||||
operator_native_thinking=native_thinking,
|
||||
last_execution_result=result,
|
||||
current_step_id="12345678-1234-5678-1234-567812345678",
|
||||
)
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.content = "Summary result."
|
||||
mock_llm = MagicMock()
|
||||
|
||||
async def mock_astream(*args, **kwargs):
|
||||
yield mock_response
|
||||
|
||||
mock_llm.astream.side_effect = mock_astream
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.summarizer.summarizer.get_llm", return_value=mock_llm
|
||||
) as mock_get_llm,
|
||||
patch("asyncio.create_task") as mock_create_task,
|
||||
):
|
||||
node = SummarizerNode(mock_context)
|
||||
await node(state)
|
||||
|
||||
# Verify task was registered in context
|
||||
assert len(mock_context.background_tasks) == 1
|
||||
assert mock_context.background_tasks[0] == mock_create_task.return_value
|
||||
|
||||
background_coroutine = mock_create_task.call_args[0][0]
|
||||
await background_coroutine
|
||||
|
||||
assert mock_llm.astream.called
|
||||
invoked_messages = mock_llm.astream.call_args[0][0]
|
||||
human_msg = invoked_messages[1].content
|
||||
|
||||
assert "Operator explicit thoughts:" in human_msg
|
||||
assert "Explicit monologue." in human_msg
|
||||
assert "Operator native thoughts:" in human_msg
|
||||
assert "Silent core reasoning." in human_msg
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_summarizer_with_chronological_trace(mock_context):
|
||||
"""Test that SummarizerNode generates a prompt containing the chronological step trace when available."""
|
||||
step_id = "12345678-1234-5678-1234-567812345678"
|
||||
state = DummyState(
|
||||
current_step_id=step_id,
|
||||
)
|
||||
|
||||
mock_step_data = {
|
||||
"step_id": step_id,
|
||||
"step_number": 1,
|
||||
"relative_time": "5.0s",
|
||||
"action_taken": [{"action": "click", "target_text": "Search"}],
|
||||
"last_execution_result": {"status": "success"},
|
||||
"interleaved_events": [
|
||||
{"type": "thought", "content": "Let's click search."},
|
||||
],
|
||||
}
|
||||
|
||||
mock_context.data_engine.get_agent_friendly_steps.return_value = [mock_step_data]
|
||||
|
||||
# Mock LLM response
|
||||
mock_response = Mock()
|
||||
mock_response.content = "Summary using chronological trace."
|
||||
mock_llm = MagicMock()
|
||||
|
||||
async def mock_astream(*args, **kwargs):
|
||||
yield mock_response
|
||||
|
||||
mock_llm.astream.side_effect = mock_astream
|
||||
|
||||
with (
|
||||
patch("artemis.agents.summarizer.summarizer.get_llm", return_value=mock_llm),
|
||||
patch("asyncio.create_task") as mock_create_task,
|
||||
):
|
||||
node = SummarizerNode(mock_context)
|
||||
await node(state)
|
||||
|
||||
# Verify task was registered in context
|
||||
assert len(mock_context.background_tasks) == 1
|
||||
assert mock_context.background_tasks[0] == mock_create_task.return_value
|
||||
|
||||
background_coroutine = mock_create_task.call_args[0][0]
|
||||
await background_coroutine
|
||||
|
||||
assert mock_llm.astream.called
|
||||
invoked_messages = mock_llm.astream.call_args[0][0]
|
||||
human_msg = invoked_messages[1].content
|
||||
|
||||
assert "--- Execution History ---" in human_msg
|
||||
assert "Let's click search." in human_msg
|
||||
@@ -0,0 +1,913 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import json
|
||||
from unittest.mock import AsyncMock, MagicMock, Mock, patch
|
||||
from langchain_core.messages import AIMessage
|
||||
from artemis.agents.validator.failure_analyzer import ValidationErrorCategory
|
||||
from artemis.agents.validator.validator import ValidatorNode
|
||||
from artemis.context import ArtemisContext
|
||||
import pytest
|
||||
|
||||
|
||||
class DummyState:
|
||||
def __init__(
|
||||
self,
|
||||
structured_decisions,
|
||||
validator_messages=[],
|
||||
current_step_id=None,
|
||||
latest_screenshot=None,
|
||||
):
|
||||
self.structured_decisions = structured_decisions
|
||||
self.validator_messages = validator_messages
|
||||
self.current_step_id = current_step_id
|
||||
self.latest_screenshot = latest_screenshot
|
||||
|
||||
async def asanitize_update(self, ctx, update, agent):
|
||||
return update
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context(tmp_path):
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
ctx.llm_config = Mock()
|
||||
ctx.data_engine = Mock()
|
||||
ctx.data_engine.base_dir = tmp_path
|
||||
ctx.data_engine.get_relative_time.return_value = 1.0
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def temp_screenshot(tmp_path):
|
||||
p = tmp_path / "screenshot.png"
|
||||
p.write_bytes(b"dummy_data")
|
||||
return str(p)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_mcp():
|
||||
with (
|
||||
patch("mcp.client.stdio.stdio_client") as mock_stdio,
|
||||
patch("artemis.agents.validator.validator.ClientSession") as mock_session_cls,
|
||||
):
|
||||
mock_ctx = AsyncMock()
|
||||
mock_ctx.__aenter__.return_value = (Mock(), Mock())
|
||||
mock_stdio.return_value = mock_ctx
|
||||
|
||||
mock_session = AsyncMock()
|
||||
mock_session.initialize = AsyncMock()
|
||||
mock_session_cls.return_value = mock_session
|
||||
|
||||
yield mock_session
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_success(mock_mcp, mock_context, temp_screenshot):
|
||||
"""Test that ValidatorNode executes actions successfully."""
|
||||
# Mock MCP calls: screenshot, tap, screenshot
|
||||
mock_mcp.call_tool.side_effect = [
|
||||
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # pre-screenshot
|
||||
Mock(content=[Mock(text="Success")]), # tap action
|
||||
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # post-screenshot
|
||||
]
|
||||
|
||||
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
|
||||
with patch("artemis.utils.image_diff.check_ui_change", return_value=True):
|
||||
node = ValidatorNode(mock_context)
|
||||
result = await node(state)
|
||||
|
||||
assert "last_execution_result" in result
|
||||
report = result["last_execution_result"]
|
||||
assert "execution" in report
|
||||
assert len(report["execution"]) == 1
|
||||
assert report["execution"][0]["action"] == "tap"
|
||||
assert "attempts" not in report["execution"][0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_failure_analysis(mock_mcp, mock_context, temp_screenshot):
|
||||
"""Test that ValidatorNode triggers failure analysis on error and handles
|
||||
cannot_fix via tool.
|
||||
"""
|
||||
tap_count = 0
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
nonlocal tap_count
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
if name == "tap":
|
||||
tap_count += 1
|
||||
return Mock(content=[Mock(text="Error: Element not found")])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
|
||||
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
|
||||
node = ValidatorNode(mock_context)
|
||||
|
||||
with (
|
||||
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
|
||||
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
|
||||
):
|
||||
mock_analyze.return_value = {
|
||||
"status": "cannot_fix",
|
||||
"analysis": "Analysis: Coordinates are likely wrong.",
|
||||
}
|
||||
|
||||
result = await node(state)
|
||||
|
||||
assert "last_execution_result" in result
|
||||
report = result["last_execution_result"]
|
||||
assert "execution" in report
|
||||
assert len(report["execution"]) == 1
|
||||
assert report["execution"][0]["action"] == "tap"
|
||||
assert report["execution"][0]["attempts"] == [
|
||||
"Error: Element not found",
|
||||
"Error: Element not found",
|
||||
]
|
||||
assert report["execution"][0]["repair"] == "Analysis: Coordinates are likely wrong."
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_repair_success(mock_mcp, mock_context, temp_screenshot):
|
||||
"""Test that ValidatorNode terminates successfully and clears remaining actions
|
||||
after successful repair.
|
||||
"""
|
||||
tap_count = 0
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
nonlocal tap_count
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
if name == "tap":
|
||||
tap_count += 1
|
||||
if tap_count <= 2: # First tap fails
|
||||
return Mock(content=[Mock(text="Error: Element not found")])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
|
||||
# Two actions planned initially
|
||||
decisions = json.dumps(
|
||||
[
|
||||
{"action": "tap", "coordinates": [105, 205]},
|
||||
{"action": "tap", "coordinates": [205, 305]},
|
||||
]
|
||||
)
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
|
||||
node = ValidatorNode(mock_context)
|
||||
|
||||
with (
|
||||
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
|
||||
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
|
||||
):
|
||||
mock_analyze.return_value = {
|
||||
"status": "fixed",
|
||||
"analysis": "Fixed by updating actions",
|
||||
}
|
||||
|
||||
result = await node(state)
|
||||
|
||||
assert "last_execution_result" in result
|
||||
report = result["last_execution_result"]
|
||||
assert "execution" in report
|
||||
assert len(report["execution"]) == 1 # Only the first failed action is in report
|
||||
|
||||
assert report["execution"][0]["coordinates"] == [105, 205]
|
||||
assert report["execution"][0]["attempts"] == [
|
||||
"Error: Element not found",
|
||||
"Error: Element not found",
|
||||
]
|
||||
assert report["execution"][0]["repair"] == "Fixed by updating actions"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_wait_for_delay(mock_mcp, mock_context, temp_screenshot):
|
||||
"""Test that wait_for_delay executes successfully."""
|
||||
|
||||
def mock_call(name, args):
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call
|
||||
|
||||
decisions = json.dumps([{"action": "wait_for_delay", "time_in_ms": 10}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
|
||||
with patch("artemis.utils.image_diff.check_ui_change", return_value=False):
|
||||
node = ValidatorNode(mock_context)
|
||||
result = await node(state)
|
||||
|
||||
assert "last_execution_result" in result
|
||||
report = result["last_execution_result"]
|
||||
assert "execution" in report
|
||||
assert len(report["execution"]) == 1
|
||||
assert report["execution"][0]["action"] == "wait_for_delay"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_focus_and_clear_text_no_ui_change(mock_mcp, mock_context, temp_screenshot):
|
||||
"""Test that focus_and_clear_text executes successfully even if no UI change is detected."""
|
||||
|
||||
def mock_call(name, args):
|
||||
if name == "focus_and_clear_text":
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call
|
||||
|
||||
decisions = json.dumps([{"action": "focus_and_clear_text", "coordinates": [400, 500]}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
|
||||
with patch("artemis.utils.image_diff.check_ui_change", return_value=False):
|
||||
node = ValidatorNode(mock_context)
|
||||
result = await node(state)
|
||||
|
||||
assert "last_execution_result" in result
|
||||
report = result["last_execution_result"]
|
||||
assert "execution" in report
|
||||
assert len(report["execution"]) == 1
|
||||
assert report["execution"][0]["action"] == "focus_and_clear_text"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_silent_failure_treated_as_success(mock_mcp, mock_context, temp_screenshot):
|
||||
"""Test that silent failure (exec succeeds but no UI change) is treated as
|
||||
success and does not trigger FailureAnalyzer.
|
||||
"""
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
if name == "tap":
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
|
||||
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
|
||||
node = ValidatorNode(mock_context)
|
||||
|
||||
with (
|
||||
patch("artemis.agents.validator.validator.VALIDATOR_POLL_TIMEOUT", 0.1),
|
||||
patch("artemis.agents.validator.validator.VALIDATOR_POLL_INTERVAL", 0.01),
|
||||
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
|
||||
patch("artemis.utils.image_diff.check_ui_change", return_value=False), # No UI change
|
||||
):
|
||||
result = await node(state)
|
||||
|
||||
# Assert FailureAnalyzer was NOT called
|
||||
mock_analyze.assert_not_called()
|
||||
|
||||
# Assert the action is considered executed successfully
|
||||
assert "last_execution_result" in result
|
||||
report = result["last_execution_result"]
|
||||
assert "execution" in report
|
||||
assert len(report["execution"]) == 1
|
||||
assert report["execution"][0]["action"] == "tap"
|
||||
assert "attempts" not in report["execution"][0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_records_skipped_actions_on_cannot_fix(
|
||||
mock_mcp, mock_context, temp_screenshot
|
||||
):
|
||||
"""Test that ValidatorNode marks unexecuted actions as Skipped when repair fails."""
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
return Mock(content=[Mock(text="Error: Connection lost")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
|
||||
# Two actions planned
|
||||
decisions = json.dumps(
|
||||
[
|
||||
{"action": "tap", "coordinates": [100, 200]},
|
||||
{"action": "press_key", "keycode": "ENTER"},
|
||||
]
|
||||
)
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
|
||||
node = ValidatorNode(mock_context)
|
||||
|
||||
with (
|
||||
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
|
||||
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
|
||||
):
|
||||
mock_analyze.return_value = {
|
||||
"status": "cannot_fix",
|
||||
"analysis": "Cannot fix: System crashed.",
|
||||
}
|
||||
|
||||
result = await node(state)
|
||||
|
||||
assert "last_execution_result" in result
|
||||
report = result["last_execution_result"]
|
||||
assert "execution" in report
|
||||
assert len(report["execution"]) == 2
|
||||
|
||||
# First action failed
|
||||
assert report["execution"][0]["action"] == "tap"
|
||||
assert report["execution"][0]["attempts"] == [
|
||||
"Error: Connection lost",
|
||||
"Error: Connection lost",
|
||||
]
|
||||
assert report["execution"][0]["repair"] == "Cannot fix: System crashed."
|
||||
|
||||
# Second action was skipped
|
||||
assert report["execution"][1]["action"] == "press_key"
|
||||
assert report["execution"][1]["attempts"] == ["Skipped"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_pre_execution_validation_and_repair(
|
||||
mock_mcp, mock_context, temp_screenshot
|
||||
):
|
||||
"""Test that pre-execution validation fails when element changes, and is
|
||||
repaired by FailureAnalyzer.
|
||||
"""
|
||||
mock_live_elements = [
|
||||
{
|
||||
"text": "Sign Up",
|
||||
"bounds": "[100,200][300,300]",
|
||||
"resource-id": "btn_signup",
|
||||
}
|
||||
]
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
if name == "get_ui_hierarchy":
|
||||
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
|
||||
if name == "tap":
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
|
||||
decisions = json.dumps(
|
||||
[
|
||||
{
|
||||
"action": "tap",
|
||||
"coordinates": [200, 250],
|
||||
"target_text": "Login",
|
||||
"target_bounds": [100, 200, 300, 300],
|
||||
"target_resource_id": "btn_login",
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
node = ValidatorNode(mock_context)
|
||||
|
||||
with (
|
||||
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
|
||||
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
|
||||
patch.object(
|
||||
ValidatorNode,
|
||||
"_validate_action_precondition_pixel",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_pixel,
|
||||
):
|
||||
mock_pixel.return_value = (
|
||||
False,
|
||||
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
|
||||
"Pixel check failed",
|
||||
)
|
||||
mock_analyze.return_value = {
|
||||
"status": "fixed",
|
||||
"analysis": ("Pre-exec failure matched and resolved by switching to Sign Up button."),
|
||||
}
|
||||
|
||||
result = await node(state)
|
||||
|
||||
mock_analyze.assert_called_once()
|
||||
args, kwargs = mock_analyze.call_args
|
||||
assert "Pre-execution validation failed" in args[2]
|
||||
assert "Login" in args[2]
|
||||
assert kwargs.get("error_category") == ValidationErrorCategory.TARGET_OCCUPIED
|
||||
|
||||
assert "last_execution_result" in result
|
||||
report = result["last_execution_result"]
|
||||
assert len(report["execution"]) == 1
|
||||
assert report["execution"][0]["action"] == "tap"
|
||||
assert "Pre-execution validation failed" in report["execution"][0]["attempts"][0]
|
||||
assert (
|
||||
report["execution"][0]["repair"]
|
||||
== "Pre-exec failure matched and resolved by switching to Sign Up"
|
||||
" button."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_pre_execution_validation_self_healing(
|
||||
mock_mcp, mock_context, temp_screenshot
|
||||
):
|
||||
"""Test that pre-execution validation succeeds and self-heals when element shifts slightly."""
|
||||
mock_live_elements = [
|
||||
{
|
||||
"text": "Login",
|
||||
"bounds": "[100,220][300,320]",
|
||||
"resource-id": "btn_login",
|
||||
}
|
||||
]
|
||||
|
||||
tapped_coordinates = None
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
nonlocal tapped_coordinates
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
if name == "get_ui_hierarchy":
|
||||
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
|
||||
if name == "tap":
|
||||
tapped_coordinates = args.get("coordinates")
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
|
||||
decisions = json.dumps(
|
||||
[
|
||||
{
|
||||
"action": "tap",
|
||||
"coordinates": [200, 250],
|
||||
"target_text": "Login",
|
||||
"target_bounds": [100, 200, 300, 300],
|
||||
"target_resource_id": "btn_login",
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
node = ValidatorNode(mock_context)
|
||||
|
||||
with patch("artemis.utils.image_diff.check_ui_change", return_value=True):
|
||||
await node(state)
|
||||
|
||||
# Center of [100, 220][300, 320] is [200, 270]
|
||||
assert tapped_coordinates == [200, 270]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_pre_execution_validation_anonymous_occupant(
|
||||
mock_mcp, mock_context, temp_screenshot
|
||||
):
|
||||
"""Test that pre-execution validation correctly flags TARGET_OCCUPIED
|
||||
when coordinates are blocked by an anonymous clickable view.
|
||||
"""
|
||||
# The live screen contains an anonymous clickable view covering
|
||||
# target coordinates [200, 250]
|
||||
mock_live_elements = [
|
||||
{
|
||||
"text": "",
|
||||
"bounds": "[150,200][250,300]",
|
||||
"clickable": True,
|
||||
"resource-id": "",
|
||||
}
|
||||
]
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
if name == "get_ui_hierarchy":
|
||||
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
|
||||
decisions = json.dumps(
|
||||
[
|
||||
{
|
||||
"action": "tap",
|
||||
"coordinates": [200, 250],
|
||||
"target_text": "Login",
|
||||
"target_bounds": [100, 200, 300, 300],
|
||||
"target_resource_id": "btn_login",
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
node = ValidatorNode(mock_context)
|
||||
|
||||
with (
|
||||
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
|
||||
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
|
||||
patch.object(
|
||||
ValidatorNode,
|
||||
"_validate_action_precondition_pixel",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_pixel,
|
||||
):
|
||||
mock_pixel.return_value = (
|
||||
False,
|
||||
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
|
||||
"Pixel check failed",
|
||||
)
|
||||
mock_analyze.return_value = {
|
||||
"status": "fixed",
|
||||
"analysis": "Pre-exec failure bypassed.",
|
||||
}
|
||||
|
||||
await node(state)
|
||||
|
||||
# Assert that the validator correctly classified this as TARGET_OCCUPIED
|
||||
# (due to the clickable anonymous view blocking the click)
|
||||
mock_analyze.assert_called_once()
|
||||
args, kwargs = mock_analyze.call_args
|
||||
assert kwargs.get("error_category") == ValidationErrorCategory.TARGET_OCCUPIED
|
||||
assert "occupied/intercepted by a different element: interactive anonymous element" in args[2]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_pixel_validation_success(mock_mcp, mock_context, temp_screenshot):
|
||||
"""Test that the pixel safety net succeeds when Gemini reports target is present."""
|
||||
mock_mcp.call_tool.side_effect = [
|
||||
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # live screenshot for pixel check
|
||||
Mock(content=[Mock(text="Success")]), # tap action
|
||||
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # post-screenshot
|
||||
]
|
||||
|
||||
decisions = json.dumps([{"action": "tap", "coordinates": [100, 200]}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
|
||||
mock_agent_cfg = Mock()
|
||||
mock_agent_cfg.provider = "google"
|
||||
mock_agent_cfg.model = "gemini-3.5-flash-lite"
|
||||
mock_context.llm_config.get_agent.return_value = mock_agent_cfg
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.text = json.dumps(
|
||||
{
|
||||
"is_present": True,
|
||||
"reason": "Button is clearly visible and clickable.",
|
||||
"confidence": 0.95,
|
||||
}
|
||||
)
|
||||
|
||||
mock_llm = MagicMock()
|
||||
mock_llm.ainvoke = AsyncMock(return_value=AIMessage(content=mock_response.text))
|
||||
|
||||
# Mock crop and draw helpers to return dummy bytes
|
||||
with (
|
||||
patch(
|
||||
"artemis.utils.visualization.crop_and_annotate_target",
|
||||
return_value=b"dummy_bytes",
|
||||
),
|
||||
patch("artemis.agents.validator.validator.get_llm", return_value=mock_llm),
|
||||
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
|
||||
):
|
||||
node = ValidatorNode(mock_context)
|
||||
result = await node(state)
|
||||
|
||||
assert "last_execution_result" in result
|
||||
report = result["last_execution_result"]
|
||||
assert len(report["execution"]) == 1
|
||||
assert "attempts" not in report["execution"][0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_pixel_validation_failure(mock_mcp, mock_context, temp_screenshot):
|
||||
"""Test that the pixel safety net fails and triggers FailureAnalyzer
|
||||
when Gemini reports target is missing.
|
||||
"""
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
|
||||
decisions = json.dumps([{"action": "tap", "coordinates": [100, 200]}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
|
||||
mock_agent_cfg = Mock()
|
||||
mock_agent_cfg.provider = "google"
|
||||
mock_agent_cfg.model = "gemini-3.5-flash-lite"
|
||||
mock_context.llm_config.get_agent.return_value = mock_agent_cfg
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.text = json.dumps(
|
||||
{
|
||||
"is_present": False,
|
||||
"reason": "The button has completely disappeared from the screen.",
|
||||
"confidence": 0.90,
|
||||
}
|
||||
)
|
||||
|
||||
mock_llm = MagicMock()
|
||||
mock_llm.ainvoke = AsyncMock(return_value=AIMessage(content=mock_response.text))
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.utils.visualization.crop_and_annotate_target",
|
||||
return_value=b"dummy_bytes",
|
||||
),
|
||||
patch("artemis.agents.validator.validator.get_llm", return_value=mock_llm),
|
||||
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
|
||||
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
|
||||
):
|
||||
mock_analyze.return_value = {
|
||||
"status": "cannot_fix",
|
||||
"analysis": "Pixel safety net flagged missing target.",
|
||||
}
|
||||
|
||||
node = ValidatorNode(mock_context)
|
||||
await node(state)
|
||||
|
||||
mock_analyze.assert_called_once()
|
||||
args, kwargs = mock_analyze.call_args
|
||||
assert "Pixel-level validation failed" in args[2]
|
||||
assert kwargs.get("error_category") == ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.agents.validator.validator.find_package")
|
||||
@patch("artemis.utils.app_launch_utils.launch_app_with_retries")
|
||||
async def test_validator_launch_app_passes_use_fallback_mcp(
|
||||
mock_launch_app, mock_find_package, mock_mcp, mock_context, temp_screenshot
|
||||
):
|
||||
"""Test that ValidatorNode calls find_package with use_fallback=False
|
||||
for launch_app action in MCP mode.
|
||||
"""
|
||||
mock_find_package.return_value = "com.example.app"
|
||||
mock_launch_app.return_value = (True, None)
|
||||
|
||||
# Mock MCP calls: screenshot (pre), screenshot (post)
|
||||
mock_mcp.call_tool.side_effect = [
|
||||
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # pre-screenshot
|
||||
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # post-screenshot
|
||||
]
|
||||
|
||||
decisions = json.dumps([{"action": "launch_app", "app_name": "My App"}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
|
||||
with patch("artemis.utils.image_diff.check_ui_change", return_value=True):
|
||||
node = ValidatorNode(mock_context)
|
||||
result = await node(state)
|
||||
|
||||
mock_find_package.assert_called_once_with(mock_context, "My App", use_fallback=False)
|
||||
mock_launch_app.assert_called_once_with(mock_context, "com.example.app")
|
||||
assert "last_execution_result" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.agents.validator.validator.find_package")
|
||||
@patch("artemis.utils.app_launch_utils.launch_app_with_retries")
|
||||
async def test_validator_launch_app_failure_no_retry(
|
||||
mock_launch_app, mock_find_package, mock_mcp, mock_context, temp_screenshot
|
||||
):
|
||||
"""Test that ValidatorNode does NOT retry launch_app action on failure."""
|
||||
mock_find_package.return_value = "com.example.app"
|
||||
mock_launch_app.return_value = (False, "Error: Force close")
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
|
||||
decisions = json.dumps([{"action": "launch_app", "app_name": "My App"}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
|
||||
with (
|
||||
patch("artemis.agents.validator.validator.VALIDATOR_POLL_TIMEOUT", 0.05),
|
||||
patch("artemis.agents.validator.validator.VALIDATOR_POLL_INTERVAL", 0.01),
|
||||
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
|
||||
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
|
||||
):
|
||||
mock_analyze.return_value = {
|
||||
"status": "cannot_fix",
|
||||
"analysis": "Cannot launch app.",
|
||||
}
|
||||
|
||||
node = ValidatorNode(mock_context)
|
||||
result = await node(state)
|
||||
|
||||
# Verify find_package was called only once
|
||||
mock_find_package.assert_called_once_with(mock_context, "My App", use_fallback=False)
|
||||
# Verify launch_app_with_retries was called only once
|
||||
# (no retry by ValidatorNode itself since it relies on launch_app_with_retries)
|
||||
mock_launch_app.assert_called_once_with(mock_context, "com.example.app")
|
||||
|
||||
# Verify attempts list has only 1 element
|
||||
assert "last_execution_result" in result
|
||||
report = result["last_execution_result"]
|
||||
assert len(report["execution"]) == 1
|
||||
assert report["execution"][0]["action"] == "launch_app"
|
||||
assert report["execution"][0]["attempts"] == ["Error: Force close"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_pre_execution_validation_disappeared_not_shifted(
|
||||
mock_mcp, mock_context, temp_screenshot
|
||||
):
|
||||
"""Test that a disappeared small button matching text in a giant container
|
||||
is classified as TARGET_DISAPPEARED, not TARGET_SHIFTED.
|
||||
"""
|
||||
# Live hierarchy only contains a giant background/container
|
||||
# containing the text "Sign Up"
|
||||
mock_live_elements = [
|
||||
{
|
||||
"text": "Sign Up",
|
||||
"bounds": "[0,0][1080,2400]",
|
||||
"resource-id": "root_container",
|
||||
}
|
||||
]
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
if name == "get_ui_hierarchy":
|
||||
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
|
||||
decisions = json.dumps(
|
||||
[
|
||||
{
|
||||
"action": "tap",
|
||||
"coordinates": [200, 250],
|
||||
"target_text": "Sign Up",
|
||||
"target_bounds": [100, 200, 300, 300],
|
||||
"target_resource_id": "btn_signup",
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
node = ValidatorNode(mock_context)
|
||||
|
||||
with (
|
||||
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
|
||||
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
|
||||
patch.object(
|
||||
ValidatorNode,
|
||||
"_validate_action_precondition_pixel",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_pixel,
|
||||
):
|
||||
mock_pixel.return_value = (
|
||||
False,
|
||||
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
|
||||
"Pixel check failed",
|
||||
)
|
||||
mock_analyze.return_value = {
|
||||
"status": "cannot_fix",
|
||||
"analysis": "Sign Up button disappeared.",
|
||||
}
|
||||
|
||||
await node(state)
|
||||
|
||||
# Assert that the validator correctly classified this as TARGET_DISAPPEARED
|
||||
# (since size mismatch prevents shift matching)
|
||||
mock_analyze.assert_called_once()
|
||||
args, kwargs = mock_analyze.call_args
|
||||
assert kwargs.get("error_category") == ValidationErrorCategory.TARGET_DISAPPEARED
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_pre_execution_validation_ocr_direct_to_pixel(
|
||||
mock_mcp, mock_context, temp_screenshot
|
||||
):
|
||||
"""Test that OCR-derived elements bypass XML validation and
|
||||
directly use pixel-based validation.
|
||||
"""
|
||||
mock_mcp.call_tool.side_effect = [
|
||||
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # live screenshot for pixel check
|
||||
Mock(content=[Mock(text="Success")]), # tap action
|
||||
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # post-screenshot
|
||||
]
|
||||
|
||||
decisions = json.dumps(
|
||||
[
|
||||
{
|
||||
"action": "tap",
|
||||
"coordinates": [200, 250],
|
||||
"target_text": "Login",
|
||||
"target_bounds": [100, 200, 300, 300],
|
||||
"target_class": "android.view.View [OCR]",
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
node = ValidatorNode(mock_context)
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
ValidatorNode,
|
||||
"_validate_action_precondition",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_xml,
|
||||
patch.object(
|
||||
ValidatorNode,
|
||||
"_validate_action_precondition_pixel",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_pixel,
|
||||
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
|
||||
):
|
||||
mock_pixel.return_value = (True, ValidationErrorCategory.NONE, "")
|
||||
|
||||
result = await node(state)
|
||||
|
||||
# Assert XML validation was NOT called
|
||||
mock_xml.assert_not_called()
|
||||
# Assert Pixel validation WAS called
|
||||
mock_pixel.assert_called_once()
|
||||
|
||||
assert "last_execution_result" in result
|
||||
report = result["last_execution_result"]
|
||||
assert report["status"] == "success"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_pre_execution_xml_failure_not_overridden_when_pixel_bypassed(
|
||||
mock_mcp, mock_context, temp_screenshot
|
||||
):
|
||||
"""Test that when XML validation fails AND Pixel validation is bypassed
|
||||
(PIXEL_BYPASSED), the XML failure is NOT overridden.
|
||||
"""
|
||||
mock_live_elements = [
|
||||
{
|
||||
"text": "Sign Up",
|
||||
"bounds": "[100,200][300,300]",
|
||||
"resource-id": "btn_signup",
|
||||
}
|
||||
]
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
if name == "get_ui_hierarchy":
|
||||
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
|
||||
decisions = json.dumps(
|
||||
[
|
||||
{
|
||||
"action": "tap",
|
||||
"coordinates": [200, 250],
|
||||
"target_text": "Login",
|
||||
"target_bounds": [100, 200, 300, 300],
|
||||
"target_resource_id": "btn_login",
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
node = ValidatorNode(mock_context)
|
||||
|
||||
with (
|
||||
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
|
||||
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
|
||||
patch.object(
|
||||
ValidatorNode,
|
||||
"_validate_action_precondition_pixel",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_pixel,
|
||||
):
|
||||
# Pixel check was bypassed due to VLM error or low confidence
|
||||
mock_pixel.return_value = (
|
||||
True,
|
||||
ValidationErrorCategory.PIXEL_BYPASSED,
|
||||
"",
|
||||
)
|
||||
mock_analyze.return_value = {
|
||||
"status": "cannot_fix",
|
||||
"analysis": ("XML validation failed and pixel bypass did not override."),
|
||||
}
|
||||
|
||||
await node(state)
|
||||
|
||||
# FailureAnalyzer SHOULD be called because XML validation failed
|
||||
# and PIXEL_BYPASSED did not override it
|
||||
mock_analyze.assert_called_once()
|
||||
args, kwargs = mock_analyze.call_args
|
||||
assert "Pre-execution validation failed" in args[2]
|
||||
@@ -0,0 +1,456 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.agents.video_analyzer.video_analyzer import VideoAnalyzer
|
||||
from artemis.context import ArtemisContext
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_analyzer_run():
|
||||
# Mock context
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = None
|
||||
mock_ctx.llm_config = MagicMock()
|
||||
mock_ctx.llm_config.utils.video_analyzer = MagicMock()
|
||||
mock_ctx.llm_config.utils.video_analyzer.model = "gemini-3.6-flash"
|
||||
|
||||
# Mock Gemini Client
|
||||
mock_client = MagicMock()
|
||||
mock_client.aio = MagicMock()
|
||||
mock_client.aio.models = MagicMock()
|
||||
mock_chunk = MagicMock()
|
||||
mock_chunk.text = "Final analysis: The action succeeded."
|
||||
mock_chunk.function_calls = []
|
||||
|
||||
async def mock_stream():
|
||||
yield mock_chunk
|
||||
|
||||
mock_client.aio.models.generate_content_stream = AsyncMock(return_value=mock_stream())
|
||||
|
||||
# Mock file reading for prompts
|
||||
mock_prompt = "# Dynamic Video Analyzer..."
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.genai.Client",
|
||||
return_value=mock_client,
|
||||
),
|
||||
patch(
|
||||
"pathlib.Path.exists",
|
||||
autospec=True,
|
||||
side_effect=lambda self: not str(self).endswith(".jpg"),
|
||||
),
|
||||
patch("pathlib.Path.read_text", return_value=mock_prompt),
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.cleanup_abandoned_gemini_files",
|
||||
AsyncMock(),
|
||||
),
|
||||
):
|
||||
analyzer = VideoAnalyzer(mock_ctx)
|
||||
result, status = await analyzer.run(
|
||||
time_description="from 10s to 15s", purpose="Verify action"
|
||||
)
|
||||
|
||||
assert result == "Final analysis: The action succeeded."
|
||||
assert status == "success"
|
||||
assert mock_client.aio.models.generate_content_stream.called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_analyzer_preserves_thought_signature():
|
||||
# Mock context
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = None
|
||||
mock_ctx.llm_config = MagicMock()
|
||||
mock_ctx.llm_config.utils.video_analyzer = MagicMock()
|
||||
mock_ctx.llm_config.utils.video_analyzer.model = "gemini-3.6-flash"
|
||||
|
||||
# Mock MobileDeviceController
|
||||
mock_controller = MagicMock()
|
||||
mock_controller.extract_segment_metadata = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
success=True,
|
||||
video_path="/tmp/video.mp4",
|
||||
actual_start_relative_time=10.0,
|
||||
)
|
||||
)
|
||||
|
||||
from google.genai import types
|
||||
|
||||
dummy_signature = b"dummy_thought_signature_bytes_56789"
|
||||
|
||||
# Turn 1: Model calls spawn_sub_agent
|
||||
spawn_call = types.FunctionCall(
|
||||
name="spawn_sub_agent",
|
||||
args={
|
||||
"start_time": 10.0,
|
||||
"end_time": 15.0,
|
||||
"specific_query": "Verify login screen",
|
||||
},
|
||||
)
|
||||
spawn_part = types.Part(
|
||||
function_call=spawn_call,
|
||||
thought_signature=dummy_signature,
|
||||
)
|
||||
|
||||
mock_chunk_1 = MagicMock()
|
||||
mock_chunk_1.text = ""
|
||||
mock_chunk_1.function_calls = [spawn_call]
|
||||
mock_chunk_1.candidates = [
|
||||
types.Candidate(content=types.Content(role="model", parts=[spawn_part]))
|
||||
]
|
||||
|
||||
# Turn 2: Model returns final answer
|
||||
mock_chunk_2 = MagicMock()
|
||||
mock_chunk_2.text = "Final analysis: The spawn succeeded and login button is visible."
|
||||
mock_chunk_2.function_calls = []
|
||||
mock_chunk_2.candidates = [
|
||||
types.Candidate(
|
||||
content=types.Content(
|
||||
role="model",
|
||||
parts=[types.Part.from_text(text=mock_chunk_2.text)],
|
||||
)
|
||||
)
|
||||
]
|
||||
|
||||
# We mock generate_content_stream
|
||||
call_count = 0
|
||||
captured_contents = []
|
||||
|
||||
async def mock_stream_1():
|
||||
yield mock_chunk_1
|
||||
|
||||
async def mock_stream_2():
|
||||
yield mock_chunk_2
|
||||
|
||||
async def mock_generate_content_stream(model, contents, config):
|
||||
nonlocal call_count
|
||||
has_submit_answer = False
|
||||
if config.tools:
|
||||
for t in config.tools:
|
||||
fd_list = getattr(t, "function_declarations", None)
|
||||
if fd_list:
|
||||
for fd in fd_list:
|
||||
if fd.name == "submit_answer":
|
||||
has_submit_answer = True
|
||||
break
|
||||
if not has_submit_answer:
|
||||
captured_contents.append(list(contents)) # copy history contents passed to call
|
||||
call_count += 1
|
||||
if call_count == 1:
|
||||
return mock_stream_1()
|
||||
else:
|
||||
return mock_stream_2()
|
||||
else:
|
||||
submit_call = types.FunctionCall(
|
||||
name="submit_answer",
|
||||
args={
|
||||
"confidence_score": 0.9,
|
||||
"summary": "Sub-agent verification finished",
|
||||
"analysis": "Analysis here",
|
||||
"verification_timestamp_secs": 10.0,
|
||||
},
|
||||
)
|
||||
mock_sub_chunk = MagicMock()
|
||||
mock_sub_chunk.text = ""
|
||||
mock_sub_chunk.function_calls = [submit_call]
|
||||
|
||||
async def sub_stream():
|
||||
yield mock_sub_chunk
|
||||
|
||||
return sub_stream()
|
||||
|
||||
# Mock Gemini Client
|
||||
mock_client = MagicMock()
|
||||
mock_client.aio = MagicMock()
|
||||
mock_client.aio.files = MagicMock()
|
||||
|
||||
mock_file = MagicMock()
|
||||
mock_file.name = "dummy_cloud_file_123"
|
||||
mock_file.uri = "https://gemini-file-api/dummy_cloud_file_123"
|
||||
mock_file.mime_type = "video/mp4"
|
||||
mock_client.aio.files.upload = AsyncMock(return_value=mock_file)
|
||||
|
||||
mock_f_state = MagicMock()
|
||||
mock_f_state.state.name = "ACTIVE"
|
||||
mock_client.aio.files.get = AsyncMock(return_value=mock_f_state)
|
||||
mock_client.aio.files.delete = AsyncMock()
|
||||
|
||||
mock_client.aio.models = MagicMock()
|
||||
mock_client.aio.models.generate_content_stream = AsyncMock(
|
||||
side_effect=mock_generate_content_stream
|
||||
)
|
||||
|
||||
# Mock file reading for prompts
|
||||
mock_prompt = "# Dynamic Video Analyzer..."
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.genai.Client",
|
||||
return_value=mock_client,
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.get_controller",
|
||||
return_value=mock_controller,
|
||||
),
|
||||
patch(
|
||||
"pathlib.Path.exists",
|
||||
autospec=True,
|
||||
side_effect=lambda self: not str(self).endswith(".jpg"),
|
||||
),
|
||||
patch(
|
||||
"pathlib.Path.stat",
|
||||
return_value=os.stat_result((0o100644, 0, 0, 0, 0, 0, 1024 * 1024, 0, 0, 0)),
|
||||
),
|
||||
patch("pathlib.Path.read_text", return_value=mock_prompt),
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.cleanup_abandoned_gemini_files",
|
||||
AsyncMock(),
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.compress_video_for_api",
|
||||
AsyncMock(return_value=Path("/tmp/video.mp4")),
|
||||
),
|
||||
):
|
||||
analyzer = VideoAnalyzer(mock_ctx)
|
||||
result, status = await analyzer.run(
|
||||
time_description="from 10s to 15s", purpose="Verify login"
|
||||
)
|
||||
|
||||
assert status == "success"
|
||||
assert "spawn succeeded" in result
|
||||
assert call_count == 2
|
||||
|
||||
# Verify that the second model call's history contains the model turn with the thought_signature
|
||||
second_call_history = captured_contents[1]
|
||||
assert len(second_call_history) >= 4
|
||||
|
||||
model_turn = second_call_history[1]
|
||||
assert model_turn.role == "model"
|
||||
assert len(model_turn.parts) == 1
|
||||
assert model_turn.parts[0].function_call.name == "spawn_sub_agent"
|
||||
assert model_turn.parts[0].thought_signature == dummy_signature
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_analyzer_sub_agent_confidence_validation():
|
||||
# Mock context
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = None
|
||||
mock_ctx.llm_config = MagicMock()
|
||||
mock_ctx.llm_config.utils.video_analyzer = MagicMock()
|
||||
mock_ctx.llm_config.utils.video_analyzer.model = "gemini-3.6-flash"
|
||||
|
||||
# Mock MobileDeviceController
|
||||
mock_controller = MagicMock()
|
||||
mock_controller.extract_segment_metadata = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
success=True,
|
||||
video_path="/tmp/video.mp4",
|
||||
actual_start_relative_time=10.0,
|
||||
)
|
||||
)
|
||||
|
||||
from google.genai import types
|
||||
|
||||
# Main agent responses
|
||||
# Turn 1: Calls spawn_sub_agent
|
||||
spawn_call = types.FunctionCall(
|
||||
name="spawn_sub_agent",
|
||||
args={
|
||||
"start_time": 10.0,
|
||||
"end_time": 15.0,
|
||||
"specific_query": "Verify login screen",
|
||||
},
|
||||
)
|
||||
mock_main_chunk_1 = MagicMock()
|
||||
mock_main_chunk_1.text = ""
|
||||
mock_main_chunk_1.function_calls = [spawn_call]
|
||||
|
||||
# Turn 2: Returns final answer
|
||||
mock_main_chunk_2 = MagicMock()
|
||||
mock_main_chunk_2.text = "Final blackboard status checked."
|
||||
mock_main_chunk_2.function_calls = []
|
||||
|
||||
# Sub-agent responses
|
||||
# Turn 1: Calls submit_answer with invalid confidence_score (string)
|
||||
invalid_submit_call = types.FunctionCall(
|
||||
name="submit_answer",
|
||||
args={
|
||||
"timeline_events": [
|
||||
{
|
||||
"confidence_score": "HIGH",
|
||||
"transcription": "Invalid confidence score call",
|
||||
"verification_timestamp_secs": 10.0,
|
||||
}
|
||||
],
|
||||
"summary": "Invalid confidence score call",
|
||||
"analysis": "Some analysis",
|
||||
},
|
||||
)
|
||||
sub_dummy_signature = b"sub_dummy_signature_9999"
|
||||
sub_part_1 = types.Part(
|
||||
function_call=invalid_submit_call,
|
||||
thought_signature=sub_dummy_signature,
|
||||
)
|
||||
mock_sub_chunk_1 = MagicMock()
|
||||
mock_sub_chunk_1.text = ""
|
||||
mock_sub_chunk_1.function_calls = [invalid_submit_call]
|
||||
mock_sub_chunk_1.candidates = [
|
||||
types.Candidate(content=types.Content(role="model", parts=[sub_part_1]))
|
||||
]
|
||||
|
||||
# Turn 2: Calls submit_answer with valid confidence_score
|
||||
valid_submit_call = types.FunctionCall(
|
||||
name="submit_answer",
|
||||
args={
|
||||
"timeline_events": [
|
||||
{
|
||||
"confidence_score": 0.8,
|
||||
"transcription": "Valid confidence score call",
|
||||
"verification_timestamp_secs": 10.0,
|
||||
}
|
||||
],
|
||||
"summary": "Valid confidence score call",
|
||||
"analysis": "Some analysis",
|
||||
},
|
||||
)
|
||||
mock_sub_chunk_2 = MagicMock()
|
||||
mock_sub_chunk_2.text = ""
|
||||
mock_sub_chunk_2.function_calls = [valid_submit_call]
|
||||
mock_sub_chunk_2.candidates = [
|
||||
types.Candidate(
|
||||
content=types.Content(role="model", parts=[types.Part(function_call=valid_submit_call)])
|
||||
)
|
||||
]
|
||||
|
||||
# Stream generator mocks
|
||||
main_calls = 0
|
||||
sub_calls = 0
|
||||
captured_sub_contents = []
|
||||
|
||||
async def mock_main_stream():
|
||||
nonlocal main_calls
|
||||
if main_calls == 0:
|
||||
main_calls += 1
|
||||
yield mock_main_chunk_1
|
||||
else:
|
||||
yield mock_main_chunk_2
|
||||
|
||||
async def mock_sub_stream():
|
||||
nonlocal sub_calls
|
||||
if sub_calls == 0:
|
||||
sub_calls += 1
|
||||
yield mock_sub_chunk_1
|
||||
else:
|
||||
yield mock_sub_chunk_2
|
||||
|
||||
async def mock_generate_content_stream(model, contents, config):
|
||||
system_instruction = config.system_instruction
|
||||
if "Dynamic Video Analyzer" in str(system_instruction):
|
||||
return mock_main_stream()
|
||||
else:
|
||||
captured_sub_contents.append(list(contents))
|
||||
return mock_sub_stream()
|
||||
|
||||
# Mock Gemini Client
|
||||
mock_client = MagicMock()
|
||||
mock_client.aio = MagicMock()
|
||||
mock_client.aio.files = MagicMock()
|
||||
|
||||
mock_file = MagicMock()
|
||||
mock_file.name = "dummy_cloud_file_123"
|
||||
mock_file.uri = "https://gemini-file-api/dummy_cloud_file_123"
|
||||
mock_file.mime_type = "video/mp4"
|
||||
mock_client.aio.files.upload = AsyncMock(return_value=mock_file)
|
||||
|
||||
mock_f_state = MagicMock()
|
||||
mock_f_state.state.name = "ACTIVE"
|
||||
mock_client.aio.files.get = AsyncMock(return_value=mock_f_state)
|
||||
mock_client.aio.files.delete = AsyncMock()
|
||||
|
||||
mock_client.aio.models = MagicMock()
|
||||
mock_client.aio.models.generate_content_stream = AsyncMock(
|
||||
side_effect=mock_generate_content_stream
|
||||
)
|
||||
|
||||
# Mock file reading for prompts
|
||||
mock_main_prompt = "# Dynamic Video Analyzer..."
|
||||
mock_sub_prompt = "# Sub Agent..."
|
||||
|
||||
def mock_read_text(self, encoding="utf-8"):
|
||||
if "video_analyzer.md" in str(self):
|
||||
return mock_main_prompt
|
||||
elif "video_sub_agent.md" in str(self):
|
||||
return mock_sub_prompt
|
||||
return ""
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.genai.Client",
|
||||
return_value=mock_client,
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.get_controller",
|
||||
return_value=mock_controller,
|
||||
),
|
||||
patch(
|
||||
"pathlib.Path.exists",
|
||||
autospec=True,
|
||||
side_effect=lambda self: not str(self).endswith(".jpg"),
|
||||
),
|
||||
patch(
|
||||
"pathlib.Path.stat",
|
||||
return_value=os.stat_result((0o100644, 0, 0, 0, 0, 0, 1024 * 1024, 0, 0, 0)),
|
||||
),
|
||||
patch("pathlib.Path.read_text", mock_read_text),
|
||||
patch("pathlib.Path.mkdir"),
|
||||
patch("pathlib.Path.touch"),
|
||||
patch("pathlib.Path.is_dir", return_value=True),
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.cleanup_abandoned_gemini_files",
|
||||
AsyncMock(),
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.compress_video_for_api",
|
||||
AsyncMock(return_value=Path("/tmp/video.mp4")),
|
||||
),
|
||||
patch(
|
||||
"asyncio.create_subprocess_exec",
|
||||
AsyncMock(return_value=MagicMock(communicate=AsyncMock(return_value=(b"", b"")))),
|
||||
),
|
||||
):
|
||||
analyzer = VideoAnalyzer(mock_ctx)
|
||||
result, status = await analyzer.run(
|
||||
time_description="from 10s to 15s", purpose="Verify login"
|
||||
)
|
||||
|
||||
assert status == "success"
|
||||
assert sub_calls == 1
|
||||
assert len(analyzer.blackboard_entries) == 1
|
||||
entry = analyzer.blackboard_entries[0]
|
||||
assert entry["confidence_score"] == 0.8
|
||||
assert entry["summary"] == "Valid confidence score call"
|
||||
|
||||
# Verify that sub-agent turn 2 received the model turn with the thought_signature
|
||||
assert len(captured_sub_contents) >= 2
|
||||
sub_turn_2_contents = captured_sub_contents[1]
|
||||
model_turns = [c for c in sub_turn_2_contents if c.role == "model"]
|
||||
assert len(model_turns) == 1
|
||||
assert model_turns[0].parts[0].thought_signature == sub_dummy_signature
|
||||
@@ -0,0 +1,234 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for Universal VideoAnalyzer dual-path execution engine."""
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import AIMessage
|
||||
|
||||
from artemis.agents.video_analyzer.video_analyzer import (
|
||||
UNIVERSAL_MAIN_TOOLS,
|
||||
UNIVERSAL_SUBMIT_ANSWER_TOOL,
|
||||
VideoAnalyzer,
|
||||
)
|
||||
from artemis.context import ArtemisContext
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context(tmp_path):
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
ctx.llm_config = Mock()
|
||||
mock_llm_cfg = Mock()
|
||||
mock_llm_cfg.model = "claude-3-7-sonnet"
|
||||
mock_llm_cfg.temperature = 0.2
|
||||
ctx.llm_config.utils = Mock()
|
||||
ctx.llm_config.utils.video_analyzer = mock_llm_cfg
|
||||
|
||||
ctx.device = Mock()
|
||||
ctx.device.device_width = 1080
|
||||
ctx.device.device_height = 2400
|
||||
|
||||
ctx.data_engine = Mock()
|
||||
ctx.data_engine.base_dir = tmp_path
|
||||
return ctx
|
||||
|
||||
|
||||
def test_video_analyzer_universal_engine_detection(mock_context):
|
||||
"""Test that VideoAnalyzer routes to Universal Engine when non-Gemini model is set."""
|
||||
from artemis.config import settings
|
||||
|
||||
with patch.object(settings, "GOOGLE_API_KEY", None):
|
||||
agent = VideoAnalyzer(mock_context)
|
||||
assert agent.use_native_gemini is False
|
||||
assert agent.model_name == "claude-3-7-sonnet"
|
||||
|
||||
|
||||
def test_universal_tool_schemas():
|
||||
"""Verify schema structures for Universal tools."""
|
||||
assert UNIVERSAL_SUBMIT_ANSWER_TOOL["type"] == "function"
|
||||
assert UNIVERSAL_SUBMIT_ANSWER_TOOL["function"]["name"] == "submit_answer"
|
||||
assert len(UNIVERSAL_MAIN_TOOLS) == 3
|
||||
tool_names = [t["function"]["name"] for t in UNIVERSAL_MAIN_TOOLS]
|
||||
assert "extract_segment_metadata" in tool_names
|
||||
assert "spawn_sub_agent" in tool_names
|
||||
assert "analyze_audio_only" in tool_names
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exec_single_chunk_universal(mock_context, tmp_path):
|
||||
"""Test keyframe extraction and LangChain tool invocation in universal sub-agent."""
|
||||
agent = VideoAnalyzer(mock_context)
|
||||
agent.use_native_gemini = False
|
||||
|
||||
fake_video = tmp_path / "compressed.mp4"
|
||||
fake_video.write_bytes(b"video_bytes")
|
||||
raw_video = tmp_path / "raw.mp4"
|
||||
raw_video.write_bytes(b"raw_bytes")
|
||||
|
||||
fake_keyframes = [
|
||||
(0.0, b"frame0"),
|
||||
(1.0, b"frame1"),
|
||||
(2.0, b"frame2"),
|
||||
]
|
||||
|
||||
mock_llm = MagicMock()
|
||||
mock_response = AIMessage(
|
||||
content="Sub-agent analyzed frames.",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "submit_answer",
|
||||
"args": {
|
||||
"summary": "Detected settings button tap.",
|
||||
"analysis": "At 1.0s user clicked Settings icon.",
|
||||
"timeline_events": [
|
||||
{
|
||||
"start_time": 0.5,
|
||||
"end_time": 1.5,
|
||||
"transcription": "Tap on Settings",
|
||||
"confidence_score": 0.95,
|
||||
"verification_timestamp_secs": 1.0,
|
||||
}
|
||||
],
|
||||
},
|
||||
"id": "sub_call_1",
|
||||
}
|
||||
],
|
||||
)
|
||||
mock_bound = MagicMock()
|
||||
mock_bound.ainvoke = AsyncMock(return_value=mock_response)
|
||||
mock_llm.bind_tools.return_value = mock_bound
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.extract_keyframes_from_video",
|
||||
return_value=fake_keyframes,
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.get_llm",
|
||||
return_value=mock_llm,
|
||||
),
|
||||
):
|
||||
result = await agent._exec_single_chunk_universal(
|
||||
compressed_path=fake_video,
|
||||
raw_path=raw_video,
|
||||
start_time=0.0,
|
||||
end_time=3.0,
|
||||
actual_start=0.0,
|
||||
prompt_with_context="Find when settings was opened.",
|
||||
specific_query="settings icon",
|
||||
)
|
||||
|
||||
assert "Detected settings button tap" in result
|
||||
assert "At 1.0s user clicked Settings icon" in result
|
||||
assert len(agent.blackboard_entries) == 1
|
||||
assert agent.blackboard_entries[0]["summary"] == "Tap on Settings"
|
||||
assert agent.blackboard_entries[0]["confidence_score"] == 0.95
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exec_analyze_audio_universal(mock_context, tmp_path):
|
||||
"""Test audio analysis in universal mode."""
|
||||
agent = VideoAnalyzer(mock_context)
|
||||
agent.use_native_gemini = False
|
||||
|
||||
fake_audio = tmp_path / "audio.aac"
|
||||
fake_audio.write_bytes(b"audio_bytes")
|
||||
|
||||
mock_llm = MagicMock()
|
||||
mock_response = AIMessage(
|
||||
content="",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "submit_answer",
|
||||
"args": {
|
||||
"summary": "Heard notification sound.",
|
||||
"analysis": "Notification bell sounded at 2.0s.",
|
||||
},
|
||||
"id": "audio_call_1",
|
||||
}
|
||||
],
|
||||
)
|
||||
mock_bound = MagicMock()
|
||||
mock_bound.ainvoke = AsyncMock(return_value=mock_response)
|
||||
mock_llm.bind_tools.return_value = mock_bound
|
||||
|
||||
with patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.get_llm",
|
||||
return_value=mock_llm,
|
||||
):
|
||||
result = await agent._exec_analyze_audio_universal(
|
||||
audio_path=fake_audio,
|
||||
start_time=1.0,
|
||||
end_time=4.0,
|
||||
actual_start=1.0,
|
||||
prompt_with_context="Listen for notification sound.",
|
||||
specific_query="notification bell",
|
||||
)
|
||||
|
||||
assert "Heard notification sound" in result
|
||||
assert len(agent.blackboard_entries) == 1
|
||||
assert agent.blackboard_entries[0]["summary"] == "Heard notification sound."
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_universal_multi_turn_reasoning(mock_context):
|
||||
"""Test main reasoning loop of VideoAnalyzer in universal mode with tool dispatch."""
|
||||
agent = VideoAnalyzer(mock_context)
|
||||
agent.use_native_gemini = False
|
||||
|
||||
# Turn 1: Call extract_segment_metadata
|
||||
msg_turn1 = AIMessage(
|
||||
content="Let me check the video metadata first.",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "extract_segment_metadata",
|
||||
"args": {"start_time": 0.0, "end_time": 10.0},
|
||||
"id": "main_call_1",
|
||||
}
|
||||
],
|
||||
)
|
||||
# Turn 2: Final response
|
||||
msg_turn2 = AIMessage(
|
||||
content="The video shows the app launched successfully at 3.5s.",
|
||||
tool_calls=[],
|
||||
)
|
||||
|
||||
mock_llm = MagicMock()
|
||||
mock_bound = MagicMock()
|
||||
mock_bound.ainvoke = AsyncMock(side_effect=[msg_turn1, msg_turn2])
|
||||
mock_llm.bind_tools.return_value = mock_bound
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.get_llm",
|
||||
return_value=mock_llm,
|
||||
),
|
||||
patch.object(
|
||||
agent,
|
||||
"exec_extract_segment_metadata",
|
||||
new_callable=AsyncMock,
|
||||
return_value='{"duration_seconds": 10.0, "file_size_mb": 2.5}',
|
||||
),
|
||||
):
|
||||
outcome, status = await agent._run_universal(
|
||||
time_description="from 0s to 10s",
|
||||
purpose="Verify app launch",
|
||||
system_prompt="You are a video analyzer.",
|
||||
)
|
||||
|
||||
assert status == "success"
|
||||
assert "launched successfully at 3.5s" in outcome
|
||||
assert agent.exec_extract_segment_metadata.call_count == 1
|
||||
@@ -0,0 +1,167 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import tempfile
|
||||
from unittest.mock import MagicMock
|
||||
from uuid import uuid4
|
||||
|
||||
from langchain_core.messages import AIMessage, HumanMessage
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.data_engine.engine import DataEngine
|
||||
from artemis.data_engine.trace import CURRENT_TRACE_ID, DataEngineCallbackHandler
|
||||
from artemis.utils.task_tree import build_plan_and_history
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def temp_workspace():
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
yield Path(tmpdir)
|
||||
shutil.rmtree(tmpdir)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_end_to_end_thoughts_non_duplication(temp_workspace):
|
||||
# 1. Setup mock context and DataEngine
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_execution_setup = MagicMock()
|
||||
mock_execution_setup.traces_path = str(temp_workspace / "traces")
|
||||
mock_ctx.execution_setup = mock_execution_setup
|
||||
mock_ctx.device = None
|
||||
mock_ctx.llm_config = MagicMock()
|
||||
|
||||
engine = DataEngine(mock_ctx)
|
||||
mock_ctx.data_engine = engine
|
||||
|
||||
# Start session
|
||||
engine.start_session("Test initial goal")
|
||||
|
||||
# 2. Allocate step ID
|
||||
step_id = engine.allocate_step_id()
|
||||
|
||||
# 3. Simulate Operator Agent run starts
|
||||
operator_trace_id = uuid4()
|
||||
engine.record_trace(
|
||||
type="agent",
|
||||
name="operator",
|
||||
payload={"args": {}},
|
||||
trace_id=operator_trace_id,
|
||||
step_id=step_id,
|
||||
status="running",
|
||||
)
|
||||
|
||||
# Set context variables for parenting
|
||||
token = CURRENT_TRACE_ID.set(operator_trace_id)
|
||||
try:
|
||||
# 4. Simulate LLM Call callback sequence
|
||||
handler = DataEngineCallbackHandler(mock_ctx)
|
||||
llm_run_id = uuid4()
|
||||
|
||||
# Start model
|
||||
handler.on_chat_model_start(
|
||||
serialized={"name": "ChatGoogleGenerativeAI"},
|
||||
messages=[[HumanMessage(content="What should I do?")]],
|
||||
run_id=llm_run_id,
|
||||
parent_run_id=None,
|
||||
)
|
||||
|
||||
# End model (generating thought + action)
|
||||
class MockGeneration:
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
|
||||
class MockLLMResult:
|
||||
def __init__(self, generations):
|
||||
self.generations = generations
|
||||
|
||||
response_message = AIMessage(
|
||||
content=[
|
||||
{
|
||||
"type": "thinking",
|
||||
"thinking": "I need to tap the search bar to find books.",
|
||||
},
|
||||
{"type": "text", "text": "Tapping search bar."},
|
||||
]
|
||||
)
|
||||
mock_response = MockLLMResult(generations=[[MockGeneration(response_message)]])
|
||||
|
||||
handler.on_llm_end(mock_response, run_id=llm_run_id)
|
||||
finally:
|
||||
CURRENT_TRACE_ID.reset(token)
|
||||
|
||||
# 5. Finish Operator Agent trace
|
||||
engine.record_trace(
|
||||
type="agent",
|
||||
name="operator",
|
||||
payload={"result": "Success"},
|
||||
trace_id=operator_trace_id,
|
||||
step_id=step_id,
|
||||
status="success",
|
||||
)
|
||||
|
||||
# 6. Record step
|
||||
engine.record_step(
|
||||
pre_screenshot_bytes=b"pre_screenshot",
|
||||
ui_tree=[],
|
||||
ocr_result=[],
|
||||
action_taken=[{"action": "tap", "coordinates": [100, 200]}],
|
||||
operator_raw_thinking="I need to tap the search bar to find books.",
|
||||
operator_native_thinking="I need to tap the search bar to find books.",
|
||||
last_execution_result={"status": "success"},
|
||||
summary="Tapped search bar",
|
||||
)
|
||||
|
||||
# Flush any pending tasks
|
||||
await engine.shutdown()
|
||||
|
||||
# 7. Retrieve agent-friendly steps
|
||||
steps = engine.get_agent_friendly_steps()
|
||||
|
||||
# Assert step was retrieved
|
||||
assert len(steps) == 1
|
||||
step_data = steps[0]
|
||||
|
||||
# Assert thoughts are present in interleaved events (since we associated step_id now!)
|
||||
events = step_data.get("interleaved_events", [])
|
||||
thought_events = [e for e in events if e.get("type") in ("thought", "native_thought")]
|
||||
assert len(thought_events) > 0
|
||||
assert any(
|
||||
e["content"] == "I need to tap the search bar to find books." for e in thought_events
|
||||
)
|
||||
|
||||
# 8. Build history and check for duplications
|
||||
plan = "- [ ] Search for books"
|
||||
history = build_plan_and_history(
|
||||
plan, steps, "default", last_n_detailed=1, for_failure_analyzer=True
|
||||
)
|
||||
|
||||
print("\n--- Generated History Output ---")
|
||||
print(history)
|
||||
print("--------------------------------")
|
||||
|
||||
# Assert thought is printed
|
||||
assert "I need to tap the search bar to find books." in history
|
||||
|
||||
# Assert thought is NOT printed multiple times
|
||||
# Count occurrences of the thought string
|
||||
count = history.count("I need to tap the search bar to find books.")
|
||||
# The thought might appear in native thinking block and raw thinking block if both were stored.
|
||||
# In our test we set BOTH operator_raw_thinking and operator_native_thinking to the same string,
|
||||
# AND the LLM trace generated both "thinking" block and "text" block.
|
||||
# Let's count how many times it was actually rendered in the Operator Decision Loop.
|
||||
# It should appear at most 2 times (once as Native Thought, once as Thought/Raw Thought).
|
||||
# If there was duplicate appending due to fallback failure, it would appear 4+ times.
|
||||
assert count <= 2
|
||||
@@ -0,0 +1,185 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.data_engine.engine import DataEngine
|
||||
|
||||
|
||||
def test_get_or_create_image_updates_missing_data(tmp_path):
|
||||
# Setup mock context
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_execution_setup = MagicMock()
|
||||
mock_execution_setup.traces_path = str(tmp_path)
|
||||
mock_ctx.execution_setup = mock_execution_setup
|
||||
mock_ctx.device = None
|
||||
|
||||
# Initialize DataEngine
|
||||
engine = DataEngine(mock_ctx)
|
||||
|
||||
image_bytes = b"fake_image_bytes"
|
||||
|
||||
# 1. Save image without OCR/UI tree
|
||||
image_name1 = engine.get_or_create_image(image_bytes)
|
||||
|
||||
# Verify it was saved with None for OCR/UI tree
|
||||
record1 = engine.storage.get_image(image_name1)
|
||||
assert record1 is not None
|
||||
assert record1.ocr_result is None
|
||||
assert record1.ui_tree is None
|
||||
|
||||
# 2. Save same image with OCR/UI tree
|
||||
fake_ocr = [{"text": "hello", "box": [0, 0, 10, 10]}]
|
||||
fake_ui = [{"resource-id": "button1", "text": "click me"}]
|
||||
|
||||
image_name2 = engine.get_or_create_image(image_bytes, ui_tree=fake_ui, ocr_result=fake_ocr)
|
||||
|
||||
assert image_name1 == image_name2
|
||||
|
||||
# Verify it now has OCR/UI tree
|
||||
record2 = engine.storage.get_image(image_name2)
|
||||
assert record2 is not None
|
||||
assert record2.ocr_result == fake_ocr
|
||||
assert record2.ui_tree == fake_ui
|
||||
|
||||
|
||||
def test_create_image_duplicate_handling(tmp_path):
|
||||
from artemis.data_engine.storage import StorageManager
|
||||
from artemis.data_engine.models import ImageRecord
|
||||
|
||||
storage = StorageManager(tmp_path / "data_engine.db", tmp_path)
|
||||
image_record = ImageRecord(
|
||||
image_name="test_image_hash",
|
||||
timestamp=123.456,
|
||||
ocr_result=None,
|
||||
ui_tree=None,
|
||||
extra_metadata={},
|
||||
)
|
||||
|
||||
# First insert
|
||||
storage.create_image(image_record)
|
||||
|
||||
# Second insert with same image_name should not raise IntegrityError due to INSERT OR IGNORE
|
||||
storage.create_image(image_record)
|
||||
|
||||
record = storage.get_image("test_image_hash")
|
||||
assert record is not None
|
||||
assert record.image_name == "test_image_hash"
|
||||
|
||||
|
||||
def test_video_recording_persistence(tmp_path):
|
||||
from uuid import uuid4
|
||||
from artemis.data_engine.storage import StorageManager
|
||||
from artemis.data_engine.models import VideoRecordingRecord
|
||||
|
||||
storage = StorageManager(tmp_path / "data_engine.db", tmp_path)
|
||||
video_id = uuid4()
|
||||
session_id = uuid4()
|
||||
|
||||
record = VideoRecordingRecord(
|
||||
video_id=video_id,
|
||||
session_id=session_id,
|
||||
device_id="emulator-5554",
|
||||
start_time=100.0,
|
||||
end_time=None,
|
||||
local_video_path="/tmp/recording.mp4",
|
||||
)
|
||||
|
||||
# Test insert
|
||||
storage.create_video_recording(record)
|
||||
|
||||
persisted = storage.get_video_recording(video_id)
|
||||
assert persisted is not None
|
||||
assert persisted.video_id == video_id
|
||||
assert persisted.session_id == session_id
|
||||
assert persisted.device_id == "emulator-5554"
|
||||
assert persisted.start_time == 100.0
|
||||
assert persisted.end_time is None
|
||||
assert persisted.local_video_path == "/tmp/recording.mp4"
|
||||
|
||||
# Test update
|
||||
record.end_time = 150.0
|
||||
record.local_video_path = "/tmp/recording_final.mp4"
|
||||
storage.update_video_recording(record)
|
||||
|
||||
updated = storage.get_video_recording(video_id)
|
||||
assert updated is not None
|
||||
assert updated.end_time == 150.0
|
||||
assert updated.local_video_path == "/tmp/recording_final.mp4"
|
||||
|
||||
|
||||
def test_record_step_suppresses_identical_post_screenshot(tmp_path):
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_execution_setup = MagicMock()
|
||||
mock_execution_setup.traces_path = str(tmp_path)
|
||||
mock_ctx.execution_setup = mock_execution_setup
|
||||
mock_ctx.device = None
|
||||
|
||||
engine = DataEngine(mock_ctx)
|
||||
engine.start_session("test_goal")
|
||||
same_image_bytes = b"identical_screen_bytes_123"
|
||||
|
||||
step_id = engine.record_step(
|
||||
pre_screenshot_bytes=same_image_bytes,
|
||||
post_screenshot_bytes=same_image_bytes,
|
||||
summary="Test step with identical screenshot",
|
||||
)
|
||||
for t in list(engine._pending_threads):
|
||||
t.join()
|
||||
|
||||
step_record = engine.storage.get_steps(engine.current_session_id)[0]
|
||||
assert step_record is not None
|
||||
assert step_record.pre_image_name is not None
|
||||
# post_image_name must be None when no visual transition occurred
|
||||
assert step_record.post_image_name is None
|
||||
|
||||
|
||||
def test_update_step_execution_result_suppresses_identical_post_screenshot(tmp_path):
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_execution_setup = MagicMock()
|
||||
mock_execution_setup.traces_path = str(tmp_path)
|
||||
mock_ctx.execution_setup = mock_execution_setup
|
||||
mock_ctx.device = None
|
||||
|
||||
engine = DataEngine(mock_ctx)
|
||||
engine.start_session("test_goal")
|
||||
image_bytes = b"pre_screen_bytes_456"
|
||||
|
||||
step_id = engine.record_step(
|
||||
pre_screenshot_bytes=image_bytes,
|
||||
summary="Initial step",
|
||||
)
|
||||
|
||||
step_record = engine.storage.get_steps(engine.current_session_id)[0]
|
||||
assert step_record.post_image_name is None
|
||||
|
||||
# Update with post_image_name matching pre_image_name
|
||||
engine.storage.update_step_execution_result(
|
||||
step_id,
|
||||
{"status": "success"},
|
||||
post_image_name=step_record.pre_image_name,
|
||||
)
|
||||
|
||||
updated_step = engine.storage.get_steps(engine.current_session_id)[0]
|
||||
assert updated_step.post_image_name is None
|
||||
|
||||
# Update with distinct post_image_name
|
||||
engine.storage.update_step_execution_result(
|
||||
step_id,
|
||||
{"status": "success"},
|
||||
post_image_name="distinct_new_hash_789",
|
||||
)
|
||||
|
||||
updated_step_distinct = engine.storage.get_steps(engine.current_session_id)[0]
|
||||
assert updated_step_distinct.post_image_name == "distinct_new_hash_789"
|
||||
@@ -0,0 +1,566 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from langchain_core.messages import HumanMessage
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.data_engine.engine import DataEngine
|
||||
from artemis.graph.state import State
|
||||
from artemis.utils.logger import DataEngineHandler
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_artemis_ctx(tmp_path):
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_execution_setup = MagicMock()
|
||||
mock_execution_setup.traces_path = str(tmp_path)
|
||||
mock_ctx.execution_setup = mock_execution_setup
|
||||
mock_ctx.device = None
|
||||
mock_ctx.task_plan_snapshot = None
|
||||
return mock_ctx
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_data_engine_handler_queue_buffering_and_background_worker(
|
||||
mock_artemis_ctx,
|
||||
):
|
||||
"""Verify DataEngineHandler.emit buffers log records via in-memory queue without spawning OS threads per log line."""
|
||||
engine = DataEngine(mock_artemis_ctx)
|
||||
session_id = engine.start_session("Test Log Queue")
|
||||
|
||||
handler = DataEngineHandler()
|
||||
|
||||
with patch("sys.modules") as mock_modules:
|
||||
mock_engine_mod = MagicMock()
|
||||
mock_engine_mod._CURRENT_DATA_ENGINE = engine
|
||||
mock_modules.get.return_value = mock_engine_mod
|
||||
|
||||
# Emit 50 rapid log records
|
||||
for i in range(50):
|
||||
record = logging.LogRecord(
|
||||
name=f"test.logger.{i}",
|
||||
level=logging.INFO,
|
||||
pathname="test.py",
|
||||
lineno=10 + i,
|
||||
msg=f"Async queue test log message {i}",
|
||||
args=(),
|
||||
exc_info=None,
|
||||
)
|
||||
handler.emit(record)
|
||||
|
||||
# Wait for the background worker thread of DataEngineHandler and DataEngine tasks to drain
|
||||
await asyncio.sleep(0.4)
|
||||
if engine._pending_tasks:
|
||||
await asyncio.gather(*engine._pending_tasks)
|
||||
|
||||
# Verify that traces were recorded and persisted accurately via direct SQLite query
|
||||
with engine.storage._get_connection() as conn:
|
||||
cursor = conn.execute(
|
||||
"SELECT * FROM traces WHERE session_id = ? AND type = 'log'",
|
||||
(str(session_id),),
|
||||
)
|
||||
log_rows = cursor.fetchall()
|
||||
|
||||
assert len(log_rows) >= 50
|
||||
messages = {json.loads(row["payload"])["message"] for row in log_rows}
|
||||
for i in range(50):
|
||||
assert f"Async queue test log message {i}" in messages
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_step_action_and_metadata_background_offloading(mock_artemis_ctx, tmp_path):
|
||||
"""Verify that update_step_* methods publish SSE events instantly in memory and offload SQLite & disk I/O to background."""
|
||||
engine = DataEngine(mock_artemis_ctx)
|
||||
session_id = engine.start_session("Test Step Offloading")
|
||||
step_id = engine.allocate_step_id()
|
||||
|
||||
# Create the initial step
|
||||
engine.record_step(summary="Initial step summary")
|
||||
if engine._pending_tasks:
|
||||
await asyncio.gather(*engine._pending_tasks)
|
||||
|
||||
# Mock _publish to verify immediate real-time SSE delivery
|
||||
published_events = []
|
||||
engine._publish = lambda event_type, payload: published_events.append((event_type, payload))
|
||||
|
||||
fake_action = {"action": "tap", "coordinates": [100, 200]}
|
||||
fake_post_jpg = b"fake_post_screenshot_jpg_bytes_12345"
|
||||
|
||||
# Call update_step_action with post screenshot bytes
|
||||
engine.update_step_action(fake_action, post_screenshot_bytes=fake_post_jpg)
|
||||
|
||||
# Verify SSE was published immediately before/during task execution
|
||||
assert len(published_events) == 1
|
||||
assert published_events[0][0] == "step_updated"
|
||||
assert published_events[0][1]["action_taken"] == fake_action
|
||||
|
||||
# Call other update_step_* methods
|
||||
engine.update_step_summary(step_id, "Updated concise summary")
|
||||
engine.update_step_thinking(step_id, "Raw thinking process...")
|
||||
engine.update_step_native_thinking(step_id, "Native thinking process...")
|
||||
engine.update_step_execution_result(
|
||||
step_id, {"status": "success"}, post_image_name="hash_post_123"
|
||||
)
|
||||
|
||||
# Now await all background pending tasks
|
||||
if engine._pending_tasks:
|
||||
await asyncio.gather(*engine._pending_tasks)
|
||||
|
||||
# Verify SQLite record has been correctly updated by background tasks
|
||||
steps = engine.storage.get_steps(session_id)
|
||||
step_record = next((s for s in steps if s.step_id == step_id), None)
|
||||
assert step_record is not None
|
||||
assert step_record.action_taken == fake_action
|
||||
assert step_record.summary == "Updated concise summary"
|
||||
assert step_record.operator_raw_thinking == "Raw thinking process..."
|
||||
assert step_record.operator_native_thinking == "Native thinking process..."
|
||||
assert step_record.last_execution_result == {"status": "success"}
|
||||
assert step_record.post_image_name == "hash_post_123"
|
||||
|
||||
# Verify post.jpg was written to disk inside step directory
|
||||
post_jpg_path = engine.current_step_dir / "post.jpg"
|
||||
assert post_jpg_path.exists()
|
||||
assert post_jpg_path.read_bytes() == fake_post_jpg
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_record_step_background_image_and_step_persistence(
|
||||
mock_artemis_ctx,
|
||||
):
|
||||
"""Verify record_step immediately calculates SHA-256 hashes and offloads get_or_create_image + create_step to background."""
|
||||
engine = DataEngine(mock_artemis_ctx)
|
||||
session_id = engine.start_session("Test Record Step Image Offloading")
|
||||
step_id = engine.allocate_step_id()
|
||||
|
||||
pre_img_bytes = b"fake_pre_screenshot_data_001"
|
||||
post_img_bytes = b"fake_post_screenshot_data_002"
|
||||
expected_pre_hash = hashlib.sha256(pre_img_bytes).hexdigest()
|
||||
expected_post_hash = hashlib.sha256(post_img_bytes).hexdigest()
|
||||
|
||||
returned_step_id = engine.record_step(
|
||||
pre_screenshot_bytes=pre_img_bytes,
|
||||
post_screenshot_bytes=post_img_bytes,
|
||||
summary="Testing background image persistence",
|
||||
)
|
||||
assert returned_step_id == step_id
|
||||
|
||||
# Await background storage tasks
|
||||
if engine._pending_tasks:
|
||||
await asyncio.gather(*engine._pending_tasks)
|
||||
|
||||
# Verify image records were created in SQLite
|
||||
pre_record = engine.storage.get_image(expected_pre_hash)
|
||||
post_record = engine.storage.get_image(expected_post_hash)
|
||||
assert pre_record is not None
|
||||
assert pre_record.image_name == expected_pre_hash
|
||||
assert post_record is not None
|
||||
assert post_record.image_name == expected_post_hash
|
||||
|
||||
# Verify image files exist on disk
|
||||
pre_path = engine.get_image_path(expected_pre_hash)
|
||||
post_path = engine.get_image_path(expected_post_hash)
|
||||
assert pre_path.exists() and pre_path.read_bytes() == pre_img_bytes
|
||||
assert post_path.exists() and post_path.read_bytes() == post_img_bytes
|
||||
|
||||
# Verify StepRecord has correct hashes and summary
|
||||
steps = engine.storage.get_steps(session_id)
|
||||
step_record = next((s for s in steps if s.step_id == step_id), None)
|
||||
assert step_record is not None
|
||||
assert step_record.pre_image_name == expected_pre_hash
|
||||
assert step_record.post_image_name == expected_post_hash
|
||||
assert step_record.summary == "Testing background image persistence"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_perception_node_async_offloading(mock_artemis_ctx, tmp_path):
|
||||
"""Verify perception_node offloads injected_instruction reading/unlinking and get_or_create_image to background."""
|
||||
from artemis.graph.perception import perception_node
|
||||
|
||||
engine = DataEngine(mock_artemis_ctx)
|
||||
engine.start_session("Test Perception Node Offloading")
|
||||
mock_artemis_ctx.data_engine = engine
|
||||
|
||||
# Create an injected_instruction.json file
|
||||
instruction_file = Path(engine.base_dir) / "injected_instruction.json"
|
||||
instruction_file.write_text(json.dumps({"instruction": "Avoid clicking Ads"}), encoding="utf-8")
|
||||
|
||||
# Prepare mock controller
|
||||
fake_screenshot_bytes = b"fake_screenshot_for_perception_node_123"
|
||||
fake_screenshot_b64 = base64.b64encode(fake_screenshot_bytes).decode("utf-8")
|
||||
expected_hash = hashlib.sha256(fake_screenshot_bytes).hexdigest()
|
||||
|
||||
mock_screen_data = MagicMock()
|
||||
mock_screen_data.base64 = fake_screenshot_b64
|
||||
mock_screen_data.elements = [{"resource-id": "app_bar", "bounds": "[0,0][1080,200]"}]
|
||||
mock_screen_data.width = 1080
|
||||
mock_screen_data.height = 2400
|
||||
|
||||
with (
|
||||
patch("artemis.graph.perception.UnifiedMobileController") as mock_controller_cls,
|
||||
patch("artemis.graph.perception.perform_ocr", new_callable=AsyncMock) as mock_ocr,
|
||||
patch("artemis.graph.perception._detect_status_bar_height", return_value=0),
|
||||
patch("artemis.graph.perception._should_skip_settling", return_value=True),
|
||||
):
|
||||
mock_controller = mock_controller_cls.return_value
|
||||
mock_controller.get_screen_data = AsyncMock(return_value=mock_screen_data)
|
||||
mock_ocr.return_value = [{"text": "Home", "box": [100, 100, 200, 200]}]
|
||||
|
||||
state = State(
|
||||
messages=[HumanMessage(content="test")],
|
||||
initial_goal="test goal",
|
||||
complete_subgoals_by_ids=[],
|
||||
validator_messages=[],
|
||||
latest_ui_hierarchy=[],
|
||||
latest_screenshot=None,
|
||||
focused_app_info=None,
|
||||
device_date=None,
|
||||
structured_decisions=None,
|
||||
)
|
||||
|
||||
update = await perception_node(state, mock_artemis_ctx)
|
||||
|
||||
# Verify perception_node returned expected update immediately
|
||||
assert update["injected_instruction"] == "Avoid clicking Ads"
|
||||
assert update["latest_screenshot"] == str(engine.get_image_path(expected_hash))
|
||||
assert update["operator_raw_data"]["width"] == 1080
|
||||
|
||||
# Verify injected_instruction.json was unlinked
|
||||
assert not instruction_file.exists()
|
||||
|
||||
# Await background tasks triggered by perception_node
|
||||
await asyncio.sleep(0.1)
|
||||
if engine._pending_tasks:
|
||||
await asyncio.gather(*engine._pending_tasks)
|
||||
|
||||
# Verify image record and file were created in background by perception_node
|
||||
img_record = engine.storage.get_image(expected_hash)
|
||||
assert img_record is not None
|
||||
assert img_record.image_name == expected_hash
|
||||
assert engine.get_image_path(expected_hash).exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_snapshot_copy_and_restore_offloading(mock_artemis_ctx, tmp_path):
|
||||
"""Verify operator create_snapshot and graph rollback/restore execute asynchronously without blocking."""
|
||||
from artemis.utils.file import create_snapshot, restore_snapshot
|
||||
|
||||
notes_dir = tmp_path / "notes"
|
||||
notes_dir.mkdir(parents=True, exist_ok=True)
|
||||
(notes_dir / "task_plan.md").write_text("# Plan\n- Step 1", encoding="utf-8")
|
||||
(notes_dir / "verification_chat_1.json").write_text('{"messages": ["hello"]}', encoding="utf-8")
|
||||
|
||||
snapshot_dir = tmp_path / "notes_snapshot"
|
||||
|
||||
# Test async offloading of create_snapshot (operator pattern)
|
||||
await asyncio.to_thread(create_snapshot, notes_dir, snapshot_dir)
|
||||
assert snapshot_dir.exists()
|
||||
assert (snapshot_dir / "task_plan.md").read_text(encoding="utf-8") == "# Plan\n- Step 1"
|
||||
|
||||
# Modify live task_plan.md
|
||||
(notes_dir / "task_plan.md").write_text("# Plan\n- Step 1 (Modified)", encoding="utf-8")
|
||||
|
||||
# Test async offloading of restore_snapshot (graph pattern when checker fails)
|
||||
def _restore_and_preserve():
|
||||
chat_contents = {}
|
||||
for p in notes_dir.glob("verification_chat_*.json"):
|
||||
chat_contents[p.name] = p.read_text(encoding="utf-8")
|
||||
restore_snapshot(snapshot_dir, notes_dir)
|
||||
for name, content in chat_contents.items():
|
||||
(notes_dir / name).write_text(content, encoding="utf-8")
|
||||
|
||||
await asyncio.to_thread(_restore_and_preserve)
|
||||
assert (notes_dir / "task_plan.md").read_text(encoding="utf-8") == "# Plan\n- Step 1"
|
||||
assert (notes_dir / "verification_chat_1.json").read_text(
|
||||
encoding="utf-8"
|
||||
) == '{"messages": ["hello"]}'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ocr_api_persistent_http_client_singleton_and_tls_reuse():
|
||||
"""Verify get_http_client returns a persistent singleton httpx.AsyncClient that reuses TLS connections across OCR calls."""
|
||||
import httpx
|
||||
from artemis.utils import ocr_api
|
||||
|
||||
# Reset singleton to test fresh creation & persistence
|
||||
ocr_api._HTTP_CLIENT = None
|
||||
client1 = ocr_api.get_http_client()
|
||||
client2 = ocr_api.get_http_client()
|
||||
|
||||
assert isinstance(client1, httpx.AsyncClient)
|
||||
assert client1 is client2
|
||||
|
||||
# Verify perform_ocr uses this exact persistent client
|
||||
fake_ocr_resp = MagicMock()
|
||||
fake_ocr_resp.status_code = 200
|
||||
fake_ocr_resp.json.return_value = {
|
||||
"responses": [
|
||||
{
|
||||
"textAnnotations": [
|
||||
{
|
||||
"description": "Combined text block",
|
||||
"boundingPoly": {"vertices": []},
|
||||
},
|
||||
{
|
||||
"description": "Hello World",
|
||||
"boundingPoly": {"vertices": [{"x": 10, "y": 20}, {"x": 100, "y": 200}]},
|
||||
},
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
with patch.object(client1, "post", new_callable=AsyncMock) as mock_post:
|
||||
mock_post.return_value = fake_ocr_resp
|
||||
result = await ocr_api.perform_ocr(screenshot_b64="ZHVtbXlfYjY0")
|
||||
|
||||
mock_post.assert_called_once()
|
||||
assert len(result) == 1
|
||||
assert result[0]["text"] == "Hello World"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ui_filter_and_ui_automator_client_pre_parsed_bounds_o1_lookup():
|
||||
"""Verify XML parsing pre-populates parsed_bounds and _parse_bounds hits this cache in O(1) without regex matches."""
|
||||
import re
|
||||
from artemis.clients.ui_automator_client import _parse_hierarchy_xml_to_elements
|
||||
from artemis.utils.ui_filter import _parse_bounds
|
||||
|
||||
# 1. Test ingestion pre-parsing in _parse_hierarchy_xml_to_elements
|
||||
xml_data = '<hierarchy><node text="Login Btn" bounds="[15,25][350,120]"/></hierarchy>'
|
||||
elements = _parse_hierarchy_xml_to_elements(xml_data)
|
||||
assert len(elements) == 1
|
||||
assert elements[0]["parsed_bounds"] == {
|
||||
"left": 15,
|
||||
"top": 25,
|
||||
"right": 350,
|
||||
"bottom": 120,
|
||||
}
|
||||
|
||||
# 2. Test _parse_bounds O(1) hit vs fallback
|
||||
with patch.object(re, "match", wraps=re.match) as spy_match:
|
||||
# Hit pre-parsed bounds -> re.match MUST NOT be called
|
||||
hit_result = _parse_bounds(elements[0])
|
||||
assert hit_result == {
|
||||
"left": 15,
|
||||
"top": 25,
|
||||
"right": 350,
|
||||
"bottom": 120,
|
||||
}
|
||||
spy_match.assert_not_called()
|
||||
|
||||
# Fallback raw string -> re.match is called exactly once
|
||||
fallback_result = _parse_bounds("[50,60][150,160]")
|
||||
assert fallback_result == {
|
||||
"left": 50,
|
||||
"top": 60,
|
||||
"right": 150,
|
||||
"bottom": 160,
|
||||
}
|
||||
assert spy_match.call_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_pre_execution_loop_reverted_to_exact_safety_contract(
|
||||
mock_artemis_ctx, tmp_path
|
||||
):
|
||||
"""Verify Validator pre-execution check runs VLM on XML failure, but retains XML error if VLM fails (reverted contract)."""
|
||||
from artemis.agents.validator.validator import ValidatorNode, ValidationErrorCategory
|
||||
|
||||
# Set mock context properties required by tracing decorators
|
||||
mock_artemis_ctx.data_engine = MagicMock()
|
||||
|
||||
node = ValidatorNode(mock_artemis_ctx)
|
||||
session = MagicMock()
|
||||
session.call_tool = AsyncMock(
|
||||
return_value=MagicMock(content=[MagicMock(text="ZHVtbXlfZGF0YQ==")])
|
||||
)
|
||||
|
||||
decisions = json.dumps(
|
||||
[
|
||||
{
|
||||
"action": "tap",
|
||||
"coordinates": [200, 250],
|
||||
"target_text": "Login",
|
||||
"target_bounds": [100, 200, 300, 300],
|
||||
"target_resource_id": "btn_login",
|
||||
}
|
||||
]
|
||||
)
|
||||
state = State(
|
||||
messages=[],
|
||||
initial_goal="test",
|
||||
complete_subgoals_by_ids=[],
|
||||
validator_messages=[],
|
||||
latest_ui_hierarchy=[],
|
||||
latest_screenshot="dummy.png",
|
||||
structured_decisions=decisions,
|
||||
focused_app_info=None,
|
||||
device_date=None,
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
ValidatorNode, "_get_initial_screenshot", new_callable=AsyncMock
|
||||
) as mock_get_screen,
|
||||
patch.object(
|
||||
ValidatorNode, "_validate_action_precondition", new_callable=AsyncMock
|
||||
) as mock_xml,
|
||||
patch.object(
|
||||
ValidatorNode,
|
||||
"_validate_action_precondition_pixel",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_pixel,
|
||||
patch(
|
||||
"artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_analyze,
|
||||
):
|
||||
mock_get_screen.return_value = ("dummy_b64", "dummy_stem")
|
||||
# 1. XML check returns failure (TARGET_OCCUPIED by 'Sign Up' button)
|
||||
mock_xml.return_value = (
|
||||
False,
|
||||
ValidationErrorCategory.TARGET_OCCUPIED,
|
||||
"Occupied by Sign Up button",
|
||||
)
|
||||
# 2. VLM check returns failure (PIXEL_TARGET_DISAPPEARED)
|
||||
mock_pixel.return_value = (
|
||||
False,
|
||||
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
|
||||
"VLM target missing",
|
||||
)
|
||||
|
||||
mock_analyze.return_value = {
|
||||
"status": "cannot_fix",
|
||||
"analysis": "Unrepairable drift",
|
||||
}
|
||||
|
||||
result = await node(state)
|
||||
|
||||
# Verify both XML check and VLM check were called (reverted double-check contract)
|
||||
mock_xml.assert_called_once()
|
||||
mock_pixel.assert_called_once()
|
||||
|
||||
# Verify FailureAnalyzer received the EXACT XML error (TARGET_OCCUPIED), NOT the VLM error
|
||||
mock_analyze.assert_called_once()
|
||||
args, kwargs = mock_analyze.call_args
|
||||
assert kwargs.get("error_category") == ValidationErrorCategory.TARGET_OCCUPIED
|
||||
assert "Occupied by Sign Up button" in args[2]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failure_analyzer_prune_intermediate_screenshots(mock_artemis_ctx):
|
||||
"""Verify FailureAnalyzer._prune_intermediate_screenshots keeps initial reference images and the latest live image, while pruning intermediate images."""
|
||||
from google.genai import types
|
||||
from artemis.agents.validator.failure_analyzer import FailureAnalyzer
|
||||
|
||||
analyzer = FailureAnalyzer(mock_artemis_ctx)
|
||||
|
||||
# Build simulated contents history:
|
||||
# 0: User (Initial reference screenshots + prompt)
|
||||
# 1: Model
|
||||
# 2: User (Intermediate tool results)
|
||||
# 3: User (Intermediate observation screenshot - SHOULD BE PRUNED)
|
||||
# 4: Model
|
||||
# 5: User (Intermediate tool results 2)
|
||||
# 6: User (Latest observation screenshot - SHOULD BE PRESERVED)
|
||||
contents = [
|
||||
types.Content(
|
||||
role="user",
|
||||
parts=[
|
||||
types.Part.from_text(text="Reference screenshot 1"),
|
||||
types.Part.from_bytes(data=b"initial_pre_image_bytes", mime_type="image/jpeg"),
|
||||
types.Part.from_bytes(data=b"initial_post_image_bytes", mime_type="image/jpeg"),
|
||||
],
|
||||
),
|
||||
types.Content(role="model", parts=[types.Part.from_text(text="Clicked something")]),
|
||||
types.Content(
|
||||
role="user",
|
||||
parts=[types.Part.from_text(text="Result text")],
|
||||
),
|
||||
types.Content(
|
||||
role="user",
|
||||
parts=[
|
||||
types.Part.from_text(text="Intermediate observation"),
|
||||
types.Part.from_bytes(
|
||||
data=b"intermediate_image_bytes_to_prune",
|
||||
mime_type="image/jpeg",
|
||||
),
|
||||
],
|
||||
),
|
||||
types.Content(role="model", parts=[types.Part.from_text(text="Clicking again")]),
|
||||
types.Content(
|
||||
role="user",
|
||||
parts=[types.Part.from_text(text="Result text 2")],
|
||||
),
|
||||
types.Content(
|
||||
role="user",
|
||||
parts=[
|
||||
types.Part.from_text(text="Latest live observation"),
|
||||
types.Part.from_bytes(
|
||||
data=b"latest_live_image_bytes_to_keep",
|
||||
mime_type="image/jpeg",
|
||||
),
|
||||
],
|
||||
),
|
||||
]
|
||||
|
||||
analyzer._prune_intermediate_screenshots(contents)
|
||||
|
||||
# 1. Verify index 0 images are untouched
|
||||
assert contents[0].parts[1].inline_data.data == b"initial_pre_image_bytes"
|
||||
assert contents[0].parts[2].inline_data.data == b"initial_post_image_bytes"
|
||||
|
||||
# 2. Verify index 3 (intermediate image) is replaced by text placeholder
|
||||
assert getattr(contents[3].parts[1], "inline_data", None) is None
|
||||
assert "[Screenshot of intermediate step omitted for performance]" in contents[3].parts[1].text
|
||||
|
||||
# 3. Verify index 6 (latest image) is fully preserved
|
||||
assert contents[6].parts[1].inline_data.data == b"latest_live_image_bytes_to_keep"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_record_step_pre_allocated_id_reset(mock_artemis_ctx):
|
||||
"""Verify that after record_step is called, the pre-allocated current_step_id is reset to None so subsequent steps generate distinct IDs."""
|
||||
from artemis.data_engine.engine import DataEngine
|
||||
|
||||
engine = DataEngine(mock_artemis_ctx)
|
||||
engine.start_session("Pre-allocated ID Reset Test")
|
||||
|
||||
# 1. Pre-allocate ID for step 1
|
||||
step_id_1 = engine.allocate_step_id()
|
||||
assert engine.current_step_id == step_id_1
|
||||
|
||||
# 2. Record step 1 (should use step_id_1 and reset current_step_id to None)
|
||||
returned_id_1 = engine.record_step(summary="Step 1")
|
||||
assert returned_id_1 == step_id_1
|
||||
assert engine.current_step_id is None
|
||||
|
||||
# 3. Pre-allocate ID for step 2 (should get a distinct new ID)
|
||||
step_id_2 = engine.allocate_step_id()
|
||||
assert step_id_2 != step_id_1
|
||||
assert engine.current_step_id == step_id_2
|
||||
|
||||
# 4. Record step 2 (should use step_id_2 and reset current_step_id to None)
|
||||
returned_id_2 = engine.record_step(summary="Step 2")
|
||||
assert returned_id_2 == step_id_2
|
||||
assert returned_id_2 != returned_id_1
|
||||
assert engine.current_step_id is None
|
||||
@@ -0,0 +1,189 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import json
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.data_engine.trace import (
|
||||
TraceSpan,
|
||||
smart_serialize,
|
||||
)
|
||||
import pytest
|
||||
|
||||
|
||||
def test_smart_serialize():
|
||||
# Test dict
|
||||
d = {"a": 1, "b": "string"}
|
||||
assert smart_serialize(d) == json.dumps(d, ensure_ascii=False)
|
||||
|
||||
# Test list
|
||||
items_list = [1, 2, "three"]
|
||||
assert smart_serialize(items_list) == json.dumps(items_list, ensure_ascii=False)
|
||||
|
||||
# Test bytes
|
||||
b = b"hello world"
|
||||
import hashlib
|
||||
|
||||
expected_sha = hashlib.sha256(b).hexdigest()[:8]
|
||||
assert smart_serialize(b) == f"<Bytes length={len(b)} sha256={expected_sha}>"
|
||||
|
||||
# Test other types
|
||||
assert smart_serialize(123) == "123"
|
||||
assert smart_serialize("just a string") == "just a string"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_trace_span():
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = MagicMock()
|
||||
|
||||
with TraceSpan(name="test_span", ctx=mock_ctx) as span:
|
||||
span.result = "span_result"
|
||||
# Simulate work
|
||||
|
||||
# TraceSpan writes twice: start (running) and end (success)
|
||||
assert mock_ctx.data_engine.record_trace.call_count == 2
|
||||
|
||||
# Verify the final success call (which is the last call)
|
||||
args, kwargs = mock_ctx.data_engine.record_trace.call_args
|
||||
assert kwargs["type"] == "span"
|
||||
assert kwargs["name"] == "test_span"
|
||||
assert kwargs["status"] == "success"
|
||||
assert kwargs["payload"]["result"] == "span_result"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_trace_span_failure():
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = MagicMock()
|
||||
|
||||
try:
|
||||
with TraceSpan(name="failed_span", ctx=mock_ctx):
|
||||
raise ValueError("Span error")
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# TraceSpan writes twice: start (running) and end (failed)
|
||||
assert mock_ctx.data_engine.record_trace.call_count == 2
|
||||
|
||||
# Verify the final failed call (which is the last call)
|
||||
args, kwargs = mock_ctx.data_engine.record_trace.call_args
|
||||
assert kwargs["type"] == "span"
|
||||
assert kwargs["name"] == "failed_span"
|
||||
assert kwargs["status"] == "failed"
|
||||
assert "error" in kwargs["payload"]
|
||||
assert "Span error" in kwargs["payload"]["error"]
|
||||
|
||||
|
||||
def test_data_engine_callback_handler():
|
||||
from uuid import uuid4
|
||||
from langchain_core.messages import HumanMessage, AIMessage
|
||||
from artemis.data_engine.trace import DataEngineCallbackHandler
|
||||
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = MagicMock()
|
||||
|
||||
mock_step_id = uuid4()
|
||||
mock_ctx.data_engine.current_step_id = mock_step_id
|
||||
|
||||
handler = DataEngineCallbackHandler(mock_ctx)
|
||||
run_id = uuid4()
|
||||
|
||||
# 1. Simulate on_chat_model_start
|
||||
serialized = {"name": "test_model"}
|
||||
messages = [[HumanMessage(content="Hello LLM")]]
|
||||
handler.on_chat_model_start(serialized, messages, run_id=run_id)
|
||||
|
||||
assert mock_ctx.data_engine.record_trace.call_count == 1
|
||||
call1_args, call1_kwargs = mock_ctx.data_engine.record_trace.call_args_list[0]
|
||||
assert call1_kwargs["type"] == "llm_call"
|
||||
assert call1_kwargs["name"] == "test_model"
|
||||
assert call1_kwargs["status"] == "running"
|
||||
assert call1_kwargs["step_id"] == mock_step_id
|
||||
assert call1_kwargs["payload"]["messages"][0]["content"] == "Hello LLM"
|
||||
|
||||
# 2. Simulate on_llm_end (successful complete response)
|
||||
class MockGeneration:
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
|
||||
class MockLLMResult:
|
||||
def __init__(self, generations):
|
||||
self.generations = generations
|
||||
|
||||
response_message = AIMessage(content="Hello User")
|
||||
mock_response = MockLLMResult(generations=[[MockGeneration(response_message)]])
|
||||
|
||||
handler.on_llm_end(mock_response, run_id=run_id)
|
||||
|
||||
assert mock_ctx.data_engine.record_trace.call_count == 3
|
||||
call2_args, call2_kwargs = mock_ctx.data_engine.record_trace.call_args_list[1]
|
||||
assert call2_kwargs["type"] == "llm_call"
|
||||
assert call2_kwargs["status"] == "success"
|
||||
assert call2_kwargs["step_id"] == mock_step_id
|
||||
assert call2_kwargs["payload"]["messages"][0]["content"] == "Hello LLM"
|
||||
assert call2_kwargs["payload"]["response"][0]["content"] == "Hello User"
|
||||
assert isinstance(call2_kwargs.get("duration"), float)
|
||||
assert call2_kwargs["duration"] >= 0.0
|
||||
|
||||
call3_args, call3_kwargs = mock_ctx.data_engine.record_trace.call_args_list[2]
|
||||
assert call3_kwargs["type"] == "raw_thinking"
|
||||
assert call3_kwargs["parent_trace_id"] == run_id
|
||||
assert call3_kwargs["payload"]["thought"] == "Hello User"
|
||||
|
||||
|
||||
def test_callback_handler_chain_parenting():
|
||||
from uuid import uuid4
|
||||
from langchain_core.messages import HumanMessage
|
||||
from artemis.data_engine.trace import DataEngineCallbackHandler, CURRENT_TRACE_ID
|
||||
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = MagicMock()
|
||||
|
||||
handler = DataEngineCallbackHandler(mock_ctx)
|
||||
|
||||
agent_run_id = uuid4()
|
||||
chain1_run_id = uuid4()
|
||||
chain2_run_id = uuid4()
|
||||
llm_run_id = uuid4()
|
||||
|
||||
token = CURRENT_TRACE_ID.set(agent_run_id)
|
||||
try:
|
||||
# Simulate chain start (top level, parent is agent_run_id)
|
||||
handler.on_chain_start({}, {}, run_id=chain1_run_id, parent_run_id=None)
|
||||
# Simulate nested chain start (parent is chain1_run_id)
|
||||
handler.on_chain_start({}, {}, run_id=chain2_run_id, parent_run_id=chain1_run_id)
|
||||
|
||||
# Simulate LLM start inside nested chain (parent is chain2_run_id)
|
||||
messages = [[HumanMessage(content="Hello Nested LLM")]]
|
||||
handler.on_chat_model_start(
|
||||
{"name": "test_nested_model"},
|
||||
messages,
|
||||
run_id=llm_run_id,
|
||||
parent_run_id=chain2_run_id,
|
||||
)
|
||||
|
||||
# Verify parent trace ID resolves to agent_run_id (since chains are not recorded)
|
||||
assert mock_ctx.data_engine.record_trace.call_count == 1
|
||||
args, kwargs = mock_ctx.data_engine.record_trace.call_args
|
||||
assert kwargs["parent_trace_id"] == agent_run_id
|
||||
|
||||
# Cleanup
|
||||
handler.on_chain_end({}, run_id=chain2_run_id)
|
||||
handler.on_chain_end({}, run_id=chain1_run_id)
|
||||
|
||||
assert len(handler.chain_parents) == 0
|
||||
finally:
|
||||
CURRENT_TRACE_ID.reset(token)
|
||||
@@ -0,0 +1,297 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.graph import wrap_note_tool, wrap_update_note_tool
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wrap_update_note_tool_valid_status_change(tmp_path):
|
||||
base_dir = tmp_path
|
||||
notes_dir = base_dir / "notes"
|
||||
notes_dir.mkdir(parents=True, exist_ok=True)
|
||||
task_plan_path = notes_dir / "task_plan.md"
|
||||
|
||||
original_plan = """# Test Plan
|
||||
- [ ] Milestone 1
|
||||
- [ ] Substep 1.1
|
||||
- [ ] Milestone 2
|
||||
"""
|
||||
task_plan_path.write_text(original_plan, encoding="utf-8")
|
||||
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = MagicMock()
|
||||
mock_ctx.data_engine.base_dir = str(base_dir)
|
||||
mock_ctx.execution_setup = None
|
||||
|
||||
mock_original_tool = MagicMock()
|
||||
mock_original_tool.name = "update_note"
|
||||
mock_original_tool.description = "Update a note"
|
||||
mock_original_tool.ainvoke = AsyncMock(return_value="Success")
|
||||
|
||||
async def fake_invoke_tool(tool, args, tool_call_id, state, record_trace=None):
|
||||
content = task_plan_path.read_text(encoding="utf-8")
|
||||
updated = content.replace(args["target"], args["replacement"])
|
||||
task_plan_path.write_text(updated, encoding="utf-8")
|
||||
return "Success"
|
||||
|
||||
target = "- [ ] Milestone 1"
|
||||
replacement = "- [x] Milestone 1"
|
||||
|
||||
with patch(
|
||||
"artemis.graph.graph.invoke_tool_with_injection",
|
||||
side_effect=fake_invoke_tool,
|
||||
):
|
||||
# Instantiate inside patch context to capture the mocked import locally
|
||||
wrapped_tool = wrap_update_note_tool(mock_ctx, mock_original_tool)
|
||||
result = await wrapped_tool.ainvoke(
|
||||
{
|
||||
"key": "task_plan",
|
||||
"target": target,
|
||||
"replacement": replacement,
|
||||
"tool_call_id": "test_call_id",
|
||||
}
|
||||
)
|
||||
|
||||
assert result == "Success"
|
||||
updated_content = task_plan_path.read_text(encoding="utf-8")
|
||||
assert "- [x] Milestone 1" in updated_content
|
||||
assert "- [ ] Milestone 2" in updated_content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wrap_update_note_tool_invalid_milestone_text_change(tmp_path):
|
||||
base_dir = tmp_path
|
||||
notes_dir = base_dir / "notes"
|
||||
notes_dir.mkdir(parents=True, exist_ok=True)
|
||||
task_plan_path = notes_dir / "task_plan.md"
|
||||
|
||||
original_plan = """# Test Plan
|
||||
- [ ] Milestone 1
|
||||
- [ ] Substep 1.1
|
||||
- [ ] Milestone 2
|
||||
"""
|
||||
task_plan_path.write_text(original_plan, encoding="utf-8")
|
||||
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = MagicMock()
|
||||
mock_ctx.data_engine.base_dir = str(base_dir)
|
||||
mock_ctx.execution_setup = None
|
||||
|
||||
mock_original_tool = MagicMock()
|
||||
mock_original_tool.name = "update_note"
|
||||
mock_original_tool.ainvoke = AsyncMock(return_value="Success")
|
||||
|
||||
async def fake_invoke_tool(tool, args, tool_call_id, state, record_trace=None):
|
||||
content = task_plan_path.read_text(encoding="utf-8")
|
||||
updated = content.replace(args["target"], args["replacement"])
|
||||
task_plan_path.write_text(updated, encoding="utf-8")
|
||||
return "Success"
|
||||
|
||||
target = "- [ ] Milestone 1"
|
||||
replacement = "- [ ] Milestone Renamed"
|
||||
|
||||
mock_validation = AsyncMock(return_value={"status": "failed", "feedback": "rejected"})
|
||||
with (
|
||||
patch(
|
||||
"artemis.graph.graph.invoke_tool_with_injection",
|
||||
side_effect=fake_invoke_tool,
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.planner.planner.run_async_planner_validation",
|
||||
mock_validation,
|
||||
),
|
||||
):
|
||||
wrapped_tool = wrap_update_note_tool(mock_ctx, mock_original_tool)
|
||||
result = await wrapped_tool.ainvoke(
|
||||
{
|
||||
"key": "task_plan",
|
||||
"target": target,
|
||||
"replacement": replacement,
|
||||
"tool_call_id": "test_call_id",
|
||||
}
|
||||
)
|
||||
|
||||
assert result == "Success"
|
||||
assert mock_ctx.planner_task is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wrap_update_note_tool_allow_subgoal_changes(tmp_path):
|
||||
base_dir = tmp_path
|
||||
notes_dir = base_dir / "notes"
|
||||
notes_dir.mkdir(parents=True, exist_ok=True)
|
||||
task_plan_path = notes_dir / "task_plan.md"
|
||||
|
||||
original_plan = """# Test Plan
|
||||
- [ ] Milestone 1
|
||||
- [ ] Substep 1.1
|
||||
- [ ] Milestone 2
|
||||
"""
|
||||
task_plan_path.write_text(original_plan, encoding="utf-8")
|
||||
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = MagicMock()
|
||||
mock_ctx.data_engine.base_dir = str(base_dir)
|
||||
mock_ctx.execution_setup = None
|
||||
|
||||
mock_original_tool = MagicMock()
|
||||
mock_original_tool.name = "update_note"
|
||||
mock_original_tool.ainvoke = AsyncMock(return_value="Success")
|
||||
|
||||
async def fake_invoke_tool(tool, args, tool_call_id, state, record_trace=None):
|
||||
content = task_plan_path.read_text(encoding="utf-8")
|
||||
updated = content.replace(args["target"], args["replacement"])
|
||||
task_plan_path.write_text(updated, encoding="utf-8")
|
||||
return "Success"
|
||||
|
||||
target = " - [ ] Substep 1.1"
|
||||
replacement = " - [x] Substep 1.1 Completed"
|
||||
|
||||
with patch(
|
||||
"artemis.graph.graph.invoke_tool_with_injection",
|
||||
side_effect=fake_invoke_tool,
|
||||
):
|
||||
wrapped_tool = wrap_update_note_tool(mock_ctx, mock_original_tool)
|
||||
result = await wrapped_tool.ainvoke(
|
||||
{
|
||||
"key": "task_plan",
|
||||
"target": target,
|
||||
"replacement": replacement,
|
||||
"tool_call_id": "test_call_id",
|
||||
}
|
||||
)
|
||||
|
||||
assert result == "Success"
|
||||
updated_content = task_plan_path.read_text(encoding="utf-8")
|
||||
assert "Substep 1.1 Completed" in updated_content
|
||||
assert "- [ ] Milestone 1" in updated_content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wrap_note_tool_invalid_rewrite(tmp_path):
|
||||
base_dir = tmp_path
|
||||
notes_dir = base_dir / "notes"
|
||||
notes_dir.mkdir(parents=True, exist_ok=True)
|
||||
task_plan_path = notes_dir / "task_plan.md"
|
||||
|
||||
original_plan = """# Test Plan
|
||||
- [ ] Milestone 1
|
||||
- [ ] Substep 1.1
|
||||
- [ ] Milestone 2
|
||||
"""
|
||||
task_plan_path.write_text(original_plan, encoding="utf-8")
|
||||
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = MagicMock()
|
||||
mock_ctx.data_engine.base_dir = str(base_dir)
|
||||
mock_ctx.execution_setup = None
|
||||
|
||||
mock_original_tool = MagicMock()
|
||||
mock_original_tool.name = "save_note"
|
||||
mock_original_tool.ainvoke = AsyncMock(return_value="Success")
|
||||
|
||||
async def fake_invoke_tool(tool, args, tool_call_id, state, record_trace=None):
|
||||
task_plan_path.write_text(args["content"], encoding="utf-8")
|
||||
return "Success"
|
||||
|
||||
corrupted_plan = """# Test Plan
|
||||
- [ ] Milestone 1 (Altered)
|
||||
- [ ] New Milestone Added
|
||||
"""
|
||||
|
||||
mock_validation = AsyncMock(return_value={"status": "failed", "feedback": "rejected"})
|
||||
with (
|
||||
patch(
|
||||
"artemis.graph.graph.invoke_tool_with_injection",
|
||||
side_effect=fake_invoke_tool,
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.planner.planner.run_async_planner_validation",
|
||||
mock_validation,
|
||||
),
|
||||
):
|
||||
wrapped_tool = wrap_note_tool(mock_ctx, mock_original_tool)
|
||||
result = await wrapped_tool.ainvoke(
|
||||
{
|
||||
"key": "task_plan",
|
||||
"content": corrupted_plan,
|
||||
"tool_call_id": "test_call_id",
|
||||
}
|
||||
)
|
||||
|
||||
assert result == "Success"
|
||||
assert mock_ctx.planner_task is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wrap_note_tool_valid_lexical_drift(tmp_path):
|
||||
from langgraph.types import Command
|
||||
from langchain_core.messages import ToolMessage
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
|
||||
base_dir = tmp_path
|
||||
notes_dir = base_dir / "notes"
|
||||
notes_dir.mkdir(parents=True, exist_ok=True)
|
||||
task_plan_path = notes_dir / "task_plan.md"
|
||||
|
||||
original_plan = """# Test Plan
|
||||
- [ ] Open the Chrome browser and navigate to youtube.com
|
||||
"""
|
||||
task_plan_path.write_text(original_plan, encoding="utf-8")
|
||||
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = MagicMock()
|
||||
mock_ctx.data_engine.base_dir = str(base_dir)
|
||||
mock_ctx.execution_setup = None
|
||||
|
||||
# Create mock Command result
|
||||
tool_msg = ToolMessage(tool_call_id="test_call_id", content="Success outcome", status="success")
|
||||
mock_command = Command(update={VALIDATOR_MESSAGES_KEY: [tool_msg]})
|
||||
|
||||
mock_original_tool = MagicMock()
|
||||
mock_original_tool.name = "save_note"
|
||||
mock_original_tool.ainvoke = AsyncMock(return_value=mock_command)
|
||||
|
||||
async def fake_invoke_tool(tool, args, tool_call_id, state, record_trace=None):
|
||||
task_plan_path.write_text(args["content"], encoding="utf-8")
|
||||
return mock_command
|
||||
|
||||
# Minor wording change that is semantically valid (similarity ~92%)
|
||||
minor_drift_plan = """# Test Plan
|
||||
- [ ] Open Chrome browser and navigate to youtube.com
|
||||
"""
|
||||
|
||||
with patch(
|
||||
"artemis.graph.graph.invoke_tool_with_injection",
|
||||
side_effect=fake_invoke_tool,
|
||||
):
|
||||
wrapped_tool = wrap_note_tool(mock_ctx, mock_original_tool)
|
||||
result = await wrapped_tool.ainvoke(
|
||||
{
|
||||
"key": "task_plan",
|
||||
"content": minor_drift_plan,
|
||||
"tool_call_id": "test_call_id",
|
||||
}
|
||||
)
|
||||
|
||||
assert isinstance(result, Command)
|
||||
assert not hasattr(mock_ctx, "planner_task") or mock_ctx.planner_task is None
|
||||
|
||||
# Verify that the drift plan is actually saved
|
||||
saved_content = task_plan_path.read_text(encoding="utf-8")
|
||||
assert "Open Chrome browser" in saved_content
|
||||
@@ -0,0 +1,146 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for Base Agent Protocol & Lifecycle Registry."""
|
||||
|
||||
import pytest
|
||||
from typing import Any
|
||||
from artemis.agents.base import AgentConfig, AgentRegistry, BaseAgent
|
||||
from artemis.agents.validator.failure_analyzer_agent import FailureAnalyzerAgent
|
||||
from artemis.agents.validator.recovery_strategies import (
|
||||
RecoveryStrategyRegistry,
|
||||
)
|
||||
from artemis.drivers.mock.mock_driver import MockDeviceDriver
|
||||
|
||||
|
||||
class DummyTestAgent(BaseAgent):
|
||||
"""Simple test agent implementing BaseAgent protocol."""
|
||||
|
||||
def __init__(self, config: AgentConfig | None = None, ctx: Any | None = None, **kwargs: Any):
|
||||
super().__init__(
|
||||
name="dummy_test_agent",
|
||||
role="Test Worker",
|
||||
description="Agent used for testing lifecycle transitions.",
|
||||
config=config,
|
||||
ctx=ctx,
|
||||
)
|
||||
|
||||
async def process(self, state: Any) -> dict[str, Any]:
|
||||
return {"status": "success", "processed_state": True}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_base_agent_lifecycle():
|
||||
"""Verify BaseAgent initialization, execution, and cleanup lifecycle."""
|
||||
agent = DummyTestAgent()
|
||||
assert agent.name == "dummy_test_agent"
|
||||
assert not agent._is_initialized
|
||||
|
||||
# 1. Initialize
|
||||
mock_ctx = object()
|
||||
await agent.initialize(mock_ctx)
|
||||
assert agent._is_initialized
|
||||
assert agent.ctx is mock_ctx
|
||||
|
||||
# 2. Process
|
||||
result = await agent.process({})
|
||||
assert result.get("status") == "success"
|
||||
assert result.get("processed_state") is True
|
||||
|
||||
# 3. Cleanup
|
||||
await agent.cleanup()
|
||||
assert not agent._is_initialized
|
||||
|
||||
|
||||
def test_agent_registry():
|
||||
"""Verify AgentRegistry registers and instantiates agents correctly."""
|
||||
registered = AgentRegistry.list_agents()
|
||||
assert "planner" in registered
|
||||
assert "summarizer" in registered
|
||||
assert "operator" in registered
|
||||
assert "failure_analyzer" in registered
|
||||
|
||||
# Instantiate PlannerAgent via registry
|
||||
planner = AgentRegistry.create_agent("planner", ctx=None)
|
||||
assert planner.name == "planner"
|
||||
assert planner.role == "Strategic Task Planner"
|
||||
|
||||
# Instantiate SummarizerAgent via registry
|
||||
summarizer = AgentRegistry.create_agent("summarizer", ctx=None)
|
||||
assert summarizer.name == "summarizer"
|
||||
assert summarizer.role == "Execution History Summarizer"
|
||||
|
||||
# Instantiate OperatorAgent via registry
|
||||
operator = AgentRegistry.create_agent("operator", ctx=None)
|
||||
assert operator.name == "operator"
|
||||
assert operator.role == "Mobile UI Interaction Operator"
|
||||
|
||||
# Instantiate FailureAnalyzerAgent via registry
|
||||
fa = AgentRegistry.create_agent("failure_analyzer", ctx=None)
|
||||
assert fa.name == "failure_analyzer"
|
||||
assert fa.role == "Autonomous Self-Healing Failure Analyzer"
|
||||
|
||||
|
||||
def test_agent_config():
|
||||
"""Verify AgentConfig parameters can be customized."""
|
||||
config = AgentConfig(
|
||||
model_name="gemini-2.5-pro",
|
||||
temperature=0.7,
|
||||
max_tokens=2048,
|
||||
enabled_tools=["click", "swipe"],
|
||||
)
|
||||
agent = DummyTestAgent(config=config)
|
||||
assert agent.config.model_name == "gemini-2.5-pro"
|
||||
assert agent.config.temperature == 0.7
|
||||
assert agent.config.enabled_tools == ["click", "swipe"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recovery_strategies_and_failure_analyzer():
|
||||
"""Verify self-healing recovery strategies and FailureAnalyzerAgent execution."""
|
||||
driver = MockDeviceDriver(device_id="mock-rec-1")
|
||||
|
||||
# 1. Test dialog dismissal strategy
|
||||
dialog_ctx = {"category": "ui_obstruction", "error_message": "Unexpected permission popup"}
|
||||
dialog_res = await RecoveryStrategyRegistry.attempt_recovery(driver, dialog_ctx)
|
||||
assert dialog_res is not None
|
||||
assert dialog_res.success
|
||||
assert dialog_res.action_type.value == "dismiss_dialog"
|
||||
assert driver.action_history[-1].get("key") == "back"
|
||||
|
||||
# 2. Test navigate back strategy
|
||||
nav_ctx = {"category": "wrong_screen"}
|
||||
nav_res = await RecoveryStrategyRegistry.attempt_recovery(driver, nav_ctx)
|
||||
assert nav_res is not None
|
||||
assert nav_res.success
|
||||
assert nav_res.action_type.value == "navigate_back"
|
||||
|
||||
# 3. Test scroll and search strategy
|
||||
scroll_ctx = {"category": "element_not_found", "scroll_direction": "down"}
|
||||
scroll_res = await RecoveryStrategyRegistry.attempt_recovery(driver, scroll_ctx)
|
||||
assert scroll_res is not None
|
||||
assert scroll_res.success
|
||||
assert scroll_res.action_type.value == "scroll_and_retry"
|
||||
|
||||
# 4. Test FailureAnalyzerAgent process
|
||||
class MockContext:
|
||||
pass
|
||||
|
||||
ctx = MockContext()
|
||||
setattr(ctx, "_active_driver", driver)
|
||||
|
||||
fa_agent = FailureAnalyzerAgent(ctx=ctx)
|
||||
fa_result = await fa_agent.process({"failure_context": {"category": "wrong_screen"}})
|
||||
assert fa_result["status"] == "recovered"
|
||||
assert fa_result["recovery_action"] == "navigate_back"
|
||||
@@ -0,0 +1,30 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import asyncio
|
||||
|
||||
|
||||
async def f():
|
||||
return [1, 2]
|
||||
|
||||
|
||||
async def m():
|
||||
try:
|
||||
val = await f()[-1]
|
||||
print(val)
|
||||
except Exception as e:
|
||||
print(repr(e))
|
||||
|
||||
|
||||
asyncio.run(m())
|
||||
@@ -0,0 +1,65 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for ARTEMIS Unified CLI application."""
|
||||
|
||||
from typer.testing import CliRunner
|
||||
from artemis.interfaces.cli.main import app
|
||||
|
||||
runner = CliRunner()
|
||||
|
||||
|
||||
def test_cli_help():
|
||||
"""Verify top-level CLI help returns status 0 and lists core subcommands."""
|
||||
result = runner.invoke(app, ["--help"])
|
||||
assert result.exit_code == 0
|
||||
assert "Artemis: Autonomous Multimodal Android Agent" in result.output
|
||||
assert "run" in result.output
|
||||
assert "batch" in result.output
|
||||
assert "bench" in result.output
|
||||
assert "server" in result.output
|
||||
assert "trace" in result.output
|
||||
assert "mcp" in result.output
|
||||
|
||||
|
||||
def test_cli_version():
|
||||
"""Verify --version returns version banner."""
|
||||
result = runner.invoke(app, ["--version"])
|
||||
assert result.exit_code == 0
|
||||
assert "Artemis Agent Platform" in result.output
|
||||
|
||||
|
||||
def test_cli_run_help():
|
||||
"""Verify 'artemis run --help' displays execution options."""
|
||||
result = runner.invoke(app, ["run", "--help"])
|
||||
assert result.exit_code == 0
|
||||
assert "--profile" in result.output
|
||||
assert "--locked-app" in result.output
|
||||
assert "--traces-path" in result.output
|
||||
|
||||
|
||||
def test_cli_batch_help():
|
||||
"""Verify 'artemis batch --help' displays batch options."""
|
||||
result = runner.invoke(app, ["batch", "--help"])
|
||||
assert result.exit_code == 0
|
||||
assert "--file" in result.output
|
||||
assert "--delay" in result.output
|
||||
|
||||
|
||||
def test_cli_trace_help():
|
||||
"""Verify 'artemis trace --help' lists trace subcommands."""
|
||||
result = runner.invoke(app, ["trace", "--help"])
|
||||
assert result.exit_code == 0
|
||||
assert "list" in result.output
|
||||
assert "view" in result.output
|
||||
@@ -0,0 +1,541 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for the unified artemis.config package."""
|
||||
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
from unittest.mock import MagicMock
|
||||
import pytest
|
||||
from pydantic import SecretStr
|
||||
|
||||
from artemis.config import (
|
||||
CONFIG_DIR,
|
||||
ROOT_DIR,
|
||||
AgentGlobalConfig,
|
||||
LLMConfig,
|
||||
OutputConfig,
|
||||
Settings,
|
||||
cleanup_temp_dir,
|
||||
clear_ipc_port,
|
||||
clear_ls_address,
|
||||
deep_merge_llm_config,
|
||||
get_app_dir,
|
||||
get_config_path,
|
||||
get_data_engine_db_path,
|
||||
get_default_llm_config,
|
||||
get_default_traces_path,
|
||||
get_temp_dir,
|
||||
get_traces_dir,
|
||||
load_agent_config,
|
||||
read_ipc_port,
|
||||
read_ls_address,
|
||||
record_events,
|
||||
write_ipc_port,
|
||||
write_ls_address,
|
||||
)
|
||||
|
||||
|
||||
def test_paths_and_directories():
|
||||
"""Verify central path resolution."""
|
||||
assert ROOT_DIR.exists()
|
||||
assert (ROOT_DIR / "artemis").exists()
|
||||
assert CONFIG_DIR.exists()
|
||||
assert (CONFIG_DIR / "artemis.jsonc").exists()
|
||||
|
||||
app_dir = get_app_dir()
|
||||
assert isinstance(app_dir, Path)
|
||||
|
||||
traces_path = get_default_traces_path()
|
||||
assert isinstance(traces_path, Path)
|
||||
assert get_traces_dir() == traces_path
|
||||
|
||||
db_path = get_data_engine_db_path()
|
||||
assert db_path.name == "data_engine.db"
|
||||
|
||||
temp_dir = get_temp_dir("test_subfolder")
|
||||
assert temp_dir.exists()
|
||||
assert temp_dir.name == "test_subfolder"
|
||||
|
||||
|
||||
def test_config_file_resolver():
|
||||
"""Test get_config_path for unified artemis.jsonc."""
|
||||
artemis_path = get_config_path("artemis.jsonc")
|
||||
assert artemis_path.exists()
|
||||
|
||||
with pytest.raises(FileNotFoundError):
|
||||
get_config_path("non_existent_config_12345.json")
|
||||
|
||||
|
||||
def test_settings_and_api_key_fallbacks(monkeypatch):
|
||||
"""Test Settings model, API key fallbacks, and get/set methods."""
|
||||
monkeypatch.delenv("GOOGLE_API_KEY", raising=False)
|
||||
monkeypatch.delenv("GEMINI_API_KEY", raising=False)
|
||||
monkeypatch.delenv("GCP_API_KEY", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
|
||||
|
||||
custom_settings = Settings(
|
||||
_env_file=None,
|
||||
GOOGLE_API_KEY=None,
|
||||
GEMINI_API_KEY=SecretStr("gemini_key_123"),
|
||||
OPENAI_API_KEY=SecretStr("openai_key_456"),
|
||||
OCR_API_KEY=SecretStr("ocr_key_789"),
|
||||
)
|
||||
# Automatic fallback should populate GOOGLE_API_KEY from GEMINI_API_KEY
|
||||
assert custom_settings.GOOGLE_API_KEY is not None
|
||||
assert custom_settings.GOOGLE_API_KEY.get_secret_value() == "gemini_key_123"
|
||||
assert custom_settings.get_api_key("ocr").get_secret_value() == "ocr_key_789"
|
||||
|
||||
assert custom_settings.get_api_key("google").get_secret_value() == "gemini_key_123"
|
||||
assert custom_settings.get_api_key("gemini").get_secret_value() == "gemini_key_123"
|
||||
assert custom_settings.get_api_key("openai").get_secret_value() == "openai_key_456"
|
||||
assert custom_settings.get_api_key("nonexistent") is None
|
||||
|
||||
# Test dynamic set_api_key
|
||||
custom_settings.set_api_key("xai", "xai_test_key", persist_to_env=False)
|
||||
assert custom_settings.get_api_key("xai").get_secret_value() == "xai_test_key"
|
||||
|
||||
|
||||
def test_llm_config_parsing_and_merging():
|
||||
"""Test LLMConfig parsing, agent querying, and deep merging."""
|
||||
llm_cfg = get_default_llm_config()
|
||||
assert isinstance(llm_cfg, LLMConfig)
|
||||
assert llm_cfg.planner.provider in ("google", "openai", "openrouter", "xai", "vertexai")
|
||||
assert llm_cfg.get_agent("planner") is not None
|
||||
assert llm_cfg.get_utils("hopper") is not None
|
||||
|
||||
# Deep merge overrides
|
||||
overrides = {
|
||||
"planner": {
|
||||
"model": "custom-model-v1",
|
||||
"temperature": 0.7,
|
||||
}
|
||||
}
|
||||
merged = deep_merge_llm_config(llm_cfg, overrides)
|
||||
assert merged.planner.model == "custom-model-v1"
|
||||
assert merged.planner.temperature == 0.7
|
||||
|
||||
|
||||
def test_agent_config_loading():
|
||||
"""Test AgentGlobalConfig parsing from agent_config.json / artemis.jsonc."""
|
||||
agent_cfg = load_agent_config()
|
||||
assert isinstance(agent_cfg, AgentGlobalConfig)
|
||||
assert "operator" in agent_cfg.explorer_versions
|
||||
assert agent_cfg.explorer.default_version == "flash"
|
||||
assert agent_cfg.explorer.flash_mode == "flash"
|
||||
assert agent_cfg.explorer.pro_mode == "flash"
|
||||
assert agent_cfg.explorer.caching is True
|
||||
assert "explorer" in agent_cfg.blacklisted_tools
|
||||
assert agent_cfg.video_analyzer.enable_ledger is True
|
||||
assert agent_cfg.planner_validation.enabled is False
|
||||
assert agent_cfg.planner_validation.similarity_threshold == 0.85
|
||||
assert agent_cfg.committee.enabled is False
|
||||
assert agent_cfg.committee.debate_rounds == 2
|
||||
assert agent_cfg.checker.enabled is False
|
||||
assert agent_cfg.checker.max_iterations == 20
|
||||
assert agent_cfg.checker.max_chat_rounds == 4
|
||||
assert agent_cfg.outputter.enabled is True
|
||||
assert agent_cfg.outputter.force_synthesis is False
|
||||
assert agent_cfg.flash.max_turns == 30
|
||||
assert agent_cfg.flash.explorer_mode == "flash"
|
||||
assert agent_cfg.pro.explorer.mode == "flash"
|
||||
assert agent_cfg.pro.checker.enabled is False
|
||||
assert agent_cfg.pro.committee.enabled is False
|
||||
assert agent_cfg.pro.planner_validation.enabled is False
|
||||
assert agent_cfg.pro.video_analyzer.enable_ledger is True
|
||||
|
||||
|
||||
def test_output_config_and_recording():
|
||||
"""Test OutputConfig behavior and event recording."""
|
||||
cfg = OutputConfig(output_description="Output a valid JSON summary")
|
||||
assert cfg.needs_structured_format() is True
|
||||
|
||||
cfg_disabled = OutputConfig(output_description="Output JSON", enable_outputter=False)
|
||||
assert cfg_disabled.needs_structured_format() is False
|
||||
|
||||
cfg_forced = OutputConfig(force_synthesis=True)
|
||||
assert cfg_forced.needs_structured_format() is True
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
output_file = Path(tmpdir) / "events.json"
|
||||
events_data = [{"event": "test_event", "status": "ok"}]
|
||||
record_events(output_file, events_data)
|
||||
assert output_file.exists()
|
||||
assert "test_event" in output_file.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_runtime_state_and_ipc():
|
||||
"""Test IPC port and LS address state helpers."""
|
||||
# Test IPC port
|
||||
write_ipc_port(49152)
|
||||
assert read_ipc_port() == 49152
|
||||
clear_ipc_port()
|
||||
|
||||
# Test LS address
|
||||
write_ls_address("localhost:12345")
|
||||
assert read_ls_address() == "localhost:12345"
|
||||
clear_ls_address()
|
||||
|
||||
# Test temp dir cleanup
|
||||
test_temp_sub = get_temp_dir("test_cleanup_dir")
|
||||
test_temp_file = test_temp_sub / "temp_marker.txt"
|
||||
test_temp_file.write_text("temporary data")
|
||||
assert test_temp_file.exists()
|
||||
|
||||
deleted = cleanup_temp_dir("test_cleanup_dir")
|
||||
assert deleted >= 1
|
||||
assert not test_temp_file.exists()
|
||||
|
||||
|
||||
def test_planner_validation_builder_and_milestones():
|
||||
"""Test AgentConfigBuilder methods for planner validation and validate_milestones threshold."""
|
||||
from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder
|
||||
from artemis.graph.graph import validate_milestones
|
||||
|
||||
# Default builder inherits from artemis.jsonc (enabled=False, similarity_threshold=0.85)
|
||||
builder = AgentConfigBuilder()
|
||||
cfg = builder.build()
|
||||
assert cfg.disable_planner_validation is True
|
||||
assert cfg.planner_validation_threshold == 0.85
|
||||
|
||||
# Fluent enabling
|
||||
cfg_enabled = (
|
||||
AgentConfigBuilder()
|
||||
.with_planner_validation(enabled=True, similarity_threshold=0.90)
|
||||
.build()
|
||||
)
|
||||
assert cfg_enabled.disable_planner_validation is False
|
||||
assert cfg_enabled.planner_validation_threshold == 0.90
|
||||
|
||||
# Fluent disabling
|
||||
cfg_disabled = AgentConfigBuilder().with_disable_planner_validation(True).build()
|
||||
assert cfg_disabled.disable_planner_validation is True
|
||||
|
||||
# Test validate_milestones with custom thresholds
|
||||
before = "- [ ] Tap Login button\n- [ ] Enter password"
|
||||
after_minor = "- [ ] Tap the Login button\n- [ ] Enter password"
|
||||
# Minor edit has ~0.94 similarity
|
||||
assert validate_milestones(before, after_minor, similarity_threshold=0.85) is False
|
||||
assert validate_milestones(before, after_minor, similarity_threshold=0.98) is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_committee_builder_and_graph_mounting():
|
||||
"""Test AgentConfigBuilder committee methods and graph mounting."""
|
||||
from artemis.context import ArtemisContext, DeviceContext, DevicePlatform, ExecutionSetup
|
||||
from artemis.graph.graph import get_graph
|
||||
from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder
|
||||
|
||||
# Default: disabled
|
||||
cfg_default = AgentConfigBuilder().build()
|
||||
assert cfg_default.enable_committee is False
|
||||
assert cfg_default.committee_debate_rounds == 2
|
||||
|
||||
# Enabled via builder
|
||||
cfg_enabled = AgentConfigBuilder().with_committee(enabled=True, debate_rounds=3).build()
|
||||
assert cfg_enabled.enable_committee is True
|
||||
assert cfg_enabled.committee_debate_rounds == 3
|
||||
|
||||
# Test mounting in graph
|
||||
device = DeviceContext(
|
||||
host_platform="LINUX",
|
||||
mobile_platform=DevicePlatform.ANDROID,
|
||||
device_id="dummy",
|
||||
device_width=1080,
|
||||
device_height=2400,
|
||||
)
|
||||
ctx_disabled = ArtemisContext(
|
||||
device=device,
|
||||
execution_setup=ExecutionSetup(enable_committee=False),
|
||||
)
|
||||
graph_disabled = await get_graph(ctx_disabled)
|
||||
op_node_disabled = graph_disabled.nodes.get("operator")
|
||||
assert op_node_disabled is not None
|
||||
# Verify ask_committee is not mounted when disabled
|
||||
op_tools_disabled = [t.name for t in op_node_disabled.bound.afunc.tools]
|
||||
assert "ask_committee" not in op_tools_disabled
|
||||
|
||||
ctx_enabled = ArtemisContext(
|
||||
device=device,
|
||||
execution_setup=ExecutionSetup(enable_committee=True, committee_debate_rounds=3),
|
||||
)
|
||||
graph_enabled = await get_graph(ctx_enabled)
|
||||
op_node_enabled = graph_enabled.nodes.get("operator")
|
||||
assert op_node_enabled is not None
|
||||
# Verify ask_committee is mounted when enabled
|
||||
op_tools_enabled = [t.name for t in op_node_enabled.bound.afunc.tools]
|
||||
assert "ask_committee" in op_tools_enabled
|
||||
|
||||
|
||||
def test_checker_builder_and_context_propagation():
|
||||
"""Test AgentConfigBuilder methods and context propagation for checker."""
|
||||
from unittest.mock import MagicMock
|
||||
from artemis.context import ArtemisContext, DeviceContext, DevicePlatform
|
||||
from artemis.sdk.agent import Agent
|
||||
from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder
|
||||
|
||||
# Default builder inherits from artemis.jsonc (enabled=False, max_iterations=20, max_chat_rounds=4)
|
||||
builder = AgentConfigBuilder()
|
||||
cfg = builder.build()
|
||||
assert cfg.disable_checker is True
|
||||
assert cfg.checker_max_iterations == 20
|
||||
assert cfg.checker_max_chat_rounds == 4
|
||||
|
||||
# Fluent enabling
|
||||
cfg_enabled = (
|
||||
AgentConfigBuilder()
|
||||
.with_checker(enabled=True, max_iterations=25, max_chat_rounds=5)
|
||||
.build()
|
||||
)
|
||||
assert cfg_enabled.disable_checker is False
|
||||
assert cfg_enabled.checker_max_iterations == 25
|
||||
assert cfg_enabled.checker_max_chat_rounds == 5
|
||||
|
||||
# Fluent disabling
|
||||
cfg_disabled = AgentConfigBuilder().with_disable_checker(True).build()
|
||||
assert cfg_disabled.disable_checker is True
|
||||
|
||||
# Test propagation to ExecutionSetup via Agent._prepare_tracing
|
||||
agent = Agent(config=cfg_enabled)
|
||||
device = DeviceContext(
|
||||
host_platform="LINUX",
|
||||
mobile_platform=DevicePlatform.ANDROID,
|
||||
device_id="dummy",
|
||||
device_width=1080,
|
||||
device_height=2400,
|
||||
)
|
||||
ctx = ArtemisContext(device=device)
|
||||
mock_task = MagicMock()
|
||||
mock_task.get_name.return_value = "test_task"
|
||||
mock_task.request.record_trace = False
|
||||
mock_task.request.name = "test_task"
|
||||
mock_task.request.profile = None
|
||||
mock_task.request.goal = "Test goal"
|
||||
agent._prepare_tracing(mock_task, ctx)
|
||||
|
||||
assert ctx.execution_setup is not None
|
||||
assert ctx.execution_setup.disable_checker is False
|
||||
assert ctx.execution_setup.checker_max_iterations == 25
|
||||
assert ctx.execution_setup.checker_max_chat_rounds == 5
|
||||
|
||||
|
||||
def test_explorer_builder_and_resolution(monkeypatch):
|
||||
"""Test AgentConfigBuilder explorer methods and multi-tier resolution logic."""
|
||||
from artemis.context import ArtemisContext, DeviceContext, DevicePlatform
|
||||
from artemis.sdk.agent import Agent
|
||||
from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder
|
||||
from artemis.tools.explorer_tool import resolve_explorer_version
|
||||
|
||||
# Default builder inherits from artemis.jsonc (default="flash", flash_mode="flash", pro_mode="flash", caching=True)
|
||||
builder = AgentConfigBuilder()
|
||||
cfg = builder.build()
|
||||
assert cfg.explorer.default_version == "flash"
|
||||
assert cfg.explorer.flash_mode == "flash"
|
||||
assert cfg.explorer.pro_mode == "flash"
|
||||
assert cfg.explorer.caching is True
|
||||
|
||||
# Fluent configuration with with_explorer
|
||||
cfg_custom = (
|
||||
AgentConfigBuilder()
|
||||
.with_explorer(
|
||||
version="pro",
|
||||
flash_mode="flash",
|
||||
pro_mode="ultra",
|
||||
caching=False,
|
||||
versions={"operator": "ultra", "validator": "pro"},
|
||||
)
|
||||
.build()
|
||||
)
|
||||
assert cfg_custom.explorer.default_version == "pro"
|
||||
assert cfg_custom.explorer.flash_mode == "flash"
|
||||
assert cfg_custom.explorer.pro_mode == "ultra"
|
||||
assert cfg_custom.explorer.caching is False
|
||||
assert cfg_custom.explorer_versions["operator"] == "ultra"
|
||||
assert cfg_custom.explorer_versions["validator"] == "pro"
|
||||
|
||||
# Fluent configuration with with_explorer_version shorthand
|
||||
cfg_shorthand = AgentConfigBuilder().with_explorer_version("ultra").build()
|
||||
assert cfg_shorthand.explorer.default_version == "ultra"
|
||||
|
||||
# Context setup for resolution tests
|
||||
device = DeviceContext(
|
||||
host_platform="LINUX",
|
||||
mobile_platform=DevicePlatform.ANDROID,
|
||||
device_id="dummy",
|
||||
device_width=1080,
|
||||
device_height=2400,
|
||||
)
|
||||
ctx = ArtemisContext(device=device, agent_config=cfg_custom)
|
||||
|
||||
# 1. Explicit override takes highest priority
|
||||
assert resolve_explorer_version(ctx, explicit_version="flash") == "flash"
|
||||
assert resolve_explorer_version(ctx, explicit_version="pro") == "pro"
|
||||
assert resolve_explorer_version(ctx, explicit_version="ultra") == "ultra"
|
||||
|
||||
# 2. Environment variable override
|
||||
monkeypatch.setenv("ARTEMIS_EXPLORER_VERSION", "ultra")
|
||||
assert resolve_explorer_version(ctx) == "ultra"
|
||||
monkeypatch.delenv("ARTEMIS_EXPLORER_VERSION", raising=False)
|
||||
|
||||
# 3. Per-agent mapping in explorer_versions
|
||||
assert resolve_explorer_version(ctx, agent_or_profile_name="operator") == "ultra"
|
||||
assert resolve_explorer_version(ctx, agent_or_profile_name="validator") == "pro"
|
||||
|
||||
# 4. Pro profile vs Flash profile resolution
|
||||
cfg_profiled = (
|
||||
AgentConfigBuilder()
|
||||
.with_explorer(flash_mode="flash", pro_mode="pro", default_version="flash", versions={})
|
||||
.build()
|
||||
)
|
||||
ctx_profiled = ArtemisContext(device=device, agent_config=cfg_profiled)
|
||||
assert resolve_explorer_version(ctx_profiled, agent_or_profile_name="flash") == "flash"
|
||||
assert resolve_explorer_version(ctx_profiled, agent_or_profile_name="flash_runner") == "flash"
|
||||
assert resolve_explorer_version(ctx_profiled, agent_or_profile_name="pro") == "pro"
|
||||
assert resolve_explorer_version(ctx_profiled, agent_or_profile_name="operator") == "pro"
|
||||
|
||||
# 5. Propagation to ExecutionSetup via Agent._prepare_tracing
|
||||
agent = Agent(config=cfg_custom)
|
||||
mock_task = MagicMock()
|
||||
mock_task.get_name.return_value = "explorer_test_task"
|
||||
mock_task.request.record_trace = False
|
||||
mock_task.request.name = "explorer_test_task"
|
||||
mock_task.request.profile = None
|
||||
mock_task.request.goal = "Test explorer propagation"
|
||||
agent._prepare_tracing(mock_task, ctx)
|
||||
|
||||
assert ctx.execution_setup is not None
|
||||
assert ctx.execution_setup.explorer_version == "pro"
|
||||
assert ctx.execution_setup.explorer_flash_mode == "flash"
|
||||
assert ctx.execution_setup.explorer_pro_mode == "ultra"
|
||||
assert ctx.execution_setup.explorer_caching is False
|
||||
|
||||
|
||||
def test_outputter_builder_and_context_propagation():
|
||||
"""Test AgentConfigBuilder outputter methods and propagation to ExecutionSetup."""
|
||||
from artemis.context import ArtemisContext, DeviceContext, DevicePlatform
|
||||
from artemis.sdk.agent import Agent
|
||||
from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder
|
||||
|
||||
# Default builder inherits from artemis.jsonc (enabled=True, force_synthesis=False)
|
||||
builder = AgentConfigBuilder()
|
||||
cfg = builder.build()
|
||||
assert cfg.disable_outputter is False
|
||||
assert cfg.outputter.enabled is True
|
||||
assert cfg.outputter.force_synthesis is False
|
||||
|
||||
# Fluent configuration with with_outputter
|
||||
cfg_custom = AgentConfigBuilder().with_outputter(enabled=True, force_synthesis=True).build()
|
||||
assert cfg_custom.disable_outputter is False
|
||||
assert cfg_custom.outputter.enabled is True
|
||||
assert cfg_custom.outputter.force_synthesis is True
|
||||
|
||||
# Fluent disabling with with_disable_outputter
|
||||
cfg_disabled = AgentConfigBuilder().with_disable_outputter(True).build()
|
||||
assert cfg_disabled.disable_outputter is True
|
||||
assert cfg_disabled.outputter.enabled is False
|
||||
|
||||
# Test propagation to ExecutionSetup via Agent._prepare_tracing
|
||||
device = DeviceContext(
|
||||
host_platform="LINUX",
|
||||
mobile_platform=DevicePlatform.ANDROID,
|
||||
device_id="dummy",
|
||||
device_width=1080,
|
||||
device_height=2400,
|
||||
)
|
||||
ctx = ArtemisContext(device=device)
|
||||
agent = Agent(config=cfg_custom)
|
||||
mock_task = MagicMock()
|
||||
mock_task.get_name.return_value = "outputter_test_task"
|
||||
mock_task.request.record_trace = False
|
||||
mock_task.request.name = "outputter_test_task"
|
||||
mock_task.request.profile = None
|
||||
mock_task.request.goal = "Test outputter propagation"
|
||||
agent._prepare_tracing(mock_task, ctx)
|
||||
|
||||
assert ctx.execution_setup is not None
|
||||
assert ctx.execution_setup.disable_outputter is False
|
||||
assert ctx.execution_setup.outputter.enabled is True
|
||||
assert ctx.execution_setup.outputter.force_synthesis is True
|
||||
|
||||
|
||||
def test_categorized_flash_and_pro_profile_builders():
|
||||
"""Test with_flash_config and with_pro_config fluent builders and bidirectional sync."""
|
||||
from artemis.config.agent import AgentGlobalConfig
|
||||
from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder
|
||||
|
||||
# 1. Test with_flash_config
|
||||
cfg_flash = AgentConfigBuilder().with_flash_config(max_turns=15, explorer_mode="flash").build()
|
||||
assert cfg_flash.flash.max_turns == 15
|
||||
assert cfg_flash.flash.explorer_mode == "flash"
|
||||
assert cfg_flash.explorer.flash_mode == "flash"
|
||||
|
||||
# 2. Test with_pro_config
|
||||
cfg_pro = (
|
||||
AgentConfigBuilder()
|
||||
.with_pro_config(
|
||||
explorer_mode="ultra",
|
||||
planner_validation=True,
|
||||
committee=True,
|
||||
checker=True,
|
||||
video_ledger=False,
|
||||
)
|
||||
.build()
|
||||
)
|
||||
assert cfg_pro.pro.explorer.mode == "ultra"
|
||||
assert cfg_pro.explorer.pro_mode == "ultra"
|
||||
assert cfg_pro.disable_planner_validation is False
|
||||
assert cfg_pro.enable_committee is True
|
||||
assert cfg_pro.disable_checker is False
|
||||
assert cfg_pro.enable_video_ledger is False
|
||||
|
||||
# 3. Test AgentGlobalConfig bidirectional synchronization
|
||||
raw_dict = {
|
||||
"flash": {"max_turns": 25, "explorer_mode": "flash"},
|
||||
"pro": {
|
||||
"explorer": {"mode": "pro", "caching": False},
|
||||
"checker": {"enabled": True, "max_iterations": 10},
|
||||
"committee": {"enabled": True, "debate_rounds": 4},
|
||||
},
|
||||
}
|
||||
synced = AgentGlobalConfig.model_validate(raw_dict)
|
||||
assert synced.flash.max_turns == 25
|
||||
assert synced.explorer.flash_mode == "flash"
|
||||
assert synced.explorer.pro_mode == "pro"
|
||||
assert synced.explorer.caching is False
|
||||
assert synced.checker.enabled is True
|
||||
assert synced.checker.max_iterations == 10
|
||||
assert synced.committee.enabled is True
|
||||
assert synced.committee.debate_rounds == 4
|
||||
|
||||
|
||||
def test_agent_config_environment_variable_overrides(monkeypatch):
|
||||
"""Test environment variable overrides for AgentGlobalConfig switches."""
|
||||
from artemis.config.agent import load_agent_config
|
||||
|
||||
monkeypatch.setenv("ARTEMIS_CHECKER_ENABLED", "true")
|
||||
monkeypatch.setenv("ARTEMIS_COMMITTEE_ENABLED", "1")
|
||||
monkeypatch.setenv("ARTEMIS_PLANNER_VALIDATION_ENABLED", "yes")
|
||||
monkeypatch.setenv("ARTEMIS_OUTPUTTER_ENABLED", "false")
|
||||
monkeypatch.setenv("ARTEMIS_VIDEO_LEDGER_ENABLED", "0")
|
||||
|
||||
cfg = load_agent_config()
|
||||
assert cfg.checker.enabled is True
|
||||
assert cfg.pro.checker.enabled is True
|
||||
assert cfg.committee.enabled is True
|
||||
assert cfg.pro.committee.enabled is True
|
||||
assert cfg.planner_validation.enabled is True
|
||||
assert cfg.pro.planner_validation.enabled is True
|
||||
assert cfg.outputter.enabled is False
|
||||
assert cfg.video_analyzer.enable_ledger is False
|
||||
assert cfg.pro.video_analyzer.enable_ledger is False
|
||||
@@ -0,0 +1,30 @@
|
||||
import asyncio
|
||||
import pytest
|
||||
from artemis.context import ArtemisContext, DeviceContext, DevicePlatform
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_artemis_context_waits_for_background_tasks():
|
||||
device = DeviceContext(
|
||||
host_platform="LINUX",
|
||||
mobile_platform=DevicePlatform.ANDROID,
|
||||
device_id="dummy",
|
||||
device_width=1080,
|
||||
device_height=2400,
|
||||
)
|
||||
|
||||
context = ArtemisContext(device=device)
|
||||
|
||||
task_run = False
|
||||
|
||||
async def dummy_task():
|
||||
nonlocal task_run
|
||||
await asyncio.sleep(0.1)
|
||||
task_run = True
|
||||
|
||||
async with context:
|
||||
task = asyncio.create_task(dummy_task())
|
||||
context.background_tasks.append(task)
|
||||
|
||||
# After exiting the context, the task should have run to completion
|
||||
assert task_run is True
|
||||
@@ -0,0 +1,91 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for Unified Base Device Driver and implementations."""
|
||||
|
||||
import pytest
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from artemis.drivers.base import KeyCode, ScreenData, SwipeDirection
|
||||
from artemis.drivers.mock.mock_driver import MockDeviceDriver
|
||||
from artemis.drivers.android.adb_driver import AndroidAdbDriver
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mock_driver_actions():
|
||||
"""Verify MockDeviceDriver records and performs all core actions."""
|
||||
driver = MockDeviceDriver(device_id="test-mock", width=1080, height=2400)
|
||||
assert driver.device_id == "test-mock"
|
||||
assert driver.screen_size == (1080, 2400)
|
||||
|
||||
# 1. Screen data
|
||||
screen_data = await driver.get_screen_data()
|
||||
assert isinstance(screen_data, ScreenData)
|
||||
assert screen_data.width == 1080
|
||||
assert screen_data.height == 2400
|
||||
assert screen_data.screenshot_base64 is not None
|
||||
|
||||
# 2. Tap
|
||||
assert await driver.tap(500, 1000) is True
|
||||
assert driver.action_history[-1]["action"] == "tap"
|
||||
assert driver.action_history[-1]["x"] == 500
|
||||
|
||||
# 3. Swipe
|
||||
assert await driver.swipe(100, 200, 300, 400) is True
|
||||
assert driver.action_history[-1]["action"] == "swipe"
|
||||
|
||||
# 4. Swipe direction
|
||||
assert await driver.swipe_direction(SwipeDirection.UP) is True
|
||||
assert driver.action_history[-1]["direction"] == "up"
|
||||
|
||||
# 5. Input text
|
||||
assert await driver.input_text("Hello Artemis") is True
|
||||
assert driver.action_history[-1]["text"] == "Hello Artemis"
|
||||
|
||||
# 6. Press key
|
||||
assert await driver.press_key(KeyCode.HOME) is True
|
||||
assert driver.action_history[-1]["key"] == "home"
|
||||
|
||||
# 7. App lifecycle
|
||||
assert await driver.launch_app("com.android.settings") is True
|
||||
assert await driver.get_current_package() == "com.android.settings"
|
||||
assert await driver.stop_app("com.android.settings") is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_android_driver_with_mock_adb():
|
||||
"""Verify AndroidAdbDriver formats shell commands properly."""
|
||||
mock_adb_client = MagicMock()
|
||||
mock_adb_device = MagicMock()
|
||||
mock_adb_client.device.return_value = mock_adb_device
|
||||
|
||||
driver = AndroidAdbDriver(
|
||||
device_id="emulator-5554",
|
||||
adb_client=mock_adb_client,
|
||||
width=1080,
|
||||
height=2400,
|
||||
)
|
||||
|
||||
# Test tap
|
||||
await driver.tap(540, 1200)
|
||||
mock_adb_device.shell.assert_called_with("input tap 540 1200")
|
||||
|
||||
# Test multi-tap
|
||||
await driver.tap(540, 1200, times=3, delay_ms=100)
|
||||
expected_multi = "input tap 540 1200 && sleep 0.100 && input tap 540 1200 && sleep 0.100 && input tap 540 1200"
|
||||
mock_adb_device.shell.assert_called_with(expected_multi)
|
||||
|
||||
# Test press key
|
||||
await driver.press_key(KeyCode.BACK)
|
||||
mock_adb_device.shell.assert_called_with("input keyevent 4")
|
||||
@@ -0,0 +1,67 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
from google.genai.errors import APIError
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
"""Initializes the Gemini client. Requires GEMINI_API_KEY in environment."""
|
||||
try:
|
||||
return genai.Client()
|
||||
except Exception as e:
|
||||
pytest.skip(f"Could not initialize genai.Client: {e}")
|
||||
|
||||
|
||||
def test_gemini_role_tool_fails(client):
|
||||
"""Confirm that using role='tool' results in a 400 APIError from Gemini."""
|
||||
contents = [
|
||||
types.Content(
|
||||
role="tool",
|
||||
parts=[types.Part.from_text(text="This is a test payload for tool execution.")],
|
||||
)
|
||||
]
|
||||
|
||||
with pytest.raises(APIError) as exc_info:
|
||||
client.models.generate_content(
|
||||
model="gemini-3.6-flash",
|
||||
contents=contents,
|
||||
)
|
||||
|
||||
# Assert it failed specifically due to the invalid role string
|
||||
error_msg = str(exc_info.value)
|
||||
assert "Invalid role string" in error_msg or "400" in error_msg
|
||||
|
||||
|
||||
def test_gemini_role_user_succeeds(client):
|
||||
"""Confirm that using role='user' (which is the correct fix) succeeds."""
|
||||
contents = [
|
||||
types.Content(
|
||||
role="user",
|
||||
parts=[
|
||||
types.Part.from_text(text="Acknowledge this message with 'ok' and nothing else.")
|
||||
],
|
||||
)
|
||||
]
|
||||
|
||||
# This should not raise any exceptions
|
||||
response = client.models.generate_content(
|
||||
model="gemini-3.6-flash",
|
||||
contents=contents,
|
||||
)
|
||||
|
||||
assert response.text is not None
|
||||
@@ -0,0 +1,211 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for Dynamic Model Router."""
|
||||
|
||||
from artemis.llm.router import ModelEndpoint, ModelProvider, ModelRouter
|
||||
|
||||
|
||||
def test_model_router_default_routes():
|
||||
"""Verify default role routing returns expected models."""
|
||||
router = ModelRouter()
|
||||
|
||||
# Planner route
|
||||
planner_ep = router.get_endpoint("planner")
|
||||
assert planner_ep.provider == ModelProvider.GEMINI
|
||||
assert "pro" in planner_ep.model_name
|
||||
|
||||
# Operator route
|
||||
operator_ep = router.get_endpoint("operator")
|
||||
assert operator_ep.provider == ModelProvider.GEMINI
|
||||
assert "flash" in operator_ep.model_name
|
||||
|
||||
|
||||
def test_model_router_custom_route_override():
|
||||
"""Verify dynamic role route override."""
|
||||
router = ModelRouter()
|
||||
custom_ep = ModelEndpoint(
|
||||
provider=ModelProvider.OPENAI,
|
||||
model_name="gpt-4o",
|
||||
api_base="https://custom.api.endpoint/v1",
|
||||
)
|
||||
router.set_role_route("operator", custom_ep)
|
||||
|
||||
ep = router.get_endpoint("operator")
|
||||
assert ep.provider == ModelProvider.OPENAI
|
||||
assert ep.model_name == "gpt-4o"
|
||||
assert ep.api_base == "https://custom.api.endpoint/v1"
|
||||
|
||||
|
||||
def test_model_router_fallbacks():
|
||||
"""Verify fallback endpoint configuration."""
|
||||
router = ModelRouter()
|
||||
fallbacks = [
|
||||
ModelEndpoint(provider=ModelProvider.GEMINI, model_name="gemini-2.5-flash"),
|
||||
ModelEndpoint(provider=ModelProvider.OPENAI, model_name="gpt-4o-mini"),
|
||||
]
|
||||
router.set_role_route("custom_worker", "gemini-2.5-pro", fallbacks=fallbacks)
|
||||
fb = router.get_fallbacks("custom_worker")
|
||||
assert len(fb) == 2
|
||||
assert fb[0].model_name == "gemini-2.5-flash"
|
||||
assert fb[1].model_name == "gpt-4o-mini"
|
||||
|
||||
|
||||
def test_model_router_anthropic_instantiation():
|
||||
"""Verify Anthropic Claude model instantiation with thinking budget."""
|
||||
router = ModelRouter()
|
||||
anth_ep = ModelEndpoint(
|
||||
provider=ModelProvider.ANTHROPIC,
|
||||
model_name="claude-3-7-sonnet-20250219",
|
||||
api_key="sk-ant-test-key",
|
||||
reasoning_effort="high",
|
||||
)
|
||||
model = router.instantiate_model(anth_ep)
|
||||
assert model.model == "claude-3-7-sonnet-20250219"
|
||||
assert getattr(model, "thinking", None) == {"type": "enabled", "budget_tokens": 32768}
|
||||
|
||||
|
||||
def test_model_router_openai_instantiation():
|
||||
"""Verify OpenAI model instantiation with reasoning effort."""
|
||||
router = ModelRouter()
|
||||
oai_ep = ModelEndpoint(
|
||||
provider=ModelProvider.OPENAI,
|
||||
model_name="o3-mini",
|
||||
api_key="sk-test-key",
|
||||
reasoning_effort="medium",
|
||||
)
|
||||
model = router.instantiate_model(oai_ep)
|
||||
assert model.model_name == "o3-mini"
|
||||
assert getattr(model, "reasoning_effort", None) == "medium"
|
||||
|
||||
|
||||
def test_robust_chat_model_wrapper_grounding_google():
|
||||
"""Verify RobustChatModelWrapper auto-injects google_search and server-side tool config for Gemini."""
|
||||
from artemis.services.llm import RobustChatModelWrapper
|
||||
from langchain_core.tools import tool
|
||||
|
||||
@tool
|
||||
def dummy_tool(query: str) -> str:
|
||||
"""A test tool."""
|
||||
return query
|
||||
|
||||
class MockGeminiModel:
|
||||
def __init__(self):
|
||||
self.bound_tools = []
|
||||
self.bound_kwargs = {}
|
||||
|
||||
def bind_tools(self, tools, *args, **kwargs):
|
||||
self.bound_tools = tools
|
||||
self.bound_kwargs = kwargs
|
||||
return self
|
||||
|
||||
# Case 1: enable_grounding=True automatically attaches google_search and tool_config
|
||||
ep_google = ModelEndpoint(
|
||||
provider=ModelProvider.GOOGLE,
|
||||
model_name="gemini-2.5-flash",
|
||||
enable_grounding=True,
|
||||
)
|
||||
wrapper = RobustChatModelWrapper(MockGeminiModel(), endpoint=ep_google)
|
||||
bound = wrapper.bind_tools([dummy_tool])
|
||||
|
||||
assert any(isinstance(t, dict) and "google_search" in t for t in bound.base_model.bound_tools)
|
||||
assert bound.base_model.bound_kwargs.get("tool_config") == {
|
||||
"include_server_side_tool_invocations": True
|
||||
}
|
||||
|
||||
# Case 2: Explicit google_search in tools automatically ensures tool_config
|
||||
ep_google_no_ground = ModelEndpoint(
|
||||
provider=ModelProvider.GOOGLE,
|
||||
model_name="gemini-2.5-flash",
|
||||
enable_grounding=False,
|
||||
)
|
||||
wrapper_explicit = RobustChatModelWrapper(MockGeminiModel(), endpoint=ep_google_no_ground)
|
||||
bound_explicit = wrapper_explicit.bind_tools([dummy_tool, {"google_search": {}}])
|
||||
|
||||
assert any(
|
||||
isinstance(t, dict) and "google_search" in t for t in bound_explicit.base_model.bound_tools
|
||||
)
|
||||
assert bound_explicit.base_model.bound_kwargs.get("tool_config") == {
|
||||
"include_server_side_tool_invocations": True
|
||||
}
|
||||
|
||||
|
||||
def test_robust_chat_model_wrapper_grounding_non_google():
|
||||
"""Verify RobustChatModelWrapper ignores grounding and strips google_search for non-Google providers."""
|
||||
from artemis.services.llm import RobustChatModelWrapper
|
||||
from langchain_core.tools import tool
|
||||
|
||||
@tool
|
||||
def dummy_tool(query: str) -> str:
|
||||
"""A test tool."""
|
||||
return query
|
||||
|
||||
class MockOpenAIModel:
|
||||
def __init__(self):
|
||||
self.bound_tools = []
|
||||
self.bound_kwargs = {}
|
||||
|
||||
def bind_tools(self, tools, *args, **kwargs):
|
||||
self.bound_tools = tools
|
||||
self.bound_kwargs = kwargs
|
||||
return self
|
||||
|
||||
ep_openai = ModelEndpoint(
|
||||
provider=ModelProvider.OPENAI,
|
||||
model_name="gpt-4o",
|
||||
enable_grounding=True,
|
||||
)
|
||||
wrapper = RobustChatModelWrapper(MockOpenAIModel(), endpoint=ep_openai)
|
||||
bound = wrapper.bind_tools([dummy_tool, {"google_search": {}}])
|
||||
|
||||
# Should ignore grounding and strip {"google_search": {}}
|
||||
assert len(bound.base_model.bound_tools) == 1
|
||||
assert bound.base_model.bound_tools[0].name == "dummy_tool"
|
||||
assert "tool_config" not in bound.base_model.bound_kwargs
|
||||
|
||||
|
||||
def test_chat_google_generative_ai_process_tool_config_patch():
|
||||
"""Verify patched ChatGoogleGenerativeAI preserves include_server_side_tool_invocations even with tool_choice."""
|
||||
from artemis.llm.router import ModelFactory
|
||||
from langchain_google_genai import ChatGoogleGenerativeAI
|
||||
from langchain_core.tools import tool
|
||||
|
||||
@tool
|
||||
def dummy_tool(query: str) -> str:
|
||||
"""A test tool."""
|
||||
return query
|
||||
|
||||
# Trigger patching via Google model creation
|
||||
ep = ModelEndpoint(provider=ModelProvider.GOOGLE, model_name="gemini-2.5-flash", api_key="fake")
|
||||
model = ModelFactory.create_model(ep)
|
||||
assert isinstance(model, ChatGoogleGenerativeAI)
|
||||
|
||||
formatted_tools = model._format_tools([dummy_tool, {"google_search": {}}], None)
|
||||
|
||||
# 1. With tool_choice="auto"
|
||||
config_auto = model._process_tool_config("auto", None, formatted_tools)
|
||||
assert config_auto is not None
|
||||
assert config_auto.include_server_side_tool_invocations is True
|
||||
|
||||
# 2. With tool_choice="any"
|
||||
config_any = model._process_tool_config("any", None, formatted_tools)
|
||||
assert config_any is not None
|
||||
assert config_any.include_server_side_tool_invocations is True
|
||||
|
||||
# 3. With explicit tool_config
|
||||
config_explicit = model._process_tool_config(
|
||||
None, {"include_server_side_tool_invocations": True}, formatted_tools
|
||||
)
|
||||
assert config_explicit is not None
|
||||
assert config_explicit.include_server_side_tool_invocations is True
|
||||
@@ -0,0 +1,46 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from artemis.agents.planner.planner import run_async_planner_validation
|
||||
|
||||
|
||||
class DummyDataEngine:
|
||||
def get_agent_friendly_steps(self):
|
||||
return []
|
||||
|
||||
|
||||
class DummyCtx:
|
||||
data_engine = DummyDataEngine()
|
||||
project_dir = str(Path(__file__).resolve().parent.parent.parent)
|
||||
|
||||
|
||||
async def test():
|
||||
content_before = "- [/] Click the button\n- [ ] Complete task"
|
||||
content_after = (
|
||||
"- [/] Click the button\n- [ ] Complete task\n- [ ] New subgoal appended at bottom"
|
||||
)
|
||||
res = await run_async_planner_validation(
|
||||
ctx=DummyCtx(),
|
||||
initial_goal="Do the task",
|
||||
content_before=content_before,
|
||||
content_after=content_after,
|
||||
operator_raw_thinking="I need to add a new subgoal",
|
||||
operator_native_thinking="Thinking...",
|
||||
)
|
||||
print(res)
|
||||
|
||||
|
||||
asyncio.run(test())
|
||||
@@ -0,0 +1,56 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for OperatorPromptBuilder and LoopDetector."""
|
||||
|
||||
from artemis.agents.operator.loop_detector import LoopDetector
|
||||
from artemis.agents.operator.prompt_builder import OperatorPromptBuilder
|
||||
|
||||
|
||||
def test_operator_prompt_builder():
|
||||
"""Verify prompt rendering formats variables correctly."""
|
||||
system_msg = OperatorPromptBuilder.build_system_message()
|
||||
assert "ARTEMIS Operator" in system_msg
|
||||
assert "click" in system_msg
|
||||
|
||||
human_msg = OperatorPromptBuilder.build_human_message(
|
||||
goal="Open Settings",
|
||||
sub_goal="Tap Network & internet",
|
||||
current_turn=2,
|
||||
history="1. Tapped Settings icon",
|
||||
)
|
||||
assert "Open Settings" in human_msg
|
||||
assert "Tap Network & internet" in human_msg
|
||||
assert "Current Turn: 2" in human_msg
|
||||
|
||||
|
||||
def test_loop_detector():
|
||||
"""Verify loop detector flags repeated action sequences."""
|
||||
detector = LoopDetector(repetition_threshold=3)
|
||||
assert not detector.is_loop_detected()
|
||||
|
||||
# Record distinct actions
|
||||
detector.record_action("click", {"target": [100, 200]})
|
||||
detector.record_action("swipe", {"action": "up"})
|
||||
assert not detector.is_loop_detected()
|
||||
|
||||
# Record 3 identical actions
|
||||
detector.record_action("click", {"target": [500, 500]})
|
||||
detector.record_action("click", {"target": [500, 500]})
|
||||
detector.record_action("click", {"target": [500, 500]})
|
||||
assert detector.is_loop_detected()
|
||||
|
||||
# Reset
|
||||
detector.reset()
|
||||
assert not detector.is_loop_detected()
|
||||
@@ -0,0 +1,66 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for the official Python SDK (ArtemisClient, Task, TaskResult)."""
|
||||
|
||||
import pytest
|
||||
from artemis import ArtemisClient, Task, TaskResult
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_artemis_client_run_mock():
|
||||
"""Verify ArtemisClient runs a task through the execution pipeline."""
|
||||
client = ArtemisClient(device_id="mock-test-dev", default_profile="pro")
|
||||
assert client.device_id == "mock-test-dev"
|
||||
assert client.default_profile == "pro"
|
||||
|
||||
result = await client.run(goal="Open Settings and enable Dark Theme")
|
||||
assert isinstance(result, TaskResult)
|
||||
assert result.status in ("completed", "running", "success")
|
||||
assert result.trace_id.startswith("trace_") or len(result.trace_id) > 0
|
||||
|
||||
|
||||
def test_task_model_validation():
|
||||
"""Verify Task validation and default arguments."""
|
||||
task = Task(goal="Search for flight prices", profile="flash", device_id="emulator-5554")
|
||||
assert task.goal == "Search for flight prices"
|
||||
assert task.profile == "flash"
|
||||
assert task.device_id == "emulator-5554"
|
||||
assert task.max_turns == 30
|
||||
assert task.locked_package is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_artemis_client_stream_run():
|
||||
"""Verify ArtemisClient.stream_run yields real-time StreamEvents."""
|
||||
from artemis import StreamEvent, StreamEventType
|
||||
|
||||
client = ArtemisClient(device_id="mock-stream-dev", default_profile="flash")
|
||||
collected_events: list[StreamEvent] = []
|
||||
|
||||
def on_event(ev: StreamEvent):
|
||||
collected_events.append(ev)
|
||||
|
||||
events: list[StreamEvent] = []
|
||||
async for event in client.stream_run(
|
||||
goal="Test streaming UI action flow",
|
||||
profile="flash",
|
||||
callbacks=[on_event],
|
||||
):
|
||||
events.append(event)
|
||||
|
||||
assert len(events) >= 2
|
||||
assert events[0].event_type == StreamEventType.STATUS
|
||||
assert events[0].payload.get("status") == "starting"
|
||||
assert len(collected_events) == len(events)
|
||||
@@ -0,0 +1,55 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for Telemetry & Span Lifecycle Tracing."""
|
||||
|
||||
import pytest
|
||||
from pathlib import Path
|
||||
from artemis.telemetry.models import SpanType
|
||||
from artemis.telemetry.tracer import TelemetryTracer
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_telemetry_span_lifecycle(tmp_path: Path):
|
||||
"""Verify TelemetryTracer properly records and persists nested spans."""
|
||||
tracer = TelemetryTracer(trace_id="test-session-001", traces_dir=tmp_path)
|
||||
|
||||
async with tracer.start_span("parent_agent_step", span_type=SpanType.AGENT) as parent_span:
|
||||
assert parent_span.status == "running"
|
||||
assert parent_span.name == "parent_agent_step"
|
||||
|
||||
async with tracer.start_span(
|
||||
"child_device_action", span_type=SpanType.DEVICE_ACTION, payload={"action": "click"}
|
||||
) as child_span:
|
||||
assert child_span.parent_id == parent_span.span_id
|
||||
assert child_span.payload["action"] == "click"
|
||||
|
||||
# Spans should be persisted in JSONL
|
||||
assert len(tracer.spans) == 2
|
||||
child_recorded = tracer.spans[0]
|
||||
parent_recorded = tracer.spans[1]
|
||||
|
||||
assert child_recorded.name == "child_device_action"
|
||||
assert child_recorded.status == "success"
|
||||
assert child_recorded.duration_ms is not None
|
||||
|
||||
assert parent_recorded.name == "parent_agent_step"
|
||||
assert parent_recorded.status == "success"
|
||||
|
||||
# File existence
|
||||
log_file = tmp_path / "telemetry_spans.jsonl"
|
||||
assert log_file.exists()
|
||||
content = log_file.read_text(encoding="utf-8")
|
||||
assert "child_device_action" in content
|
||||
assert "parent_agent_step" in content
|
||||
@@ -0,0 +1,70 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for Universal Tool Protocol and Actions."""
|
||||
|
||||
import pytest
|
||||
from artemis.drivers.mock.mock_driver import MockDeviceDriver
|
||||
from artemis.tools.base import ToolRegistry
|
||||
import artemis.tools.actions.device_actions # noqa: F401 - Register action tools
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tool_registry_registration():
|
||||
"""Verify standard device actions are registered in ToolRegistry."""
|
||||
click_tool = ToolRegistry.get("click")
|
||||
assert click_tool is not None
|
||||
assert click_tool.name == "click"
|
||||
assert click_tool.category == "action"
|
||||
|
||||
swipe_tool = ToolRegistry.get("swipe")
|
||||
assert swipe_tool is not None
|
||||
assert swipe_tool.name == "swipe"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tool_execution_with_driver():
|
||||
"""Verify tool execution correctly dispatches to MockDeviceDriver."""
|
||||
driver = MockDeviceDriver(width=1000, height=2000)
|
||||
|
||||
# Click normalized [500, 500] -> pixel (500, 1000)
|
||||
result = await ToolRegistry.execute("click", {"target": [500, 500]}, driver=driver, ctx=None)
|
||||
assert "Click" in result or "Clicked" in result
|
||||
assert driver.action_history[-1]["action"] == "tap"
|
||||
assert driver.action_history[-1]["x"] == 500
|
||||
assert driver.action_history[-1]["y"] == 1000
|
||||
|
||||
# Swipe cardinal "up"
|
||||
result = await ToolRegistry.execute("swipe", {"action": "up"}, driver=driver, ctx=None)
|
||||
assert "Swipe" in result or "Swiped" in result
|
||||
assert driver.action_history[-1]["action"] == "swipe_direction"
|
||||
|
||||
|
||||
def test_tool_to_genai_declaration():
|
||||
"""Verify tools export to Google GenAI FunctionDeclaration properly."""
|
||||
declarations = ToolRegistry.get_genai_declarations(["click", "swipe", "input_text"])
|
||||
assert len(declarations) == 3
|
||||
|
||||
decl_names = [d.name for d in declarations]
|
||||
assert "click" in decl_names
|
||||
assert "swipe" in decl_names
|
||||
assert "input_text" in decl_names
|
||||
|
||||
|
||||
def test_tool_to_langchain_tool():
|
||||
"""Verify tools export to LangChain BaseTool."""
|
||||
lc_tools = ToolRegistry.get_langchain_tools(ctx=None, names=["click", "press_key"])
|
||||
assert len(lc_tools) == 2
|
||||
assert lc_tools[0].name == "click"
|
||||
assert lc_tools[1].name == "press_key"
|
||||
@@ -0,0 +1,279 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for UnifiedMobileController video recording and playback features."""
|
||||
|
||||
import time
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.controllers.unified_controller import UnifiedMobileController
|
||||
from artemis.drivers.mock.mock_driver import MockDeviceDriver
|
||||
from artemis.utils.video import (
|
||||
RecordingSession,
|
||||
get_active_session,
|
||||
remove_active_session,
|
||||
set_active_session,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_ctx(tmp_path):
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.device = MagicMock()
|
||||
ctx.device.device_id = "emulator-5554"
|
||||
ctx.device.mobile_platform = "android"
|
||||
|
||||
# DataEngine mock
|
||||
ctx.data_engine = MagicMock()
|
||||
ctx.data_engine.current_session_id = uuid4()
|
||||
ctx.data_engine.session_start_time = time.time()
|
||||
ctx.data_engine.storage = MagicMock()
|
||||
|
||||
# Mock driver
|
||||
mock_driver = MockDeviceDriver(device_id="emulator-5554")
|
||||
ctx._active_driver = mock_driver
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unified_controller_start_recording(mock_ctx, tmp_path):
|
||||
controller = UnifiedMobileController(mock_ctx)
|
||||
remove_active_session("emulator-5554")
|
||||
|
||||
# Mock scrcpy subprocess
|
||||
mock_proc = MagicMock()
|
||||
mock_proc.returncode = None
|
||||
|
||||
with patch("asyncio.create_subprocess_exec", AsyncMock(return_value=mock_proc)):
|
||||
with patch("asyncio.sleep", AsyncMock()):
|
||||
res = await controller.start_video_recording(output_dir=tmp_path)
|
||||
|
||||
assert res.success is True
|
||||
assert get_active_session("emulator-5554") is not None
|
||||
assert mock_ctx.data_engine.record_video_start.called
|
||||
|
||||
# Calling start again should report already in progress
|
||||
res2 = await controller.start_video_recording(output_dir=tmp_path)
|
||||
assert res2.success is False
|
||||
assert "already in progress" in res2.message
|
||||
|
||||
remove_active_session("emulator-5554")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unified_controller_stop_recording(mock_ctx, tmp_path):
|
||||
controller = UnifiedMobileController(mock_ctx)
|
||||
remove_active_session("emulator-5554")
|
||||
|
||||
# Create dummy recording.mkv
|
||||
mkv_path = tmp_path / "recording.mkv"
|
||||
mkv_path.write_bytes(b"dummy mkv content")
|
||||
|
||||
mock_proc = MagicMock()
|
||||
mock_proc.returncode = None
|
||||
mock_proc.terminate = MagicMock()
|
||||
mock_proc.wait = AsyncMock()
|
||||
|
||||
session = RecordingSession(
|
||||
video_id=uuid4(),
|
||||
device_id="emulator-5554",
|
||||
start_time=time.time() - 10.0,
|
||||
data_engine_start_time=time.time() - 10.0,
|
||||
local_video_path=mkv_path,
|
||||
process=mock_proc,
|
||||
)
|
||||
set_active_session("emulator-5554", session)
|
||||
|
||||
with patch.object(
|
||||
controller,
|
||||
"_convert_mkv_to_mp4",
|
||||
AsyncMock(side_effect=lambda src, dst: dst.write_bytes(b"mp4 content") or True),
|
||||
):
|
||||
res = await controller.stop_video_recording()
|
||||
|
||||
assert res.success is True
|
||||
assert res.video_path is not None
|
||||
assert str(res.video_path).endswith("recording.mp4")
|
||||
assert get_active_session("emulator-5554") is None
|
||||
assert mock_ctx.data_engine.record_video_stop.called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unified_controller_extract_segment_metadata(mock_ctx, tmp_path):
|
||||
controller = UnifiedMobileController(mock_ctx)
|
||||
remove_active_session("emulator-5554")
|
||||
|
||||
mkv_path = tmp_path / "recording.mkv"
|
||||
mkv_path.write_bytes(b"dummy mkv content")
|
||||
|
||||
mock_proc = MagicMock()
|
||||
mock_proc.returncode = None
|
||||
|
||||
session = RecordingSession(
|
||||
video_id=uuid4(),
|
||||
device_id="emulator-5554",
|
||||
start_time=time.time() - 20.0,
|
||||
data_engine_start_time=time.time() - 20.0,
|
||||
local_video_path=mkv_path,
|
||||
process=mock_proc,
|
||||
)
|
||||
set_active_session("emulator-5554", session)
|
||||
|
||||
with patch(
|
||||
"artemis.controllers.unified_controller.trim_video",
|
||||
AsyncMock(side_effect=lambda src, s, e, dst: dst.write_bytes(b"segment mp4") or True),
|
||||
):
|
||||
res = await controller.extract_segment_metadata(start_time=2.0, end_time=8.0)
|
||||
|
||||
assert res.success is True
|
||||
assert res.video_path is not None
|
||||
assert res.video_path.exists()
|
||||
# Ensure session is still active (not stopped by extraction!)
|
||||
assert get_active_session("emulator-5554") is not None
|
||||
remove_active_session("emulator-5554")
|
||||
|
||||
|
||||
def test_data_engine_video_lifecycle(tmp_path):
|
||||
from artemis.data_engine.engine import DataEngine
|
||||
from artemis.context import ArtemisContext
|
||||
|
||||
mock_c = MagicMock(spec=ArtemisContext)
|
||||
mock_c.execution_setup = MagicMock(traces_path=str(tmp_path / "traces"))
|
||||
mock_c.device = None
|
||||
engine = DataEngine(mock_c)
|
||||
session_id = engine.start_session(goal="Test Video Engine")
|
||||
|
||||
vid = uuid4()
|
||||
vpath = tmp_path / "recording.mp4"
|
||||
vpath.write_bytes(b"video bytes")
|
||||
|
||||
# Start video recording
|
||||
engine.record_video_start(vid, "device-1", vpath)
|
||||
rec = engine.storage.get_video_recording(vid)
|
||||
assert rec is not None
|
||||
assert rec.device_id == "device-1"
|
||||
|
||||
# Stop video recording
|
||||
engine.record_video_stop(vid, "device-1", vpath, time.time() - 10.0, time.time())
|
||||
sess = engine.storage.get_session(session_id)
|
||||
assert sess.video_filepath == str(vpath)
|
||||
|
||||
# Move video path on finalize
|
||||
new_vpath = tmp_path / "archived" / "recording.mp4"
|
||||
new_vpath.parent.mkdir(parents=True, exist_ok=True)
|
||||
new_vpath.write_bytes(b"video bytes")
|
||||
engine.update_video_path(new_vpath)
|
||||
|
||||
sess_updated = engine.storage.get_session(session_id)
|
||||
assert sess_updated.video_filepath == str(new_vpath)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unified_controller_crash_recovery_and_multi_segment(mock_ctx, tmp_path):
|
||||
"""Verify that if recording is interrupted, segments are auto-recovered and concatenated seamlessly."""
|
||||
controller = UnifiedMobileController(mock_ctx)
|
||||
remove_active_session("emulator-5554")
|
||||
|
||||
seg1 = tmp_path / "recording_0.mkv"
|
||||
seg1.write_bytes(b"segment 0")
|
||||
seg2 = tmp_path / "recording_1.mkv"
|
||||
seg2.write_bytes(b"segment 1")
|
||||
|
||||
mock_proc = MagicMock()
|
||||
mock_proc.returncode = 1 # Simulated crash
|
||||
|
||||
session = RecordingSession(
|
||||
video_id=uuid4(),
|
||||
device_id="emulator-5554",
|
||||
start_time=time.time() - 30.0,
|
||||
data_engine_start_time=time.time() - 30.0,
|
||||
local_video_path=seg2,
|
||||
android_video_segments=[seg1],
|
||||
android_segment_index=1,
|
||||
process=mock_proc,
|
||||
)
|
||||
set_active_session("emulator-5554", session)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.controllers.unified_controller.concatenate_videos",
|
||||
AsyncMock(side_effect=lambda segs, out: out.write_bytes(b"combined mkv") or True),
|
||||
),
|
||||
patch.object(
|
||||
controller,
|
||||
"_convert_mkv_to_mp4",
|
||||
AsyncMock(side_effect=lambda src, dst: dst.write_bytes(b"final mp4") or True),
|
||||
),
|
||||
):
|
||||
res = await controller.stop_video_recording()
|
||||
|
||||
assert res.success is True
|
||||
assert res.video_path is not None
|
||||
assert res.video_path.exists()
|
||||
assert str(res.video_path).endswith("recording.mp4")
|
||||
assert get_active_session("emulator-5554") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unified_controller_timeline_alignment(mock_ctx, tmp_path):
|
||||
"""Verify that relative time offset between DataEngine T0 and video start is precisely calculated."""
|
||||
controller = UnifiedMobileController(mock_ctx)
|
||||
remove_active_session("emulator-5554")
|
||||
|
||||
mkv_path = tmp_path / "recording.mkv"
|
||||
mkv_path.write_bytes(b"dummy mkv content")
|
||||
|
||||
now = time.time()
|
||||
# Assume session started 15s ago, and video started 12s ago (offset = 3.0s)
|
||||
session_start_t0 = now - 15.0
|
||||
video_start_t = now - 12.0
|
||||
|
||||
session = RecordingSession(
|
||||
video_id=uuid4(),
|
||||
device_id="emulator-5554",
|
||||
start_time=video_start_t,
|
||||
data_engine_start_time=session_start_t0,
|
||||
local_video_path=mkv_path,
|
||||
process=MagicMock(returncode=None),
|
||||
)
|
||||
set_active_session("emulator-5554", session)
|
||||
|
||||
trimmed_ranges = []
|
||||
|
||||
async def mock_trim(src, s_time, e_time, dst):
|
||||
trimmed_ranges.append((s_time, e_time))
|
||||
dst.write_bytes(b"trimmed")
|
||||
return True
|
||||
|
||||
with patch(
|
||||
"artemis.controllers.unified_controller.trim_video", AsyncMock(side_effect=mock_trim)
|
||||
):
|
||||
# Agent asks for system time range [5.0s, 10.0s]
|
||||
res = await controller.extract_segment_metadata(start_time=5.0, end_time=10.0)
|
||||
|
||||
assert res.success is True
|
||||
# Since offset = 3.0s, video_start_relative_time should be 5.0 - 3.0 = 2.0s
|
||||
# and video_end_relative_time should be 10.0 - 3.0 = 7.0s
|
||||
assert len(trimmed_ranges) == 1
|
||||
s_time, e_time = trimmed_ranges[0]
|
||||
assert abs(s_time - 2.0) < 0.1
|
||||
assert abs(e_time - 7.0) < 0.1
|
||||
# actual_start_relative_time reported back to agent must align with system T0 (5.0s)
|
||||
assert abs(res.actual_start_relative_time - 5.0) < 0.1
|
||||
|
||||
remove_active_session("emulator-5554")
|
||||
@@ -0,0 +1,36 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
from artemis.utils.notes import update_note_content
|
||||
|
||||
base_dir = "/tmp/artemis_test"
|
||||
os.makedirs(f"{base_dir}/notes", exist_ok=True)
|
||||
task_plan = """- [x] Initial task
|
||||
- [/] Active task
|
||||
- [ ] Future task"""
|
||||
with open(f"{base_dir}/notes/task_plan.md", "w") as f:
|
||||
f.write(task_plan)
|
||||
|
||||
try:
|
||||
res = update_note_content(
|
||||
base_dir,
|
||||
"task_plan",
|
||||
"- [/] Active task",
|
||||
"- [/] Active task\n - [ ] New subgoal",
|
||||
)
|
||||
print("Update successful!")
|
||||
print(open(f"{base_dir}/notes/task_plan.md").read())
|
||||
except Exception as e:
|
||||
print("Error:", e)
|
||||
@@ -0,0 +1,132 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.scratchpad import (
|
||||
AppendNoteArgs,
|
||||
AppendNotePure,
|
||||
AppendNotePureTool,
|
||||
append_note_pure,
|
||||
get_append_note_tool_pure,
|
||||
)
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_ctx(tmp_path):
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.data_engine = MagicMock()
|
||||
ctx.data_engine.base_dir = str(tmp_path)
|
||||
return ctx
|
||||
|
||||
|
||||
def test_append_note_pure_tool_subclass_and_registry():
|
||||
"""Verify AppendNotePureTool is a subclass of ArtemisTool and properly registered."""
|
||||
assert issubclass(AppendNotePureTool, ArtemisTool)
|
||||
assert issubclass(AppendNotePure, ArtemisTool)
|
||||
assert isinstance(append_note_pure, ArtemisTool)
|
||||
assert isinstance(append_note_pure, AppendNotePureTool)
|
||||
|
||||
assert append_note_pure.name == "append_note_pure"
|
||||
assert append_note_pure.category == "memory"
|
||||
assert append_note_pure.args_schema == AppendNoteArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("append_note_pure")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, AppendNotePureTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = append_note_pure.to_genai_declaration()
|
||||
assert declaration.name == "append_note_pure"
|
||||
assert "key" in declaration.parameters.properties
|
||||
assert "content" in declaration.parameters.properties
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_append_note_pure_direct_execution_success(mock_ctx, tmp_path):
|
||||
"""Verify direct execution of AppendNotePureTool.execute appends to note file."""
|
||||
# First append creates note
|
||||
result1 = await append_note_pure.execute(
|
||||
ctx=mock_ctx,
|
||||
key="pure_append",
|
||||
content="First line",
|
||||
)
|
||||
assert result1 == "Successfully appended note to pure_append.md."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "pure_append.md"
|
||||
assert note_path.exists()
|
||||
assert note_path.read_text(encoding="utf-8") == "First line\n"
|
||||
|
||||
# Second append appends line
|
||||
result2 = await append_note_pure.execute(
|
||||
ctx=mock_ctx,
|
||||
key="pure_append",
|
||||
content="Second line",
|
||||
)
|
||||
assert result2 == "Successfully appended note to pure_append.md."
|
||||
assert note_path.read_text(encoding="utf-8") == "First line\nSecond line\n"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_append_note_pure_callable_execution(mock_ctx, tmp_path):
|
||||
"""Verify invoking append_note_pure directly as a callable."""
|
||||
result = await append_note_pure(
|
||||
ctx=mock_ctx,
|
||||
key="callable_pure_append",
|
||||
content="Pure callable append",
|
||||
)
|
||||
assert result == "Successfully appended note to callable_pure_append.md."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "callable_pure_append.md"
|
||||
assert note_path.exists()
|
||||
assert note_path.read_text(encoding="utf-8") == "Pure callable append\n"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_append_note_pure_execution_failure(mock_ctx):
|
||||
"""Verify error handling when append_note_content fails."""
|
||||
with patch(
|
||||
"artemis.tools.scratchpad.append_note_content",
|
||||
side_effect=PermissionError("Permission denied"),
|
||||
):
|
||||
result = await append_note_pure.execute(
|
||||
ctx=mock_ctx,
|
||||
key="fail_append",
|
||||
content="Some content",
|
||||
)
|
||||
assert "Failed to append note fail_append.md: Permission denied" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_append_note_tool_pure_langchain_ainvoke(mock_ctx, tmp_path):
|
||||
"""Verify get_append_note_tool_pure exports a LangChain tool named 'append_note'."""
|
||||
pure_tool = get_append_note_tool_pure(mock_ctx)
|
||||
assert pure_tool.name == "append_note"
|
||||
|
||||
result = await pure_tool.ainvoke(
|
||||
{
|
||||
"key": "lc_pure_append",
|
||||
"content": "Saved via pure append tool",
|
||||
}
|
||||
)
|
||||
assert result == "Successfully appended note to lc_pure_append.md."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "lc_pure_append.md"
|
||||
assert note_path.exists()
|
||||
assert note_path.read_text(encoding="utf-8") == "Saved via pure append tool\n"
|
||||
@@ -0,0 +1,190 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.scratchpad import (
|
||||
AppendNote,
|
||||
AppendNoteArgs,
|
||||
AppendNoteTool,
|
||||
get_append_note_tool,
|
||||
get_append_note_tool_pure,
|
||||
append_note,
|
||||
append_note_wrapper,
|
||||
)
|
||||
from langgraph.types import Command
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_ctx(tmp_path):
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.data_engine = MagicMock()
|
||||
ctx.data_engine.base_dir = str(tmp_path)
|
||||
return ctx
|
||||
|
||||
|
||||
def test_append_note_tool_subclass_and_registry():
|
||||
"""Verify AppendNoteTool is a subclass of ArtemisTool and properly registered."""
|
||||
assert issubclass(AppendNoteTool, ArtemisTool)
|
||||
assert issubclass(AppendNote, ArtemisTool)
|
||||
assert isinstance(append_note, ArtemisTool)
|
||||
assert isinstance(append_note, AppendNoteTool)
|
||||
|
||||
assert append_note.name == "append_note"
|
||||
assert append_note.category == "memory"
|
||||
assert append_note.args_schema == AppendNoteArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("append_note")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, AppendNoteTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = append_note.to_genai_declaration()
|
||||
assert declaration.name == "append_note"
|
||||
assert "key" in declaration.parameters.properties
|
||||
assert "content" in declaration.parameters.properties
|
||||
|
||||
# Wrapper check
|
||||
assert append_note_wrapper is not None
|
||||
assert append_note_wrapper.tool_fn_getter == get_append_note_tool
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_append_note_direct_execution_success(mock_ctx, tmp_path):
|
||||
"""Verify direct execution of AppendNoteTool.execute appends to note file."""
|
||||
# First append creates the note
|
||||
result1 = await append_note.execute(
|
||||
ctx=mock_ctx,
|
||||
key="test_append",
|
||||
content="First line",
|
||||
)
|
||||
assert result1 == "Successfully appended note to test_append.md."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "test_append.md"
|
||||
assert note_path.exists()
|
||||
assert note_path.read_text(encoding="utf-8") == "First line\n"
|
||||
|
||||
# Second append appends with newline
|
||||
result2 = await append_note.execute(
|
||||
ctx=mock_ctx,
|
||||
key="test_append",
|
||||
content="Second line",
|
||||
)
|
||||
assert result2 == "Successfully appended note to test_append.md."
|
||||
assert note_path.read_text(encoding="utf-8") == "First line\nSecond line\n"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_append_note_callable_execution(mock_ctx, tmp_path):
|
||||
"""Verify invoking append_note directly as a callable."""
|
||||
result = await append_note(
|
||||
ctx=mock_ctx,
|
||||
key="callable_append",
|
||||
content="Callable append test",
|
||||
)
|
||||
assert result == "Successfully appended note to callable_append.md."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "callable_append.md"
|
||||
assert note_path.exists()
|
||||
assert note_path.read_text(encoding="utf-8") == "Callable append test\n"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_append_note_with_state_command(mock_ctx, tmp_path):
|
||||
"""Verify AppendNoteTool.execute with state returns Command updating VALIDATOR_MESSAGES_KEY."""
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.current_agent = "operator"
|
||||
mock_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update)
|
||||
|
||||
result = await append_note.execute(
|
||||
ctx=mock_ctx,
|
||||
key="state_append_note",
|
||||
content="Appended with state",
|
||||
state=mock_state,
|
||||
tool_call_id="call_append_999",
|
||||
)
|
||||
|
||||
assert isinstance(result, Command)
|
||||
assert VALIDATOR_MESSAGES_KEY in result.update
|
||||
tool_messages = result.update[VALIDATOR_MESSAGES_KEY]
|
||||
assert len(tool_messages) == 1
|
||||
assert tool_messages[0].tool_call_id == "call_append_999"
|
||||
assert tool_messages[0].content == "Successfully appended note to state_append_note.md."
|
||||
assert tool_messages[0].status == "success"
|
||||
|
||||
mock_state.asanitize_update.assert_called_once()
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "state_append_note.md"
|
||||
assert note_path.exists()
|
||||
assert note_path.read_text(encoding="utf-8") == "Appended with state\n"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_append_note_execution_failure(mock_ctx):
|
||||
"""Verify error handling when appending note fails."""
|
||||
with patch(
|
||||
"artemis.tools.scratchpad.append_note_content",
|
||||
side_effect=PermissionError("Permission denied"),
|
||||
):
|
||||
result = await append_note.execute(
|
||||
ctx=mock_ctx,
|
||||
key="fail_append_note",
|
||||
content="Fail content",
|
||||
)
|
||||
assert "Failed to append note fail_append_note.md: Permission denied" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_append_note_tool_langchain_ainvoke(mock_ctx, tmp_path):
|
||||
"""Verify get_append_note_tool exports a LangChain tool that works with ainvoke."""
|
||||
lc_tool = get_append_note_tool(mock_ctx)
|
||||
assert lc_tool.name == "append_note"
|
||||
|
||||
result = await lc_tool.ainvoke(
|
||||
{
|
||||
"key": "lc_append_note",
|
||||
"content": "Appended via LangChain BaseTool",
|
||||
}
|
||||
)
|
||||
assert result == "Successfully appended note to lc_append_note.md."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "lc_append_note.md"
|
||||
assert note_path.exists()
|
||||
assert note_path.read_text(encoding="utf-8") == "Appended via LangChain BaseTool\n"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_append_note_tool_pure(mock_ctx, tmp_path):
|
||||
"""Verify get_append_note_tool_pure returns a pure LangChain tool."""
|
||||
pure_tool = get_append_note_tool_pure(mock_ctx)
|
||||
assert pure_tool.name == "append_note"
|
||||
|
||||
result = await pure_tool.ainvoke(
|
||||
{
|
||||
"key": "pure_append_note",
|
||||
"content": "Appended via pure tool",
|
||||
}
|
||||
)
|
||||
assert result == "Successfully appended note to pure_append_note.md."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "pure_append_note.md"
|
||||
assert note_path.exists()
|
||||
assert note_path.read_text(encoding="utf-8") == "Appended via pure tool\n"
|
||||
@@ -0,0 +1,158 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.mobile.exec_tools import (
|
||||
Click,
|
||||
ClickArgs,
|
||||
ClickTool,
|
||||
click,
|
||||
click_wrapper,
|
||||
get_click_tool,
|
||||
)
|
||||
from langgraph.types import Command
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_ctx():
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.device = MagicMock()
|
||||
ctx.device.device_width = 1080
|
||||
ctx.device.device_height = 2400
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_driver():
|
||||
driver = MagicMock(spec=BaseDeviceDriver)
|
||||
driver.screen_size = (1080, 2400)
|
||||
driver.tap = AsyncMock(return_value=True)
|
||||
return driver
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_state():
|
||||
state = MagicMock(spec=State)
|
||||
|
||||
async def _mock_asanitize_update(ctx, update, agent):
|
||||
return update
|
||||
|
||||
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
|
||||
return state
|
||||
|
||||
|
||||
def test_click_tool_subclass_and_registry():
|
||||
"""Verify ClickTool is a subclass of ArtemisTool and properly registered."""
|
||||
assert issubclass(ClickTool, ArtemisTool)
|
||||
assert issubclass(Click, ArtemisTool)
|
||||
assert isinstance(click, ArtemisTool)
|
||||
assert isinstance(click, ClickTool)
|
||||
|
||||
assert click.name == "click"
|
||||
assert click.category == "action"
|
||||
assert click.args_schema == ClickArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("click")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, ClickTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = click.to_genai_declaration()
|
||||
assert declaration.name == "click"
|
||||
assert "target" in declaration.parameters.properties
|
||||
|
||||
# Wrapper check
|
||||
assert click_wrapper is not None
|
||||
assert click_wrapper.tool_fn_getter == get_click_tool
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_click_direct_execution_with_driver(mock_driver):
|
||||
"""Verify direct execution of ClickTool with BaseDeviceDriver."""
|
||||
result = await click.execute(driver=mock_driver, target=[500, 500])
|
||||
mock_driver.tap.assert_called_once_with(540, 1200)
|
||||
assert "Clicked at [500, 500] (normalized) successfully." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_click_direct_execution_with_ctx(mock_ctx):
|
||||
"""Verify direct execution of ClickTool with ArtemisContext."""
|
||||
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
|
||||
mock_controller_inst = MagicMock()
|
||||
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
|
||||
mock_controller_cls.return_value = mock_controller_inst
|
||||
|
||||
result = await click.execute(ctx=mock_ctx, target=[250, 750])
|
||||
mock_controller_inst.tap_at.assert_called_once_with(270, 1800)
|
||||
assert "Clicked at [250, 750] (normalized) successfully." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_click_with_state_command(mock_ctx, mock_state):
|
||||
"""Verify ClickTool returns Command when state is provided."""
|
||||
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
|
||||
mock_controller_inst = MagicMock()
|
||||
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
|
||||
mock_controller_cls.return_value = mock_controller_inst
|
||||
|
||||
cmd = await click.execute(
|
||||
ctx=mock_ctx,
|
||||
target=[100, 200],
|
||||
tool_call_id="call_click_1",
|
||||
state=mock_state,
|
||||
)
|
||||
assert isinstance(cmd, Command)
|
||||
assert VALIDATOR_MESSAGES_KEY in cmd.update
|
||||
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
|
||||
assert len(messages) == 1
|
||||
assert messages[0].tool_call_id == "call_click_1"
|
||||
assert messages[0].status == "success"
|
||||
assert "Clicked at [100, 200]" in messages[0].content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_click_callable_execution(mock_driver):
|
||||
"""Verify invoking click directly as a callable."""
|
||||
result = await click(driver=mock_driver, target=[100, 100])
|
||||
assert "Clicked at [100, 100] (normalized) successfully." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_click_invalid_target(mock_ctx):
|
||||
"""Verify error handling on invalid target coordinates."""
|
||||
result = await click.execute(ctx=mock_ctx, target=[])
|
||||
assert "Error during click: Invalid target coordinates" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_click_tool_langchain_ainvoke(mock_ctx):
|
||||
"""Verify get_click_tool exports a LangChain tool that works with ainvoke."""
|
||||
click_tool = get_click_tool(mock_ctx)
|
||||
assert click_tool.name == "click"
|
||||
|
||||
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
|
||||
mock_controller_inst = MagicMock()
|
||||
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
|
||||
mock_controller_cls.return_value = mock_controller_inst
|
||||
|
||||
result = await click_tool.ainvoke({"target": [500, 500]})
|
||||
assert "Clicked at [500, 500] (normalized) successfully." in result
|
||||
@@ -0,0 +1,603 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import asyncio
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.tools.command_tool import (
|
||||
_BACKGROUND_TASKS,
|
||||
_PERSISTENT_ENVIRONMENTS,
|
||||
get_manage_task_tool,
|
||||
get_run_adb_command_tool,
|
||||
get_run_short_adb_command_tool,
|
||||
)
|
||||
import pytest
|
||||
|
||||
|
||||
class MockProcess:
|
||||
def __init__(
|
||||
self,
|
||||
output_bytes: bytes,
|
||||
exit_code: int = 0,
|
||||
pid: int = 12345,
|
||||
delay: float = 0,
|
||||
):
|
||||
self.output_bytes = output_bytes
|
||||
self.exit_code = exit_code
|
||||
self.pid = pid
|
||||
self.returncode = None
|
||||
self._read_offset = 0
|
||||
self.delay = delay
|
||||
|
||||
self.stdin = MagicMock()
|
||||
self.stdin.write = MagicMock()
|
||||
self.stdin.drain = AsyncMock()
|
||||
self.stdin.close = MagicMock()
|
||||
|
||||
self.stdout = MagicMock()
|
||||
self.stdout.readline = AsyncMock(side_effect=self._readline)
|
||||
|
||||
async def _readline(self) -> bytes:
|
||||
if self.delay > 0:
|
||||
await asyncio.sleep(self.delay)
|
||||
self.delay = 0 # Only delay the first read
|
||||
|
||||
if self._read_offset >= len(self.output_bytes):
|
||||
self.returncode = self.exit_code
|
||||
return b""
|
||||
|
||||
idx = self.output_bytes.find(b"\n", self._read_offset)
|
||||
if idx == -1:
|
||||
line = self.output_bytes[self._read_offset :]
|
||||
self._read_offset = len(self.output_bytes)
|
||||
else:
|
||||
line = self.output_bytes[self._read_offset : idx + 1]
|
||||
self._read_offset = idx + 1
|
||||
return line
|
||||
|
||||
async def wait(self) -> int:
|
||||
self.returncode = self.exit_code
|
||||
return self.exit_code
|
||||
|
||||
def terminate(self):
|
||||
self.returncode = -15
|
||||
self.exit_code = -15
|
||||
|
||||
async def communicate(self) -> tuple[bytes, bytes]:
|
||||
lines = []
|
||||
while True:
|
||||
line = await self._readline()
|
||||
if not line:
|
||||
break
|
||||
lines.append(line)
|
||||
self.returncode = self.exit_code
|
||||
return b"".join(lines), b""
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_ctx():
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.data_engine = None
|
||||
ctx.device = MagicMock()
|
||||
ctx.device.device_id = "test_device_1234"
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("asyncio.create_subprocess_exec")
|
||||
async def test_run_command_sync(mock_exec, mock_ctx):
|
||||
# Mock subprocess output
|
||||
mock_process = MockProcess(
|
||||
output_bytes=b"hello world\n===EXIT_CODE===0\n===ENV_START===\n",
|
||||
exit_code=0,
|
||||
)
|
||||
mock_exec.return_value = mock_process
|
||||
|
||||
run_command = get_run_adb_command_tool(mock_ctx)
|
||||
|
||||
result = await run_command.ainvoke(
|
||||
{
|
||||
"CommandLine": "echo 'hello world'",
|
||||
"Cwd": "/data/local/tmp",
|
||||
"RunPersistent": False,
|
||||
"WaitMsBeforeAsync": 500,
|
||||
}
|
||||
)
|
||||
|
||||
assert "completed with exit code 0" in result
|
||||
assert "hello world" in result
|
||||
mock_exec.assert_called_once()
|
||||
# Check that adb was called with device ID and "shell"
|
||||
args, kwargs = mock_exec.call_args
|
||||
assert args[0] == "adb"
|
||||
assert args[1] == "-s"
|
||||
assert args[2] == "test_device_1234"
|
||||
assert args[3] == "shell"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("asyncio.create_subprocess_exec")
|
||||
async def test_run_command_persistent_env(mock_exec, mock_ctx):
|
||||
run_command = get_run_adb_command_tool(mock_ctx)
|
||||
terminal_id = "test_term_android"
|
||||
|
||||
# 1. Set environment variable in persistent terminal
|
||||
mock_process_1 = MockProcess(
|
||||
output_bytes=(b"===EXIT_CODE===0\n===ENV_START===\nARTEMIS_TEST_ENV=coffee\n"),
|
||||
exit_code=0,
|
||||
)
|
||||
mock_exec.return_value = mock_process_1
|
||||
|
||||
res1 = await run_command.ainvoke(
|
||||
{
|
||||
"CommandLine": "export ARTEMIS_TEST_ENV=coffee",
|
||||
"Cwd": "/data/local/tmp",
|
||||
"RunPersistent": True,
|
||||
"RequestedTerminalID": terminal_id,
|
||||
"WaitMsBeforeAsync": 500,
|
||||
}
|
||||
)
|
||||
assert "completed with exit code 0" in res1
|
||||
assert terminal_id in _PERSISTENT_ENVIRONMENTS
|
||||
assert _PERSISTENT_ENVIRONMENTS[terminal_id].get("ARTEMIS_TEST_ENV") == "coffee"
|
||||
|
||||
# 2. Query environment variable in subsequent command on same terminal
|
||||
mock_process_2 = MockProcess(
|
||||
output_bytes=b"coffee\n===EXIT_CODE===0\n===ENV_START===\nARTEMIS_TEST_ENV=coffee\n",
|
||||
exit_code=0,
|
||||
)
|
||||
mock_exec.return_value = mock_process_2
|
||||
|
||||
res2 = await run_command.ainvoke(
|
||||
{
|
||||
"CommandLine": "echo $ARTEMIS_TEST_ENV",
|
||||
"Cwd": "/data/local/tmp",
|
||||
"RunPersistent": True,
|
||||
"RequestedTerminalID": terminal_id,
|
||||
"WaitMsBeforeAsync": 500,
|
||||
}
|
||||
)
|
||||
assert "completed with exit code 0" in res2
|
||||
assert "coffee" in res2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("asyncio.create_subprocess_exec")
|
||||
async def test_run_command_async_and_status(mock_exec, mock_ctx):
|
||||
# Mock a long running command by delaying stdout response
|
||||
mock_process = MockProcess(
|
||||
output_bytes=(b"background task finished\n===EXIT_CODE===0\n===ENV_START===\n"),
|
||||
exit_code=0,
|
||||
delay=1.0, # 1.0s delay will trigger timeout
|
||||
)
|
||||
mock_exec.return_value = mock_process
|
||||
|
||||
run_command = get_run_adb_command_tool(mock_ctx)
|
||||
manage_task = get_manage_task_tool(mock_ctx)
|
||||
|
||||
_BACKGROUND_TASKS.clear()
|
||||
|
||||
res = await run_command.ainvoke(
|
||||
{
|
||||
"CommandLine": "sleep 2 && echo 'background task finished'",
|
||||
"Cwd": "/data/local/tmp",
|
||||
"RunPersistent": False,
|
||||
"WaitMsBeforeAsync": 100, # 100ms wait threshold
|
||||
}
|
||||
)
|
||||
|
||||
assert "sent to the background as a task" in res
|
||||
assert "TaskId: task_" in res
|
||||
|
||||
lines = res.splitlines()
|
||||
task_id_line = [line for line in lines if line.startswith("TaskId: ")][0]
|
||||
task_id = task_id_line.split(": ")[1]
|
||||
|
||||
assert task_id in _BACKGROUND_TASKS
|
||||
|
||||
# Check status (should be running)
|
||||
status_res = await manage_task.ainvoke(
|
||||
{
|
||||
"Action": "status",
|
||||
"TaskId": task_id,
|
||||
}
|
||||
)
|
||||
assert "Status: running" in status_res
|
||||
|
||||
# Wait for the background listener task to finish reading delayed output
|
||||
await asyncio.sleep(1.2)
|
||||
|
||||
# Status should be completed (and task removed from active map)
|
||||
assert task_id not in _BACKGROUND_TASKS
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("asyncio.create_subprocess_exec")
|
||||
async def test_run_command_kill(mock_exec, mock_ctx):
|
||||
mock_process = MockProcess(output_bytes=b"", exit_code=0, delay=5.0)
|
||||
mock_exec.return_value = mock_process
|
||||
|
||||
run_command = get_run_adb_command_tool(mock_ctx)
|
||||
manage_task = get_manage_task_tool(mock_ctx)
|
||||
|
||||
_BACKGROUND_TASKS.clear()
|
||||
|
||||
res = await run_command.ainvoke(
|
||||
{
|
||||
"CommandLine": "sleep 10",
|
||||
"Cwd": "/data/local/tmp",
|
||||
"RunPersistent": False,
|
||||
"WaitMsBeforeAsync": 100,
|
||||
}
|
||||
)
|
||||
|
||||
lines = res.splitlines()
|
||||
task_id_line = [line for line in lines if line.startswith("TaskId: ")][0]
|
||||
task_id = task_id_line.split(": ")[1]
|
||||
|
||||
# Kill the task
|
||||
kill_res = await manage_task.ainvoke(
|
||||
{
|
||||
"Action": "kill",
|
||||
"TaskId": task_id,
|
||||
}
|
||||
)
|
||||
assert f"Task {task_id} successfully terminated" in kill_res
|
||||
assert task_id not in _BACKGROUND_TASKS
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("asyncio.create_subprocess_exec")
|
||||
async def test_run_short_command_success(mock_exec, mock_ctx):
|
||||
mock_process = MockProcess(output_bytes=b"short command success output", exit_code=0)
|
||||
mock_exec.return_value = mock_process
|
||||
|
||||
run_short_command = get_run_short_adb_command_tool(mock_ctx)
|
||||
|
||||
result = await run_short_command.ainvoke(
|
||||
{
|
||||
"CommandLine": "echo 'short command'",
|
||||
}
|
||||
)
|
||||
|
||||
assert "completed with exit code 0" in result
|
||||
assert "short command success output" in result
|
||||
mock_exec.assert_called_once()
|
||||
args, kwargs = mock_exec.call_args
|
||||
assert args[0] == "adb"
|
||||
assert args[1] == "-s"
|
||||
assert args[2] == "test_device_1234"
|
||||
assert args[3] == "shell"
|
||||
# Verify it cd'd to /data/local/tmp by default
|
||||
assert "cd /data/local/tmp" in args[4]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("asyncio.create_subprocess_exec")
|
||||
async def test_run_short_command_timeout(mock_exec, mock_ctx):
|
||||
mock_process = MockProcess(output_bytes=b"some output", exit_code=0, delay=5.0)
|
||||
mock_exec.return_value = mock_process
|
||||
|
||||
run_short_command = get_run_short_adb_command_tool(mock_ctx)
|
||||
|
||||
result = await run_short_command.ainvoke(
|
||||
{
|
||||
"CommandLine": "sleep 5",
|
||||
}
|
||||
)
|
||||
|
||||
assert "Error: Command timed out after 3 seconds." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("asyncio.create_subprocess_exec")
|
||||
async def test_run_command_sync_long_output(mock_exec, mock_ctx):
|
||||
# Generate long output (> 500 lines)
|
||||
long_output = "\n".join(f"Output line {i}" for i in range(1, 600))
|
||||
mock_process = MockProcess(
|
||||
output_bytes=long_output.encode() + b"\n===EXIT_CODE===0\n===ENV_START===\n",
|
||||
exit_code=0,
|
||||
)
|
||||
mock_exec.return_value = mock_process
|
||||
|
||||
run_command = get_run_adb_command_tool(mock_ctx)
|
||||
result = await run_command.ainvoke(
|
||||
{
|
||||
"CommandLine": "generate_long_output",
|
||||
"Cwd": "/data/local/tmp",
|
||||
"RunPersistent": False,
|
||||
"WaitMsBeforeAsync": 500,
|
||||
}
|
||||
)
|
||||
|
||||
assert "ADB command completed with exit code 0." in result
|
||||
assert "has been truncated" in result
|
||||
assert "Output line 599" in result
|
||||
assert "Output line 1" not in result # since it was truncated to last 200 lines
|
||||
assert "analyze_task_output" in result
|
||||
assert "TaskId: task_sync_" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.agents.log_analyzer.output_analyzer.get_llm")
|
||||
async def test_analyze_task_output_tool(mock_get_llm, mock_ctx):
|
||||
# Set up mock LLM
|
||||
mock_llm = MagicMock()
|
||||
mock_response = MagicMock()
|
||||
mock_response.content = "This is the analysis result: everything looks good."
|
||||
mock_response.tool_calls = []
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
mock_llm.ainvoke = AsyncMock(return_value=mock_response)
|
||||
mock_get_llm.return_value = mock_llm
|
||||
|
||||
# Set up finished tasks logs cache manually
|
||||
from artemis.tools.command_tool import (
|
||||
_FINISHED_TASKS_LOGS,
|
||||
_register_finished_task,
|
||||
get_analyze_task_output_tool,
|
||||
)
|
||||
|
||||
task_id = "test_task_long_output"
|
||||
_register_finished_task(
|
||||
task_id=task_id,
|
||||
command="cat long_log.txt",
|
||||
cwd="/data/local/tmp",
|
||||
terminal_id=None,
|
||||
status="completed",
|
||||
exit_code=0,
|
||||
output="line1\nline2\n" + "\n".join(f"line{i}" for i in range(3, 50)),
|
||||
)
|
||||
|
||||
analyze_tool = get_analyze_task_output_tool(mock_ctx)
|
||||
result = await analyze_tool.ainvoke({"TaskId": task_id, "Query": "Summarize the log"})
|
||||
|
||||
assert "Analysis result for Task test_task_long_output:" in result
|
||||
assert "This is the analysis result: everything looks good." in result
|
||||
mock_get_llm.assert_called_once_with(ctx=mock_ctx, name="output_analyzer")
|
||||
|
||||
# Clean up registry
|
||||
if task_id in _FINISHED_TASKS_LOGS:
|
||||
del _FINISHED_TASKS_LOGS[task_id]
|
||||
|
||||
|
||||
def test_run_adb_command_tool_subclass_and_registry():
|
||||
"""Verify RunAdbCommandTool is a subclass of ArtemisTool and properly registered."""
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.command_tool import (
|
||||
RunAdbCommand,
|
||||
RunAdbCommandArgs,
|
||||
RunAdbCommandTool,
|
||||
run_adb_command,
|
||||
)
|
||||
|
||||
assert issubclass(RunAdbCommandTool, ArtemisTool)
|
||||
assert issubclass(RunAdbCommand, ArtemisTool)
|
||||
assert isinstance(run_adb_command, ArtemisTool)
|
||||
assert isinstance(run_adb_command, RunAdbCommandTool)
|
||||
|
||||
assert run_adb_command.name == "run_adb_command"
|
||||
assert run_adb_command.category == "system"
|
||||
assert run_adb_command.args_schema == RunAdbCommandArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("run_adb_command")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, RunAdbCommandTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = run_adb_command.to_genai_declaration()
|
||||
assert declaration.name == "run_adb_command"
|
||||
assert "CommandLine" in declaration.parameters.properties
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_adb_command_tool_mock_driver_execute():
|
||||
"""Verify RunAdbCommandTool.execute dispatches to MockDeviceDriver when ctx is None."""
|
||||
from artemis.drivers.mock.mock_driver import MockDeviceDriver
|
||||
from artemis.tools.command_tool import RunAdbCommandTool
|
||||
|
||||
driver = MockDeviceDriver(width=1080, height=2400)
|
||||
tool_inst = RunAdbCommandTool()
|
||||
|
||||
result = await tool_inst.execute(
|
||||
driver=driver,
|
||||
ctx=None,
|
||||
CommandLine="pm list packages",
|
||||
)
|
||||
assert "mock_output: pm list packages" in result
|
||||
assert "pm list packages" in driver.action_history[-1]["command"]
|
||||
|
||||
|
||||
def test_manage_task_tool_subclass_and_registry():
|
||||
"""Verify ManageTaskTool is a subclass of ArtemisTool and properly registered."""
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.command_tool import (
|
||||
ManageTask,
|
||||
ManageTaskArgs,
|
||||
ManageTaskTool,
|
||||
manage_task,
|
||||
)
|
||||
|
||||
assert issubclass(ManageTaskTool, ArtemisTool)
|
||||
assert issubclass(ManageTask, ArtemisTool)
|
||||
assert isinstance(manage_task, ArtemisTool)
|
||||
assert isinstance(manage_task, ManageTaskTool)
|
||||
|
||||
assert manage_task.name == "manage_task"
|
||||
assert manage_task.category == "system"
|
||||
assert manage_task.args_schema == ManageTaskArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("manage_task")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, ManageTaskTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = manage_task.to_genai_declaration()
|
||||
assert declaration.name == "manage_task"
|
||||
assert "Action" in declaration.parameters.properties
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manage_task_tool_direct_execute(mock_ctx):
|
||||
"""Verify direct ManageTaskTool.execute execution."""
|
||||
from artemis.tools.command_tool import (
|
||||
_BACKGROUND_TASKS,
|
||||
BackgroundTask,
|
||||
manage_task,
|
||||
)
|
||||
|
||||
_BACKGROUND_TASKS.clear()
|
||||
list_res = await manage_task.execute(ctx=mock_ctx, Action="list")
|
||||
assert "No active background tasks" in list_res
|
||||
|
||||
# Create dummy task
|
||||
dummy_proc = MagicMock()
|
||||
bg_task = BackgroundTask(
|
||||
task_id="dummy_task_99",
|
||||
command="sleep 100",
|
||||
process=dummy_proc,
|
||||
terminal_id="term_1",
|
||||
cwd="/data/local/tmp",
|
||||
)
|
||||
_BACKGROUND_TASKS["dummy_task_99"] = bg_task
|
||||
|
||||
list_res2 = await manage_task.execute(ctx=mock_ctx, Action="list")
|
||||
assert "dummy_task_99" in list_res2
|
||||
assert "sleep 100" in list_res2
|
||||
|
||||
# Status check
|
||||
status_res = await manage_task.execute(ctx=mock_ctx, Action="status", TaskId="dummy_task_99")
|
||||
assert "Status: running" in status_res
|
||||
assert "dummy_task_99" in status_res
|
||||
|
||||
_BACKGROUND_TASKS.clear()
|
||||
|
||||
|
||||
def test_analyze_task_output_tool_subclass_and_registry():
|
||||
"""Verify AnalyzeTaskOutputTool is a subclass of ArtemisTool and properly registered."""
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.command_tool import (
|
||||
AnalyzeTaskOutput,
|
||||
AnalyzeTaskOutputArgs,
|
||||
AnalyzeTaskOutputTool,
|
||||
analyze_task_output,
|
||||
)
|
||||
|
||||
assert issubclass(AnalyzeTaskOutputTool, ArtemisTool)
|
||||
assert issubclass(AnalyzeTaskOutput, ArtemisTool)
|
||||
assert isinstance(analyze_task_output, ArtemisTool)
|
||||
assert isinstance(analyze_task_output, AnalyzeTaskOutputTool)
|
||||
|
||||
assert analyze_task_output.name == "analyze_task_output"
|
||||
assert analyze_task_output.category == "system"
|
||||
assert analyze_task_output.args_schema == AnalyzeTaskOutputArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("analyze_task_output")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, AnalyzeTaskOutputTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = analyze_task_output.to_genai_declaration()
|
||||
assert declaration.name == "analyze_task_output"
|
||||
assert "TaskId" in declaration.parameters.properties
|
||||
assert "Query" in declaration.parameters.properties
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.agents.log_analyzer.output_analyzer.get_llm")
|
||||
async def test_analyze_task_output_tool_direct_execute(mock_get_llm, mock_ctx):
|
||||
"""Verify direct AnalyzeTaskOutputTool.execute execution."""
|
||||
from artemis.tools.command_tool import (
|
||||
_FINISHED_TASKS_LOGS,
|
||||
_register_finished_task,
|
||||
analyze_task_output,
|
||||
)
|
||||
|
||||
mock_llm = MagicMock()
|
||||
mock_response = MagicMock()
|
||||
mock_response.content = "Summary of task output."
|
||||
mock_response.tool_calls = []
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
mock_llm.ainvoke = AsyncMock(return_value=mock_response)
|
||||
mock_get_llm.return_value = mock_llm
|
||||
|
||||
task_id = "test_direct_analyze"
|
||||
_register_finished_task(
|
||||
task_id=task_id,
|
||||
command="cat file.txt",
|
||||
cwd="/data/local/tmp",
|
||||
terminal_id=None,
|
||||
status="completed",
|
||||
exit_code=0,
|
||||
output="Sample log content",
|
||||
)
|
||||
|
||||
result = await analyze_task_output.execute(
|
||||
ctx=mock_ctx, TaskId=task_id, Query="What is the content?"
|
||||
)
|
||||
assert "Analysis result for Task test_direct_analyze:" in result
|
||||
assert "Summary of task output." in result
|
||||
|
||||
if task_id in _FINISHED_TASKS_LOGS:
|
||||
del _FINISHED_TASKS_LOGS[task_id]
|
||||
|
||||
|
||||
def test_run_short_adb_command_tool_subclass_and_registry():
|
||||
"""Verify RunShortAdbCommandTool is a subclass of ArtemisTool and properly registered."""
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.command_tool import (
|
||||
RunShortAdbCommand,
|
||||
RunShortAdbCommandArgs,
|
||||
RunShortAdbCommandTool,
|
||||
run_short_adb_command,
|
||||
)
|
||||
|
||||
assert issubclass(RunShortAdbCommandTool, ArtemisTool)
|
||||
assert issubclass(RunShortAdbCommand, ArtemisTool)
|
||||
assert isinstance(run_short_adb_command, ArtemisTool)
|
||||
assert isinstance(run_short_adb_command, RunShortAdbCommandTool)
|
||||
|
||||
assert run_short_adb_command.name == "run_short_adb_command"
|
||||
assert run_short_adb_command.category == "system"
|
||||
assert run_short_adb_command.args_schema == RunShortAdbCommandArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("run_short_adb_command")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, RunShortAdbCommandTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = run_short_adb_command.to_genai_declaration()
|
||||
assert declaration.name == "run_short_adb_command"
|
||||
assert "CommandLine" in declaration.parameters.properties
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_short_adb_command_tool_mock_driver_execute():
|
||||
"""Verify RunShortAdbCommandTool.execute dispatches to MockDeviceDriver when ctx is None."""
|
||||
from artemis.drivers.mock.mock_driver import MockDeviceDriver
|
||||
from artemis.tools.command_tool import RunShortAdbCommandTool
|
||||
|
||||
driver = MockDeviceDriver(width=1080, height=2400)
|
||||
tool_inst = RunShortAdbCommandTool()
|
||||
|
||||
result = await tool_inst.execute(
|
||||
driver=driver,
|
||||
ctx=None,
|
||||
CommandLine="getprop ro.build.version.release",
|
||||
)
|
||||
assert "mock_output: getprop ro.build.version.release" in result
|
||||
assert "getprop ro.build.version.release" in driver.action_history[-1]["command"]
|
||||
@@ -0,0 +1,222 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.tools.committee_tool import get_ask_committee_tool
|
||||
|
||||
|
||||
class TestCommitteeTool(unittest.IsolatedAsyncioTestCase):
|
||||
async def test_ask_committee_tool_with_directive(self):
|
||||
# Create a temp directory for testing
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
|
||||
try:
|
||||
notes_dir = Path(temp_dir) / "notes"
|
||||
notes_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Create a mock screenshot file
|
||||
mock_screenshot_path = Path(temp_dir) / "mock_screenshot.jpg"
|
||||
mock_screenshot_path.write_bytes(b"fake image bytes")
|
||||
|
||||
# Create a mock failed plans history file
|
||||
mock_failed_plans_history_path = notes_dir / "failed_plans_history.md"
|
||||
mock_failed_plans_history_path.write_text(
|
||||
"## Failed Plan Attempt #1\n### Plan:\n- [ ] Step 1\n###"
|
||||
" Failure Reason:\nTimeout waiting for element",
|
||||
encoding="utf-8",
|
||||
)
|
||||
# Mock context and state
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.device = MagicMock()
|
||||
mock_ctx.device.device_width = 1080
|
||||
mock_ctx.device.device_height = 2400
|
||||
mock_ctx.data_engine = MagicMock()
|
||||
mock_ctx.data_engine.base_dir = temp_dir
|
||||
mock_ctx.data_engine.get_agent_friendly_steps.return_value = [
|
||||
{"pre_image_name": "test_image", "step_number": 1}
|
||||
]
|
||||
|
||||
mock_state = MagicMock()
|
||||
mock_state.initial_goal = "Test Goal"
|
||||
mock_state.messages = []
|
||||
mock_state.latest_ui_hierarchy = None
|
||||
mock_state.latest_screenshot = str(mock_screenshot_path)
|
||||
mock_state.focused_app_info = None
|
||||
mock_state.device_date = None
|
||||
mock_state.structured_decisions = None
|
||||
mock_state.complete_subgoals_by_ids = []
|
||||
mock_state.validator_messages = []
|
||||
mock_state.operator_replan_reason = "Mock Immediate Replan Reason"
|
||||
|
||||
# Mock LLM responses for the members
|
||||
mock_llm_pl = MagicMock()
|
||||
mock_llm_pl.bind_tools.return_value = mock_llm_pl
|
||||
|
||||
m1 = MagicMock(content="Op: I think we should tap.")
|
||||
m1.tool_calls = []
|
||||
m2 = MagicMock(content="Op: I agree with Exp.")
|
||||
m2.tool_calls = []
|
||||
m3 = MagicMock(content="Op: Final conclusion: Tap at [50, 50]")
|
||||
m3.tool_calls = []
|
||||
|
||||
mock_llm_pl.ainvoke = AsyncMock(side_effect=[m1, m2, m3])
|
||||
|
||||
mock_llm_diag = MagicMock()
|
||||
mock_llm_diag.bind_tools.return_value = mock_llm_diag
|
||||
m_diag = MagicMock(content="Diag: I see an error.")
|
||||
m_diag.tool_calls = []
|
||||
mock_llm_diag.ainvoke = AsyncMock(return_value=m_diag)
|
||||
|
||||
# Mock History Analyzer to call a tool first
|
||||
mock_llm_hist = MagicMock()
|
||||
mock_llm_hist.bind_tools.return_value = mock_llm_hist
|
||||
|
||||
mock_response_tool_call = MagicMock()
|
||||
mock_response_tool_call.tool_calls = [
|
||||
{
|
||||
"name": "get_step_details",
|
||||
"args": {"start_step": 1, "end_step": 1},
|
||||
"id": "call_1",
|
||||
}
|
||||
]
|
||||
mock_response_tool_call.content = ""
|
||||
|
||||
mock_response_final = MagicMock()
|
||||
mock_response_final.tool_calls = []
|
||||
mock_response_final.content = "Hist: I found step 1 details."
|
||||
|
||||
m_hist_agree = MagicMock(content="Hist: Still agree.")
|
||||
m_hist_agree.tool_calls = []
|
||||
|
||||
mock_llm_hist.ainvoke = AsyncMock(
|
||||
side_effect=[
|
||||
mock_response_tool_call,
|
||||
mock_response_final,
|
||||
# For second round
|
||||
m_hist_agree,
|
||||
]
|
||||
)
|
||||
|
||||
def mock_get_llm(ctx, name, temperature=None):
|
||||
if name == "planner_avatar":
|
||||
return mock_llm_pl
|
||||
elif name == "diagnoser_expert":
|
||||
return mock_llm_diag
|
||||
elif name == "history_analyzer_expert":
|
||||
return mock_llm_hist
|
||||
return MagicMock()
|
||||
|
||||
# Patch get_llm and the actual tool functions to avoid real calls
|
||||
with (
|
||||
patch(
|
||||
"artemis.tools.committee_tool.get_llm",
|
||||
side_effect=mock_get_llm,
|
||||
),
|
||||
patch(
|
||||
"artemis.tools.committee_tool.trace_langchain_tool",
|
||||
side_effect=lambda t, ctx: t,
|
||||
),
|
||||
):
|
||||
tool = get_ask_committee_tool(mock_ctx)
|
||||
|
||||
# Invoke tool with avatar_directive
|
||||
from artemis.data_engine.trace import CURRENT_TRACE_ID
|
||||
|
||||
token = CURRENT_TRACE_ID.set("test_trace_id")
|
||||
try:
|
||||
result = await tool.ainvoke(
|
||||
{
|
||||
"avatar_directive": "Advocate for paying",
|
||||
"state": mock_state,
|
||||
}
|
||||
)
|
||||
finally:
|
||||
CURRENT_TRACE_ID.reset(token)
|
||||
|
||||
# Verify result
|
||||
self.assertIn("Final conclusion: Tap at [50, 50]", result)
|
||||
|
||||
# Verify file was created and contains content
|
||||
notes_dir = Path(temp_dir) / "notes"
|
||||
blackboard_files = list(notes_dir.glob("blackboard_*.md"))
|
||||
self.assertTrue(len(blackboard_files) > 0, "No blackboard file created")
|
||||
blackboard_path = blackboard_files[0]
|
||||
content = blackboard_path.read_text(encoding="utf-8")
|
||||
self.assertIn("Advocate for paying", content) # Check directive is in blackboard
|
||||
self.assertIn("Hist: I found step 1 details.", content)
|
||||
self.assertIn(
|
||||
"Timeout waiting for element", content
|
||||
) # Check failed history is in blackboard
|
||||
self.assertIn("Test Goal", content) # Check initial goal is in blackboard
|
||||
self.assertIn(
|
||||
"Mock Immediate Replan Reason", content
|
||||
) # Check current replan reason is in blackboard
|
||||
|
||||
finally:
|
||||
# Clean up temp directory
|
||||
shutil.rmtree(temp_dir)
|
||||
|
||||
def test_ask_committee_tool_subclass_and_registry(self):
|
||||
"""Verify AskCommitteeTool is a subclass of ArtemisTool and properly registered."""
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.committee_tool import (
|
||||
AskCommittee,
|
||||
AskCommitteeArgs,
|
||||
AskCommitteeTool,
|
||||
AskCommitteeToolAlias,
|
||||
ask_committee,
|
||||
ask_committee_wrapper,
|
||||
get_ask_committee_tool,
|
||||
)
|
||||
|
||||
self.assertTrue(issubclass(AskCommitteeTool, ArtemisTool))
|
||||
self.assertTrue(issubclass(AskCommittee, ArtemisTool))
|
||||
self.assertTrue(issubclass(AskCommitteeToolAlias, ArtemisTool))
|
||||
self.assertIsInstance(ask_committee, ArtemisTool)
|
||||
self.assertIsInstance(ask_committee, AskCommitteeTool)
|
||||
|
||||
self.assertEqual(ask_committee.name, "ask_committee")
|
||||
self.assertEqual(ask_committee.category, "custom")
|
||||
self.assertEqual(ask_committee.args_schema, AskCommitteeArgs)
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("ask_committee")
|
||||
self.assertIsNotNone(reg_tool)
|
||||
self.assertIsInstance(reg_tool, AskCommitteeTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = ask_committee.to_genai_declaration()
|
||||
self.assertEqual(declaration.name, "ask_committee")
|
||||
self.assertIn("avatar_directive", declaration.parameters.properties)
|
||||
|
||||
# Wrapper check
|
||||
self.assertIsNotNone(ask_committee_wrapper)
|
||||
self.assertEqual(ask_committee_wrapper.tool_fn_getter, get_ask_committee_tool)
|
||||
|
||||
async def test_ask_committee_no_ctx(self):
|
||||
"""Verify executing without ctx returns an error message."""
|
||||
from artemis.tools.committee_tool import ask_committee
|
||||
|
||||
result = await ask_committee.execute(avatar_directive="Test")
|
||||
self.assertEqual(result, "Error: ArtemisContext is required for ask_committee.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,375 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import json
|
||||
from unittest.mock import AsyncMock, MagicMock, mock_open, patch
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.explorer_tool import (
|
||||
AskExplorer,
|
||||
AskExplorerArgs,
|
||||
AskExplorerTool,
|
||||
AskExplorerToolAlias,
|
||||
ask_explorer,
|
||||
ask_explorer_wrapper,
|
||||
get_ask_explorer_tool,
|
||||
)
|
||||
import pytest
|
||||
|
||||
|
||||
def test_ask_explorer_tool_subclass_and_registry():
|
||||
"""Verify AskExplorerTool is a subclass of ArtemisTool and properly registered."""
|
||||
assert issubclass(AskExplorerTool, ArtemisTool)
|
||||
assert issubclass(AskExplorer, ArtemisTool)
|
||||
assert issubclass(AskExplorerToolAlias, ArtemisTool)
|
||||
assert isinstance(ask_explorer, ArtemisTool)
|
||||
assert isinstance(ask_explorer, AskExplorerTool)
|
||||
|
||||
assert ask_explorer.name == "ask_explorer"
|
||||
assert ask_explorer.category == "explorer"
|
||||
assert ask_explorer.args_schema == AskExplorerArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("ask_explorer")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, AskExplorerTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = ask_explorer.to_genai_declaration()
|
||||
assert declaration.name == "ask_explorer"
|
||||
assert "query" in declaration.parameters.properties
|
||||
|
||||
# Wrapper check
|
||||
assert ask_explorer_wrapper is not None
|
||||
assert ask_explorer_wrapper.tool_fn_getter == get_ask_explorer_tool
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ask_explorer_multiple_candidates():
|
||||
# Mock Context and State
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = None
|
||||
mock_ctx.agent_config = None
|
||||
mock_ctx.llm_config = None
|
||||
mock_ctx.execution_setup = None
|
||||
mock_ctx.device = MagicMock()
|
||||
mock_ctx.device.device_width = 1080
|
||||
mock_ctx.device.device_height = 2400
|
||||
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.latest_screenshot = "/tmp/test.jpg"
|
||||
mock_state.indexed_points = [[100, 200]] # Pre-existing point
|
||||
mock_state.indexed_elements = [
|
||||
{
|
||||
"index": 1,
|
||||
"center": [100, 200],
|
||||
"text": "Pre-existing",
|
||||
"bounds": None,
|
||||
"class": "Test",
|
||||
"resource_id": None,
|
||||
"is_ocr": False,
|
||||
}
|
||||
]
|
||||
|
||||
# Mock Explorer Agent Outcome with multiple candidates and fallback message
|
||||
explorer_outcome = {
|
||||
"candidates": [
|
||||
{
|
||||
"label": "S1",
|
||||
"coords": [500, 500],
|
||||
"description": "First Button",
|
||||
},
|
||||
{
|
||||
"label": "S2",
|
||||
"coords": [250, 750],
|
||||
"description": "Second Button",
|
||||
},
|
||||
],
|
||||
"fallback_message": "Please choose carefully.",
|
||||
}
|
||||
explorer_output_str = json.dumps(explorer_outcome)
|
||||
|
||||
# We need to patch the Explorer class inside the tool
|
||||
mock_explorer_instance = MagicMock()
|
||||
mock_explorer_instance.run = AsyncMock(return_value=explorer_output_str)
|
||||
|
||||
with patch(
|
||||
"artemis.tools.explorer_tool.Explorer",
|
||||
return_value=mock_explorer_instance,
|
||||
):
|
||||
# Retrieve the tool
|
||||
ask_explorer_tool = get_ask_explorer_tool(mock_ctx, version="pro")
|
||||
|
||||
result = await ask_explorer_tool.ainvoke(
|
||||
{
|
||||
"query": "Find buttons",
|
||||
"context_feedback": "Attempt 1",
|
||||
"state": mock_state,
|
||||
}
|
||||
)
|
||||
|
||||
assert mock_state.indexed_points == [
|
||||
[100, 200],
|
||||
[540, 1200],
|
||||
[270, 1800],
|
||||
]
|
||||
|
||||
assert len(mock_state.indexed_elements) == 3
|
||||
assert mock_state.indexed_elements[1]["index"] == 2
|
||||
assert mock_state.indexed_elements[1]["center"] == [540, 1200]
|
||||
assert mock_state.indexed_elements[1]["text"] == "First Button"
|
||||
assert mock_state.indexed_elements[2]["index"] == 3
|
||||
assert mock_state.indexed_elements[2]["center"] == [270, 1800]
|
||||
assert mock_state.indexed_elements[2]["text"] == "Second Button"
|
||||
|
||||
assert isinstance(result, str)
|
||||
assert "Explorer successfully located the following candidate(s):" in result
|
||||
assert "- [2] 'First Button' at coordinate [500, 500]" in result
|
||||
assert "- [3] 'Second Button' at coordinate [250, 750]" in result
|
||||
assert (
|
||||
"You can click/act on them directly by calling perform_action with"
|
||||
" their respective index." in result
|
||||
)
|
||||
assert "Additional Notes from Explorer: Please choose carefully." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ask_explorer_only_fallback_message():
|
||||
# Mock Context and State
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = None
|
||||
mock_ctx.agent_config = None
|
||||
mock_ctx.llm_config = None
|
||||
mock_ctx.execution_setup = None
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.latest_screenshot = "/tmp/test.jpg"
|
||||
mock_state.indexed_points = []
|
||||
mock_state.indexed_elements = []
|
||||
|
||||
explorer_outcome = {
|
||||
"candidates": [],
|
||||
"fallback_message": ("Element not found because it is covered by a keyboard."),
|
||||
}
|
||||
explorer_output_str = json.dumps(explorer_outcome)
|
||||
|
||||
mock_explorer_instance = MagicMock()
|
||||
mock_explorer_instance.run = AsyncMock(return_value=explorer_output_str)
|
||||
|
||||
with patch(
|
||||
"artemis.tools.explorer_tool.Explorer",
|
||||
return_value=mock_explorer_instance,
|
||||
):
|
||||
ask_explorer_tool = get_ask_explorer_tool(mock_ctx, version="pro")
|
||||
|
||||
result = await ask_explorer_tool.ainvoke(
|
||||
{
|
||||
"query": "Find hidden button",
|
||||
"context_feedback": "",
|
||||
"state": mock_state,
|
||||
}
|
||||
)
|
||||
|
||||
assert mock_state.indexed_points == []
|
||||
assert mock_state.indexed_elements == []
|
||||
|
||||
assert (
|
||||
"Explorer could not locate the element. Message: Element not found"
|
||||
" because it is covered by a keyboard." in result
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ask_explorer_multimodal_success():
|
||||
# Mock Context and State
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = None
|
||||
mock_ctx.agent_config = None
|
||||
mock_ctx.llm_config = None
|
||||
mock_ctx.execution_setup = None
|
||||
mock_ctx.device = MagicMock()
|
||||
mock_ctx.device.device_width = 1080
|
||||
mock_ctx.device.device_height = 2400
|
||||
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.latest_screenshot = "/tmp/test.jpg"
|
||||
mock_state.indexed_points = [[100, 200]]
|
||||
mock_state.indexed_elements = [
|
||||
{
|
||||
"index": 1,
|
||||
"center": [100, 200],
|
||||
"text": "Pre-existing",
|
||||
"bounds": None,
|
||||
"class": "Test",
|
||||
"resource_id": None,
|
||||
"is_ocr": False,
|
||||
}
|
||||
]
|
||||
|
||||
explorer_outcome = {
|
||||
"candidates": [
|
||||
{
|
||||
"label": "S1",
|
||||
"coords": [500, 500],
|
||||
"description": "First Button",
|
||||
},
|
||||
],
|
||||
"fallback_message": "",
|
||||
}
|
||||
explorer_output_str = json.dumps(explorer_outcome)
|
||||
|
||||
mock_explorer_instance = MagicMock()
|
||||
mock_explorer_instance.run = AsyncMock(return_value=explorer_output_str)
|
||||
|
||||
m_open = mock_open(read_data=b"fake_annotated_image_bytes")
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.tools.explorer_tool.Explorer",
|
||||
return_value=mock_explorer_instance,
|
||||
),
|
||||
patch("artemis.tools.explorer_tool.draw_dots") as mock_draw_dots,
|
||||
patch("builtins.open", m_open),
|
||||
patch("pathlib.Path.mkdir"),
|
||||
):
|
||||
ask_explorer_tool = get_ask_explorer_tool(mock_ctx, version="pro")
|
||||
|
||||
result = await ask_explorer_tool.ainvoke(
|
||||
{
|
||||
"query": "Find buttons",
|
||||
"context_feedback": "",
|
||||
"state": mock_state,
|
||||
}
|
||||
)
|
||||
|
||||
assert isinstance(result, list)
|
||||
assert len(result) == 2
|
||||
assert result[0]["type"] == "text"
|
||||
assert "Explorer successfully located" in result[0]["text"]
|
||||
assert result[1]["type"] == "image_url"
|
||||
assert result[1]["image_url"]["url"].startswith("data:image/jpeg;base64,")
|
||||
|
||||
mock_draw_dots.assert_called_once()
|
||||
points_arg = mock_draw_dots.call_args[0][1]
|
||||
assert points_arg == [[100, 200], [540, 1200]]
|
||||
labels_arg = mock_draw_dots.call_args[0][2]
|
||||
assert labels_arg == ["1", "2"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ask_explorer_direct_execution():
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = None
|
||||
mock_ctx.agent_config = None
|
||||
mock_ctx.llm_config = None
|
||||
mock_ctx.execution_setup = None
|
||||
mock_ctx.device = None
|
||||
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.latest_screenshot = "/tmp/test.jpg"
|
||||
mock_state.indexed_points = []
|
||||
mock_state.indexed_elements = []
|
||||
|
||||
mock_explorer_instance = MagicMock()
|
||||
mock_explorer_instance.run = AsyncMock(
|
||||
return_value=json.dumps(
|
||||
{
|
||||
"candidates": [],
|
||||
"fallback_message": "Direct execution not found.",
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
with patch(
|
||||
"artemis.tools.explorer_tool.Explorer",
|
||||
return_value=mock_explorer_instance,
|
||||
):
|
||||
result = await ask_explorer.execute(
|
||||
ctx=mock_ctx, state=mock_state, query="Find icon", version="pro"
|
||||
)
|
||||
assert "Direct execution not found." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ask_explorer_callable_execution():
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = None
|
||||
mock_ctx.agent_config = None
|
||||
mock_ctx.llm_config = None
|
||||
mock_ctx.execution_setup = None
|
||||
mock_ctx.device = None
|
||||
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.latest_screenshot = "/tmp/test.jpg"
|
||||
mock_state.indexed_points = []
|
||||
mock_state.indexed_elements = []
|
||||
|
||||
mock_explorer_instance = MagicMock()
|
||||
mock_explorer_instance.run = AsyncMock(
|
||||
return_value=json.dumps(
|
||||
{
|
||||
"candidates": [],
|
||||
"fallback_message": "Callable test message.",
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
with patch(
|
||||
"artemis.tools.explorer_tool.Explorer",
|
||||
return_value=mock_explorer_instance,
|
||||
):
|
||||
result = await ask_explorer(
|
||||
ctx=mock_ctx, state=mock_state, query="Search bar", version="pro"
|
||||
)
|
||||
assert "Callable test message." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ask_explorer_flash_mode():
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = None
|
||||
mock_ctx.agent_config = None
|
||||
mock_ctx.llm_config = None
|
||||
mock_ctx.execution_setup = None
|
||||
mock_ctx.device = MagicMock()
|
||||
mock_ctx.device.device_width = 1080
|
||||
mock_ctx.device.device_height = 2400
|
||||
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.latest_screenshot = "/tmp/test.jpg"
|
||||
mock_state.indexed_points = []
|
||||
mock_state.indexed_elements = []
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.tools.explorer_tool._run_object_detection",
|
||||
new_callable=AsyncMock,
|
||||
return_value={
|
||||
"detected": [
|
||||
{"label": "Icon", "point": [500, 500]},
|
||||
],
|
||||
"failed": [],
|
||||
},
|
||||
),
|
||||
patch(
|
||||
"builtins.open",
|
||||
mock_open(read_data=json.dumps({"templates": [], "instructions": ""})),
|
||||
),
|
||||
):
|
||||
result = await ask_explorer.execute(
|
||||
ctx=mock_ctx, state=mock_state, query="Icon", version="flash"
|
||||
)
|
||||
assert "Explorer successfully located" in result
|
||||
assert "Icon" in result
|
||||
@@ -0,0 +1,99 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.image_processor_tool import get_ask_image_processor_tool
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ask_vision_coder_success():
|
||||
# Mock Context and State
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = None
|
||||
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.latest_screenshot = "/tmp/test_screen.jpg"
|
||||
|
||||
# Mock VisionCoder Agent Outcome
|
||||
vision_coder_outcome = {
|
||||
"new_image_path": "/tmp/coder_output.jpg",
|
||||
"summary": "Found red button at pixel coords (100, 200)",
|
||||
}
|
||||
|
||||
# We need to patch the ImageProcessor class inside the tool
|
||||
mock_coder_instance = MagicMock()
|
||||
mock_coder_instance.run = AsyncMock(return_value=vision_coder_outcome)
|
||||
|
||||
with patch(
|
||||
"artemis.agents.image_processor.image_processor.ImageProcessor",
|
||||
return_value=mock_coder_instance,
|
||||
):
|
||||
# Retrieve the tool
|
||||
ask_vision_coder_tool = get_ask_image_processor_tool(mock_ctx)
|
||||
|
||||
# Invoke the tool
|
||||
result = await ask_vision_coder_tool.ainvoke(
|
||||
{
|
||||
"instruction": "Find the red button",
|
||||
"target_image_id": "img_0",
|
||||
"state": mock_state,
|
||||
}
|
||||
)
|
||||
|
||||
# Verify the agent was called with correct arguments
|
||||
mock_coder_instance.run.assert_called_once_with(
|
||||
"Find the red button", "/tmp/test_screen.jpg"
|
||||
)
|
||||
|
||||
# Verify the output matches
|
||||
assert result == vision_coder_outcome
|
||||
|
||||
|
||||
def test_ask_image_processor_tool_subclass_and_registry():
|
||||
"""Verify AskImageProcessorTool is an ArtemisTool subclass and registered in ToolRegistry."""
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.image_processor_tool import (
|
||||
AskImageProcessor,
|
||||
AskImageProcessorTool,
|
||||
AskVisionCoderArgs,
|
||||
AskVisionCoderTool,
|
||||
ImageProcessorTool,
|
||||
ask_image_processor,
|
||||
)
|
||||
|
||||
assert issubclass(AskImageProcessorTool, ArtemisTool)
|
||||
assert issubclass(AskImageProcessor, ArtemisTool)
|
||||
assert issubclass(AskVisionCoderTool, ArtemisTool)
|
||||
assert issubclass(ImageProcessorTool, ArtemisTool)
|
||||
assert isinstance(ask_image_processor, ArtemisTool)
|
||||
assert isinstance(ask_image_processor, AskImageProcessorTool)
|
||||
|
||||
assert ask_image_processor.name == "ask_image_processor"
|
||||
assert ask_image_processor.category == "custom"
|
||||
assert ask_image_processor.args_schema == AskVisionCoderArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("ask_image_processor")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, AskImageProcessorTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = ask_image_processor.to_genai_declaration()
|
||||
assert declaration.name == "ask_image_processor"
|
||||
assert "instruction" in declaration.parameters.properties
|
||||
assert "target_image_id" in declaration.parameters.properties
|
||||
@@ -0,0 +1,182 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.mobile.exec_tools import (
|
||||
InputText,
|
||||
InputTextArgs,
|
||||
InputTextTool,
|
||||
get_input_text_tool,
|
||||
input_text,
|
||||
input_text_wrapper,
|
||||
)
|
||||
from langgraph.types import Command
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_ctx():
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.device = MagicMock()
|
||||
ctx.device.device_width = 1080
|
||||
ctx.device.device_height = 2400
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_driver():
|
||||
driver = MagicMock(spec=BaseDeviceDriver)
|
||||
driver.screen_size = (1080, 2400)
|
||||
driver.tap = AsyncMock(return_value=True)
|
||||
driver.input_text = AsyncMock(return_value=True)
|
||||
return driver
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_state():
|
||||
state = MagicMock(spec=State)
|
||||
|
||||
async def _mock_asanitize_update(ctx, update, agent):
|
||||
return update
|
||||
|
||||
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
|
||||
return state
|
||||
|
||||
|
||||
def test_input_text_tool_subclass_and_registry():
|
||||
"""Verify InputTextTool is a subclass of ArtemisTool and properly registered."""
|
||||
assert issubclass(InputTextTool, ArtemisTool)
|
||||
assert issubclass(InputText, ArtemisTool)
|
||||
assert isinstance(input_text, ArtemisTool)
|
||||
assert isinstance(input_text, InputTextTool)
|
||||
|
||||
assert input_text.name == "input_text"
|
||||
assert input_text.category == "action"
|
||||
assert input_text.args_schema == InputTextArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("input_text")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, InputTextTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = input_text.to_genai_declaration()
|
||||
assert declaration.name == "input_text"
|
||||
assert "text" in declaration.parameters.properties
|
||||
assert "target" in declaration.parameters.properties
|
||||
assert "clear_exist" in declaration.parameters.properties
|
||||
|
||||
# Wrapper check
|
||||
assert input_text_wrapper is not None
|
||||
assert input_text_wrapper.tool_fn_getter == get_input_text_tool
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_input_text_direct_execution_with_driver(mock_driver):
|
||||
"""Verify direct execution of InputTextTool with BaseDeviceDriver."""
|
||||
result = await input_text.execute(
|
||||
driver=mock_driver, text="hello world", target=[500, 500], clear_exist=True
|
||||
)
|
||||
mock_driver.tap.assert_called_once_with(540, 1200)
|
||||
mock_driver.input_text.assert_called_once_with("hello world", clear_existing=True)
|
||||
assert "Typed 'hello world' successfully." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_input_text_direct_execution_with_ctx(mock_ctx):
|
||||
"""Verify direct execution of InputTextTool with ArtemisContext."""
|
||||
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
|
||||
mock_controller_inst = MagicMock()
|
||||
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
|
||||
mock_controller_inst.erase_text = AsyncMock(return_value=True)
|
||||
mock_controller_inst.type_text = AsyncMock(return_value=True)
|
||||
mock_controller_cls.return_value = mock_controller_inst
|
||||
|
||||
result = await input_text.execute(
|
||||
ctx=mock_ctx, text="test input", target=[250, 750], clear_exist=True
|
||||
)
|
||||
mock_controller_inst.tap_at.assert_called_once_with(270, 1800)
|
||||
mock_controller_inst.erase_text.assert_called_once()
|
||||
mock_controller_inst.type_text.assert_called_once_with("test input", clear_existing=False)
|
||||
assert "Typed 'test input' successfully." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_input_text_with_state_command(mock_ctx, mock_state):
|
||||
"""Verify InputTextTool returns Command when state is provided."""
|
||||
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
|
||||
mock_controller_inst = MagicMock()
|
||||
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
|
||||
mock_controller_inst.erase_text = AsyncMock(return_value=True)
|
||||
mock_controller_inst.type_text = AsyncMock(return_value=True)
|
||||
mock_controller_cls.return_value = mock_controller_inst
|
||||
|
||||
cmd = await input_text.execute(
|
||||
ctx=mock_ctx,
|
||||
text="query",
|
||||
target=[100, 200],
|
||||
tool_call_id="call_it_1",
|
||||
state=mock_state,
|
||||
)
|
||||
assert isinstance(cmd, Command)
|
||||
assert VALIDATOR_MESSAGES_KEY in cmd.update
|
||||
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
|
||||
assert len(messages) == 1
|
||||
assert messages[0].tool_call_id == "call_it_1"
|
||||
assert messages[0].status == "success"
|
||||
assert "Typed 'query'" in messages[0].content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_input_text_callable_execution(mock_driver):
|
||||
"""Verify invoking input_text directly as a callable."""
|
||||
result = await input_text(driver=mock_driver, text="call text", target=[100, 100])
|
||||
assert "Typed 'call text' successfully." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_input_text_invalid_target(mock_ctx):
|
||||
"""Verify error handling on invalid target coordinates."""
|
||||
result = await input_text.execute(ctx=mock_ctx, text="bad", target=[])
|
||||
assert "Error during input text: Invalid target coordinates" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_input_text_tool_langchain_ainvoke(mock_ctx):
|
||||
"""Verify get_input_text_tool exports a LangChain tool that works with ainvoke."""
|
||||
it_tool = get_input_text_tool(mock_ctx)
|
||||
assert it_tool.name == "input_text"
|
||||
|
||||
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
|
||||
mock_controller_inst = MagicMock()
|
||||
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
|
||||
mock_controller_inst.press_key = AsyncMock(return_value=True)
|
||||
mock_controller_inst.type_text = AsyncMock(return_value=True)
|
||||
mock_controller_cls.return_value = mock_controller_inst
|
||||
|
||||
result = await it_tool.ainvoke(
|
||||
{
|
||||
"text": "abc",
|
||||
"target": [500, 500],
|
||||
"clear_exist": False,
|
||||
}
|
||||
)
|
||||
mock_controller_inst.press_key.assert_called_once_with("123")
|
||||
assert "Typed 'abc' successfully." in result
|
||||
@@ -0,0 +1,92 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
from artemis.agents.hopper.hopper import HopperOutput
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.tools.mobile.launch_app import find_package
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context():
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.package_cache = {}
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.tools.mobile.launch_app.list_packages_async")
|
||||
@patch("artemis.tools.mobile.launch_app.hopper")
|
||||
async def test_find_package_cache_hit(mock_hopper, mock_list_packages, mock_context):
|
||||
mock_list_packages.return_value = "com.example.app1\ncom.example.app2"
|
||||
|
||||
# First call: not in cache, calls hopper
|
||||
mock_hopper.return_value = HopperOutput(
|
||||
found=True, output="com.example.app1", reason="Found it"
|
||||
)
|
||||
|
||||
pkg1 = await find_package(mock_context, "App 1")
|
||||
assert pkg1 == "com.example.app1"
|
||||
assert mock_hopper.call_count == 1
|
||||
assert mock_context.package_cache["App 1"] == "com.example.app1"
|
||||
|
||||
# Second call: cache hit, does not call hopper or list_packages again
|
||||
mock_hopper.reset_mock()
|
||||
mock_list_packages.reset_mock()
|
||||
|
||||
pkg2 = await find_package(mock_context, "App 1")
|
||||
assert pkg2 == "com.example.app1"
|
||||
mock_hopper.assert_not_called()
|
||||
mock_list_packages.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.tools.mobile.launch_app.list_packages_async")
|
||||
@patch("artemis.tools.mobile.launch_app.hopper")
|
||||
async def test_find_package_not_found_caches_none(mock_hopper, mock_list_packages, mock_context):
|
||||
mock_list_packages.return_value = "com.example.app1\ncom.example.app2"
|
||||
|
||||
# Hopper doesn't find it
|
||||
mock_hopper.return_value = HopperOutput(found=False, output=None, reason="Not found")
|
||||
|
||||
pkg1 = await find_package(mock_context, "App 3")
|
||||
assert pkg1 is None
|
||||
assert mock_hopper.call_count == 1
|
||||
assert mock_context.package_cache["App 3"] is None
|
||||
|
||||
# Second call: cache hit (None), does not call hopper again
|
||||
mock_hopper.reset_mock()
|
||||
pkg2 = await find_package(mock_context, "App 3")
|
||||
assert pkg2 is None
|
||||
mock_hopper.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.tools.mobile.launch_app.list_packages_async")
|
||||
@patch("artemis.tools.mobile.launch_app.hopper")
|
||||
async def test_find_package_passes_use_fallback(mock_hopper, mock_list_packages, mock_context):
|
||||
mock_list_packages.return_value = "com.example.app1\ncom.example.app2"
|
||||
mock_hopper.return_value = HopperOutput(
|
||||
found=True, output="com.example.app1", reason="Found it"
|
||||
)
|
||||
|
||||
# Pass use_fallback=False
|
||||
await find_package(mock_context, "App 1", use_fallback=False)
|
||||
mock_hopper.assert_called_once_with(
|
||||
ctx=mock_context,
|
||||
request=("I'm looking for the package name of the following app: 'App 1'"),
|
||||
data="com.example.app1\ncom.example.app2",
|
||||
use_fallback=False,
|
||||
)
|
||||
@@ -0,0 +1,208 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.mobile.launch_app import (
|
||||
LaunchApp,
|
||||
LaunchAppArgs,
|
||||
LaunchAppTool,
|
||||
get_launch_app_tool,
|
||||
launch_app,
|
||||
launch_app_wrapper,
|
||||
)
|
||||
from langgraph.types import Command
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_ctx():
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.package_cache = {}
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_driver():
|
||||
driver = MagicMock(spec=BaseDeviceDriver)
|
||||
driver.launch_app = AsyncMock(return_value=True)
|
||||
return driver
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_state():
|
||||
state = MagicMock(spec=State)
|
||||
|
||||
async def _mock_asanitize_update(ctx, update, agent):
|
||||
return update
|
||||
|
||||
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
|
||||
return state
|
||||
|
||||
|
||||
def test_launch_app_tool_subclass_and_registry():
|
||||
"""Verify LaunchAppTool is a subclass of ArtemisTool and properly registered."""
|
||||
assert issubclass(LaunchAppTool, ArtemisTool)
|
||||
assert issubclass(LaunchApp, ArtemisTool)
|
||||
assert isinstance(launch_app, ArtemisTool)
|
||||
assert isinstance(launch_app, LaunchAppTool)
|
||||
|
||||
assert launch_app.name == "launch_app"
|
||||
assert launch_app.category == "action"
|
||||
assert launch_app.args_schema == LaunchAppArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("launch_app")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, LaunchAppTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = launch_app.to_genai_declaration()
|
||||
assert declaration.name == "launch_app"
|
||||
assert "app_name" in declaration.parameters.properties
|
||||
|
||||
# Wrapper check
|
||||
assert launch_app_wrapper is not None
|
||||
assert launch_app_wrapper.tool_fn_getter == get_launch_app_tool
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_launch_app_direct_execution_with_driver(mock_driver):
|
||||
"""Verify direct execution with BaseDeviceDriver."""
|
||||
result = await launch_app.execute(driver=mock_driver, app_name="com.android.settings")
|
||||
mock_driver.launch_app.assert_called_once_with("com.android.settings")
|
||||
assert "App 'com.android.settings' launched successfully." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_launch_app_direct_execution_with_ctx(mock_ctx):
|
||||
"""Verify direct execution with ArtemisContext."""
|
||||
with (
|
||||
patch(
|
||||
"artemis.tools.mobile.launch_app.find_package",
|
||||
new_callable=AsyncMock,
|
||||
return_value="com.example.myapp",
|
||||
) as mock_find,
|
||||
patch(
|
||||
"artemis.tools.mobile.launch_app.launch_app_with_retries",
|
||||
new_callable=AsyncMock,
|
||||
return_value=(True, None),
|
||||
) as mock_launch,
|
||||
):
|
||||
result = await launch_app.execute(ctx=mock_ctx, app_name="My App")
|
||||
mock_find.assert_called_once_with(ctx=mock_ctx, app_name="My App")
|
||||
mock_launch.assert_called_once_with(ctx=mock_ctx, app_package="com.example.myapp")
|
||||
assert "App 'My App' launched successfully." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_launch_app_package_not_found(mock_ctx):
|
||||
"""Verify outcome when package is not found."""
|
||||
with patch(
|
||||
"artemis.tools.mobile.launch_app.find_package",
|
||||
new_callable=AsyncMock,
|
||||
return_value=None,
|
||||
):
|
||||
result = await launch_app.execute(ctx=mock_ctx, app_name="Nonexistent App")
|
||||
assert "Failed to launch app 'Nonexistent App': Package not found." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_launch_app_execution_failure(mock_ctx):
|
||||
"""Verify outcome when launch_app_with_retries fails."""
|
||||
with (
|
||||
patch(
|
||||
"artemis.tools.mobile.launch_app.find_package",
|
||||
new_callable=AsyncMock,
|
||||
return_value="com.example.failapp",
|
||||
),
|
||||
patch(
|
||||
"artemis.tools.mobile.launch_app.launch_app_with_retries",
|
||||
new_callable=AsyncMock,
|
||||
return_value=(False, "App crashed on startup"),
|
||||
),
|
||||
):
|
||||
result = await launch_app.execute(ctx=mock_ctx, app_name="Crash App")
|
||||
assert "Failed to launch app 'Crash App': App crashed on startup" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_launch_app_with_state_command(mock_ctx, mock_state):
|
||||
"""Verify LaunchAppTool returns Command when state is provided."""
|
||||
with (
|
||||
patch(
|
||||
"artemis.tools.mobile.launch_app.find_package",
|
||||
new_callable=AsyncMock,
|
||||
return_value="com.example.stateapp",
|
||||
),
|
||||
patch(
|
||||
"artemis.tools.mobile.launch_app.launch_app_with_retries",
|
||||
new_callable=AsyncMock,
|
||||
return_value=(True, None),
|
||||
),
|
||||
):
|
||||
cmd = await launch_app.execute(
|
||||
ctx=mock_ctx,
|
||||
app_name="State App",
|
||||
tool_call_id="call_la_1",
|
||||
state=mock_state,
|
||||
)
|
||||
assert isinstance(cmd, Command)
|
||||
assert VALIDATOR_MESSAGES_KEY in cmd.update
|
||||
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
|
||||
assert len(messages) == 1
|
||||
assert messages[0].tool_call_id == "call_la_1"
|
||||
assert messages[0].status == "success"
|
||||
assert "App 'State App' launched successfully." in messages[0].content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_launch_app_callable_execution(mock_driver):
|
||||
"""Verify invoking launch_app directly as a callable."""
|
||||
result = await launch_app(driver=mock_driver, app_name="com.android.chrome")
|
||||
assert "App 'com.android.chrome' launched successfully." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_launch_app_missing_name(mock_ctx):
|
||||
"""Verify error handling when app_name is empty."""
|
||||
result = await launch_app.execute(ctx=mock_ctx, app_name="")
|
||||
assert "Error during launch app: app_name parameter is required." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_launch_app_tool_langchain_ainvoke(mock_ctx):
|
||||
"""Verify get_launch_app_tool exports a LangChain tool that works with ainvoke."""
|
||||
la_tool = get_launch_app_tool(mock_ctx)
|
||||
assert la_tool.name == "launch_app"
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.tools.mobile.launch_app.find_package",
|
||||
new_callable=AsyncMock,
|
||||
return_value="com.example.ainvoke",
|
||||
),
|
||||
patch(
|
||||
"artemis.tools.mobile.launch_app.launch_app_with_retries",
|
||||
new_callable=AsyncMock,
|
||||
return_value=(True, None),
|
||||
),
|
||||
):
|
||||
result = await la_tool.ainvoke({"app_name": "Invoke App"})
|
||||
assert "App 'Invoke App' launched successfully." in result
|
||||
@@ -0,0 +1,109 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.scratchpad import (
|
||||
ListNotesArgs,
|
||||
ListNotesPure,
|
||||
ListNotesPureTool,
|
||||
get_list_notes_tool_pure,
|
||||
list_notes_pure,
|
||||
)
|
||||
from artemis.utils.notes import save_note_content
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_ctx(tmp_path):
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.data_engine = MagicMock()
|
||||
ctx.data_engine.base_dir = str(tmp_path)
|
||||
return ctx
|
||||
|
||||
|
||||
def test_list_notes_pure_tool_subclass_and_registry():
|
||||
"""Verify ListNotesPureTool is a subclass of ArtemisTool and properly registered."""
|
||||
assert issubclass(ListNotesPureTool, ArtemisTool)
|
||||
assert issubclass(ListNotesPure, ArtemisTool)
|
||||
assert isinstance(list_notes_pure, ArtemisTool)
|
||||
assert isinstance(list_notes_pure, ListNotesPureTool)
|
||||
|
||||
assert list_notes_pure.name == "list_notes_pure"
|
||||
assert list_notes_pure.category == "memory"
|
||||
assert list_notes_pure.args_schema == ListNotesArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("list_notes_pure")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, ListNotesPureTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = list_notes_pure.to_genai_declaration()
|
||||
assert declaration.name == "list_notes_pure"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_notes_pure_direct_execution_empty(mock_ctx):
|
||||
"""Verify listing notes when no notes are stored."""
|
||||
result = await list_notes_pure.execute(ctx=mock_ctx)
|
||||
assert "No notes saved yet." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_notes_pure_direct_execution_success(mock_ctx, tmp_path):
|
||||
"""Verify direct execution of ListNotesPureTool.execute lists stored notes."""
|
||||
save_note_content(tmp_path, "note_1", "line 1\nline 2")
|
||||
save_note_content(tmp_path, "note_2", "single line")
|
||||
|
||||
result = await list_notes_pure.execute(ctx=mock_ctx)
|
||||
assert "Here are all the notes:" in result
|
||||
assert "- note_1 (2 lines)" in result
|
||||
assert "- note_2 (1 lines)" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_notes_pure_callable_execution(mock_ctx, tmp_path):
|
||||
"""Verify invoking list_notes_pure directly as a callable."""
|
||||
save_note_content(tmp_path, "callable_note", "line A\nline B")
|
||||
|
||||
result = await list_notes_pure(ctx=mock_ctx)
|
||||
assert "Here are all the notes:" in result
|
||||
assert "- callable_note (2 lines)" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_notes_pure_execution_failure(mock_ctx):
|
||||
"""Verify error handling when list_notes_pure fails."""
|
||||
with patch(
|
||||
"artemis.tools.scratchpad.list_notes_info",
|
||||
side_effect=RuntimeError("Disk failure"),
|
||||
):
|
||||
result = await list_notes_pure.execute(ctx=mock_ctx)
|
||||
assert "Failed to list notes: Disk failure" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_list_notes_tool_pure_langchain_ainvoke(mock_ctx, tmp_path):
|
||||
"""Verify get_list_notes_tool_pure exports a LangChain tool named 'list_notes'."""
|
||||
save_note_content(tmp_path, "pure_export_note", "content")
|
||||
|
||||
pure_tool = get_list_notes_tool_pure(mock_ctx)
|
||||
assert pure_tool.name == "list_notes"
|
||||
|
||||
result = await pure_tool.ainvoke({})
|
||||
assert "Here are all the notes:" in result
|
||||
assert "- pure_export_note (1 lines)" in result
|
||||
@@ -0,0 +1,158 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.scratchpad import (
|
||||
ListNotes,
|
||||
ListNotesArgs,
|
||||
ListNotesTool,
|
||||
get_list_notes_tool,
|
||||
get_list_notes_tool_pure,
|
||||
list_notes,
|
||||
list_notes_wrapper,
|
||||
)
|
||||
from artemis.utils.notes import save_note_content
|
||||
from langgraph.types import Command
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_ctx(tmp_path):
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.data_engine = MagicMock()
|
||||
ctx.data_engine.base_dir = str(tmp_path)
|
||||
return ctx
|
||||
|
||||
|
||||
def test_list_notes_tool_subclass_and_registry():
|
||||
"""Verify ListNotesTool is a subclass of ArtemisTool and properly registered."""
|
||||
assert issubclass(ListNotesTool, ArtemisTool)
|
||||
assert issubclass(ListNotes, ArtemisTool)
|
||||
assert isinstance(list_notes, ArtemisTool)
|
||||
assert isinstance(list_notes, ListNotesTool)
|
||||
|
||||
assert list_notes.name == "list_notes"
|
||||
assert list_notes.category == "memory"
|
||||
assert list_notes.args_schema == ListNotesArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("list_notes")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, ListNotesTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = list_notes.to_genai_declaration()
|
||||
assert declaration.name == "list_notes"
|
||||
|
||||
# Wrapper check
|
||||
assert list_notes_wrapper is not None
|
||||
assert list_notes_wrapper.tool_fn_getter == get_list_notes_tool
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_notes_direct_execution_empty(mock_ctx):
|
||||
"""Verify listing notes when no notes are stored."""
|
||||
result = await list_notes.execute(ctx=mock_ctx)
|
||||
assert "No notes saved yet." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_notes_direct_execution_success(mock_ctx, tmp_path):
|
||||
"""Verify direct execution of ListNotesTool.execute lists stored notes."""
|
||||
save_note_content(tmp_path, "note_alpha", "line 1\nline 2")
|
||||
save_note_content(tmp_path, "note_beta", "single line")
|
||||
|
||||
result = await list_notes.execute(ctx=mock_ctx)
|
||||
assert "Here are all the notes:" in result
|
||||
assert "- note_alpha (2 lines)" in result
|
||||
assert "- note_beta (1 lines)" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_notes_callable_execution(mock_ctx, tmp_path):
|
||||
"""Verify invoking list_notes directly as a callable."""
|
||||
save_note_content(tmp_path, "note_gamma", "line A\nline B\nline C")
|
||||
|
||||
result = await list_notes(ctx=mock_ctx)
|
||||
assert "Here are all the notes:" in result
|
||||
assert "- note_gamma (3 lines)" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_notes_with_state_command(mock_ctx, tmp_path):
|
||||
"""Verify ListNotesTool.execute with state returns Command updating VALIDATOR_MESSAGES_KEY."""
|
||||
save_note_content(tmp_path, "state_note", "content")
|
||||
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.current_agent = "operator"
|
||||
mock_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update)
|
||||
|
||||
result = await list_notes.execute(
|
||||
ctx=mock_ctx,
|
||||
state=mock_state,
|
||||
tool_call_id="call_list_999",
|
||||
)
|
||||
|
||||
assert isinstance(result, Command)
|
||||
assert VALIDATOR_MESSAGES_KEY in result.update
|
||||
tool_messages = result.update[VALIDATOR_MESSAGES_KEY]
|
||||
assert len(tool_messages) == 1
|
||||
assert tool_messages[0].tool_call_id == "call_list_999"
|
||||
assert "Here are all the notes:" in tool_messages[0].content
|
||||
assert "- state_note (1 lines)" in tool_messages[0].content
|
||||
assert tool_messages[0].status == "success"
|
||||
|
||||
mock_state.asanitize_update.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_notes_execution_failure(mock_ctx):
|
||||
"""Verify error handling when list_notes fails."""
|
||||
with patch(
|
||||
"artemis.tools.scratchpad.list_notes_info",
|
||||
side_effect=RuntimeError("Disk failure"),
|
||||
):
|
||||
result = await list_notes.execute(ctx=mock_ctx)
|
||||
assert "Failed to list notes: Disk failure" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_list_notes_tool_langchain_ainvoke(mock_ctx, tmp_path):
|
||||
"""Verify get_list_notes_tool exports a LangChain tool that works with ainvoke."""
|
||||
save_note_content(tmp_path, "lc_note", "hello")
|
||||
|
||||
lc_tool = get_list_notes_tool(mock_ctx)
|
||||
assert lc_tool.name == "list_notes"
|
||||
|
||||
result = await lc_tool.ainvoke({})
|
||||
assert "Here are all the notes:" in result
|
||||
assert "- lc_note (1 lines)" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_list_notes_tool_pure(mock_ctx, tmp_path):
|
||||
"""Verify get_list_notes_tool_pure returns a pure LangChain tool."""
|
||||
save_note_content(tmp_path, "pure_note", "world")
|
||||
|
||||
pure_tool = get_list_notes_tool_pure(mock_ctx)
|
||||
assert pure_tool.name == "list_notes"
|
||||
|
||||
result = await pure_tool.ainvoke({})
|
||||
assert "Here are all the notes:" in result
|
||||
assert "- pure_note (1 lines)" in result
|
||||
@@ -0,0 +1,115 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.log_tool import (
|
||||
AnalyzeLogs,
|
||||
AnalyzeLogsArgs,
|
||||
AnalyzeLogsTool,
|
||||
LogTool,
|
||||
analyze_logs,
|
||||
analyze_logs_wrapper,
|
||||
get_analyze_logs_tool,
|
||||
)
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_analyze_logs_tool_execution():
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = None
|
||||
|
||||
mock_state = MagicMock(spec=State)
|
||||
|
||||
expected_output = "Found 2 errors in system log: NullPointerException at line 42."
|
||||
|
||||
mock_analyst = MagicMock()
|
||||
mock_analyst.run = AsyncMock(return_value=expected_output)
|
||||
|
||||
with patch(
|
||||
"artemis.tools.log_tool.LogAnalyzerNode",
|
||||
return_value=mock_analyst,
|
||||
):
|
||||
tool = get_analyze_logs_tool(mock_ctx)
|
||||
|
||||
result = await tool.ainvoke(
|
||||
{
|
||||
"specific_query": "Check for crash exceptions",
|
||||
"state": mock_state,
|
||||
}
|
||||
)
|
||||
|
||||
mock_analyst.run.assert_called_once_with("Check for crash exceptions", mock_state)
|
||||
assert result == expected_output
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_analyze_logs_tool_direct_call():
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_state = MagicMock(spec=State)
|
||||
|
||||
mock_analyst = MagicMock()
|
||||
mock_analyst.run = AsyncMock(return_value="Log scan complete.")
|
||||
|
||||
with patch(
|
||||
"artemis.tools.log_tool.LogAnalyzerNode",
|
||||
return_value=mock_analyst,
|
||||
):
|
||||
result = await analyze_logs(
|
||||
ctx=mock_ctx,
|
||||
specific_query="Find ANR",
|
||||
state=mock_state,
|
||||
)
|
||||
mock_analyst.run.assert_called_once_with("Find ANR", mock_state)
|
||||
assert result == "Log scan complete."
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_analyze_logs_tool_no_context():
|
||||
result = await analyze_logs(
|
||||
ctx=None,
|
||||
specific_query="Find crash",
|
||||
)
|
||||
assert result == "Error: ArtemisContext is required for LogAnalyzer."
|
||||
|
||||
|
||||
def test_analyze_logs_tool_subclass_and_registry():
|
||||
"""Verify AnalyzeLogsTool is an ArtemisTool subclass and registered in ToolRegistry."""
|
||||
assert issubclass(AnalyzeLogsTool, ArtemisTool)
|
||||
assert issubclass(AnalyzeLogs, ArtemisTool)
|
||||
assert issubclass(LogTool, ArtemisTool)
|
||||
assert isinstance(analyze_logs, ArtemisTool)
|
||||
assert isinstance(analyze_logs, AnalyzeLogsTool)
|
||||
|
||||
assert analyze_logs.name == "analyze_logs"
|
||||
assert analyze_logs.category == "custom"
|
||||
assert analyze_logs.args_schema == AnalyzeLogsArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("analyze_logs")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, AnalyzeLogsTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = analyze_logs.to_genai_declaration()
|
||||
assert declaration.name == "analyze_logs"
|
||||
assert "specific_query" in declaration.parameters.properties
|
||||
|
||||
# Wrapper check
|
||||
assert analyze_logs_wrapper is not None
|
||||
assert analyze_logs_wrapper.tool_fn_getter == get_analyze_logs_tool
|
||||
@@ -0,0 +1,116 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from datetime import datetime
|
||||
from unittest.mock import Mock, patch
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.tools.mobile.log_utils import fetch_and_filter_logs
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context():
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
ctx.data_engine = Mock()
|
||||
ctx.data_engine.session_start_time = 1700000000.0 # Example timestamp
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_device():
|
||||
return Mock()
|
||||
|
||||
|
||||
@patch("artemis.tools.mobile.log_utils.get_adb_device")
|
||||
def test_fetch_and_filter_logs_basic(mock_get_adb_device, mock_context, mock_device):
|
||||
mock_get_adb_device.return_value = mock_device
|
||||
mock_device.shell.return_value = "line 1\nline 2\n"
|
||||
|
||||
result = fetch_and_filter_logs(mock_context, lines=10)
|
||||
|
||||
mock_device.shell.assert_called_once_with("logcat -v threadtime -t 10")
|
||||
assert result == "line 1\nline 2\n"
|
||||
|
||||
|
||||
@patch("artemis.tools.mobile.log_utils.get_adb_device")
|
||||
@patch("artemis.tools.mobile.log_utils.resolve_time")
|
||||
def test_fetch_and_filter_logs_since_time_success(
|
||||
mock_resolve_time, mock_get_adb_device, mock_context, mock_device
|
||||
):
|
||||
mock_get_adb_device.return_value = mock_device
|
||||
since_dt = datetime(datetime.now().year, 5, 3, 11, 11, 11, 111000)
|
||||
mock_resolve_time.side_effect = [
|
||||
("05-03 11:11:11.111", since_dt), # for since_time
|
||||
(None, None), # for until_time
|
||||
]
|
||||
|
||||
mock_device.shell.return_value = "05-03 11:11:12.000 line 1\n05-03 11:11:13.000 line 2\n"
|
||||
|
||||
result = fetch_and_filter_logs(mock_context, since_time="05-03 11:11:11.111")
|
||||
|
||||
mock_device.shell.assert_called_once_with('logcat -v threadtime -t "05-03 11:11:11.111"')
|
||||
assert "line 1" in result
|
||||
assert "line 2" in result
|
||||
|
||||
|
||||
@patch("artemis.tools.mobile.log_utils.get_adb_device")
|
||||
@patch("artemis.tools.mobile.log_utils.resolve_time")
|
||||
def test_fetch_and_filter_logs_since_time_fallback(
|
||||
mock_resolve_time, mock_get_adb_device, mock_context, mock_device
|
||||
):
|
||||
mock_get_adb_device.return_value = mock_device
|
||||
current_year = datetime.now().year
|
||||
since_dt = datetime(current_year, 5, 4, 12, 0, 0)
|
||||
mock_resolve_time.side_effect = [
|
||||
("05-04 12:00:00.000", since_dt), # for since_time
|
||||
(None, None), # for until_time
|
||||
]
|
||||
|
||||
# Direct fetch fails
|
||||
mock_device.shell.side_effect = [
|
||||
Exception("logcat failed"), # first call to direct fetch
|
||||
"05-04 11:59:00.000 line old\n05-04 12:00:01.000 line new\n", # second call (fallback)
|
||||
]
|
||||
|
||||
result = fetch_and_filter_logs(mock_context, lines=100, since_time="05-04 12:00:00.000")
|
||||
|
||||
assert mock_device.shell.call_count == 2
|
||||
mock_device.shell.assert_any_call('logcat -v threadtime -t "05-04 12:00:00.000"')
|
||||
mock_device.shell.assert_any_call("logcat -v threadtime -t 100")
|
||||
|
||||
# Should filter out "line old" because it's before since_dt
|
||||
assert "line old" not in result
|
||||
assert "line new" in result
|
||||
|
||||
|
||||
@patch("artemis.tools.mobile.log_utils.get_adb_device")
|
||||
@patch("artemis.tools.mobile.log_utils.resolve_time")
|
||||
def test_fetch_and_filter_logs_until_time(
|
||||
mock_resolve_time, mock_get_adb_device, mock_context, mock_device
|
||||
):
|
||||
mock_get_adb_device.return_value = mock_device
|
||||
current_year = datetime.now().year
|
||||
until_dt = datetime(current_year, 5, 4, 12, 0, 0)
|
||||
mock_resolve_time.side_effect = [
|
||||
(None, None), # for since_time
|
||||
("05-04 12:00:00.000", until_dt), # for until_time
|
||||
]
|
||||
|
||||
mock_device.shell.return_value = "05-04 11:59:00.000 line old\n05-04 12:00:01.000 line new\n"
|
||||
|
||||
result = fetch_and_filter_logs(mock_context, lines=100, until_time="05-04 12:00:00.000")
|
||||
|
||||
mock_device.shell.assert_called_once_with("logcat -v threadtime -t 100")
|
||||
assert "line old" in result
|
||||
assert "line new" not in result
|
||||
@@ -0,0 +1,162 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.mobile.exec_tools import (
|
||||
LongPress,
|
||||
LongPressArgs,
|
||||
LongPressTool,
|
||||
get_long_press_tool,
|
||||
long_press,
|
||||
long_press_wrapper,
|
||||
)
|
||||
from langgraph.types import Command
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_ctx():
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.device = MagicMock()
|
||||
ctx.device.device_width = 1080
|
||||
ctx.device.device_height = 2400
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_driver():
|
||||
driver = MagicMock(spec=BaseDeviceDriver)
|
||||
driver.screen_size = (1080, 2400)
|
||||
driver.long_press = AsyncMock(return_value=True)
|
||||
return driver
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_state():
|
||||
state = MagicMock(spec=State)
|
||||
|
||||
async def _mock_asanitize_update(ctx, update, agent):
|
||||
return update
|
||||
|
||||
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
|
||||
return state
|
||||
|
||||
|
||||
def test_long_press_tool_subclass_and_registry():
|
||||
"""Verify LongPressTool is a subclass of ArtemisTool and properly registered."""
|
||||
assert issubclass(LongPressTool, ArtemisTool)
|
||||
assert issubclass(LongPress, ArtemisTool)
|
||||
assert isinstance(long_press, ArtemisTool)
|
||||
assert isinstance(long_press, LongPressTool)
|
||||
|
||||
assert long_press.name == "long_press"
|
||||
assert long_press.category == "action"
|
||||
assert long_press.args_schema == LongPressArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("long_press")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, LongPressTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = long_press.to_genai_declaration()
|
||||
assert declaration.name == "long_press"
|
||||
assert "target" in declaration.parameters.properties
|
||||
assert "duration" in declaration.parameters.properties
|
||||
|
||||
# Wrapper check
|
||||
assert long_press_wrapper is not None
|
||||
assert long_press_wrapper.tool_fn_getter == get_long_press_tool
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_long_press_direct_execution_with_driver(mock_driver):
|
||||
"""Verify direct execution of LongPressTool with BaseDeviceDriver."""
|
||||
result = await long_press.execute(driver=mock_driver, target=[500, 500], duration=1500)
|
||||
mock_driver.long_press.assert_called_once_with(540, 1200, duration_ms=1500)
|
||||
assert "Long pressed at [500, 500] (normalized) successfully." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_long_press_direct_execution_with_ctx(mock_ctx):
|
||||
"""Verify direct execution of LongPressTool with ArtemisContext."""
|
||||
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
|
||||
mock_controller_inst = MagicMock()
|
||||
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
|
||||
mock_controller_cls.return_value = mock_controller_inst
|
||||
|
||||
result = await long_press.execute(ctx=mock_ctx, target=[250, 750], duration=2000)
|
||||
mock_controller_inst.tap_at.assert_called_once_with(
|
||||
270, 1800, long_press=True, long_press_duration=2000
|
||||
)
|
||||
assert "Long pressed at [250, 750] (normalized) successfully." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_long_press_with_state_command(mock_ctx, mock_state):
|
||||
"""Verify LongPressTool returns Command when state is provided."""
|
||||
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
|
||||
mock_controller_inst = MagicMock()
|
||||
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
|
||||
mock_controller_cls.return_value = mock_controller_inst
|
||||
|
||||
cmd = await long_press.execute(
|
||||
ctx=mock_ctx,
|
||||
target=[100, 200],
|
||||
duration=1200,
|
||||
tool_call_id="call_lp_1",
|
||||
state=mock_state,
|
||||
)
|
||||
assert isinstance(cmd, Command)
|
||||
assert VALIDATOR_MESSAGES_KEY in cmd.update
|
||||
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
|
||||
assert len(messages) == 1
|
||||
assert messages[0].tool_call_id == "call_lp_1"
|
||||
assert messages[0].status == "success"
|
||||
assert "Long pressed at [100, 200]" in messages[0].content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_long_press_callable_execution(mock_driver):
|
||||
"""Verify invoking long_press directly as a callable."""
|
||||
result = await long_press(driver=mock_driver, target=[100, 100])
|
||||
assert "Long pressed at [100, 100] (normalized) successfully." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_long_press_invalid_target(mock_ctx):
|
||||
"""Verify error handling on invalid target coordinates."""
|
||||
result = await long_press.execute(ctx=mock_ctx, target=[])
|
||||
assert "Error during long press: Invalid target coordinates" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_long_press_tool_langchain_ainvoke(mock_ctx):
|
||||
"""Verify get_long_press_tool exports a LangChain tool that works with ainvoke."""
|
||||
lp_tool = get_long_press_tool(mock_ctx)
|
||||
assert lp_tool.name == "long_press"
|
||||
|
||||
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
|
||||
mock_controller_inst = MagicMock()
|
||||
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
|
||||
mock_controller_cls.return_value = mock_controller_inst
|
||||
|
||||
result = await lp_tool.ainvoke({"target": [500, 500], "duration": 1500})
|
||||
assert "Long pressed at [500, 500] (normalized) successfully." in result
|
||||
@@ -0,0 +1,186 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for ObjectDetectionTool and OperatorObjectDetectionTool."""
|
||||
|
||||
import json
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.object_detection_tool import (
|
||||
ObjectDetection,
|
||||
ObjectDetectionArgs,
|
||||
ObjectDetectionTool,
|
||||
ObjectDetectorTool,
|
||||
OperatorObjectDetection,
|
||||
OperatorObjectDetectionArgs,
|
||||
OperatorObjectDetectionTool,
|
||||
OperatorObjectDetectorTool,
|
||||
get_object_detector_tool,
|
||||
get_operator_object_detector_tool,
|
||||
object_detection,
|
||||
object_detection_wrapper,
|
||||
operator_object_detection,
|
||||
operator_object_detection_wrapper,
|
||||
)
|
||||
import pytest
|
||||
|
||||
|
||||
def test_object_detection_tool_subclass_and_registry():
|
||||
"""Verify ObjectDetectionTool and OperatorObjectDetectionTool subclass ArtemisTool and register."""
|
||||
assert issubclass(ObjectDetectionTool, ArtemisTool)
|
||||
assert issubclass(ObjectDetection, ArtemisTool)
|
||||
assert issubclass(ObjectDetectorTool, ArtemisTool)
|
||||
assert issubclass(OperatorObjectDetectionTool, ArtemisTool)
|
||||
assert issubclass(OperatorObjectDetectionTool, ObjectDetectionTool)
|
||||
assert issubclass(OperatorObjectDetection, ArtemisTool)
|
||||
assert issubclass(OperatorObjectDetectorTool, ArtemisTool)
|
||||
|
||||
assert isinstance(object_detection, ArtemisTool)
|
||||
assert isinstance(object_detection, ObjectDetectionTool)
|
||||
assert isinstance(operator_object_detection, ArtemisTool)
|
||||
assert isinstance(operator_object_detection, OperatorObjectDetectionTool)
|
||||
|
||||
# Properties
|
||||
assert object_detection.name == "object_detection"
|
||||
assert object_detection.category == "perception"
|
||||
assert object_detection.args_schema == ObjectDetectionArgs
|
||||
|
||||
assert operator_object_detection.name == "object_detection"
|
||||
assert operator_object_detection.category == "perception"
|
||||
assert operator_object_detection.args_schema == OperatorObjectDetectionArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("object_detection")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, ObjectDetectionTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = object_detection.to_genai_declaration()
|
||||
assert declaration.name == "object_detection"
|
||||
assert "image_path" in declaration.parameters.properties
|
||||
assert "queries" in declaration.parameters.properties
|
||||
|
||||
op_declaration = operator_object_detection.to_genai_declaration()
|
||||
assert op_declaration.name == "object_detection"
|
||||
assert "queries" in op_declaration.parameters.properties
|
||||
|
||||
# Wrappers
|
||||
assert object_detection_wrapper.tool_fn_getter == get_object_detector_tool
|
||||
assert operator_object_detection_wrapper.tool_fn_getter == get_operator_object_detector_tool
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_object_detection_direct_execution_success():
|
||||
"""Verify direct async call to object_detection with image_path."""
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = None
|
||||
|
||||
expected_result = {
|
||||
"detected": [{"label": "button", "point": [100, 200]}],
|
||||
"failed": [],
|
||||
}
|
||||
|
||||
with patch(
|
||||
"artemis.tools.object_detection_tool._run_object_detection",
|
||||
AsyncMock(return_value=expected_result),
|
||||
) as mock_run:
|
||||
result = await object_detection(
|
||||
ctx=mock_ctx,
|
||||
image_path="/path/to/screenshot.jpg",
|
||||
queries=["button"],
|
||||
)
|
||||
|
||||
mock_run.assert_called_once()
|
||||
assert json.loads(result) == expected_result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_operator_object_detection_direct_execution_with_state():
|
||||
"""Verify operator_object_detection reads screenshot from state."""
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = None
|
||||
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.latest_screenshot = "/path/to/state_screen.jpg"
|
||||
mock_state.asanitize_update = None # Non-langgraph direct call
|
||||
|
||||
expected_result = {
|
||||
"detected": [{"label": "search bar", "point": [500, 100]}],
|
||||
"failed": [],
|
||||
}
|
||||
|
||||
with patch(
|
||||
"artemis.tools.object_detection_tool._run_object_detection",
|
||||
AsyncMock(return_value=expected_result),
|
||||
) as mock_run:
|
||||
result = await operator_object_detection(
|
||||
ctx=mock_ctx,
|
||||
state=mock_state,
|
||||
queries=["search bar"],
|
||||
)
|
||||
|
||||
mock_run.assert_called_once()
|
||||
# Verify it used the latest_screenshot from state
|
||||
call_args = mock_run.call_args[0]
|
||||
assert call_args[1] == "/path/to/state_screen.jpg"
|
||||
assert json.loads(result) == expected_result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_object_detection_no_image_error():
|
||||
"""Verify error when no image path is given and no state screenshot is available."""
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = None
|
||||
|
||||
result = await object_detection(
|
||||
ctx=mock_ctx,
|
||||
image_path=None,
|
||||
queries=["icon"],
|
||||
)
|
||||
assert "Object Detection failed" in result
|
||||
assert "No image path provided" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_object_detection_exception_handling():
|
||||
"""Verify graceful handling when _run_object_detection raises an exception."""
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = None
|
||||
|
||||
with patch(
|
||||
"artemis.tools.object_detection_tool._run_object_detection",
|
||||
AsyncMock(side_effect=RuntimeError("Model inference timeout")),
|
||||
):
|
||||
result = await object_detection(
|
||||
ctx=mock_ctx,
|
||||
image_path="/path/to/screen.jpg",
|
||||
queries=["icon"],
|
||||
)
|
||||
assert "Object Detection failed: Model inference timeout" in result
|
||||
|
||||
|
||||
def test_get_langchain_tools():
|
||||
"""Verify LangChain BaseTool exports."""
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
tool = get_object_detector_tool(mock_ctx)
|
||||
op_tool = get_operator_object_detector_tool(mock_ctx)
|
||||
|
||||
assert tool.name == "object_detection"
|
||||
assert tool.args_schema == ObjectDetectionArgs
|
||||
|
||||
assert op_tool.name == "object_detection"
|
||||
assert op_tool.args_schema == OperatorObjectDetectionArgs
|
||||
@@ -0,0 +1,70 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.object_detection_tool import get_operator_object_detector_tool
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_object_detection_trace():
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = MagicMock()
|
||||
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.latest_screenshot = "/path/to/screenshot.jpg"
|
||||
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
|
||||
|
||||
# We need to mock the decorator in object_detection_tool.py if we want to avoid it calling real get_llm
|
||||
# Or we can just mock get_llm.
|
||||
# In the tool definition: @trace(type="tool", ctx=ctx)
|
||||
# This calls record_trace immediately on entry/exit.
|
||||
|
||||
mock_tool = get_operator_object_detector_tool(mock_ctx)
|
||||
|
||||
# Mock the internal function called by the tool
|
||||
with patch(
|
||||
"artemis.tools.object_detection_tool._run_object_detection",
|
||||
AsyncMock(return_value="{}"),
|
||||
) as mock_run:
|
||||
# We need to handle the fact that ainvoke might fail if LangChain internals are not fully mocked.
|
||||
# But let's try it.
|
||||
|
||||
await mock_tool.ainvoke(
|
||||
{
|
||||
"type": "tool_call",
|
||||
"name": "object_detection",
|
||||
"id": "call_123",
|
||||
"args": {"queries": ["test_label"], "state": mock_state},
|
||||
}
|
||||
)
|
||||
|
||||
# Verify _run_object_detection was called
|
||||
mock_run.assert_called_once()
|
||||
|
||||
# Verify record_trace was called for the span
|
||||
assert mock_ctx.data_engine.record_trace.call_count >= 1
|
||||
|
||||
calls = mock_ctx.data_engine.record_trace.call_args_list
|
||||
span_called = False
|
||||
for call in calls:
|
||||
kwargs = call.kwargs
|
||||
if kwargs.get("type") == "span" and kwargs.get("name") == "run_object_detection":
|
||||
span_called = True
|
||||
break
|
||||
|
||||
assert span_called, "TraceSpan 'run_object_detection' was not called"
|
||||
@@ -0,0 +1,238 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import base64
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.mobile.ocr import (
|
||||
OCRRecognition,
|
||||
OcrArgs,
|
||||
OcrRecognition,
|
||||
OcrRecognitionTool,
|
||||
OcrTool,
|
||||
get_ocr_tool,
|
||||
ocr_recognition,
|
||||
ocr_recognition_wrapper,
|
||||
)
|
||||
from langgraph.types import Command
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_ctx():
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.device = MagicMock()
|
||||
ctx.device.device_width = 1080
|
||||
ctx.device.device_height = 2400
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_driver():
|
||||
driver = MagicMock(spec=BaseDeviceDriver)
|
||||
driver.screen_size = (1080, 2400)
|
||||
screen_data = MagicMock()
|
||||
screen_data.screenshot_base64 = base64.b64encode(b"dummy_image_data").decode("utf-8")
|
||||
screen_data.screenshot_bytes = b"dummy_image_data"
|
||||
driver.get_screen_data = AsyncMock(return_value=screen_data)
|
||||
return driver
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_state(tmp_path):
|
||||
screenshot_file = tmp_path / "test_screenshot.png"
|
||||
screenshot_file.write_bytes(b"dummy_image_content")
|
||||
|
||||
state = MagicMock(spec=State)
|
||||
state.latest_screenshot = str(screenshot_file)
|
||||
|
||||
async def _mock_asanitize_update(ctx, update, agent):
|
||||
return update
|
||||
|
||||
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
|
||||
return state
|
||||
|
||||
|
||||
def test_ocr_tool_subclass_and_registry():
|
||||
"""Verify OcrRecognitionTool is a subclass of ArtemisTool and properly registered."""
|
||||
assert issubclass(OcrRecognitionTool, ArtemisTool)
|
||||
assert issubclass(OcrRecognition, ArtemisTool)
|
||||
assert issubclass(OCRRecognition, ArtemisTool)
|
||||
assert issubclass(OcrTool, ArtemisTool)
|
||||
assert isinstance(ocr_recognition, ArtemisTool)
|
||||
assert isinstance(ocr_recognition, OcrRecognitionTool)
|
||||
|
||||
assert ocr_recognition.name == "ocr_recognition"
|
||||
assert ocr_recognition.category == "explorer"
|
||||
assert ocr_recognition.args_schema == OcrArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("ocr_recognition")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, OcrRecognitionTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = ocr_recognition.to_genai_declaration()
|
||||
assert declaration.name == "ocr_recognition"
|
||||
|
||||
# Wrapper check
|
||||
assert ocr_recognition_wrapper is not None
|
||||
assert ocr_recognition_wrapper.tool_fn_getter == get_ocr_tool
|
||||
|
||||
|
||||
class SimpleState:
|
||||
def __init__(self, screenshot_path: str):
|
||||
self.latest_screenshot = screenshot_path
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ocr_direct_execution_with_state(mock_ctx, tmp_path):
|
||||
"""Verify direct execution with state and screenshot file."""
|
||||
screenshot_file = tmp_path / "test_screenshot.png"
|
||||
screenshot_file.write_bytes(b"dummy_image_content")
|
||||
simple_state = SimpleState(str(screenshot_file))
|
||||
|
||||
mock_results = [
|
||||
{
|
||||
"text": "Submit",
|
||||
"position": [
|
||||
{"x": 100, "y": 200},
|
||||
{"x": 200, "y": 200},
|
||||
{"x": 200, "y": 300},
|
||||
{"x": 100, "y": 300},
|
||||
],
|
||||
}
|
||||
]
|
||||
with patch(
|
||||
"artemis.tools.mobile.ocr.perform_ocr",
|
||||
new_callable=AsyncMock,
|
||||
return_value=mock_results,
|
||||
):
|
||||
result = await ocr_recognition.execute(ctx=mock_ctx, state=simple_state)
|
||||
assert "OCR Recognition successful." in result
|
||||
assert "Submit" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ocr_direct_execution_with_driver(mock_ctx, mock_driver):
|
||||
"""Verify direct execution with driver."""
|
||||
mock_results = [
|
||||
{
|
||||
"text": "Settings",
|
||||
"position": [
|
||||
{"x": 500, "y": 500},
|
||||
{"x": 600, "y": 600},
|
||||
],
|
||||
}
|
||||
]
|
||||
with patch(
|
||||
"artemis.tools.mobile.ocr.perform_ocr",
|
||||
new_callable=AsyncMock,
|
||||
return_value=mock_results,
|
||||
):
|
||||
result = await ocr_recognition.execute(ctx=mock_ctx, driver=mock_driver)
|
||||
assert "OCR Recognition successful." in result
|
||||
assert "Settings" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ocr_no_text_detected(mock_ctx, tmp_path):
|
||||
"""Verify outcome when no text is detected."""
|
||||
screenshot_file = tmp_path / "test_empty_screenshot.png"
|
||||
screenshot_file.write_bytes(b"dummy_empty_content")
|
||||
simple_state = SimpleState(str(screenshot_file))
|
||||
|
||||
with patch(
|
||||
"artemis.tools.mobile.ocr.perform_ocr",
|
||||
new_callable=AsyncMock,
|
||||
return_value=[],
|
||||
):
|
||||
result = await ocr_recognition.execute(ctx=mock_ctx, state=simple_state)
|
||||
assert "No text detected on the screen." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ocr_with_state_command(mock_ctx, mock_state):
|
||||
"""Verify OcrRecognitionTool returns Command when state is provided."""
|
||||
mock_results = [{"text": "Hello", "position": [{"x": 100, "y": 100}]}]
|
||||
with patch(
|
||||
"artemis.tools.mobile.ocr.perform_ocr",
|
||||
new_callable=AsyncMock,
|
||||
return_value=mock_results,
|
||||
):
|
||||
cmd = await ocr_recognition.execute(
|
||||
ctx=mock_ctx,
|
||||
tool_call_id="call_ocr_1",
|
||||
state=mock_state,
|
||||
)
|
||||
assert isinstance(cmd, Command)
|
||||
assert VALIDATOR_MESSAGES_KEY in cmd.update
|
||||
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
|
||||
assert len(messages) == 1
|
||||
assert messages[0].tool_call_id == "call_ocr_1"
|
||||
assert messages[0].status == "success"
|
||||
assert "Hello" in messages[0].content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ocr_failure_missing_screenshot(mock_ctx):
|
||||
"""Verify error handling when screenshot path is missing."""
|
||||
empty_state = MagicMock(spec=State)
|
||||
empty_state.latest_screenshot = None
|
||||
empty_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update)
|
||||
|
||||
cmd = await ocr_recognition.execute(
|
||||
ctx=mock_ctx,
|
||||
tool_call_id="call_ocr_err",
|
||||
state=empty_state,
|
||||
)
|
||||
assert isinstance(cmd, Command)
|
||||
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
|
||||
assert messages[0].status == "error"
|
||||
assert "OCR Recognition failed" in messages[0].content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ocr_callable_execution(mock_ctx, mock_driver):
|
||||
"""Verify invoking ocr_recognition directly as a callable."""
|
||||
with patch(
|
||||
"artemis.tools.mobile.ocr.perform_ocr",
|
||||
new_callable=AsyncMock,
|
||||
return_value=[{"text": "Callable", "position": []}],
|
||||
):
|
||||
result = await ocr_recognition(ctx=mock_ctx, driver=mock_driver)
|
||||
assert "OCR Recognition successful." in result
|
||||
assert "Callable" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_ocr_tool_langchain_ainvoke(mock_ctx, mock_driver):
|
||||
"""Verify get_ocr_tool exports a LangChain tool that works with ainvoke."""
|
||||
mock_ctx._active_driver = mock_driver
|
||||
ocr_tool = get_ocr_tool(mock_ctx)
|
||||
assert ocr_tool.name == "ocr_recognition"
|
||||
|
||||
with patch(
|
||||
"artemis.tools.mobile.ocr.perform_ocr",
|
||||
new_callable=AsyncMock,
|
||||
return_value=[{"text": "LangChain", "position": []}],
|
||||
):
|
||||
result = await ocr_tool.ainvoke({})
|
||||
assert "OCR Recognition successful." in result
|
||||
assert "LangChain" in result
|
||||
@@ -0,0 +1,174 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.mobile.exec_tools import (
|
||||
PressKey,
|
||||
PressKeyArgs,
|
||||
PressKeyTool,
|
||||
get_press_key_tool,
|
||||
press_key,
|
||||
press_key_wrapper,
|
||||
)
|
||||
from langgraph.types import Command
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_ctx():
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.device = MagicMock()
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_driver():
|
||||
driver = MagicMock(spec=BaseDeviceDriver)
|
||||
driver.press_key = AsyncMock(return_value=True)
|
||||
return driver
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_state():
|
||||
state = MagicMock(spec=State)
|
||||
|
||||
async def _mock_asanitize_update(ctx, update, agent):
|
||||
return update
|
||||
|
||||
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
|
||||
return state
|
||||
|
||||
|
||||
def test_press_key_tool_subclass_and_registry():
|
||||
"""Verify PressKeyTool is a subclass of ArtemisTool and properly registered."""
|
||||
assert issubclass(PressKeyTool, ArtemisTool)
|
||||
assert issubclass(PressKey, ArtemisTool)
|
||||
assert isinstance(press_key, ArtemisTool)
|
||||
assert isinstance(press_key, PressKeyTool)
|
||||
|
||||
assert press_key.name == "press_key"
|
||||
assert press_key.category == "action"
|
||||
assert press_key.args_schema == PressKeyArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("press_key")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, PressKeyTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = press_key.to_genai_declaration()
|
||||
assert declaration.name == "press_key"
|
||||
assert "key" in declaration.parameters.properties
|
||||
|
||||
# Wrapper check
|
||||
assert press_key_wrapper is not None
|
||||
assert press_key_wrapper.tool_fn_getter == get_press_key_tool
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_press_key_direct_execution_with_driver(mock_driver):
|
||||
"""Verify direct execution with BaseDeviceDriver."""
|
||||
result = await press_key.execute(driver=mock_driver, key="BACK")
|
||||
mock_driver.press_key.assert_called_once_with("BACK")
|
||||
assert "Pressed key 'BACK' successfully." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_press_key_direct_execution_with_ctx(mock_ctx):
|
||||
"""Verify direct execution with ArtemisContext for various keys."""
|
||||
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
|
||||
mock_controller_inst = MagicMock()
|
||||
mock_controller_inst.press_enter = AsyncMock(return_value=True)
|
||||
mock_controller_inst.go_back = AsyncMock(return_value=True)
|
||||
mock_controller_inst.go_home = AsyncMock(return_value=True)
|
||||
mock_controller_inst.press_key = AsyncMock(return_value=True)
|
||||
mock_controller_cls.return_value = mock_controller_inst
|
||||
|
||||
# Test ENTER
|
||||
res_enter = await press_key.execute(ctx=mock_ctx, key="ENTER")
|
||||
mock_controller_inst.press_enter.assert_called_once()
|
||||
assert "Pressed key 'ENTER' successfully." in res_enter
|
||||
|
||||
# Test BACK
|
||||
res_back = await press_key.execute(ctx=mock_ctx, key="BACK")
|
||||
mock_controller_inst.go_back.assert_called_once()
|
||||
assert "Pressed key 'BACK' successfully." in res_back
|
||||
|
||||
# Test HOME
|
||||
res_home = await press_key.execute(ctx=mock_ctx, key="HOME")
|
||||
mock_controller_inst.go_home.assert_called_once()
|
||||
assert "Pressed key 'HOME' successfully." in res_home
|
||||
|
||||
# Test APP_SWITCH
|
||||
res_app_switch = await press_key.execute(ctx=mock_ctx, key="APP_SWITCH")
|
||||
mock_controller_inst.press_key.assert_called_with("KEYCODE_APP_SWITCH")
|
||||
assert "Pressed key 'APP_SWITCH' successfully." in res_app_switch
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_press_key_with_state_command(mock_ctx, mock_state):
|
||||
"""Verify PressKeyTool returns Command when state is provided."""
|
||||
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
|
||||
mock_controller_inst = MagicMock()
|
||||
mock_controller_inst.go_home = AsyncMock(return_value=True)
|
||||
mock_controller_cls.return_value = mock_controller_inst
|
||||
|
||||
cmd = await press_key.execute(
|
||||
ctx=mock_ctx,
|
||||
key="HOME",
|
||||
tool_call_id="call_pk_1",
|
||||
state=mock_state,
|
||||
)
|
||||
assert isinstance(cmd, Command)
|
||||
assert VALIDATOR_MESSAGES_KEY in cmd.update
|
||||
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
|
||||
assert len(messages) == 1
|
||||
assert messages[0].tool_call_id == "call_pk_1"
|
||||
assert messages[0].status == "success"
|
||||
assert "Pressed key 'HOME' successfully." in messages[0].content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_press_key_callable_execution(mock_driver):
|
||||
"""Verify invoking press_key directly as a callable."""
|
||||
result = await press_key(driver=mock_driver, key="ENTER")
|
||||
assert "Pressed key 'ENTER' successfully." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_press_key_missing_key(mock_ctx):
|
||||
"""Verify error handling when key is empty or missing."""
|
||||
result = await press_key.execute(ctx=mock_ctx, key="")
|
||||
assert "Error during press key: Key parameter is required." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_press_key_tool_langchain_ainvoke(mock_ctx):
|
||||
"""Verify get_press_key_tool exports a LangChain tool that works with ainvoke."""
|
||||
pk_tool = get_press_key_tool(mock_ctx)
|
||||
assert pk_tool.name == "press_key"
|
||||
|
||||
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
|
||||
mock_controller_inst = MagicMock()
|
||||
mock_controller_inst.go_back = AsyncMock(return_value=True)
|
||||
mock_controller_cls.return_value = mock_controller_inst
|
||||
|
||||
result = await pk_tool.ainvoke({"key": "BACK"})
|
||||
assert "Pressed key 'BACK' successfully." in result
|
||||
@@ -0,0 +1,145 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.mobile.read_hierarchy import (
|
||||
GetUIHierarchy,
|
||||
GetUiHierarchy,
|
||||
GetUiHierarchyArgs,
|
||||
GetUiHierarchyTool,
|
||||
get_ui_hierarchy,
|
||||
get_ui_hierarchy_tool,
|
||||
ui_hierarchy_wrapper,
|
||||
)
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_ctx():
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.ui_adb_client = MagicMock()
|
||||
ctx.ui_adb_client.get_hierarchy.return_value = "<hierarchy><node text='Home'/></hierarchy>"
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_driver():
|
||||
driver = MagicMock(spec=BaseDeviceDriver)
|
||||
screen_data = MagicMock()
|
||||
screen_data.ui_hierarchy_xml = "<hierarchy><node text='Settings'/></hierarchy>"
|
||||
driver.get_screen_data = AsyncMock(return_value=screen_data)
|
||||
return driver
|
||||
|
||||
|
||||
def test_get_ui_hierarchy_tool_subclass_and_registry():
|
||||
"""Verify GetUiHierarchyTool is a subclass of ArtemisTool and properly registered."""
|
||||
assert issubclass(GetUiHierarchyTool, ArtemisTool)
|
||||
assert issubclass(GetUiHierarchy, ArtemisTool)
|
||||
assert issubclass(GetUIHierarchy, ArtemisTool)
|
||||
assert isinstance(get_ui_hierarchy, ArtemisTool)
|
||||
assert isinstance(get_ui_hierarchy, GetUiHierarchyTool)
|
||||
|
||||
assert get_ui_hierarchy.name == "get_ui_hierarchy"
|
||||
assert get_ui_hierarchy.category == "diagnostic"
|
||||
assert get_ui_hierarchy.args_schema == GetUiHierarchyArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("get_ui_hierarchy")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, GetUiHierarchyTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = get_ui_hierarchy.to_genai_declaration()
|
||||
assert declaration.name == "get_ui_hierarchy"
|
||||
|
||||
# Wrapper check
|
||||
assert ui_hierarchy_wrapper is not None
|
||||
assert ui_hierarchy_wrapper.tool_fn_getter == get_ui_hierarchy_tool
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_ui_hierarchy_direct_execution_with_ctx(mock_ctx):
|
||||
"""Verify direct execution with ArtemisContext."""
|
||||
result = await get_ui_hierarchy.execute(ctx=mock_ctx)
|
||||
mock_ctx.ui_adb_client.get_hierarchy.assert_called_once()
|
||||
assert "<hierarchy><node text='Home'/></hierarchy>" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_ui_hierarchy_direct_execution_with_driver_screen_data(mock_driver):
|
||||
"""Verify direct execution with driver's get_screen_data."""
|
||||
result = await get_ui_hierarchy.execute(driver=mock_driver)
|
||||
mock_driver.get_screen_data.assert_called_once()
|
||||
assert "<hierarchy><node text='Settings'/></hierarchy>" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_ui_hierarchy_direct_execution_with_driver_get_hierarchy():
|
||||
"""Verify direct execution with driver's get_ui_hierarchy."""
|
||||
driver = MagicMock(spec=BaseDeviceDriver)
|
||||
del driver.get_screen_data
|
||||
driver.get_ui_hierarchy = AsyncMock(
|
||||
return_value="<hierarchy><node text='DirectDriver'/></hierarchy>"
|
||||
)
|
||||
result = await get_ui_hierarchy.execute(driver=driver)
|
||||
driver.get_ui_hierarchy.assert_called_once()
|
||||
assert "<hierarchy><node text='DirectDriver'/></hierarchy>" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_ui_hierarchy_with_state(mock_ctx):
|
||||
"""Verify GetUiHierarchyTool returns string when state is provided."""
|
||||
state = MagicMock()
|
||||
result = await get_ui_hierarchy.execute(
|
||||
ctx=mock_ctx,
|
||||
tool_call_id="call_hier_1",
|
||||
state=state,
|
||||
)
|
||||
assert "<hierarchy><node text='Home'/></hierarchy>" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_ui_hierarchy_no_client_or_driver():
|
||||
"""Verify error handling when no client or driver is provided."""
|
||||
result = await get_ui_hierarchy.execute()
|
||||
assert "Error retrieving UI hierarchy: No UI Automator client or driver provided." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_ui_hierarchy_exception_handling(mock_ctx):
|
||||
"""Verify error handling when ui_adb_client raises an exception."""
|
||||
mock_ctx.ui_adb_client.get_hierarchy.side_effect = RuntimeError("ADB server died")
|
||||
result = await get_ui_hierarchy.execute(ctx=mock_ctx)
|
||||
assert "Error retrieving UI hierarchy: ADB server died" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_ui_hierarchy_callable_execution(mock_ctx):
|
||||
"""Verify invoking get_ui_hierarchy directly as a callable."""
|
||||
result = await get_ui_hierarchy(ctx=mock_ctx)
|
||||
assert "<node text='Home'/>" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_ui_hierarchy_tool_langchain_ainvoke(mock_ctx):
|
||||
"""Verify get_ui_hierarchy_tool exports a LangChain tool that works with ainvoke."""
|
||||
tool = get_ui_hierarchy_tool(mock_ctx)
|
||||
assert tool.name == "get_ui_hierarchy"
|
||||
|
||||
result = await tool.ainvoke({})
|
||||
assert "<node text='Home'/>" in result
|
||||
@@ -0,0 +1,140 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.mobile.read_logs import (
|
||||
ReadLogs,
|
||||
ReadLogsArgs,
|
||||
ReadLogsTool,
|
||||
ReadLogsToolAlias,
|
||||
get_read_logs_tool,
|
||||
read_logs,
|
||||
)
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_ctx():
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
return ctx
|
||||
|
||||
|
||||
def test_read_logs_tool_subclass_and_registry():
|
||||
"""Verify ReadLogsTool is a subclass of ArtemisTool and properly registered."""
|
||||
assert issubclass(ReadLogsTool, ArtemisTool)
|
||||
assert issubclass(ReadLogs, ArtemisTool)
|
||||
assert issubclass(ReadLogsToolAlias, ArtemisTool)
|
||||
assert isinstance(read_logs, ArtemisTool)
|
||||
assert isinstance(read_logs, ReadLogsTool)
|
||||
|
||||
assert read_logs.name == "read_logs"
|
||||
assert read_logs.category == "diagnostic"
|
||||
assert read_logs.args_schema == ReadLogsArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("read_logs")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, ReadLogsTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = read_logs.to_genai_declaration()
|
||||
assert declaration.name == "read_logs"
|
||||
assert "lines" in declaration.parameters.properties
|
||||
assert "since_time" in declaration.parameters.properties
|
||||
assert "until_time" in declaration.parameters.properties
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_logs_direct_execution(mock_ctx):
|
||||
"""Verify direct execution of ReadLogsTool."""
|
||||
mock_log_output = "08-12 10:00:00.000 1000 1000 I Test: Hello Logs\n"
|
||||
with patch(
|
||||
"artemis.tools.mobile.read_logs.fetch_and_filter_logs",
|
||||
return_value=mock_log_output,
|
||||
) as mock_fetch:
|
||||
result = await read_logs.execute(
|
||||
ctx=mock_ctx,
|
||||
lines=100,
|
||||
since_time="08-12 09:59:00",
|
||||
until_time="08-12 10:01:00",
|
||||
)
|
||||
mock_fetch.assert_called_once_with(
|
||||
ctx=mock_ctx,
|
||||
lines=100,
|
||||
since_time="08-12 09:59:00",
|
||||
until_time="08-12 10:01:00",
|
||||
)
|
||||
assert result == mock_log_output
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_logs_default_lines(mock_ctx):
|
||||
"""Verify read_logs defaults lines to 200 when None."""
|
||||
with patch(
|
||||
"artemis.tools.mobile.read_logs.fetch_and_filter_logs",
|
||||
return_value="logs",
|
||||
) as mock_fetch:
|
||||
result = await read_logs.execute(ctx=mock_ctx, lines=None)
|
||||
mock_fetch.assert_called_once_with(
|
||||
ctx=mock_ctx,
|
||||
lines=200,
|
||||
since_time=None,
|
||||
until_time=None,
|
||||
)
|
||||
assert result == "logs"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_logs_exception_handling(mock_ctx):
|
||||
"""Verify error handling when fetching logs raises an exception."""
|
||||
with patch(
|
||||
"artemis.tools.mobile.read_logs.fetch_and_filter_logs",
|
||||
side_effect=RuntimeError("Device disconnected"),
|
||||
):
|
||||
result = await read_logs.execute(ctx=mock_ctx)
|
||||
assert "Failed to read logs: Device disconnected" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_logs_callable_execution(mock_ctx):
|
||||
"""Verify invoking read_logs directly as a callable."""
|
||||
with patch(
|
||||
"artemis.tools.mobile.read_logs.fetch_and_filter_logs",
|
||||
return_value="sample log stream",
|
||||
):
|
||||
result = await read_logs(ctx=mock_ctx, lines=50)
|
||||
assert result == "sample log stream"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_read_logs_tool_langchain_ainvoke(mock_ctx):
|
||||
"""Verify get_read_logs_tool exports a LangChain tool that works with ainvoke."""
|
||||
tool = get_read_logs_tool(mock_ctx)
|
||||
assert tool.name == "read_logs"
|
||||
|
||||
with patch(
|
||||
"artemis.tools.mobile.read_logs.fetch_and_filter_logs",
|
||||
return_value="langchain log content",
|
||||
) as mock_fetch:
|
||||
result = await tool.ainvoke({"lines": 300})
|
||||
mock_fetch.assert_called_once_with(
|
||||
ctx=mock_ctx,
|
||||
lines=300,
|
||||
since_time=None,
|
||||
until_time=None,
|
||||
)
|
||||
assert result == "langchain log content"
|
||||
@@ -0,0 +1,127 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.scratchpad import (
|
||||
ReadNoteArgs,
|
||||
ReadNotePure,
|
||||
ReadNotePureTool,
|
||||
get_read_note_tool_pure,
|
||||
read_note_pure,
|
||||
)
|
||||
from artemis.utils.notes import save_note_content
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_ctx(tmp_path):
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.data_engine = MagicMock()
|
||||
ctx.data_engine.base_dir = str(tmp_path)
|
||||
return ctx
|
||||
|
||||
|
||||
def test_read_note_pure_tool_subclass_and_registry():
|
||||
"""Verify ReadNotePureTool is a subclass of ArtemisTool and properly registered."""
|
||||
assert issubclass(ReadNotePureTool, ArtemisTool)
|
||||
assert issubclass(ReadNotePure, ArtemisTool)
|
||||
assert isinstance(read_note_pure, ArtemisTool)
|
||||
assert isinstance(read_note_pure, ReadNotePureTool)
|
||||
|
||||
assert read_note_pure.name == "read_note_pure"
|
||||
assert read_note_pure.category == "memory"
|
||||
assert read_note_pure.args_schema == ReadNoteArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("read_note_pure")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, ReadNotePureTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = read_note_pure.to_genai_declaration()
|
||||
assert declaration.name == "read_note_pure"
|
||||
assert "key" in declaration.parameters.properties
|
||||
assert "start_line" in declaration.parameters.properties
|
||||
assert "end_line" in declaration.parameters.properties
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_note_pure_direct_execution_success(mock_ctx, tmp_path):
|
||||
"""Verify direct execution of ReadNotePureTool.execute reads saved note."""
|
||||
save_note_content(tmp_path, "sample_pure", "line 1\nline 2\nline 3")
|
||||
|
||||
result = await read_note_pure.execute(ctx=mock_ctx, key="sample_pure")
|
||||
assert "Successfully read note 'sample_pure'." in result
|
||||
assert "line 1\nline 2\nline 3" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_note_pure_slice(mock_ctx, tmp_path):
|
||||
"""Verify reading a line slice with start_line and end_line via ReadNotePureTool."""
|
||||
save_note_content(tmp_path, "slice_pure", "line 1\nline 2\nline 3\nline 4")
|
||||
|
||||
result = await read_note_pure.execute(
|
||||
ctx=mock_ctx,
|
||||
key="slice_pure",
|
||||
start_line=2,
|
||||
end_line=3,
|
||||
)
|
||||
assert "Successfully read note 'slice_pure' (lines 2 to 3)." in result
|
||||
assert "line 1" not in result
|
||||
assert "line 2\nline 3" in result
|
||||
assert "line 4" not in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_note_pure_callable_execution(mock_ctx, tmp_path):
|
||||
"""Verify invoking read_note_pure directly as a callable."""
|
||||
save_note_content(tmp_path, "callable_pure", "hello from pure callable")
|
||||
|
||||
result = await read_note_pure(ctx=mock_ctx, key="callable_pure")
|
||||
assert "Successfully read note 'callable_pure'." in result
|
||||
assert "hello from pure callable" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_note_pure_not_found(mock_ctx):
|
||||
"""Verify error handling when note is missing."""
|
||||
result = await read_note_pure.execute(ctx=mock_ctx, key="nonexistent_pure")
|
||||
assert "Note 'nonexistent_pure' not found" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_note_pure_other_error(mock_ctx):
|
||||
"""Verify error handling when read_note_content raises an exception."""
|
||||
with patch(
|
||||
"artemis.tools.scratchpad.read_note_content",
|
||||
side_effect=OSError("Disk error"),
|
||||
):
|
||||
result = await read_note_pure.execute(ctx=mock_ctx, key="err_note")
|
||||
assert "Failed to read note 'err_note': Disk error" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_read_note_tool_pure_langchain_ainvoke(mock_ctx, tmp_path):
|
||||
"""Verify get_read_note_tool_pure exports a LangChain tool named 'read_note'."""
|
||||
save_note_content(tmp_path, "pure_export", "exported pure note")
|
||||
|
||||
pure_tool = get_read_note_tool_pure(mock_ctx)
|
||||
assert pure_tool.name == "read_note"
|
||||
|
||||
result = await pure_tool.ainvoke({"key": "pure_export"})
|
||||
assert "Successfully read note 'pure_export'." in result
|
||||
assert "exported pure note" in result
|
||||
@@ -0,0 +1,177 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.scratchpad import (
|
||||
ReadNote,
|
||||
ReadNoteArgs,
|
||||
ReadNoteTool,
|
||||
get_read_note_tool,
|
||||
get_read_note_tool_pure,
|
||||
read_note,
|
||||
read_note_wrapper,
|
||||
)
|
||||
from artemis.utils.notes import save_note_content
|
||||
from langgraph.types import Command
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_ctx(tmp_path):
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.data_engine = MagicMock()
|
||||
ctx.data_engine.base_dir = str(tmp_path)
|
||||
return ctx
|
||||
|
||||
|
||||
def test_read_note_tool_subclass_and_registry():
|
||||
"""Verify ReadNoteTool is a subclass of ArtemisTool and properly registered."""
|
||||
assert issubclass(ReadNoteTool, ArtemisTool)
|
||||
assert issubclass(ReadNote, ArtemisTool)
|
||||
assert isinstance(read_note, ArtemisTool)
|
||||
assert isinstance(read_note, ReadNoteTool)
|
||||
|
||||
assert read_note.name == "read_note"
|
||||
assert read_note.category == "memory"
|
||||
assert read_note.args_schema == ReadNoteArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("read_note")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, ReadNoteTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = read_note.to_genai_declaration()
|
||||
assert declaration.name == "read_note"
|
||||
assert "key" in declaration.parameters.properties
|
||||
assert "start_line" in declaration.parameters.properties
|
||||
assert "end_line" in declaration.parameters.properties
|
||||
|
||||
# Wrapper check
|
||||
assert read_note_wrapper is not None
|
||||
assert read_note_wrapper.tool_fn_getter == get_read_note_tool
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_note_direct_execution_success(mock_ctx, tmp_path):
|
||||
"""Verify direct execution of ReadNoteTool.execute reads saved note."""
|
||||
save_note_content(tmp_path, "sample", "line 1\nline 2\nline 3\nline 4\nline 5")
|
||||
|
||||
result = await read_note.execute(ctx=mock_ctx, key="sample")
|
||||
assert "Successfully read note 'sample'." in result
|
||||
assert "line 1\nline 2\nline 3\nline 4\nline 5" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_note_slice(mock_ctx, tmp_path):
|
||||
"""Verify reading a line slice with start_line and end_line."""
|
||||
save_note_content(tmp_path, "sample", "line 1\nline 2\nline 3\nline 4\nline 5")
|
||||
|
||||
result = await read_note.execute(
|
||||
ctx=mock_ctx,
|
||||
key="sample",
|
||||
start_line=2,
|
||||
end_line=4,
|
||||
)
|
||||
assert "Successfully read note 'sample' (lines 2 to 4)." in result
|
||||
assert "line 1" not in result
|
||||
assert "line 2\nline 3\nline 4" in result
|
||||
assert "line 5" not in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_note_callable_execution(mock_ctx, tmp_path):
|
||||
"""Verify invoking read_note directly as a callable."""
|
||||
save_note_content(tmp_path, "callable_sample", "hello from callable")
|
||||
|
||||
result = await read_note(ctx=mock_ctx, key="callable_sample")
|
||||
assert "Successfully read note 'callable_sample'." in result
|
||||
assert "hello from callable" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_note_with_state_command(mock_ctx, tmp_path):
|
||||
"""Verify ReadNoteTool.execute with state returns Command updating VALIDATOR_MESSAGES_KEY."""
|
||||
save_note_content(tmp_path, "state_sample", "state note content")
|
||||
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.current_agent = "operator"
|
||||
mock_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update)
|
||||
|
||||
result = await read_note.execute(
|
||||
ctx=mock_ctx,
|
||||
key="state_sample",
|
||||
state=mock_state,
|
||||
tool_call_id="call_read_999",
|
||||
)
|
||||
|
||||
assert isinstance(result, Command)
|
||||
assert VALIDATOR_MESSAGES_KEY in result.update
|
||||
tool_messages = result.update[VALIDATOR_MESSAGES_KEY]
|
||||
assert len(tool_messages) == 1
|
||||
assert tool_messages[0].tool_call_id == "call_read_999"
|
||||
assert "Successfully read note 'state_sample'." in tool_messages[0].content
|
||||
assert "state note content" in tool_messages[0].content
|
||||
assert tool_messages[0].status == "success"
|
||||
|
||||
mock_state.asanitize_update.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_note_not_found(mock_ctx):
|
||||
"""Verify error handling when a note is not found."""
|
||||
result = await read_note.execute(ctx=mock_ctx, key="nonexistent")
|
||||
assert "Note 'nonexistent' not found" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_note_other_error(mock_ctx):
|
||||
"""Verify error handling for unexpected exceptions."""
|
||||
with patch(
|
||||
"artemis.tools.scratchpad.read_note_content",
|
||||
side_effect=ValueError("Corrupted data"),
|
||||
):
|
||||
result = await read_note.execute(ctx=mock_ctx, key="bad_note")
|
||||
assert "Failed to read note 'bad_note': Corrupted data" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_read_note_tool_langchain_ainvoke(mock_ctx, tmp_path):
|
||||
"""Verify get_read_note_tool exports a LangChain tool that works with ainvoke."""
|
||||
save_note_content(tmp_path, "lc_sample", "langchain content")
|
||||
|
||||
lc_tool = get_read_note_tool(mock_ctx)
|
||||
assert lc_tool.name == "read_note"
|
||||
|
||||
result = await lc_tool.ainvoke({"key": "lc_sample"})
|
||||
assert "Successfully read note 'lc_sample'." in result
|
||||
assert "langchain content" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_read_note_tool_pure(mock_ctx, tmp_path):
|
||||
"""Verify get_read_note_tool_pure returns a pure LangChain tool."""
|
||||
save_note_content(tmp_path, "pure_sample", "pure content")
|
||||
|
||||
pure_tool = get_read_note_tool_pure(mock_ctx)
|
||||
assert pure_tool.name == "read_note"
|
||||
|
||||
result = await pure_tool.ainvoke({"key": "pure_sample"})
|
||||
assert "Successfully read note 'pure_sample'." in result
|
||||
assert "pure content" in result
|
||||
@@ -0,0 +1,122 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.scratchpad import (
|
||||
SaveNoteArgs,
|
||||
SaveNotePure,
|
||||
SaveNotePureTool,
|
||||
get_save_note_tool_pure,
|
||||
save_note_pure,
|
||||
)
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_ctx(tmp_path):
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.data_engine = MagicMock()
|
||||
ctx.data_engine.base_dir = str(tmp_path)
|
||||
return ctx
|
||||
|
||||
|
||||
def test_save_note_pure_tool_subclass_and_registry():
|
||||
"""Verify SaveNotePureTool is a subclass of ArtemisTool and properly registered."""
|
||||
assert issubclass(SaveNotePureTool, ArtemisTool)
|
||||
assert issubclass(SaveNotePure, ArtemisTool)
|
||||
assert isinstance(save_note_pure, ArtemisTool)
|
||||
assert isinstance(save_note_pure, SaveNotePureTool)
|
||||
|
||||
assert save_note_pure.name == "save_note_pure"
|
||||
assert save_note_pure.category == "memory"
|
||||
assert save_note_pure.args_schema == SaveNoteArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("save_note_pure")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, SaveNotePureTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = save_note_pure.to_genai_declaration()
|
||||
assert declaration.name == "save_note_pure"
|
||||
assert "key" in declaration.parameters.properties
|
||||
assert "content" in declaration.parameters.properties
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_save_note_pure_direct_execution_success(mock_ctx, tmp_path):
|
||||
"""Verify direct execution of SaveNotePureTool.execute creates note file."""
|
||||
result = await save_note_pure.execute(
|
||||
ctx=mock_ctx,
|
||||
key="test_pure_save",
|
||||
content="Pure saved content",
|
||||
)
|
||||
assert result == "Successfully saved note to test_pure_save.md."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "test_pure_save.md"
|
||||
assert note_path.exists()
|
||||
assert note_path.read_text(encoding="utf-8") == "Pure saved content"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_save_note_pure_callable_execution(mock_ctx, tmp_path):
|
||||
"""Verify invoking save_note_pure directly as a callable."""
|
||||
result = await save_note_pure(
|
||||
ctx=mock_ctx,
|
||||
key="callable_pure_save",
|
||||
content="Callable pure content",
|
||||
)
|
||||
assert result == "Successfully saved note to callable_pure_save.md."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "callable_pure_save.md"
|
||||
assert note_path.exists()
|
||||
assert note_path.read_text(encoding="utf-8") == "Callable pure content"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_save_note_pure_execution_failure(mock_ctx):
|
||||
"""Verify error handling when save_note_content fails."""
|
||||
with patch(
|
||||
"artemis.tools.scratchpad.save_note_content",
|
||||
side_effect=PermissionError("Permission denied"),
|
||||
):
|
||||
result = await save_note_pure.execute(
|
||||
ctx=mock_ctx,
|
||||
key="fail_save",
|
||||
content="Some content",
|
||||
)
|
||||
assert "Failed to save note fail_save.md: Permission denied" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_save_note_tool_pure_langchain_ainvoke(mock_ctx, tmp_path):
|
||||
"""Verify get_save_note_tool_pure exports a LangChain tool named 'save_note'."""
|
||||
pure_tool = get_save_note_tool_pure(mock_ctx)
|
||||
assert pure_tool.name == "save_note"
|
||||
|
||||
result = await pure_tool.ainvoke(
|
||||
{
|
||||
"key": "lc_pure_save",
|
||||
"content": "Saved via LangChain pure tool",
|
||||
}
|
||||
)
|
||||
assert result == "Successfully saved note to lc_pure_save.md."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "lc_pure_save.md"
|
||||
assert note_path.exists()
|
||||
assert note_path.read_text(encoding="utf-8") == "Saved via LangChain pure tool"
|
||||
@@ -0,0 +1,180 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.scratchpad import (
|
||||
SaveNote,
|
||||
SaveNoteArgs,
|
||||
SaveNoteTool,
|
||||
get_save_note_tool,
|
||||
get_save_note_tool_pure,
|
||||
save_note,
|
||||
save_note_wrapper,
|
||||
)
|
||||
from langgraph.types import Command
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_ctx(tmp_path):
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.data_engine = MagicMock()
|
||||
ctx.data_engine.base_dir = str(tmp_path)
|
||||
return ctx
|
||||
|
||||
|
||||
def test_save_note_tool_subclass_and_registry():
|
||||
"""Verify SaveNoteTool is a subclass of ArtemisTool and properly registered in ToolRegistry."""
|
||||
assert issubclass(SaveNoteTool, ArtemisTool)
|
||||
assert issubclass(SaveNote, ArtemisTool)
|
||||
assert isinstance(save_note, ArtemisTool)
|
||||
assert isinstance(save_note, SaveNoteTool)
|
||||
|
||||
assert save_note.name == "save_note"
|
||||
assert save_note.category == "memory"
|
||||
assert save_note.args_schema == SaveNoteArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("save_note")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, SaveNoteTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = save_note.to_genai_declaration()
|
||||
assert declaration.name == "save_note"
|
||||
assert "key" in declaration.parameters.properties
|
||||
assert "content" in declaration.parameters.properties
|
||||
|
||||
# Wrapper check
|
||||
assert save_note_wrapper is not None
|
||||
assert save_note_wrapper.tool_fn_getter == get_save_note_tool
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_save_note_direct_execution_success(mock_ctx, tmp_path):
|
||||
"""Verify direct execution of SaveNoteTool.execute creates note file."""
|
||||
result = await save_note.execute(
|
||||
ctx=mock_ctx,
|
||||
key="test_note",
|
||||
content="Hello Artemis memory",
|
||||
)
|
||||
assert result == "Successfully saved note to test_note.md."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "test_note.md"
|
||||
assert note_path.exists()
|
||||
assert note_path.read_text(encoding="utf-8") == "Hello Artemis memory"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_save_note_callable_execution(mock_ctx, tmp_path):
|
||||
"""Verify invoking save_note directly as a callable."""
|
||||
result = await save_note(
|
||||
ctx=mock_ctx,
|
||||
key="callable_note",
|
||||
content="Callable invocation test",
|
||||
)
|
||||
assert result == "Successfully saved note to callable_note.md."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "callable_note.md"
|
||||
assert note_path.exists()
|
||||
assert note_path.read_text(encoding="utf-8") == "Callable invocation test"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_save_note_with_state_command(mock_ctx, tmp_path):
|
||||
"""Verify SaveNoteTool.execute with state returns Command updating VALIDATOR_MESSAGES_KEY."""
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.current_agent = "operator"
|
||||
mock_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update)
|
||||
|
||||
result = await save_note.execute(
|
||||
ctx=mock_ctx,
|
||||
key="state_note",
|
||||
content="Note with state",
|
||||
state=mock_state,
|
||||
tool_call_id="call_save_999",
|
||||
)
|
||||
|
||||
assert isinstance(result, Command)
|
||||
assert VALIDATOR_MESSAGES_KEY in result.update
|
||||
tool_messages = result.update[VALIDATOR_MESSAGES_KEY]
|
||||
assert len(tool_messages) == 1
|
||||
assert tool_messages[0].tool_call_id == "call_save_999"
|
||||
assert tool_messages[0].content == "Successfully saved note to state_note.md."
|
||||
assert tool_messages[0].status == "success"
|
||||
|
||||
mock_state.asanitize_update.assert_called_once()
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "state_note.md"
|
||||
assert note_path.exists()
|
||||
assert note_path.read_text(encoding="utf-8") == "Note with state"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_save_note_execution_failure(mock_ctx):
|
||||
"""Verify error handling when saving note fails."""
|
||||
with patch(
|
||||
"artemis.tools.scratchpad.save_note_content",
|
||||
side_effect=PermissionError("Permission denied"),
|
||||
):
|
||||
result = await save_note.execute(
|
||||
ctx=mock_ctx,
|
||||
key="fail_note",
|
||||
content="Fail content",
|
||||
)
|
||||
assert "Failed to save note fail_note.md: Permission denied" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_save_note_tool_langchain_ainvoke(mock_ctx, tmp_path):
|
||||
"""Verify get_save_note_tool exports a LangChain tool that works with ainvoke."""
|
||||
lc_tool = get_save_note_tool(mock_ctx)
|
||||
assert lc_tool.name == "save_note"
|
||||
|
||||
result = await lc_tool.ainvoke(
|
||||
{
|
||||
"key": "lc_note",
|
||||
"content": "Saved via LangChain BaseTool",
|
||||
}
|
||||
)
|
||||
assert result == "Successfully saved note to lc_note.md."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "lc_note.md"
|
||||
assert note_path.exists()
|
||||
assert note_path.read_text(encoding="utf-8") == "Saved via LangChain BaseTool"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_save_note_tool_pure(mock_ctx, tmp_path):
|
||||
"""Verify get_save_note_tool_pure returns a pure LangChain tool."""
|
||||
pure_tool = get_save_note_tool_pure(mock_ctx)
|
||||
assert pure_tool.name == "save_note"
|
||||
|
||||
result = await pure_tool.ainvoke(
|
||||
{
|
||||
"key": "pure_note",
|
||||
"content": "Saved via pure tool",
|
||||
}
|
||||
)
|
||||
assert result == "Successfully saved note to pure_note.md."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "pure_note.md"
|
||||
assert note_path.exists()
|
||||
assert note_path.read_text(encoding="utf-8") == "Saved via pure tool"
|
||||
@@ -0,0 +1,262 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.mobile.search_logs import (
|
||||
SearchLogs,
|
||||
SearchLogsArgs,
|
||||
SearchLogsTool,
|
||||
SearchLogsToolAlias,
|
||||
get_search_logs_tool,
|
||||
search_and_merge_logs,
|
||||
search_logs,
|
||||
)
|
||||
import pytest
|
||||
|
||||
|
||||
def test_search_and_merge_logs_no_matches():
|
||||
logs = "line 0: info\nline 1: info\nline 2: info"
|
||||
result = search_and_merge_logs(logs, "ERROR", context_lines=2)
|
||||
assert result == "No matches found for 'ERROR'."
|
||||
|
||||
|
||||
def test_search_and_merge_logs_single_match():
|
||||
logs = "\n".join([f"line {i}" if i != 5 else "ERROR: failure" for i in range(10)])
|
||||
result = search_and_merge_logs(logs, "ERROR", context_lines=2)
|
||||
|
||||
expected = (
|
||||
"--- Match at line(s) 5 ---\n"
|
||||
" 3: line 3\n"
|
||||
" 4: line 4\n"
|
||||
" 5: [MATCH] ERROR: failure\n"
|
||||
" 6: line 6\n"
|
||||
" 7: line 7\n"
|
||||
"--------------------"
|
||||
)
|
||||
assert result == expected
|
||||
|
||||
|
||||
def test_search_and_merge_logs_overlapping_matches():
|
||||
logs = "\n".join([f"line {i}" if i not in [4, 6] else "ERROR: failure" for i in range(10)])
|
||||
result = search_and_merge_logs(logs, "ERROR", context_lines=2)
|
||||
|
||||
# Match at 4 (range 2 to 6 inclusive)
|
||||
# Match at 6 (range 4 to 8 inclusive)
|
||||
# Merged range: 2 to 8 inclusive
|
||||
expected = (
|
||||
"--- Match at line(s) 4, 6 ---\n"
|
||||
" 2: line 2\n"
|
||||
" 3: line 3\n"
|
||||
" 4: [MATCH] ERROR: failure\n"
|
||||
" 5: line 5\n"
|
||||
" 6: [MATCH] ERROR: failure\n"
|
||||
" 7: line 7\n"
|
||||
" 8: line 8\n"
|
||||
"--------------------"
|
||||
)
|
||||
assert result == expected
|
||||
|
||||
|
||||
def test_search_and_merge_logs_adjacent_matches():
|
||||
# Match at 3 (range 1 to 5 inclusive)
|
||||
# Match at 6 (range 4 to 8 inclusive)
|
||||
# Overlap is at lines 4-5. They are adjacent/overlapping and should merge.
|
||||
logs = "\n".join([f"line {i}" if i not in [3, 6] else "ERROR: failure" for i in range(10)])
|
||||
result = search_and_merge_logs(logs, "ERROR", context_lines=2)
|
||||
|
||||
expected = (
|
||||
"--- Match at line(s) 3, 6 ---\n"
|
||||
" 1: line 1\n"
|
||||
" 2: line 2\n"
|
||||
" 3: [MATCH] ERROR: failure\n"
|
||||
" 4: line 4\n"
|
||||
" 5: line 5\n"
|
||||
" 6: [MATCH] ERROR: failure\n"
|
||||
" 7: line 7\n"
|
||||
" 8: line 8\n"
|
||||
"--------------------"
|
||||
)
|
||||
assert result == expected
|
||||
|
||||
|
||||
def test_search_and_merge_logs_multiple_separate_blocks():
|
||||
# Match at 2 (range 0 to 4 inclusive)
|
||||
# Match at 8 (range 6 to 9 inclusive since len is 10)
|
||||
# Non-overlapping since end of first is 4, start of second is 6.
|
||||
logs = "\n".join([f"line {i}" if i not in [2, 8] else "ERROR: failure" for i in range(10)])
|
||||
result = search_and_merge_logs(logs, "ERROR", context_lines=2)
|
||||
|
||||
expected = (
|
||||
"--- Match at line(s) 2 ---\n"
|
||||
" 0: line 0\n"
|
||||
" 1: line 1\n"
|
||||
" 2: [MATCH] ERROR: failure\n"
|
||||
" 3: line 3\n"
|
||||
" 4: line 4\n"
|
||||
"--------------------\n"
|
||||
"--- Match at line(s) 8 ---\n"
|
||||
" 6: line 6\n"
|
||||
" 7: line 7\n"
|
||||
" 8: [MATCH] ERROR: failure\n"
|
||||
" 9: line 9\n"
|
||||
"--------------------"
|
||||
)
|
||||
assert result == expected
|
||||
|
||||
|
||||
def test_search_and_merge_logs_regex():
|
||||
logs = "line 0\nERR_123: failed\nline 2\nERR_456: critical"
|
||||
result = search_and_merge_logs(logs, r"ERR_\d+", context_lines=1, is_regex=True)
|
||||
|
||||
expected = (
|
||||
"--- Match at line(s) 1, 3 ---\n"
|
||||
" 0: line 0\n"
|
||||
" 1: [MATCH] ERR_123: failed\n"
|
||||
" 2: line 2\n"
|
||||
" 3: [MATCH] ERR_456: critical\n"
|
||||
"--------------------"
|
||||
)
|
||||
assert result == expected
|
||||
|
||||
|
||||
def test_search_and_merge_logs_invalid_regex():
|
||||
logs = "line 0"
|
||||
result = search_and_merge_logs(logs, "[invalid", is_regex=True)
|
||||
assert result.startswith("Error: Invalid regex:")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SearchLogsTool Unit Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
@pytest.fixture
|
||||
def mock_ctx():
|
||||
return MagicMock(spec=ArtemisContext)
|
||||
|
||||
|
||||
def test_search_logs_tool_subclass_and_registry():
|
||||
"""Verify SearchLogsTool is a subclass of ArtemisTool and properly registered."""
|
||||
assert issubclass(SearchLogsTool, ArtemisTool)
|
||||
assert issubclass(SearchLogs, ArtemisTool)
|
||||
assert issubclass(SearchLogsToolAlias, ArtemisTool)
|
||||
assert isinstance(search_logs, ArtemisTool)
|
||||
assert isinstance(search_logs, SearchLogsTool)
|
||||
|
||||
assert search_logs.name == "search_logs"
|
||||
assert search_logs.category == "diagnostic"
|
||||
assert search_logs.args_schema == SearchLogsArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("search_logs")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, SearchLogsTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = search_logs.to_genai_declaration()
|
||||
assert declaration.name == "search_logs"
|
||||
assert "keyword" in declaration.parameters.properties
|
||||
assert "is_regex" in declaration.parameters.properties
|
||||
assert "lines" in declaration.parameters.properties
|
||||
assert "since_time" in declaration.parameters.properties
|
||||
assert "until_time" in declaration.parameters.properties
|
||||
assert "context_lines" in declaration.parameters.properties
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_logs_direct_execution(mock_ctx):
|
||||
"""Verify direct execution of SearchLogsTool."""
|
||||
mock_log_output = "line 1: FATAL: crash occurred\nline 2: normal"
|
||||
with patch(
|
||||
"artemis.tools.mobile.search_logs.fetch_and_filter_logs",
|
||||
return_value=mock_log_output,
|
||||
) as mock_fetch:
|
||||
result = await search_logs.execute(
|
||||
ctx=mock_ctx,
|
||||
keyword="FATAL",
|
||||
is_regex=False,
|
||||
lines=5000,
|
||||
since_time="08-12 10:00:00",
|
||||
until_time="08-12 10:05:00",
|
||||
context_lines=1,
|
||||
)
|
||||
mock_fetch.assert_called_once_with(
|
||||
ctx=mock_ctx,
|
||||
lines=5000,
|
||||
since_time="08-12 10:00:00",
|
||||
until_time="08-12 10:05:00",
|
||||
)
|
||||
assert "[MATCH] line 1: FATAL: crash occurred" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_logs_default_args(mock_ctx):
|
||||
"""Verify search_logs defaults lines to 10000 and context_lines to 0."""
|
||||
with patch(
|
||||
"artemis.tools.mobile.search_logs.fetch_and_filter_logs",
|
||||
return_value="info line\nerror line",
|
||||
) as mock_fetch:
|
||||
result = await search_logs.execute(
|
||||
ctx=mock_ctx, keyword="error", lines=None, context_lines=None
|
||||
)
|
||||
mock_fetch.assert_called_once_with(
|
||||
ctx=mock_ctx,
|
||||
lines=10000,
|
||||
since_time=None,
|
||||
until_time=None,
|
||||
)
|
||||
assert "[MATCH] error line" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_logs_exception_handling(mock_ctx):
|
||||
"""Verify error handling when searching logs raises an exception."""
|
||||
with patch(
|
||||
"artemis.tools.mobile.search_logs.fetch_and_filter_logs",
|
||||
side_effect=RuntimeError("Device unreachable"),
|
||||
):
|
||||
result = await search_logs.execute(ctx=mock_ctx, keyword="test")
|
||||
assert "Error searching logs: Device unreachable" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_logs_callable_execution(mock_ctx):
|
||||
"""Verify invoking search_logs directly as a callable."""
|
||||
with patch(
|
||||
"artemis.tools.mobile.search_logs.fetch_and_filter_logs",
|
||||
return_value="NullPointerException in activity",
|
||||
):
|
||||
result = await search_logs(ctx=mock_ctx, keyword="NullPointer", context_lines=0)
|
||||
assert "[MATCH] NullPointerException in activity" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_search_logs_tool_langchain_ainvoke(mock_ctx):
|
||||
"""Verify get_search_logs_tool exports a LangChain tool that works with ainvoke."""
|
||||
tool = get_search_logs_tool(mock_ctx)
|
||||
assert tool.name == "search_logs"
|
||||
|
||||
with patch(
|
||||
"artemis.tools.mobile.search_logs.fetch_and_filter_logs",
|
||||
return_value="critical error detected",
|
||||
) as mock_fetch:
|
||||
result = await tool.ainvoke({"keyword": "critical", "lines": 2000})
|
||||
mock_fetch.assert_called_once_with(
|
||||
ctx=mock_ctx,
|
||||
lines=2000,
|
||||
since_time=None,
|
||||
until_time=None,
|
||||
)
|
||||
assert "[MATCH] critical error detected" in result
|
||||
@@ -0,0 +1,255 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.mobile.exec_tools import (
|
||||
Swipe,
|
||||
SwipeArgs,
|
||||
SwipeTool,
|
||||
get_swipe_tool,
|
||||
swipe,
|
||||
swipe_wrapper,
|
||||
)
|
||||
from langgraph.types import Command
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_ctx():
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.device = MagicMock()
|
||||
ctx.device.device_width = 1080
|
||||
ctx.device.device_height = 2400
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_driver():
|
||||
driver = MagicMock(spec=BaseDeviceDriver)
|
||||
driver.screen_size = (1080, 2400)
|
||||
driver.swipe_direction = AsyncMock(return_value=True)
|
||||
driver.swipe = AsyncMock(return_value=True)
|
||||
return driver
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_state():
|
||||
state = MagicMock(spec=State)
|
||||
|
||||
async def _mock_asanitize_update(ctx, update, agent):
|
||||
return update
|
||||
|
||||
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
|
||||
return state
|
||||
|
||||
|
||||
def test_swipe_tool_subclass_and_registry():
|
||||
"""Verify SwipeTool is a subclass of ArtemisTool and properly registered."""
|
||||
assert issubclass(SwipeTool, ArtemisTool)
|
||||
assert issubclass(Swipe, ArtemisTool)
|
||||
assert isinstance(swipe, ArtemisTool)
|
||||
assert isinstance(swipe, SwipeTool)
|
||||
|
||||
assert swipe.name == "swipe"
|
||||
assert swipe.category == "action"
|
||||
assert swipe.args_schema == SwipeArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("swipe")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, ArtemisTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = swipe.to_genai_declaration()
|
||||
assert declaration.name == "swipe"
|
||||
assert "action" in declaration.parameters.properties
|
||||
assert "duration" in declaration.parameters.properties
|
||||
|
||||
# Wrapper check
|
||||
assert swipe_wrapper is not None
|
||||
assert swipe_wrapper.tool_fn_getter == get_swipe_tool
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swipe_direct_execution_with_driver_direction(mock_driver):
|
||||
"""Verify direct execution with direction on BaseDeviceDriver."""
|
||||
result = await swipe.execute(driver=mock_driver, action="up", duration=500)
|
||||
mock_driver.swipe_direction.assert_called_once_with("up", duration_ms=500)
|
||||
assert "Swipe completed successfully." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swipe_direct_execution_with_driver_coords(mock_driver):
|
||||
"""Verify direct execution with coordinates on BaseDeviceDriver."""
|
||||
result = await swipe.execute(driver=mock_driver, action=[100, 200, 300, 400], duration=600)
|
||||
mock_driver.swipe.assert_called_once_with(108, 480, 324, 960, duration_ms=600)
|
||||
assert "Swipe completed successfully." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swipe_direct_execution_with_ctx_direction(mock_ctx):
|
||||
"""Verify direct execution with direction on ArtemisContext."""
|
||||
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
|
||||
mock_controller_inst = MagicMock()
|
||||
mock_controller_inst.swipe_coords = AsyncMock(return_value=None)
|
||||
mock_controller_cls.return_value = mock_controller_inst
|
||||
|
||||
result = await swipe.execute(ctx=mock_ctx, action="down", duration=400)
|
||||
mock_controller_inst.swipe_coords.assert_called_once_with(540, 480, 540, 1920, 400)
|
||||
assert "Swipe completed successfully." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swipe_direct_execution_with_ctx_coords(mock_ctx):
|
||||
"""Verify direct execution with coordinates on ArtemisContext."""
|
||||
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
|
||||
mock_controller_inst = MagicMock()
|
||||
mock_controller_inst.swipe_coords = AsyncMock(return_value=None)
|
||||
mock_controller_cls.return_value = mock_controller_inst
|
||||
|
||||
result = await swipe.execute(ctx=mock_ctx, action=[200, 300, 800, 900], duration=800)
|
||||
mock_controller_inst.swipe_coords.assert_called_once_with(216, 720, 864, 2160, 800)
|
||||
assert "Swipe completed successfully." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swipe_with_state_command(mock_ctx, mock_state):
|
||||
"""Verify SwipeTool returns Command when state is provided."""
|
||||
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
|
||||
mock_controller_inst = MagicMock()
|
||||
mock_controller_inst.swipe_coords = AsyncMock(return_value=None)
|
||||
mock_controller_cls.return_value = mock_controller_inst
|
||||
|
||||
cmd = await swipe.execute(
|
||||
ctx=mock_ctx,
|
||||
action="left",
|
||||
tool_call_id="call_sw_1",
|
||||
state=mock_state,
|
||||
)
|
||||
assert isinstance(cmd, Command)
|
||||
assert VALIDATOR_MESSAGES_KEY in cmd.update
|
||||
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
|
||||
assert len(messages) == 1
|
||||
assert messages[0].tool_call_id == "call_sw_1"
|
||||
assert messages[0].status == "success"
|
||||
assert "Swipe completed successfully." in messages[0].content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swipe_callable_execution(mock_driver):
|
||||
"""Verify invoking swipe directly as a callable."""
|
||||
result = await swipe(driver=mock_driver, action="right")
|
||||
assert "Swipe completed successfully." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swipe_invalid_action(mock_ctx):
|
||||
"""Verify error handling on invalid action."""
|
||||
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController"):
|
||||
result = await swipe.execute(ctx=mock_ctx, action="diagonal")
|
||||
assert "Error during swipe: Invalid" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_swipe_tool_langchain_ainvoke(mock_ctx):
|
||||
"""Verify get_swipe_tool exports a LangChain tool that works with ainvoke."""
|
||||
sw_tool = get_swipe_tool(mock_ctx)
|
||||
assert sw_tool.name == "swipe"
|
||||
|
||||
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
|
||||
mock_controller_inst = MagicMock()
|
||||
mock_controller_inst.swipe_coords = AsyncMock(return_value=None)
|
||||
mock_controller_cls.return_value = mock_controller_inst
|
||||
|
||||
result = await sw_tool.ainvoke({"action": "up"})
|
||||
assert "Swipe completed successfully." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swipe_with_direction_and_start_end(mock_driver):
|
||||
"""Verify swipe works with explicit direction and start/end parameters."""
|
||||
# Direction parameter
|
||||
result_dir = await swipe.execute(driver=mock_driver, direction="left", duration=500)
|
||||
mock_driver.swipe_direction.assert_called_with("left", duration_ms=500)
|
||||
assert "Swipe completed successfully." in result_dir
|
||||
|
||||
# Start and End parameters
|
||||
result_drag = await swipe.execute(
|
||||
driver=mock_driver, start=[100, 200], end=[300, 400], duration=1500
|
||||
)
|
||||
mock_driver.swipe.assert_called_with(108, 480, 324, 960, duration_ms=1500)
|
||||
assert "Swipe completed successfully." in result_drag
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swipe_with_coordinates_alias(mock_driver):
|
||||
"""Verify swipe works with coordinates alias parameter."""
|
||||
result = await swipe.execute(driver=mock_driver, coordinates=[200, 300, 400, 500], duration=800)
|
||||
mock_driver.swipe.assert_called_with(216, 720, 432, 1200, duration_ms=800)
|
||||
assert "Swipe completed successfully." in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swipe_with_string_coordinates(mock_driver):
|
||||
"""Verify swipe works with string-formatted coordinates."""
|
||||
# List format in string
|
||||
result1 = await swipe.execute(driver=mock_driver, action="[920, 290, 920, 180]", duration=1000)
|
||||
mock_driver.swipe.assert_called_with(993, 696, 993, 432, duration_ms=1000)
|
||||
assert "Swipe completed successfully." in result1
|
||||
|
||||
# Space-separated format in string
|
||||
result2 = await swipe.execute(driver=mock_driver, action="920 290 920 180", duration=1000)
|
||||
assert "Swipe completed successfully." in result2
|
||||
|
||||
|
||||
def test_parse_swipe_parameters_direct():
|
||||
"""Verify parse_swipe_parameters with various parameter formats."""
|
||||
from artemis.utils.coordinates import parse_swipe_parameters
|
||||
|
||||
# List
|
||||
k, t, d = parse_swipe_parameters([100, 200, 300, 400])
|
||||
assert k == "coords"
|
||||
assert t == [100, 200, 300, 400]
|
||||
|
||||
# String list
|
||||
k, t, d = parse_swipe_parameters("[920, 290, 920, 180]")
|
||||
assert k == "coords"
|
||||
assert t == [920, 290, 920, 180]
|
||||
|
||||
# String space separated
|
||||
k, t, d = parse_swipe_parameters("920 290 920 180")
|
||||
assert k == "coords"
|
||||
assert t == [920, 290, 920, 180]
|
||||
|
||||
# Dict with action string
|
||||
k, t, d = parse_swipe_parameters({"action": "[920, 290, 920, 180]", "duration": 1000})
|
||||
assert k == "coords"
|
||||
assert t == [920, 290, 920, 180]
|
||||
assert d == 1000
|
||||
|
||||
# Dict with start and end
|
||||
k, t, d = parse_swipe_parameters({"start": [100, 200], "end": [300, 400]})
|
||||
assert k == "coords"
|
||||
assert t == [100, 200, 300, 400]
|
||||
|
||||
# Direction string
|
||||
k, t, d = parse_swipe_parameters({"action": "up"})
|
||||
assert k == "direction"
|
||||
assert t == "up"
|
||||
@@ -0,0 +1,146 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.scratchpad import (
|
||||
UpdateNoteArgs,
|
||||
UpdateNotePure,
|
||||
UpdateNotePureTool,
|
||||
get_update_note_tool_pure,
|
||||
update_note_pure,
|
||||
)
|
||||
from artemis.utils.notes import save_note_content
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_ctx(tmp_path):
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.data_engine = MagicMock()
|
||||
ctx.data_engine.base_dir = str(tmp_path)
|
||||
return ctx
|
||||
|
||||
|
||||
def test_update_note_pure_tool_subclass_and_registry():
|
||||
"""Verify UpdateNotePureTool is a subclass of ArtemisTool and properly registered."""
|
||||
assert issubclass(UpdateNotePureTool, ArtemisTool)
|
||||
assert issubclass(UpdateNotePure, ArtemisTool)
|
||||
assert isinstance(update_note_pure, ArtemisTool)
|
||||
assert isinstance(update_note_pure, UpdateNotePureTool)
|
||||
|
||||
assert update_note_pure.name == "update_note_pure"
|
||||
assert update_note_pure.category == "memory"
|
||||
assert update_note_pure.args_schema == UpdateNoteArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("update_note_pure")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, UpdateNotePureTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = update_note_pure.to_genai_declaration()
|
||||
assert declaration.name == "update_note_pure"
|
||||
assert "key" in declaration.parameters.properties
|
||||
assert "target" in declaration.parameters.properties
|
||||
assert "replacement" in declaration.parameters.properties
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_note_pure_direct_execution_success(mock_ctx, tmp_path):
|
||||
"""Verify direct execution of UpdateNotePureTool.execute updates existing note."""
|
||||
save_note_content(tmp_path, "plan", "Step 1: Pending\nStep 2: Done")
|
||||
|
||||
result = await update_note_pure.execute(
|
||||
ctx=mock_ctx,
|
||||
key="plan",
|
||||
target="Step 1: Pending",
|
||||
replacement="Step 1: Completed",
|
||||
)
|
||||
assert result == "Successfully updated note 'plan'."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "plan.md"
|
||||
assert note_path.read_text(encoding="utf-8") == "Step 1: Completed\nStep 2: Done"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_note_pure_with_warning(mock_ctx, tmp_path):
|
||||
"""Verify update note pure handles warning from fuzzy matching."""
|
||||
save_note_content(tmp_path, "fuzzy_note", "Target String")
|
||||
|
||||
with patch(
|
||||
"artemis.tools.scratchpad.update_note_content",
|
||||
return_value="Fuzzy match applied",
|
||||
):
|
||||
result = await update_note_pure.execute(
|
||||
ctx=mock_ctx,
|
||||
key="fuzzy_note",
|
||||
target="target string",
|
||||
replacement="New String",
|
||||
)
|
||||
assert "Successfully updated note 'fuzzy_note'." in result
|
||||
assert "WARNING: Fuzzy match applied" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_note_pure_callable_execution(mock_ctx, tmp_path):
|
||||
"""Verify invoking update_note_pure directly as a callable."""
|
||||
save_note_content(tmp_path, "callable_plan", "alpha beta")
|
||||
|
||||
result = await update_note_pure(
|
||||
ctx=mock_ctx,
|
||||
key="callable_plan",
|
||||
target="beta",
|
||||
replacement="gamma",
|
||||
)
|
||||
assert result == "Successfully updated note 'callable_plan'."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "callable_plan.md"
|
||||
assert note_path.read_text(encoding="utf-8") == "alpha gamma"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_note_pure_execution_failure(mock_ctx):
|
||||
"""Verify error handling when note update fails."""
|
||||
result = await update_note_pure.execute(
|
||||
ctx=mock_ctx,
|
||||
key="missing_note",
|
||||
target="old",
|
||||
replacement="new",
|
||||
)
|
||||
assert "Failed to update note 'missing_note':" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_update_note_tool_pure_langchain_ainvoke(mock_ctx, tmp_path):
|
||||
"""Verify get_update_note_tool_pure exports a LangChain tool named 'update_note'."""
|
||||
save_note_content(tmp_path, "export_plan", "initial value")
|
||||
|
||||
pure_tool = get_update_note_tool_pure(mock_ctx)
|
||||
assert pure_tool.name == "update_note"
|
||||
|
||||
result = await pure_tool.ainvoke(
|
||||
{
|
||||
"key": "export_plan",
|
||||
"target": "initial",
|
||||
"replacement": "updated",
|
||||
}
|
||||
)
|
||||
assert result == "Successfully updated note 'export_plan'."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "export_plan.md"
|
||||
assert note_path.read_text(encoding="utf-8") == "updated value"
|
||||
@@ -0,0 +1,208 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.scratchpad import (
|
||||
UpdateNote,
|
||||
UpdateNoteArgs,
|
||||
UpdateNoteTool,
|
||||
get_update_note_tool,
|
||||
get_update_note_tool_pure,
|
||||
update_note,
|
||||
update_note_wrapper,
|
||||
)
|
||||
from artemis.utils.notes import save_note_content
|
||||
from langgraph.types import Command
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_ctx(tmp_path):
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.data_engine = MagicMock()
|
||||
ctx.data_engine.base_dir = str(tmp_path)
|
||||
return ctx
|
||||
|
||||
|
||||
def test_update_note_tool_subclass_and_registry():
|
||||
"""Verify UpdateNoteTool is a subclass of ArtemisTool and properly registered."""
|
||||
assert issubclass(UpdateNoteTool, ArtemisTool)
|
||||
assert issubclass(UpdateNote, ArtemisTool)
|
||||
assert isinstance(update_note, ArtemisTool)
|
||||
assert isinstance(update_note, UpdateNoteTool)
|
||||
|
||||
assert update_note.name == "update_note"
|
||||
assert update_note.category == "memory"
|
||||
assert update_note.args_schema == UpdateNoteArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("update_note")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, UpdateNoteTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = update_note.to_genai_declaration()
|
||||
assert declaration.name == "update_note"
|
||||
assert "key" in declaration.parameters.properties
|
||||
assert "target" in declaration.parameters.properties
|
||||
assert "replacement" in declaration.parameters.properties
|
||||
|
||||
# Wrapper check
|
||||
assert update_note_wrapper is not None
|
||||
assert update_note_wrapper.tool_fn_getter == get_update_note_tool
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_note_direct_execution_success(mock_ctx, tmp_path):
|
||||
"""Verify direct execution of UpdateNoteTool.execute updates existing note."""
|
||||
save_note_content(tmp_path, "plan", "- [ ] Step 1\n- [ ] Step 2")
|
||||
|
||||
result = await update_note.execute(
|
||||
ctx=mock_ctx,
|
||||
key="plan",
|
||||
target="- [ ] Step 1",
|
||||
replacement="- [x] Step 1",
|
||||
)
|
||||
assert result == "Successfully updated note 'plan'."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "plan.md"
|
||||
assert note_path.read_text(encoding="utf-8") == "- [x] Step 1\n- [ ] Step 2"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_note_with_warning(mock_ctx, tmp_path):
|
||||
"""Verify update note handles warning from fuzzy/relaxed matching."""
|
||||
save_note_content(tmp_path, "fuzzy_plan", "- [ ] Step 1")
|
||||
|
||||
with patch(
|
||||
"artemis.tools.scratchpad.update_note_content",
|
||||
return_value="Fuzzy match applied",
|
||||
):
|
||||
result = await update_note.execute(
|
||||
ctx=mock_ctx,
|
||||
key="fuzzy_plan",
|
||||
target="Step 1",
|
||||
replacement="Step 1 Done",
|
||||
)
|
||||
assert "Successfully updated note 'fuzzy_plan'." in result
|
||||
assert "WARNING: Fuzzy match applied" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_note_callable_execution(mock_ctx, tmp_path):
|
||||
"""Verify invoking update_note directly as a callable."""
|
||||
save_note_content(tmp_path, "callable_plan", "- [ ] Task 1\n- [ ] Task 2")
|
||||
|
||||
result = await update_note(
|
||||
ctx=mock_ctx,
|
||||
key="callable_plan",
|
||||
target="- [ ] Task 2",
|
||||
replacement="- [x] Task 2",
|
||||
)
|
||||
assert result == "Successfully updated note 'callable_plan'."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "callable_plan.md"
|
||||
assert note_path.read_text(encoding="utf-8") == "- [ ] Task 1\n- [x] Task 2"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_note_with_state_command(mock_ctx, tmp_path):
|
||||
"""Verify UpdateNoteTool.execute with state returns Command updating VALIDATOR_MESSAGES_KEY."""
|
||||
save_note_content(tmp_path, "state_plan", "- [ ] Step 1")
|
||||
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.current_agent = "operator"
|
||||
mock_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update)
|
||||
|
||||
result = await update_note.execute(
|
||||
ctx=mock_ctx,
|
||||
key="state_plan",
|
||||
target="- [ ] Step 1",
|
||||
replacement="- [x] Step 1",
|
||||
state=mock_state,
|
||||
tool_call_id="call_update_999",
|
||||
)
|
||||
|
||||
assert isinstance(result, Command)
|
||||
assert VALIDATOR_MESSAGES_KEY in result.update
|
||||
tool_messages = result.update[VALIDATOR_MESSAGES_KEY]
|
||||
assert len(tool_messages) == 1
|
||||
assert tool_messages[0].tool_call_id == "call_update_999"
|
||||
assert tool_messages[0].content == "Successfully updated note 'state_plan'."
|
||||
assert tool_messages[0].status == "success"
|
||||
|
||||
mock_state.asanitize_update.assert_called_once()
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "state_plan.md"
|
||||
assert note_path.read_text(encoding="utf-8") == "- [x] Step 1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_note_execution_failure(mock_ctx):
|
||||
"""Verify error handling when updating a note fails."""
|
||||
result = await update_note.execute(
|
||||
ctx=mock_ctx,
|
||||
key="non_existent_note",
|
||||
target="old",
|
||||
replacement="new",
|
||||
)
|
||||
assert "Failed to update note 'non_existent_note':" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_update_note_tool_langchain_ainvoke(mock_ctx, tmp_path):
|
||||
"""Verify get_update_note_tool exports a LangChain tool that works with ainvoke."""
|
||||
save_note_content(tmp_path, "lc_plan", "alpha beta gamma")
|
||||
|
||||
lc_tool = get_update_note_tool(mock_ctx)
|
||||
assert lc_tool.name == "update_note"
|
||||
|
||||
result = await lc_tool.ainvoke(
|
||||
{
|
||||
"key": "lc_plan",
|
||||
"target": "beta",
|
||||
"replacement": "delta",
|
||||
}
|
||||
)
|
||||
assert result == "Successfully updated note 'lc_plan'."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "lc_plan.md"
|
||||
assert note_path.read_text(encoding="utf-8") == "alpha delta gamma"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_update_note_tool_pure(mock_ctx, tmp_path):
|
||||
"""Verify get_update_note_tool_pure returns a pure LangChain tool."""
|
||||
save_note_content(tmp_path, "pure_plan", "hello world")
|
||||
|
||||
pure_tool = get_update_note_tool_pure(mock_ctx)
|
||||
assert pure_tool.name == "update_note"
|
||||
|
||||
result = await pure_tool.ainvoke(
|
||||
{
|
||||
"key": "pure_plan",
|
||||
"target": "world",
|
||||
"replacement": "Artemis",
|
||||
}
|
||||
)
|
||||
assert result == "Successfully updated note 'pure_plan'."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "pure_plan.md"
|
||||
assert note_path.read_text(encoding="utf-8") == "hello Artemis"
|
||||
@@ -0,0 +1,457 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
# Mock the problematic langgraph import at module level
|
||||
_orig_chat_agent_executor = sys.modules.get("langgraph.prebuilt.chat_agent_executor")
|
||||
_orig_state = sys.modules.get("artemis.graph.state")
|
||||
|
||||
sys.modules["langgraph.prebuilt.chat_agent_executor"] = Mock()
|
||||
sys.modules["artemis.graph.state"] = Mock()
|
||||
|
||||
from artemis.context import DeviceContext, DevicePlatform, ArtemisContext # noqa: E402
|
||||
from artemis.tools.types import Target # noqa: E402
|
||||
from artemis.tools.utils import ( # noqa: E402
|
||||
IdSelectorRequest,
|
||||
SelectorRequestWithCoordinates,
|
||||
focus_element_if_needed,
|
||||
move_cursor_to_end_if_bounds,
|
||||
)
|
||||
from artemis.utils.ui_hierarchy import ElementBounds # noqa: E402
|
||||
|
||||
# Restore original modules to prevent contaminating other tests
|
||||
if _orig_chat_agent_executor is not None:
|
||||
sys.modules["langgraph.prebuilt.chat_agent_executor"] = _orig_chat_agent_executor
|
||||
else:
|
||||
del sys.modules["langgraph.prebuilt.chat_agent_executor"]
|
||||
|
||||
if _orig_state is not None:
|
||||
sys.modules["artemis.graph.state"] = _orig_state
|
||||
else:
|
||||
del sys.modules["artemis.graph.state"]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context():
|
||||
"""Create a mock ArtemisContext for testing."""
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
|
||||
# Create device context with necessary attributes
|
||||
ctx.device = Mock(spec=DeviceContext)
|
||||
ctx.device.mobile_platform = DevicePlatform.ANDROID
|
||||
ctx.device.device_id = "test_device_123"
|
||||
ctx.device.device_width = 1080
|
||||
ctx.device.device_height = 2340
|
||||
ctx.device.host_platform = "LINUX"
|
||||
|
||||
ctx.ui_adb_client = Mock()
|
||||
|
||||
# Mock the ADB client for Android
|
||||
ctx.adb_client = Mock()
|
||||
mock_device = Mock()
|
||||
mock_device.shell = Mock(return_value="")
|
||||
ctx.adb_client.device = Mock(return_value=mock_device)
|
||||
|
||||
# Mock the ADB client for Android
|
||||
mock_response = Mock()
|
||||
mock_response.json.return_value = {"elements": []}
|
||||
ctx.ui_adb_client.get_screen_data = Mock(return_value=mock_response)
|
||||
ctx.ui_adb_client.get_hierarchy = Mock(
|
||||
return_value=(
|
||||
b'<hierarchy><node bounds="[0,0][1000,1000]"><node'
|
||||
b' bounds="[0,0][100,100]"/></node></hierarchy>'
|
||||
)
|
||||
)
|
||||
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_state():
|
||||
"""Create a mock State for testing."""
|
||||
state = Mock()
|
||||
state.latest_ui_hierarchy = []
|
||||
return state
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_element():
|
||||
"""Create a sample UI element for testing."""
|
||||
return {
|
||||
"resourceId": "com.example:id/text_input",
|
||||
"text": "Sample text",
|
||||
"bounds": {"x": 100, "y": 200, "width": 300, "height": 50},
|
||||
"focused": "false",
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_rich_element():
|
||||
"""Create a sample rich UI element for testing."""
|
||||
return {
|
||||
"attributes": {
|
||||
"resource-id": "com.example:id/text_input",
|
||||
"focused": "false",
|
||||
"text": "Sample text",
|
||||
"bounds": {"x": 100, "y": 200, "width": 300, "height": 50},
|
||||
},
|
||||
"children": [],
|
||||
}
|
||||
|
||||
|
||||
class TestMoveCursorToEndIfBounds:
|
||||
"""Test cases for move_cursor_to_end_if_bounds function."""
|
||||
|
||||
@patch("artemis.tools.utils.tap")
|
||||
@patch("artemis.tools.utils.find_element_by_resource_id")
|
||||
def test_move_cursor_with_resource_id(
|
||||
self,
|
||||
mock_find_element,
|
||||
mock_tap,
|
||||
mock_context,
|
||||
mock_state,
|
||||
sample_element,
|
||||
):
|
||||
"""Test moving cursor using resource_id (highest priority)."""
|
||||
mock_state.latest_ui_hierarchy = [sample_element]
|
||||
mock_find_element.return_value = sample_element
|
||||
|
||||
target = Target(
|
||||
resource_id="com.example:id/text_input",
|
||||
resource_id_index=None,
|
||||
text=None,
|
||||
text_index=None,
|
||||
bounds=None,
|
||||
)
|
||||
result = asyncio.run(
|
||||
move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target)
|
||||
)
|
||||
|
||||
mock_find_element.assert_called_once_with(
|
||||
ui_hierarchy=[sample_element],
|
||||
resource_id="com.example:id/text_input",
|
||||
index=0,
|
||||
)
|
||||
mock_tap.assert_called_once()
|
||||
call_args = mock_tap.call_args[1]
|
||||
selector_request = call_args["selector_request"]
|
||||
assert isinstance(selector_request, SelectorRequestWithCoordinates)
|
||||
coords = selector_request.coordinates
|
||||
assert coords.x == 397 # 100 + 300 * 0.99
|
||||
assert coords.y == 249 # 200 + 50 * 0.99
|
||||
assert result == sample_element
|
||||
|
||||
@patch("artemis.tools.utils.tap")
|
||||
@patch("artemis.tools.utils.find_element_by_resource_id")
|
||||
def test_move_cursor_with_coordinates_only(
|
||||
self, mock_find_element, mock_tap, mock_context, mock_state
|
||||
):
|
||||
"""Test moving cursor when only coordinates are provided."""
|
||||
bounds = ElementBounds(x=50, y=150, width=200, height=40)
|
||||
target = Target(
|
||||
resource_id=None,
|
||||
resource_id_index=None,
|
||||
text=None,
|
||||
text_index=None,
|
||||
bounds=bounds,
|
||||
)
|
||||
|
||||
result = asyncio.run(
|
||||
move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target)
|
||||
)
|
||||
|
||||
mock_find_element.assert_not_called()
|
||||
mock_tap.assert_called_once()
|
||||
call_args = mock_tap.call_args[1]
|
||||
selector_request = call_args["selector_request"]
|
||||
coords = selector_request.coordinates
|
||||
assert coords.x == 248 # 50 + 200 * 0.99
|
||||
assert coords.y == 189 # 150 + 40 * 0.99
|
||||
assert result is None # No element is returned when using coords directly
|
||||
|
||||
@patch("artemis.tools.utils.tap")
|
||||
@patch("artemis.tools.utils.find_element_by_text")
|
||||
def test_move_cursor_with_text_only_success(
|
||||
self, mock_find_text, mock_tap, mock_context, mock_state, sample_element
|
||||
):
|
||||
"""Test moving cursor when only text is provided and succeeds."""
|
||||
mock_state.latest_ui_hierarchy = [sample_element]
|
||||
mock_find_text.return_value = sample_element
|
||||
|
||||
target = Target(
|
||||
resource_id=None,
|
||||
resource_id_index=None,
|
||||
text="Sample text",
|
||||
text_index=0,
|
||||
bounds=None,
|
||||
)
|
||||
result = asyncio.run(
|
||||
move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target)
|
||||
)
|
||||
|
||||
mock_find_text.assert_called_once_with([sample_element], "Sample text", index=0)
|
||||
mock_tap.assert_called_once()
|
||||
assert result == sample_element
|
||||
|
||||
@patch("artemis.tools.utils.tap")
|
||||
@patch("artemis.tools.utils.find_element_by_text")
|
||||
def test_move_cursor_with_text_only_element_not_found(
|
||||
self, mock_find_text, mock_tap, mock_context, mock_state
|
||||
):
|
||||
"""Test when searching by text finds no element."""
|
||||
mock_state.latest_ui_hierarchy = []
|
||||
mock_find_text.return_value = None
|
||||
|
||||
target = Target(
|
||||
resource_id=None,
|
||||
resource_id_index=None,
|
||||
text="Nonexistent text",
|
||||
text_index=None,
|
||||
bounds=None,
|
||||
)
|
||||
result = asyncio.run(
|
||||
move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target)
|
||||
)
|
||||
|
||||
mock_tap.assert_not_called()
|
||||
assert result is None
|
||||
|
||||
@patch("artemis.tools.utils.tap")
|
||||
@patch("artemis.tools.utils.find_element_by_text")
|
||||
def test_move_cursor_with_text_only_no_bounds(
|
||||
self, mock_find_text, mock_tap, mock_context, mock_state
|
||||
):
|
||||
"""Test when element is found by text but has no bounds."""
|
||||
element_no_bounds = {"text": "Text without bounds"}
|
||||
mock_state.latest_ui_hierarchy = [element_no_bounds]
|
||||
mock_find_text.return_value = element_no_bounds
|
||||
|
||||
target = Target(
|
||||
resource_id=None,
|
||||
resource_id_index=None,
|
||||
text="Text without bounds",
|
||||
text_index=None,
|
||||
bounds=None,
|
||||
)
|
||||
result = asyncio.run(
|
||||
move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target)
|
||||
)
|
||||
|
||||
mock_tap.assert_not_called()
|
||||
assert result is None # Should return None as no action was taken
|
||||
|
||||
@patch("artemis.tools.utils.find_element_by_resource_id")
|
||||
def test_move_cursor_element_not_found_by_id(self, mock_find_element, mock_context, mock_state):
|
||||
"""Test when element is not found by resource_id."""
|
||||
mock_find_element.return_value = None
|
||||
|
||||
target = Target(
|
||||
resource_id="com.example:id/nonexistent",
|
||||
resource_id_index=None,
|
||||
text=None,
|
||||
text_index=None,
|
||||
bounds=None,
|
||||
)
|
||||
result = asyncio.run(
|
||||
move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target)
|
||||
)
|
||||
|
||||
assert result is None
|
||||
|
||||
|
||||
class TestFocusElementIfNeeded:
|
||||
"""Test cases for focus_element_if_needed function."""
|
||||
|
||||
@patch("artemis.tools.utils.tap")
|
||||
@patch("artemis.tools.utils.find_element_by_resource_id")
|
||||
def test_focus_element_already_focused(
|
||||
self, mock_find_element, mock_tap, mock_context, sample_rich_element
|
||||
):
|
||||
"""Test when element is already focused."""
|
||||
focused_element = sample_rich_element.copy()
|
||||
focused_element["attributes"]["focused"] = "true"
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.json.return_value = {"elements": [focused_element]}
|
||||
mock_context.ui_adb_client.get_screen_data = Mock(return_value=mock_response)
|
||||
mock_find_element.return_value = focused_element["attributes"]
|
||||
|
||||
target = Target(
|
||||
resource_id="com.example:id/text_input",
|
||||
resource_id_index=None,
|
||||
text=None,
|
||||
text_index=None,
|
||||
bounds=None,
|
||||
)
|
||||
result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target))
|
||||
|
||||
mock_tap.assert_not_called()
|
||||
assert result == "resource_id"
|
||||
mock_context.ui_adb_client.get_hierarchy.assert_called_once()
|
||||
|
||||
@patch("artemis.tools.utils.tap")
|
||||
@patch("artemis.tools.utils.find_element_by_resource_id")
|
||||
def test_focus_element_needs_focus_success(
|
||||
self, mock_find_element, mock_tap, mock_context, sample_rich_element
|
||||
):
|
||||
"""Test when element needs focus and focusing succeeds."""
|
||||
unfocused_element = sample_rich_element
|
||||
focused_element = {
|
||||
"attributes": {
|
||||
"resource-id": "com.example:id/text_input",
|
||||
"focused": "true",
|
||||
},
|
||||
"children": [],
|
||||
}
|
||||
|
||||
mock_find_element.side_effect = [
|
||||
unfocused_element["attributes"],
|
||||
focused_element["attributes"],
|
||||
]
|
||||
|
||||
target = Target(
|
||||
resource_id="com.example:id/text_input",
|
||||
resource_id_index=None,
|
||||
text=None,
|
||||
text_index=None,
|
||||
bounds=None,
|
||||
)
|
||||
result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target))
|
||||
|
||||
mock_tap.assert_called_once_with(
|
||||
ctx=mock_context,
|
||||
selector_request=IdSelectorRequest(id="com.example:id/text_input"),
|
||||
index=0,
|
||||
)
|
||||
assert mock_context.ui_adb_client.get_hierarchy.call_count == 2
|
||||
assert result == "resource_id"
|
||||
|
||||
@patch("artemis.tools.utils.tap")
|
||||
@patch("artemis.tools.utils.logger")
|
||||
@patch("artemis.tools.utils.find_element_by_resource_id")
|
||||
def test_focus_id_and_text_mismatch_fallback_to_text(
|
||||
self,
|
||||
mock_find_id,
|
||||
mock_logger,
|
||||
mock_tap,
|
||||
mock_context,
|
||||
sample_rich_element,
|
||||
):
|
||||
"""Test fallback when resource_id and text point to different elements."""
|
||||
element_from_id = sample_rich_element["attributes"].copy()
|
||||
element_from_id["text"] = "Different text"
|
||||
|
||||
element_from_text = sample_rich_element.copy()
|
||||
element_from_text["attributes"]["bounds"] = {
|
||||
"x": 10,
|
||||
"y": 20,
|
||||
"width": 100,
|
||||
"height": 30,
|
||||
}
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.json.return_value = {"elements": [element_from_text]}
|
||||
mock_context.ui_adb_client.get_screen_data = Mock(return_value=mock_response)
|
||||
mock_find_id.return_value = element_from_id
|
||||
|
||||
with patch("artemis.tools.utils.find_element_by_text") as mock_find_text:
|
||||
mock_find_text.return_value = element_from_text["attributes"]
|
||||
|
||||
target = Target(
|
||||
resource_id="com.example:id/text_input",
|
||||
resource_id_index=None,
|
||||
text="Sample text",
|
||||
text_index=None,
|
||||
bounds=None,
|
||||
)
|
||||
result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target))
|
||||
|
||||
mock_logger.warning.assert_called_once()
|
||||
mock_tap.assert_called_once()
|
||||
assert result == "text"
|
||||
|
||||
@patch("artemis.tools.utils.tap")
|
||||
@patch("artemis.tools.utils.find_element_by_text")
|
||||
def test_focus_fallback_to_text(
|
||||
self, mock_find_text, mock_tap, mock_context, sample_rich_element
|
||||
):
|
||||
"""Test fallback to focusing using text."""
|
||||
element_with_bounds = sample_rich_element.copy()
|
||||
element_with_bounds["attributes"]["bounds"] = {
|
||||
"x": 10,
|
||||
"y": 20,
|
||||
"width": 100,
|
||||
"height": 30,
|
||||
}
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.json.return_value = {"elements": [element_with_bounds]}
|
||||
mock_context.ui_adb_client.get_screen_data = Mock(return_value=mock_response)
|
||||
mock_find_text.return_value = element_with_bounds["attributes"]
|
||||
|
||||
target = Target(
|
||||
resource_id=None,
|
||||
resource_id_index=None,
|
||||
text="Sample text",
|
||||
text_index=None,
|
||||
bounds=None,
|
||||
)
|
||||
result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target))
|
||||
|
||||
mock_find_text.assert_called_once()
|
||||
mock_tap.assert_called_once()
|
||||
call_args = mock_tap.call_args[1]
|
||||
selector = call_args["selector_request"]
|
||||
assert isinstance(selector, SelectorRequestWithCoordinates)
|
||||
assert selector.coordinates.x == 60 # 10 + 100/2
|
||||
assert selector.coordinates.y == 35 # 20 + 30/2
|
||||
assert result == "text"
|
||||
|
||||
@patch("artemis.tools.utils.logger")
|
||||
def test_focus_all_locators_fail(self, mock_logger, mock_context):
|
||||
"""Test failure when no locator can find an element."""
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.json.return_value = {"elements": []}
|
||||
mock_context.ui_adb_client.get_screen_data = Mock(return_value=mock_response)
|
||||
with (
|
||||
patch("artemis.tools.utils.find_element_by_resource_id") as mock_find_id,
|
||||
patch("artemis.tools.utils.find_element_by_text") as mock_find_text,
|
||||
):
|
||||
mock_find_id.return_value = None
|
||||
mock_find_text.return_value = None
|
||||
|
||||
target = Target(
|
||||
resource_id="nonexistent",
|
||||
resource_id_index=None,
|
||||
text="nonexistent",
|
||||
text_index=None,
|
||||
bounds=None,
|
||||
)
|
||||
result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target))
|
||||
|
||||
mock_logger.error.assert_called_once_with(
|
||||
"Failed to focus element."
|
||||
+ " No valid locator (resource_id, coordinates, or text) succeeded."
|
||||
)
|
||||
assert result is None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__])
|
||||
@@ -0,0 +1,139 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.video_tool import (
|
||||
VideoAnalyzerArgs,
|
||||
VideoAnalyzerPure,
|
||||
VideoAnalyzerPureTool,
|
||||
get_video_analyzer_tool_pure,
|
||||
video_analyzer_pure,
|
||||
)
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_ctx():
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
return ctx
|
||||
|
||||
|
||||
def test_video_analyzer_pure_tool_subclass_and_registry():
|
||||
"""Verify VideoAnalyzerPureTool is a subclass of ArtemisTool and properly registered."""
|
||||
assert issubclass(VideoAnalyzerPureTool, ArtemisTool)
|
||||
assert issubclass(VideoAnalyzerPure, ArtemisTool)
|
||||
assert isinstance(video_analyzer_pure, ArtemisTool)
|
||||
assert isinstance(video_analyzer_pure, VideoAnalyzerPureTool)
|
||||
|
||||
assert video_analyzer_pure.name == "video_analyzer_pure"
|
||||
assert video_analyzer_pure.category == "perception"
|
||||
assert video_analyzer_pure.args_schema == VideoAnalyzerArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("video_analyzer_pure")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, VideoAnalyzerPureTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = video_analyzer_pure.to_genai_declaration()
|
||||
assert declaration.name == "video_analyzer_pure"
|
||||
assert "time_description" in declaration.parameters.properties
|
||||
assert "purpose" in declaration.parameters.properties
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_analyzer_pure_direct_execution_success(mock_ctx):
|
||||
"""Verify direct execution of VideoAnalyzerPureTool.execute returns outcome directly."""
|
||||
with patch(
|
||||
"artemis.tools.video_tool.VideoAnalyzer.run",
|
||||
new_callable=AsyncMock,
|
||||
return_value=("Pure video analysis success", "success"),
|
||||
):
|
||||
result = await video_analyzer_pure.execute(
|
||||
ctx=mock_ctx,
|
||||
time_description="from 0s to 5s",
|
||||
purpose="Verify video playback",
|
||||
)
|
||||
assert result == "Pure video analysis success"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_analyzer_pure_direct_execution_failed(mock_ctx):
|
||||
"""Verify direct execution handles failed status properly."""
|
||||
with patch(
|
||||
"artemis.tools.video_tool.VideoAnalyzer.run",
|
||||
new_callable=AsyncMock,
|
||||
return_value=("Pure could not find element", "failed"),
|
||||
):
|
||||
result = await video_analyzer_pure.execute(
|
||||
ctx=mock_ctx,
|
||||
time_description="from 10s to 15s",
|
||||
purpose="Check button click",
|
||||
)
|
||||
assert result == "Video analysis failed: Pure could not find element"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_analyzer_pure_callable_execution(mock_ctx):
|
||||
"""Verify invoking video_analyzer_pure directly as a callable."""
|
||||
with patch(
|
||||
"artemis.tools.video_tool.VideoAnalyzer.run",
|
||||
new_callable=AsyncMock,
|
||||
return_value=("Pure callable output", "success"),
|
||||
):
|
||||
result = await video_analyzer_pure(
|
||||
ctx=mock_ctx,
|
||||
time_description="from 1s to 2s",
|
||||
purpose="Callable check",
|
||||
)
|
||||
assert result == "Pure callable output"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_analyzer_pure_exception_handling(mock_ctx):
|
||||
"""Verify VideoAnalyzerPureTool gracefully handles exceptions."""
|
||||
with patch(
|
||||
"artemis.tools.video_tool.VideoAnalyzer.run",
|
||||
new_callable=AsyncMock,
|
||||
side_effect=RuntimeError("Video processing error"),
|
||||
):
|
||||
result = await video_analyzer_pure.execute(
|
||||
ctx=mock_ctx,
|
||||
time_description="from 0s to 5s",
|
||||
purpose="Test error",
|
||||
)
|
||||
assert "Error running video analyzer: Video processing error" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_video_analyzer_tool_pure_ainvoke(mock_ctx):
|
||||
"""Verify get_video_analyzer_tool_pure exports a LangChain tool named 'video_analyzer'."""
|
||||
pure_tool = get_video_analyzer_tool_pure(mock_ctx)
|
||||
assert pure_tool.name == "video_analyzer"
|
||||
|
||||
with patch(
|
||||
"artemis.tools.video_tool.VideoAnalyzer.run",
|
||||
new_callable=AsyncMock,
|
||||
return_value=("Exported pure tool output", "success"),
|
||||
):
|
||||
result = await pure_tool.ainvoke(
|
||||
{
|
||||
"time_description": "from 2s to 4s",
|
||||
"purpose": "Pure invoke check",
|
||||
}
|
||||
)
|
||||
assert result == "Exported pure tool output"
|
||||
@@ -0,0 +1,204 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.video_tool import (
|
||||
DIAGNOSER_VIDEO_ANALYZER_DOCSTRING,
|
||||
OPERATOR_VIDEO_ANALYZER_DOCSTRING,
|
||||
VideoAnalyzerArgs,
|
||||
VideoAnalyzerTool,
|
||||
get_video_analyzer_tool,
|
||||
get_video_analyzer_tool_pure,
|
||||
video_analyzer,
|
||||
)
|
||||
from langgraph.types import Command
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_ctx():
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_state(mock_ctx):
|
||||
state = MagicMock(spec=State)
|
||||
|
||||
async def _mock_asanitize_update(ctx, update, agent):
|
||||
return update
|
||||
|
||||
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
|
||||
return state
|
||||
|
||||
|
||||
def test_video_analyzer_tool_subclass_and_registry():
|
||||
"""Verify VideoAnalyzerTool is a subclass of ArtemisTool and properly registered."""
|
||||
assert issubclass(VideoAnalyzerTool, ArtemisTool)
|
||||
assert isinstance(video_analyzer, ArtemisTool)
|
||||
assert isinstance(video_analyzer, VideoAnalyzerTool)
|
||||
|
||||
assert video_analyzer.name == "video_analyzer"
|
||||
assert video_analyzer.category == "perception"
|
||||
assert video_analyzer.args_schema == VideoAnalyzerArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("video_analyzer")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, VideoAnalyzerTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = video_analyzer.to_genai_declaration()
|
||||
assert declaration.name == "video_analyzer"
|
||||
assert "time_description" in declaration.parameters.properties
|
||||
assert "purpose" in declaration.parameters.properties
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_analyzer_direct_execution_success(mock_ctx):
|
||||
"""Verify direct execution of VideoAnalyzerTool.execute returns outcome directly."""
|
||||
with patch(
|
||||
"artemis.tools.video_tool.VideoAnalyzer.run",
|
||||
new_callable=AsyncMock,
|
||||
return_value=("Video analysis success details", "success"),
|
||||
):
|
||||
result = await video_analyzer.execute(
|
||||
ctx=mock_ctx,
|
||||
time_description="from 0s to 5s",
|
||||
purpose="Verify video playback",
|
||||
)
|
||||
assert result == "Video analysis success details"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_analyzer_direct_execution_failed(mock_ctx):
|
||||
"""Verify direct execution handles failed status properly."""
|
||||
with patch(
|
||||
"artemis.tools.video_tool.VideoAnalyzer.run",
|
||||
new_callable=AsyncMock,
|
||||
return_value=("Could not find target element", "failed"),
|
||||
):
|
||||
result = await video_analyzer.execute(
|
||||
ctx=mock_ctx,
|
||||
time_description="from 10s to 15s",
|
||||
purpose="Check button click",
|
||||
)
|
||||
assert result == "Video analysis failed: Could not find target element"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_analyzer_with_state_command(mock_ctx, mock_state):
|
||||
"""Verify execute returns a Command with ToolMessage when state is provided."""
|
||||
with patch(
|
||||
"artemis.tools.video_tool.VideoAnalyzer.run",
|
||||
new_callable=AsyncMock,
|
||||
return_value=("Action verified in video", "success"),
|
||||
):
|
||||
cmd = await video_analyzer.execute(
|
||||
ctx=mock_ctx,
|
||||
time_description="from 5s to 10s",
|
||||
purpose="Verify action",
|
||||
tool_call_id="call_video_123",
|
||||
state=mock_state,
|
||||
)
|
||||
assert isinstance(cmd, Command)
|
||||
assert VALIDATOR_MESSAGES_KEY in cmd.update
|
||||
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
|
||||
assert len(messages) == 1
|
||||
assert messages[0].tool_call_id == "call_video_123"
|
||||
assert messages[0].content == "Action verified in video"
|
||||
assert messages[0].status == "success"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_analyzer_exception_handling(mock_ctx, mock_state):
|
||||
"""Verify execute gracefully handles exceptions."""
|
||||
with patch(
|
||||
"artemis.tools.video_tool.VideoAnalyzer.run",
|
||||
new_callable=AsyncMock,
|
||||
side_effect=RuntimeError("Video processing error"),
|
||||
):
|
||||
# Without state
|
||||
res = await video_analyzer.execute(
|
||||
ctx=mock_ctx,
|
||||
time_description="from 0s to 5s",
|
||||
purpose="Test error",
|
||||
)
|
||||
assert "Error running video analyzer: Video processing error" in res
|
||||
|
||||
# With state
|
||||
cmd = await video_analyzer.execute(
|
||||
ctx=mock_ctx,
|
||||
time_description="from 0s to 5s",
|
||||
purpose="Test error",
|
||||
tool_call_id="call_err",
|
||||
state=mock_state,
|
||||
)
|
||||
assert isinstance(cmd, Command)
|
||||
assert VALIDATOR_MESSAGES_KEY in cmd.update
|
||||
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
|
||||
assert messages[0].status == "error"
|
||||
assert "Error running video analyzer" in messages[0].content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_analyzer_callable_execution(mock_ctx):
|
||||
"""Verify invoking video_analyzer directly as a callable."""
|
||||
with patch(
|
||||
"artemis.tools.video_tool.VideoAnalyzer.run",
|
||||
new_callable=AsyncMock,
|
||||
return_value=("Callable output", "success"),
|
||||
):
|
||||
result = await video_analyzer(
|
||||
ctx=mock_ctx,
|
||||
time_description="from 1s to 2s",
|
||||
purpose="Callable check",
|
||||
)
|
||||
assert result == "Callable output"
|
||||
|
||||
|
||||
def test_get_video_analyzer_tool_roles(mock_ctx):
|
||||
"""Verify get_video_analyzer_tool returns tool with role-specific description."""
|
||||
op_tool = get_video_analyzer_tool(mock_ctx, role="operator")
|
||||
assert op_tool.name == "video_analyzer"
|
||||
assert op_tool.description.strip() == OPERATOR_VIDEO_ANALYZER_DOCSTRING.strip()
|
||||
|
||||
diag_tool = get_video_analyzer_tool(mock_ctx, role="diagnoser")
|
||||
assert diag_tool.name == "video_analyzer"
|
||||
assert diag_tool.description.strip() == DIAGNOSER_VIDEO_ANALYZER_DOCSTRING.strip()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_video_analyzer_tool_pure_ainvoke(mock_ctx):
|
||||
"""Verify get_video_analyzer_tool_pure exports a LangChain tool."""
|
||||
pure_tool = get_video_analyzer_tool_pure(mock_ctx)
|
||||
assert pure_tool.name == "video_analyzer"
|
||||
|
||||
with patch(
|
||||
"artemis.tools.video_tool.VideoAnalyzer.run",
|
||||
new_callable=AsyncMock,
|
||||
return_value=("Pure tool output", "success"),
|
||||
):
|
||||
result = await pure_tool.ainvoke(
|
||||
{
|
||||
"time_description": "from 2s to 4s",
|
||||
"purpose": "Pure invoke check",
|
||||
}
|
||||
)
|
||||
assert result == "Pure tool output"
|
||||
@@ -0,0 +1,114 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.wait_tool import (
|
||||
Wait,
|
||||
WaitArgs,
|
||||
WaitTool,
|
||||
get_wait_tool,
|
||||
wait,
|
||||
wait_wrapper,
|
||||
)
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_ctx():
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
return ctx
|
||||
|
||||
|
||||
def test_wait_tool_subclass_and_registry():
|
||||
"""Verify WaitTool is a subclass of ArtemisTool and properly registered."""
|
||||
assert issubclass(WaitTool, ArtemisTool)
|
||||
assert issubclass(Wait, ArtemisTool)
|
||||
assert isinstance(wait, ArtemisTool)
|
||||
assert isinstance(wait, WaitTool)
|
||||
|
||||
assert wait.name == "wait"
|
||||
assert wait.category == "system"
|
||||
assert wait.args_schema == WaitArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("wait")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, WaitTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = wait.to_genai_declaration()
|
||||
assert declaration.name == "wait"
|
||||
assert "seconds" in declaration.parameters.properties
|
||||
|
||||
# Wrapper check
|
||||
assert wait_wrapper is not None
|
||||
assert wait_wrapper.tool_fn_getter == get_wait_tool
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wait_direct_execution(mock_ctx):
|
||||
"""Verify direct execution of WaitTool.execute."""
|
||||
with pytest.MonkeyPatch.context() as mp:
|
||||
mock_sleep = AsyncMock()
|
||||
mp.setattr(asyncio, "sleep", mock_sleep)
|
||||
|
||||
result = await wait.execute(ctx=mock_ctx, seconds=5)
|
||||
mock_sleep.assert_called_once_with(5)
|
||||
assert result == "Successfully waited for 5 seconds."
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wait_callable_execution(mock_ctx):
|
||||
"""Verify invoking wait directly as a callable."""
|
||||
with pytest.MonkeyPatch.context() as mp:
|
||||
mock_sleep = AsyncMock()
|
||||
mp.setattr(asyncio, "sleep", mock_sleep)
|
||||
|
||||
result = await wait(ctx=mock_ctx, seconds=3)
|
||||
mock_sleep.assert_called_once_with(3)
|
||||
assert result == "Successfully waited for 3 seconds."
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wait_tool(mock_ctx):
|
||||
wait_tool = get_wait_tool(mock_ctx)
|
||||
|
||||
start_time = time.time()
|
||||
result = await wait_tool.ainvoke({"seconds": 2})
|
||||
end_time = time.time()
|
||||
|
||||
duration = end_time - start_time
|
||||
assert duration >= 1.9 # Allow slight timing variations
|
||||
assert "Successfully waited for 2 seconds" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wait_tool_bounds(mock_ctx):
|
||||
wait_tool = get_wait_tool(mock_ctx)
|
||||
|
||||
with pytest.MonkeyPatch.context() as mp:
|
||||
mock_sleep = AsyncMock()
|
||||
mp.setattr(asyncio, "sleep", mock_sleep)
|
||||
|
||||
await wait_tool.ainvoke({"seconds": 120})
|
||||
mock_sleep.assert_called_once_with(60)
|
||||
|
||||
mock_sleep.reset_mock()
|
||||
await wait_tool.ainvoke({"seconds": -5})
|
||||
mock_sleep.assert_called_once_with(1)
|
||||
@@ -0,0 +1,148 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.utils.app_launch_utils import launch_app_with_retries
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context():
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
ctx.device = Mock()
|
||||
ctx.device.mobile_platform = "android"
|
||||
ctx.device.device_id = "emulator-5554"
|
||||
ctx.data_engine = None
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.utils.app_launch_utils.UnifiedMobileController")
|
||||
@patch("artemis.utils.app_launch_utils.get_current_foreground_package_async")
|
||||
async def test_launch_app_success_immediate(mock_get_foreground, mock_controller_cls, mock_context):
|
||||
"""Test successful immediate app launch."""
|
||||
mock_controller = Mock()
|
||||
mock_controller.launch_app = AsyncMock(return_value=True)
|
||||
mock_controller_cls.return_value = mock_controller
|
||||
|
||||
# App immediately in foreground
|
||||
mock_get_foreground.return_value = "com.google.android.youtube"
|
||||
|
||||
success, error_msg = await launch_app_with_retries(
|
||||
mock_context,
|
||||
"com.google.android.youtube",
|
||||
max_retries=1,
|
||||
max_poll_seconds=2,
|
||||
)
|
||||
|
||||
assert success is True
|
||||
assert error_msg is None
|
||||
mock_controller.launch_app.assert_called_once_with("com.google.android.youtube")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.utils.app_launch_utils.get_focused_task_package")
|
||||
@patch("artemis.utils.app_launch_utils.UnifiedMobileController")
|
||||
@patch("artemis.utils.app_launch_utils.get_current_foreground_package_async")
|
||||
async def test_launch_app_success_permission_overlay(
|
||||
mock_get_foreground,
|
||||
mock_controller_cls,
|
||||
mock_get_focused_task,
|
||||
mock_context,
|
||||
):
|
||||
"""Test successful launch if overlayed by system permission controller and task is matching."""
|
||||
mock_controller = Mock()
|
||||
mock_controller.launch_app = AsyncMock(return_value=True)
|
||||
mock_controller_cls.return_value = mock_controller
|
||||
|
||||
# System permission controller is focused foreground
|
||||
mock_get_foreground.return_value = "com.google.android.permissioncontroller"
|
||||
# Second layer check: Task stack shows com.google.android.youtube is focused
|
||||
mock_get_focused_task.return_value = "com.google.android.youtube"
|
||||
|
||||
success, error_msg = await launch_app_with_retries(
|
||||
mock_context,
|
||||
"com.google.android.youtube",
|
||||
max_retries=1,
|
||||
max_poll_seconds=2,
|
||||
)
|
||||
|
||||
assert success is True
|
||||
assert error_msg is None
|
||||
mock_controller.launch_app.assert_called_once_with("com.google.android.youtube")
|
||||
mock_get_focused_task.assert_called_once_with(mock_context)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.utils.app_launch_utils.get_focused_task_package")
|
||||
@patch("artemis.utils.app_launch_utils.UnifiedMobileController")
|
||||
@patch("artemis.utils.app_launch_utils.get_current_foreground_package_async")
|
||||
async def test_launch_app_failure_permission_overlay_wrong_task(
|
||||
mock_get_foreground,
|
||||
mock_controller_cls,
|
||||
mock_get_focused_task,
|
||||
mock_context,
|
||||
):
|
||||
"""Test launch failure if overlayed by system permission controller but task does not match."""
|
||||
mock_controller = Mock()
|
||||
mock_controller.launch_app = AsyncMock(return_value=True)
|
||||
mock_controller.terminate_app = AsyncMock()
|
||||
mock_controller_cls.return_value = mock_controller
|
||||
|
||||
# System permission controller is focused foreground
|
||||
mock_get_foreground.return_value = "com.google.android.permissioncontroller"
|
||||
# Second layer check: Task stack is some other app (e.g. com.android.settings)
|
||||
mock_get_focused_task.return_value = "com.android.settings"
|
||||
|
||||
success, error_msg = await launch_app_with_retries(
|
||||
mock_context,
|
||||
"com.google.android.youtube",
|
||||
max_retries=1,
|
||||
max_poll_seconds=1,
|
||||
)
|
||||
|
||||
assert success is False
|
||||
assert "Failed to launch com.google.android.youtube" in error_msg
|
||||
mock_get_focused_task.assert_called_once_with(mock_context)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.utils.app_launch_utils.UnifiedMobileController")
|
||||
@patch("artemis.utils.app_launch_utils.get_current_foreground_package_async")
|
||||
async def test_launch_app_failure_wrong_app(mock_get_foreground, mock_controller_cls, mock_context):
|
||||
"""Test launch failure and retry when wrong app remains in foreground."""
|
||||
mock_controller = Mock()
|
||||
mock_controller.launch_app = AsyncMock(return_value=True)
|
||||
mock_controller.terminate_app = AsyncMock()
|
||||
mock_controller_cls.return_value = mock_controller
|
||||
|
||||
# A different app (e.g., settings or home screen) is in foreground
|
||||
mock_get_foreground.return_value = "com.android.settings"
|
||||
|
||||
success, error_msg = await launch_app_with_retries(
|
||||
mock_context,
|
||||
"com.google.android.youtube",
|
||||
max_retries=2,
|
||||
max_poll_seconds=1,
|
||||
)
|
||||
|
||||
assert success is False
|
||||
assert "Failed to launch com.google.android.youtube" in error_msg
|
||||
|
||||
# Should have attempted launching twice
|
||||
assert mock_controller.launch_app.call_count == 2
|
||||
# Should have attempted force-stopping once before retry
|
||||
assert mock_controller.terminate_app.call_count == 1
|
||||
@@ -0,0 +1,178 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import cv2
|
||||
from artemis.utils.image_diff import check_ui_change
|
||||
import numpy as np
|
||||
|
||||
|
||||
def test_no_change():
|
||||
img = np.zeros((1000, 1000), dtype=np.uint8)
|
||||
_, img_bytes = cv2.imencode(".jpg", img)
|
||||
|
||||
action = {"action": "tap", "coordinates": [500, 500]}
|
||||
assert not check_ui_change(img_bytes.tobytes(), img_bytes.tobytes(), action)
|
||||
print("test_no_change passed")
|
||||
|
||||
|
||||
def test_global_change():
|
||||
img1 = np.zeros((1000, 1000), dtype=np.uint8)
|
||||
img2 = np.ones((1000, 1000), dtype=np.uint8) * 255
|
||||
|
||||
_, img1_bytes = cv2.imencode(".jpg", img1)
|
||||
_, img2_bytes = cv2.imencode(".jpg", img2)
|
||||
|
||||
action = {"action": "swipe", "coordinates": [500, 500, 500, 200]}
|
||||
assert check_ui_change(img1_bytes.tobytes(), img2_bytes.tobytes(), action)
|
||||
print("test_global_change passed")
|
||||
|
||||
|
||||
def test_roi_change():
|
||||
img1 = np.zeros((1000, 1000), dtype=np.uint8)
|
||||
img2 = img1.copy()
|
||||
img2[480:520, 480:520] = 255
|
||||
|
||||
_, img1_bytes = cv2.imencode(".jpg", img1)
|
||||
_, img2_bytes = cv2.imencode(".jpg", img2)
|
||||
|
||||
action = {"action": "tap", "coordinates": [500, 500]}
|
||||
assert check_ui_change(img1_bytes.tobytes(), img2_bytes.tobytes(), action)
|
||||
print("test_roi_change passed")
|
||||
|
||||
|
||||
def test_status_bar_ignore():
|
||||
img1 = np.zeros((1000, 1000), dtype=np.uint8)
|
||||
img2 = img1.copy()
|
||||
img2[0:50, 0:1000] = 255
|
||||
|
||||
_, img1_bytes = cv2.imencode(".jpg", img1)
|
||||
_, img2_bytes = cv2.imencode(".jpg", img2)
|
||||
|
||||
action = {"action": "tap", "coordinates": [500, 500]}
|
||||
assert not check_ui_change(img1_bytes.tobytes(), img2_bytes.tobytes(), action)
|
||||
print("test_status_bar_ignore passed")
|
||||
|
||||
|
||||
def test_roi_threshold():
|
||||
# Use standard 1080x2400 resolution to keep scale_factor = 1.0 (roi_size = 50)
|
||||
img1 = np.zeros((2400, 1080), dtype=np.uint8)
|
||||
img2 = img1.copy()
|
||||
# Center of 1080x2400 is (540, 1200). Draw 10x10 square at center.
|
||||
img2[1195:1205, 535:545] = 255
|
||||
|
||||
_, img1_bytes = cv2.imencode(".jpg", img1)
|
||||
_, img2_bytes = cv2.imencode(".jpg", img2)
|
||||
|
||||
action = {"action": "tap", "coordinates": [540, 1200]}
|
||||
assert not check_ui_change(
|
||||
img1_bytes.tobytes(),
|
||||
img2_bytes.tobytes(),
|
||||
action,
|
||||
full_screen_threshold=0.001,
|
||||
roi_threshold=0.05,
|
||||
)
|
||||
print("test_roi_threshold passed")
|
||||
|
||||
|
||||
def test_wait_for_screen_stability():
|
||||
from artemis.utils.image_diff import wait_for_screen_stability
|
||||
|
||||
img_static = np.zeros((1000, 1000), dtype=np.uint8)
|
||||
_, static_bytes = cv2.imencode(".jpg", img_static)
|
||||
static_b64 = base64.b64encode(static_bytes).decode("utf-8")
|
||||
|
||||
# 1. Test immediate stability (already static)
|
||||
calls_immediate = 0
|
||||
|
||||
async def mock_screenshot_immediate():
|
||||
nonlocal calls_immediate
|
||||
calls_immediate += 1
|
||||
return static_b64
|
||||
|
||||
res_immediate = asyncio.run(
|
||||
wait_for_screen_stability(mock_screenshot_immediate, max_timeout=1.0, interval=0.05)
|
||||
)
|
||||
assert res_immediate == static_b64
|
||||
assert calls_immediate == 2
|
||||
print("test_wait_for_screen_stability: immediate stability subtest passed")
|
||||
|
||||
# 2. Test gradual stability (settles after 3 moving frames)
|
||||
calls_gradual = 0
|
||||
|
||||
async def mock_screenshot_gradual():
|
||||
nonlocal calls_gradual
|
||||
calls_gradual += 1
|
||||
if calls_gradual <= 3:
|
||||
img = np.zeros((1000, 1000), dtype=np.uint8)
|
||||
img[500:, :] = calls_gradual * 50
|
||||
_, bytes_data = cv2.imencode(".jpg", img)
|
||||
return base64.b64encode(bytes_data).decode("utf-8")
|
||||
return static_b64
|
||||
|
||||
res_gradual = asyncio.run(
|
||||
wait_for_screen_stability(mock_screenshot_gradual, max_timeout=1.0, interval=0.05)
|
||||
)
|
||||
assert res_gradual == static_b64
|
||||
assert calls_gradual > 3
|
||||
print("test_wait_for_screen_stability: gradual stability subtest passed")
|
||||
print("test_wait_for_screen_stability passed")
|
||||
|
||||
|
||||
def test_wait_for_screen_data_stability():
|
||||
from artemis.utils.image_diff import wait_for_screen_data_stability
|
||||
from artemis.controllers.device_controller import ScreenDataResponse
|
||||
|
||||
img_static = np.zeros((1000, 1000), dtype=np.uint8)
|
||||
_, static_bytes = cv2.imencode(".jpg", img_static)
|
||||
static_b64 = base64.b64encode(static_bytes).decode("utf-8")
|
||||
|
||||
# Test stability returning ScreenDataResponse
|
||||
calls = 0
|
||||
|
||||
async def mock_get_screen_data():
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return ScreenDataResponse(
|
||||
base64=static_b64,
|
||||
elements=[],
|
||||
width=1000,
|
||||
height=1000,
|
||||
platform="android",
|
||||
)
|
||||
|
||||
res = asyncio.run(
|
||||
wait_for_screen_data_stability(mock_get_screen_data, max_timeout=1.0, interval=0.05)
|
||||
)
|
||||
assert res.base64 == static_b64
|
||||
assert res.width == 1000
|
||||
assert calls == 2
|
||||
print("test_wait_for_screen_data_stability passed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
test_no_change()
|
||||
test_global_change()
|
||||
test_roi_change()
|
||||
test_status_bar_ignore()
|
||||
test_roi_threshold()
|
||||
test_wait_for_screen_stability()
|
||||
test_wait_for_screen_data_stability()
|
||||
print("All tests passed successfully!")
|
||||
except AssertionError as e:
|
||||
print(f"Test failed: {e}")
|
||||
except Exception as e:
|
||||
print(f"Error running tests: {e}")
|
||||
@@ -0,0 +1,165 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from artemis.utils.notes import (
|
||||
append_note_content,
|
||||
list_notes_info,
|
||||
read_note_content,
|
||||
save_note_content,
|
||||
update_note_content,
|
||||
)
|
||||
import pytest
|
||||
|
||||
|
||||
def test_list_notes_info(tmp_path):
|
||||
base_dir = tmp_path
|
||||
|
||||
# Create some notes
|
||||
save_note_content(base_dir, "note1", "line1\nline2\nline3")
|
||||
save_note_content(base_dir, "note2", "line1\nline2")
|
||||
|
||||
info = list_notes_info(base_dir)
|
||||
assert info == {"note1": 3, "note2": 2}
|
||||
|
||||
|
||||
def test_read_note_content_range(tmp_path):
|
||||
base_dir = tmp_path
|
||||
save_note_content(base_dir, "note1", "line1\nline2\nline3\nline4")
|
||||
|
||||
# Read full
|
||||
assert read_note_content(base_dir, "note1") == "line1\nline2\nline3\nline4"
|
||||
|
||||
# Read range
|
||||
assert read_note_content(base_dir, "note1", start_line=2, end_line=3) == "line2\nline3"
|
||||
|
||||
# Read from start to line
|
||||
assert read_note_content(base_dir, "note1", end_line=2) == "line1\nline2"
|
||||
|
||||
# Read from line to end
|
||||
assert read_note_content(base_dir, "note1", start_line=3) == "line3\nline4"
|
||||
|
||||
|
||||
def test_update_note_exact_match(tmp_path):
|
||||
base_dir = tmp_path
|
||||
original_content = 'def demo():\n print("Hello, World!")\n return True\n'
|
||||
save_note_content(base_dir, "task_plan", original_content)
|
||||
|
||||
# Exact match
|
||||
warning = update_note_content(
|
||||
base_dir,
|
||||
"task_plan",
|
||||
' print("Hello, World!")',
|
||||
' print("Hello, Jetski!")',
|
||||
)
|
||||
assert warning is None
|
||||
|
||||
updated_content = read_note_content(base_dir, "task_plan")
|
||||
assert 'print("Hello, Jetski!")' in updated_content
|
||||
assert 'print("Hello, World!")' not in updated_content
|
||||
|
||||
|
||||
def test_update_note_relaxed_match(tmp_path):
|
||||
base_dir = tmp_path
|
||||
# Multi-line content with checkboxes and indentation
|
||||
original_content = (
|
||||
"- [ ] Step 1: Research the issue\n - [ ] Step 2: Write code\n - [ ] Step 3: Run tests\n"
|
||||
)
|
||||
save_note_content(base_dir, "task_plan", original_content)
|
||||
|
||||
# Relaxed match: case difference, quote difference, checkbox bullet difference, trailing whitespace
|
||||
target = " * [ ] step 2: write code "
|
||||
replacement = " - [/] Step 2: In progress"
|
||||
|
||||
warning = update_note_content(base_dir, "task_plan", target, replacement)
|
||||
|
||||
# Warning should be returned
|
||||
assert warning is not None
|
||||
assert "We did our best to apply changes despite some inaccuracies" in warning
|
||||
|
||||
updated_content = read_note_content(base_dir, "task_plan")
|
||||
assert "Step 2: In progress" in updated_content
|
||||
assert "Step 2: Write code" not in updated_content
|
||||
# Check indentation is preserved from replacement
|
||||
assert " - [/] Step 2: In progress\n" in updated_content
|
||||
|
||||
|
||||
def test_update_note_fuzzy_match(tmp_path):
|
||||
base_dir = tmp_path
|
||||
original_content = 'print("Alpha message")\nprint("Gamma message")\nprint("Delta message")\n'
|
||||
save_note_content(base_dir, "task_plan", original_content)
|
||||
|
||||
# Fuzzy match: typo (Gemma vs Gamma)
|
||||
target = 'print("Gemma message")'
|
||||
replacement = 'print("Gamma updated!")'
|
||||
|
||||
warning = update_note_content(base_dir, "task_plan", target, replacement)
|
||||
assert warning is not None
|
||||
assert "We did our best to apply changes despite some inaccuracies" in warning
|
||||
|
||||
updated_content = read_note_content(base_dir, "task_plan")
|
||||
assert 'print("Gamma updated!")' in updated_content
|
||||
assert 'print("Gamma message")' not in updated_content
|
||||
|
||||
|
||||
def test_update_note_ambiguous_match(tmp_path):
|
||||
base_dir = tmp_path
|
||||
# Multiple lines that could match target
|
||||
original_content = 'print("Beta message")\nprint("Zeta message")\nprint("Eta message")\n'
|
||||
save_note_content(base_dir, "task_plan", original_content)
|
||||
|
||||
# Target is highly ambiguous (Xeta is almost equally close to Beta and Zeta)
|
||||
target = 'print("Xeta message")'
|
||||
replacement = 'print("Ambiguous match!")'
|
||||
|
||||
# Should raise ValueError due to ambiguity or inability to find unique match
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
update_note_content(base_dir, "task_plan", target, replacement)
|
||||
|
||||
assert "Target string 'print(\"Xeta message\")' not found" in str(
|
||||
exc_info.value
|
||||
) or "not unique" in str(exc_info.value)
|
||||
|
||||
|
||||
def test_update_note_not_found(tmp_path):
|
||||
base_dir = tmp_path
|
||||
original_content = 'print("Alpha message")\n'
|
||||
save_note_content(base_dir, "task_plan", original_content)
|
||||
|
||||
# Totally different target
|
||||
target = 'print("Something totally different")'
|
||||
replacement = 'print("Hello")'
|
||||
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
update_note_content(base_dir, "task_plan", target, replacement)
|
||||
assert "not found" in str(exc_info.value)
|
||||
|
||||
|
||||
def test_append_note_content(tmp_path):
|
||||
base_dir = tmp_path
|
||||
|
||||
# Case 1: File does not exist
|
||||
append_note_content(base_dir, "new_note", "hello")
|
||||
assert read_note_content(base_dir, "new_note") == "hello\n"
|
||||
|
||||
# Case 2: File exists and ends with a newline
|
||||
append_note_content(base_dir, "new_note", "world")
|
||||
assert read_note_content(base_dir, "new_note") == "hello\nworld\n"
|
||||
|
||||
# Case 3: File exists and does not end with a newline
|
||||
file_path = base_dir / "notes" / "no_newline.md"
|
||||
file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
file_path.write_text("no newline", encoding="utf-8")
|
||||
|
||||
append_note_content(base_dir, "no_newline", "added text")
|
||||
assert read_note_content(base_dir, "no_newline") == "no newline\nadded text\n"
|
||||
@@ -0,0 +1,248 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import base64
|
||||
from io import BytesIO
|
||||
import unittest
|
||||
from artemis.utils.ocr_xml_fusion import (
|
||||
_calculate_overlap_ratio,
|
||||
_crop_image_remove_status_bar,
|
||||
_detect_status_bar_height,
|
||||
_is_low_value_text,
|
||||
_map_coordinates_back,
|
||||
_parse_ocr_position,
|
||||
fuse_ocr_with_xml,
|
||||
)
|
||||
from PIL import Image
|
||||
|
||||
|
||||
class TestOCRFusion(unittest.TestCase):
|
||||
def test_is_low_value_text(self):
|
||||
# Test length 1
|
||||
self.assertTrue(_is_low_value_text("a"))
|
||||
self.assertFalse(_is_low_value_text("Yes"))
|
||||
self.assertFalse(_is_low_value_text("1"))
|
||||
|
||||
# Test placeholders
|
||||
self.assertTrue(_is_low_value_text("search"))
|
||||
|
||||
# Test default markers
|
||||
self.assertTrue(_is_low_value_text("[Icon]"))
|
||||
|
||||
# Test valid text
|
||||
self.assertFalse(_is_low_value_text("Click Me"))
|
||||
|
||||
def test_parse_ocr_position(self):
|
||||
position = [
|
||||
{"x": 10, "y": 20},
|
||||
{"x": 50, "y": 20},
|
||||
{"x": 50, "y": 60},
|
||||
{"x": 10, "y": 60},
|
||||
]
|
||||
bounds = _parse_ocr_position(position)
|
||||
self.assertEqual(bounds, {"left": 10, "top": 20, "right": 50, "bottom": 60})
|
||||
|
||||
def test_calculate_overlap_ratio(self):
|
||||
ocr_bounds = {"left": 10, "top": 10, "right": 30, "bottom": 30}
|
||||
xml_bounds = {"left": 0, "top": 0, "right": 40, "bottom": 40}
|
||||
# OCR is fully inside XML
|
||||
self.assertEqual(_calculate_overlap_ratio(ocr_bounds, xml_bounds), 1.0)
|
||||
|
||||
xml_bounds = {"left": 20, "top": 20, "right": 40, "bottom": 40}
|
||||
# Intersection is 10x10 = 100. OCR area is 20x20 = 400. Ratio = 0.25
|
||||
self.assertEqual(_calculate_overlap_ratio(ocr_bounds, xml_bounds), 0.25)
|
||||
|
||||
def test_detect_status_bar_height(self):
|
||||
xml = [
|
||||
{"package": "com.android.systemui", "bounds": "[0,0][1080,120]"},
|
||||
{"package": "com.example.app", "bounds": "[0,120][1080,2400]"},
|
||||
]
|
||||
# Pass screen_height=2400, should find 120 from XML
|
||||
self.assertEqual(_detect_status_bar_height(xml, screen_height=2400), 120)
|
||||
|
||||
# Fallback: should be 4% of 2400 = 96
|
||||
xml = [{"package": "com.example.app", "bounds": "[0,0][1080,2400]"}]
|
||||
self.assertEqual(_detect_status_bar_height(xml, screen_height=2400), 96)
|
||||
|
||||
def test_crop_image_remove_status_bar(self):
|
||||
# Create a dummy image
|
||||
img = Image.new("RGB", (100, 100), color="red")
|
||||
buffered = BytesIO()
|
||||
img.save(buffered, format="PNG")
|
||||
img_b64 = base64.b64encode(buffered.getvalue()).decode("utf-8")
|
||||
|
||||
cropped_b64, w, h = _crop_image_remove_status_bar(img_b64, crop_height=20)
|
||||
self.assertEqual(w, 100)
|
||||
self.assertEqual(h, 100)
|
||||
|
||||
# Verify cropped size
|
||||
cropped_img = Image.open(BytesIO(base64.b64decode(cropped_b64)))
|
||||
self.assertEqual(cropped_img.size, (100, 80))
|
||||
|
||||
def test_map_coordinates_back(self):
|
||||
ocr_results = [
|
||||
{
|
||||
"text": "test",
|
||||
"position": [{"x": 10, "y": 10}, {"x": 20, "y": 20}],
|
||||
}
|
||||
]
|
||||
mapped = _map_coordinates_back(ocr_results, status_bar_height=100)
|
||||
self.assertEqual(mapped[0]["position"][0]["y"], 110)
|
||||
self.assertEqual(mapped[0]["position"][1]["y"], 120)
|
||||
|
||||
def test_fuse_ocr_with_xml(self):
|
||||
xml = [
|
||||
{
|
||||
"class": "android.widget.Button",
|
||||
"bounds": "[10,100][200,150]",
|
||||
"text": "Submit",
|
||||
},
|
||||
{
|
||||
"class": "android.widget.TextView",
|
||||
"bounds": "[10,200][200,250]",
|
||||
"text": "",
|
||||
},
|
||||
]
|
||||
ocr = [
|
||||
# Should be discarded (similar to Submit)
|
||||
{
|
||||
"text": "Submit",
|
||||
"position": [{"x": 15, "y": 105}, {"x": 95, "y": 145}],
|
||||
},
|
||||
# Multiple OCRs matching the same node (TextView at [10,200])
|
||||
# They are on the same line, should be merged in order
|
||||
{
|
||||
"text": "Result",
|
||||
"position": [{"x": 15, "y": 205}, {"x": 95, "y": 245}],
|
||||
},
|
||||
{
|
||||
"text": "Found",
|
||||
"position": [{"x": 105, "y": 205}, {"x": 185, "y": 245}],
|
||||
},
|
||||
# Unmatched OCRs that are close to each other (should be merged into one virtual node)
|
||||
{
|
||||
"text": "Floating",
|
||||
"position": [{"x": 15, "y": 305}, {"x": 95, "y": 345}],
|
||||
},
|
||||
{
|
||||
"text": "Text",
|
||||
"position": [{"x": 105, "y": 305}, {"x": 185, "y": 345}],
|
||||
},
|
||||
# Unmatched OCR far away (should be separate virtual node)
|
||||
{
|
||||
"text": "Far",
|
||||
"position": [{"x": 15, "y": 405}, {"x": 95, "y": 445}],
|
||||
},
|
||||
]
|
||||
|
||||
fused = fuse_ocr_with_xml(xml, ocr)
|
||||
|
||||
# Check first node (Submit) - should NOT have ocr_text
|
||||
self.assertNotIn("ocr_text", fused[0])
|
||||
|
||||
# Check second node (Result Found) - should have aggregated ocr_elements
|
||||
self.assertEqual(fused[1]["ocr_elements"][0]["text"], "Result Found")
|
||||
self.assertIn("[OCR]", fused[1]["class"])
|
||||
|
||||
# We expect 0 virtual nodes now as per new strategy.
|
||||
# Total nodes should be just the original 2 XML nodes.
|
||||
self.assertEqual(len(fused), 2)
|
||||
|
||||
def test_banded_overlap_matching(self):
|
||||
# Screen is 1080x2400
|
||||
# XML has a giant container and a small correct button.
|
||||
xml = [
|
||||
# Giant container (FrameLayout, covers a lot of screen)
|
||||
{
|
||||
"class": "android.widget.FrameLayout",
|
||||
"bounds": "[0,80][1080,1000]",
|
||||
"text": "",
|
||||
},
|
||||
# Small correct button
|
||||
{
|
||||
"class": "android.widget.Button",
|
||||
"bounds": "[185,84][512,114]",
|
||||
"text": "",
|
||||
},
|
||||
]
|
||||
|
||||
# OCR Search is slightly shifted to the left of the Button.
|
||||
# Button: [185,84][512,114] -> area 327 * 30 = 9810
|
||||
# OCR: [174,89][326,109] -> area 152 * 20 = 3040
|
||||
# Overlap with Button:
|
||||
# Intersection: [185,89][326,109] -> area 141 * 20 = 2820
|
||||
# Ratio: 2820 / 3040 = 0.927 (Band 1)
|
||||
# Overlap with FrameLayout:
|
||||
# OCR is fully inside FrameLayout [0,80][1080,1000]
|
||||
# Ratio: 1.0 (Band 1)
|
||||
# Both are in Band 1 (>=0.9).
|
||||
# The Button has MUCH smaller area than FrameLayout (9810 < 993600).
|
||||
# So Button should win despite having lower overlap (0.927 < 1.0).
|
||||
ocr = [
|
||||
{
|
||||
"text": "Search",
|
||||
"position": [
|
||||
{"x": 174, "y": 89},
|
||||
{"x": 326, "y": 89},
|
||||
{"x": 326, "y": 109},
|
||||
{"x": 174, "y": 109},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
fused = fuse_ocr_with_xml(xml, ocr)
|
||||
|
||||
# Check that OCR text "Search" is mounted on the Button node (index 1)
|
||||
self.assertIn("ocr_elements", fused[1])
|
||||
self.assertEqual(fused[1]["ocr_elements"][0]["text"], "Search")
|
||||
|
||||
# And NOT on the FrameLayout node (index 0)
|
||||
self.assertNotIn("ocr_elements", fused[0])
|
||||
|
||||
def test_giant_node_exclusion(self):
|
||||
# Screen is 1080x2400. Area = 2,592,000.
|
||||
# 10% area = 259,200.
|
||||
# FrameLayout is 1080x1000 -> area 1,080,000 (>10%).
|
||||
# It should be excluded from matching entirely.
|
||||
# Since there is no other node, the OCR should be discarded.
|
||||
xml = [
|
||||
{
|
||||
"class": "android.widget.FrameLayout",
|
||||
"bounds": "[0,0][1080,1000]",
|
||||
"text": "",
|
||||
},
|
||||
]
|
||||
ocr = [
|
||||
# Some floating developer text "dX : 0.0"
|
||||
{
|
||||
"text": "dX : 0.0",
|
||||
"position": [
|
||||
{"x": 144, "y": 55},
|
||||
{"x": 215, "y": 55},
|
||||
{"x": 215, "y": 63},
|
||||
{"x": 144, "y": 63},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
fused = fuse_ocr_with_xml(xml, ocr)
|
||||
|
||||
# The giant FrameLayout should NOT have ocr_elements
|
||||
self.assertNotIn("ocr_elements", fused[0])
|
||||
# The class should NOT be updated with [OCR]
|
||||
self.assertNotIn("[OCR]", fused[0].get("class", ""))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,650 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import hashlib
|
||||
from artemis.utils.task_tree import (
|
||||
build_plan_and_history,
|
||||
get_active_subgoal_hashes,
|
||||
get_recent_subgoal_hashes,
|
||||
)
|
||||
|
||||
|
||||
def test_build_plan_and_history_separated_layout():
|
||||
plan = """- [ ] Open Settings app
|
||||
- [ ] Navigate to System settings"""
|
||||
|
||||
# No steps executed
|
||||
output = build_plan_and_history(plan, [], "default")
|
||||
assert plan in output
|
||||
assert "--- Execution History ---" in output
|
||||
assert "No steps executed yet." in output
|
||||
|
||||
# With steps
|
||||
steps = [
|
||||
{
|
||||
"step_id": "step_1",
|
||||
"step_number": 1,
|
||||
"relative_time": "2.5s",
|
||||
"summary": "Launched settings app",
|
||||
"operator_raw_thinking": "Thought 1",
|
||||
"action_taken": [{"action": "launch_app"}],
|
||||
"last_execution_result": {"status": "success"},
|
||||
}
|
||||
]
|
||||
output = build_plan_and_history(plan, steps, "default", last_n_detailed=1)
|
||||
assert plan in output
|
||||
assert "--- Execution History ---" in output
|
||||
assert "- **Step 1 (Most Recent Step, Start: 2.5s)**" in output
|
||||
assert " - Thought 1" in output
|
||||
assert " * [Planned Action]: Launched app 'None'" in output
|
||||
|
||||
|
||||
def test_build_plan_and_history_granularities():
|
||||
plan = """- [x] Open Settings app
|
||||
- [/] Navigate to System settings
|
||||
- [ ] Navigate to Languages & input"""
|
||||
|
||||
subgoal_hash_1 = hashlib.md5(b"Open Settings app").hexdigest()
|
||||
subgoal_hash_2 = hashlib.md5(b"Navigate to System settings").hexdigest()
|
||||
|
||||
steps = [
|
||||
{
|
||||
"step_id": "step_1",
|
||||
"step_number": 1,
|
||||
"relative_time": "2.5s",
|
||||
"summary": "Launched settings",
|
||||
"operator_raw_thinking": "Thought 1",
|
||||
"action_taken": [{"action": "launch_app"}],
|
||||
"last_execution_result": {"status": "success"},
|
||||
"extra_metadata": {"subgoal_hash": subgoal_hash_1},
|
||||
},
|
||||
{
|
||||
"step_id": "step_2",
|
||||
"step_number": 2,
|
||||
"relative_time": "8.0s",
|
||||
"summary": "Swiped down",
|
||||
"operator_raw_thinking": "Thought 2",
|
||||
"action_taken": [{"action": "swipe"}],
|
||||
"last_execution_result": {"status": "success"},
|
||||
"extra_metadata": {"subgoal_hash": subgoal_hash_2},
|
||||
},
|
||||
{
|
||||
"step_id": "step_3",
|
||||
"step_number": 3,
|
||||
"relative_time": "15.0s",
|
||||
"summary": "Clicked System settings",
|
||||
"operator_raw_thinking": "Thought 3",
|
||||
"action_taken": [{"action": "click"}],
|
||||
"last_execution_result": {"status": "success"},
|
||||
"extra_metadata": {"subgoal_hash": subgoal_hash_2},
|
||||
},
|
||||
]
|
||||
|
||||
# Test last_n_detailed = 0 (high-level analysis context, all summarized)
|
||||
output_0 = build_plan_and_history(plan, steps, subgoal_hash_2, last_n_detailed=0)
|
||||
assert "- *Step 1 (Start: 2.5s): Launched settings*" in output_0
|
||||
assert "- *Step 2 (Start: 8.0s): Swiped down*" in output_0
|
||||
assert "- *Step 3 (Start: 15.0s): Clicked System settings*" in output_0
|
||||
assert "*Thought*: Thought 3" not in output_0
|
||||
|
||||
# Test last_n_detailed = 1 (normal execution context, only last detailed)
|
||||
output_1 = build_plan_and_history(plan, steps, subgoal_hash_2, last_n_detailed=1)
|
||||
assert "- *Step 1 (Start: 2.5s): Launched settings*" in output_1
|
||||
assert "- *Step 2 (Start: 8.0s): Swiped down*" in output_1
|
||||
assert "- **Step 3 (Most Recent Step, Start: 15.0s)**" in output_1
|
||||
assert " - Thought 3" in output_1
|
||||
|
||||
# Test last_n_detailed = 3 (committee context, last 3 detailed)
|
||||
output_3 = build_plan_and_history(plan, steps, subgoal_hash_2, last_n_detailed=3)
|
||||
assert "- **Step 1 (Start: 2.5s)**" in output_3
|
||||
assert " - Thought 1" in output_3
|
||||
assert "- **Step 2 (Start: 8.0s)**" in output_3
|
||||
assert " - Thought 2" in output_3
|
||||
assert "- **Step 3 (Most Recent Step, Start: 15.0s)**" in output_3
|
||||
assert " - Thought 3" in output_3
|
||||
|
||||
# Test keep_subgoal_hashes (milestone filtering for checker/diagnoser)
|
||||
# Only keep steps for subgoal_hash_2
|
||||
output_filter = build_plan_and_history(
|
||||
plan, steps, subgoal_hash_2, keep_subgoal_hashes={subgoal_hash_2}
|
||||
)
|
||||
assert "Launched settings" not in output_filter # Step 1 is filtered out!
|
||||
assert "Swiped down" in output_filter # Step 2 belongs to subgoal_hash_2, kept!
|
||||
assert " - Thought 3" in output_filter # Step 3 is the last step, kept detailed!
|
||||
|
||||
|
||||
def test_get_active_subgoal_hashes_fallback():
|
||||
# Scenario 1: Normal Active Subgoal (Level 1 [/] active)
|
||||
plan_normal = """- [x] Open Settings app
|
||||
- [/] Navigate to System settings
|
||||
- [ ] Navigate to Languages & input"""
|
||||
|
||||
h_settings = hashlib.md5(b"Open Settings app").hexdigest()
|
||||
h_system = hashlib.md5(b"Navigate to System settings").hexdigest()
|
||||
|
||||
parent, sub = get_active_subgoal_hashes(plan_normal)
|
||||
assert parent == h_system
|
||||
assert sub is None
|
||||
|
||||
# Scenario 2: Indented Sub-subgoal Active (Level 2 [/] active, consolidated to parent)
|
||||
plan_indented = """- [x] Open Settings app
|
||||
- [/] Navigate to System settings
|
||||
- [/] Scroll down to find System settings
|
||||
- [ ] Navigate to Languages & input"""
|
||||
|
||||
parent, sub = get_active_subgoal_hashes(plan_indented)
|
||||
assert parent == h_system
|
||||
assert sub is None
|
||||
|
||||
# Scenario 3: Safe Fallback (All pending "[ ]", no active subgoal)
|
||||
plan_fallback = """- [ ] Open Settings app
|
||||
- [ ] Navigate to System settings
|
||||
- [ ] Navigate to Languages & input"""
|
||||
|
||||
parent, sub = get_active_subgoal_hashes(plan_fallback)
|
||||
assert parent == h_settings # Successfully fell back to the first subgoal's hash!
|
||||
assert sub is None
|
||||
|
||||
# Scenario 4: All Completed (All "[x]", no active subgoal)
|
||||
plan_all_done = """- [x] Open Settings app
|
||||
- [x] Navigate to System settings
|
||||
- [x] Navigate to Languages & input"""
|
||||
|
||||
parent, sub = get_active_subgoal_hashes(plan_all_done)
|
||||
assert parent == "default"
|
||||
assert sub is None
|
||||
|
||||
|
||||
def test_build_plan_and_history_sliding_window():
|
||||
plan = """- [x] Subgoal 1
|
||||
- [x] Subgoal 2
|
||||
- [/] Subgoal 3"""
|
||||
|
||||
subgoal_hash_1 = hashlib.md5(b"Subgoal 1").hexdigest()
|
||||
subgoal_hash_2 = hashlib.md5(b"Subgoal 2").hexdigest()
|
||||
subgoal_hash_3 = hashlib.md5(b"Subgoal 3").hexdigest()
|
||||
|
||||
steps = [
|
||||
{
|
||||
"step_id": "step_1",
|
||||
"step_number": 1,
|
||||
"summary": "Step 1 summary",
|
||||
"extra_metadata": {"subgoal_hash": subgoal_hash_1},
|
||||
},
|
||||
{
|
||||
"step_id": "step_2",
|
||||
"step_number": 2,
|
||||
"summary": "Step 2 summary",
|
||||
"extra_metadata": {"subgoal_hash": subgoal_hash_1},
|
||||
},
|
||||
{
|
||||
"step_id": "step_3",
|
||||
"step_number": 3,
|
||||
"summary": "Step 3 summary",
|
||||
"extra_metadata": {"subgoal_hash": subgoal_hash_2},
|
||||
},
|
||||
{
|
||||
"step_id": "step_4",
|
||||
"step_number": 4,
|
||||
"summary": "Step 4 summary",
|
||||
"extra_metadata": {"subgoal_hash": subgoal_hash_3},
|
||||
},
|
||||
{
|
||||
"step_id": "step_5",
|
||||
"step_number": 5,
|
||||
"summary": "Step 5 summary",
|
||||
"extra_metadata": {"subgoal_hash": subgoal_hash_3},
|
||||
},
|
||||
{
|
||||
"step_id": "step_6",
|
||||
"step_number": 6,
|
||||
"summary": "Step 6 summary",
|
||||
"extra_metadata": {"subgoal_hash": subgoal_hash_3},
|
||||
},
|
||||
{
|
||||
"step_id": "step_7",
|
||||
"step_number": 7,
|
||||
"summary": "Step 7 summary",
|
||||
"extra_metadata": {"subgoal_hash": subgoal_hash_3},
|
||||
},
|
||||
]
|
||||
|
||||
# Case 1: min_summaries = 2
|
||||
# subgoal 3 has 4 steps (4, 5, 6, 7). This already exceeds min_summaries=2.
|
||||
# So older steps (1, 2, 3) from completed subgoals should be compressed/hidden!
|
||||
output_c1 = build_plan_and_history(
|
||||
plan, steps, subgoal_hash_3, min_summaries=2, last_n_detailed=0
|
||||
)
|
||||
assert "Step 1 summary" not in output_c1
|
||||
assert "Step 2 summary" not in output_c1
|
||||
assert "Step 3 summary" not in output_c1
|
||||
assert "Step 4 summary" in output_c1
|
||||
assert "Step 5 summary" in output_c1
|
||||
assert "Step 6 summary" in output_c1
|
||||
assert "Step 7 summary" in output_c1
|
||||
|
||||
# Case 2: min_summaries = 5
|
||||
# subgoal 3 has 4 steps (4, 5, 6, 7), which translates to 3 non-last steps.
|
||||
# To reach min_summaries=5 non-last steps, the sliding window slides back to keep Step 3 and Step 2 visible!
|
||||
# But Step 1 should still be compressed/hidden.
|
||||
output_c2 = build_plan_and_history(
|
||||
plan, steps, subgoal_hash_3, min_summaries=5, last_n_detailed=0
|
||||
)
|
||||
assert "Step 1 summary" not in output_c2
|
||||
assert "Step 2 summary" in output_c2 # Kept visible by sliding window!
|
||||
assert "Step 3 summary" in output_c2 # Kept visible by sliding window!
|
||||
assert "Step 4 summary" in output_c2
|
||||
assert "Step 5 summary" in output_c2
|
||||
assert "Step 6 summary" in output_c2
|
||||
assert "Step 7 summary" in output_c2
|
||||
|
||||
|
||||
def test_build_plan_and_history_clean_success_result():
|
||||
plan = "- [ ] Subgoal 1"
|
||||
steps = [
|
||||
{
|
||||
"step_id": "step_1",
|
||||
"step_number": 1,
|
||||
"relative_time": "2.5s",
|
||||
"summary": "Swiped screen",
|
||||
"action_taken": [{"action": "swipe"}],
|
||||
"last_execution_result": {
|
||||
"executed_actions": [{"action": "swipe"}],
|
||||
"status": "success",
|
||||
},
|
||||
}
|
||||
]
|
||||
output = build_plan_and_history(plan, steps, "default", last_n_detailed=1)
|
||||
|
||||
# The output should contain "status": "success" but NOT the redundant "executed_actions"
|
||||
# Success results are stripped under Factual Realism
|
||||
assert "* [Validator Execution Result]" not in output
|
||||
assert "executed_actions" not in output
|
||||
|
||||
|
||||
def test_build_plan_and_history_with_tool_calls():
|
||||
plan = "- [ ] Subgoal 1"
|
||||
steps = [
|
||||
{
|
||||
"step_id": "step_1",
|
||||
"step_number": 1,
|
||||
"relative_time": "2.5s",
|
||||
"summary": "Read note and swiped",
|
||||
"action_taken": [{"action": "swipe"}],
|
||||
"last_execution_result": {"status": "success"},
|
||||
"tool_calls": [
|
||||
{
|
||||
"name": "read_note",
|
||||
"payload": {
|
||||
"args": {"key": "other_note"},
|
||||
"result": "- [ ] Milestone",
|
||||
},
|
||||
"status": "success",
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
output = build_plan_and_history(plan, steps, "default", last_n_detailed=1)
|
||||
|
||||
assert "* [Operator Decision Loop]:" in output
|
||||
assert '`read_note({"key": "other_note"})` -> - [ ] Milestone' in output
|
||||
|
||||
|
||||
def test_build_plan_and_history_with_interleaved_events():
|
||||
plan = "- [ ] Subgoal 1"
|
||||
steps = [
|
||||
{
|
||||
"step_id": "step_1",
|
||||
"step_number": 1,
|
||||
"relative_time": "2.5s",
|
||||
"summary": "Read note and thought",
|
||||
"action_taken": [{"action": "swipe"}],
|
||||
"last_execution_result": {"status": "success"},
|
||||
"interleaved_events": [
|
||||
{"type": "thought", "content": "I want to search for files."},
|
||||
{
|
||||
"type": "tool_call",
|
||||
"name": "list_notes",
|
||||
"args": {"dir": "/notes"},
|
||||
"result": ["notes.txt"],
|
||||
},
|
||||
{"type": "thought", "content": "I see notes.txt, let's read it."},
|
||||
{
|
||||
"type": "tool_call",
|
||||
"name": "read_note",
|
||||
"args": {"file": "notes.txt"},
|
||||
"result": "Milestone plan",
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
output = build_plan_and_history(plan, steps, "default", last_n_detailed=1)
|
||||
|
||||
# It should render interleaved thoughts and tool calls in natural order
|
||||
assert " - I want to search for files." in output
|
||||
assert ' - [Tool Call]: `list_notes({"dir": "/notes"})` -> [\'notes.txt\']' in output
|
||||
assert " - I see notes.txt, let's read it." in output
|
||||
assert ' - [Tool Call]: `read_note({"file": "notes.txt"})` -> Milestone plan' in output
|
||||
|
||||
# It should use the new section header
|
||||
assert "* [Operator Decision Loop]:" in output
|
||||
|
||||
|
||||
def test_build_plan_and_history_concatenates_thoughts():
|
||||
plan = "- [ ] Subgoal 1"
|
||||
steps = [
|
||||
{
|
||||
"step_id": "step_1",
|
||||
"step_number": 1,
|
||||
"relative_time": "2.5s",
|
||||
"summary": "Step with multiple thoughts",
|
||||
"action_taken": [{"action": "click", "thought": "Action thought"}],
|
||||
"operator_raw_thinking": "Raw thought",
|
||||
"last_execution_result": {"status": "success"},
|
||||
"interleaved_events": [
|
||||
{"type": "thought", "content": "Event thought"},
|
||||
],
|
||||
}
|
||||
]
|
||||
output = build_plan_and_history(plan, steps, "default", last_n_detailed=1)
|
||||
|
||||
# Check that all thoughts are concatenated and visible in the output
|
||||
assert " - Action thought" in output
|
||||
assert " - Event thought" in output
|
||||
assert " - Raw thought" in output
|
||||
|
||||
|
||||
def test_build_plan_and_history_chronological_last_step():
|
||||
plan = "- [ ] Subgoal 1"
|
||||
steps = [
|
||||
{
|
||||
"step_id": "step_1",
|
||||
"step_number": 1,
|
||||
"relative_time": "2.5s",
|
||||
"summary": "Completed search",
|
||||
"action_taken": [{"action": "click", "target_text": "Search"}],
|
||||
"last_execution_result": {"status": "success"},
|
||||
"interleaved_events": [
|
||||
{"type": "thought", "content": "I should find search icon."},
|
||||
{
|
||||
"type": "tool_call",
|
||||
"name": "ask_explorer",
|
||||
"args": {"query": "search icon"},
|
||||
"result": {"center": [500, 600]},
|
||||
},
|
||||
{
|
||||
"type": "native_thought",
|
||||
"content": "Clicking coordinates [500, 600] now.",
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
output = build_plan_and_history(
|
||||
plan, steps, "default", last_n_detailed=1, chronological_last_step=True
|
||||
)
|
||||
|
||||
assert plan in output
|
||||
assert "--- Execution History ---" in output
|
||||
assert "- **Step 1 (Most Recent Step, Start: 2.5s)**" in output
|
||||
assert "* [Operator Decision Loop]:" in output
|
||||
assert " - I should find search icon." in output
|
||||
assert (
|
||||
'- [Tool Call]: `ask_explorer({"query": "search icon"})` -> {"center":'
|
||||
" [500, 600]}" in output
|
||||
)
|
||||
assert " - Clicking coordinates [500, 600] now." in output
|
||||
assert "* [Planned Action]: Tapped 'Search' at None" in output
|
||||
assert "* [Validator Execution Result]" not in output
|
||||
|
||||
|
||||
def test_build_plan_and_history_safety_net_and_failure_analyzer():
|
||||
plan = "- [ ] Subgoal 1"
|
||||
steps = [
|
||||
{
|
||||
"step_id": "step_1",
|
||||
"step_number": 1,
|
||||
"relative_time": "5.0s",
|
||||
"summary": "Repaired click search",
|
||||
"action_taken": [{"action": "click", "target_text": "Search"}],
|
||||
"last_execution_result": {"status": "success"},
|
||||
"interleaved_events": [
|
||||
{"type": "thought", "content": "Let's click search."},
|
||||
# Safety net failure
|
||||
{
|
||||
"type": "tool_call",
|
||||
"name": "safety_net_validation",
|
||||
"args": {"target": "Search"},
|
||||
"result": [
|
||||
False,
|
||||
"TARGET_DISAPPEARED",
|
||||
"Target button is not visible",
|
||||
],
|
||||
},
|
||||
# Failure Analyzer intervention
|
||||
{
|
||||
"type": "failure_analyzer_thought",
|
||||
"content": ("The search button is covered by keyboard, need to go BACK first."),
|
||||
},
|
||||
{
|
||||
"type": "tool_call",
|
||||
"name": "_exec_press_key",
|
||||
"args": {"key": "BACK", "state": "<State>"},
|
||||
"result": ["Pressed key BACK", "<Bytes>", "/path/to/shot.jpg"],
|
||||
},
|
||||
{
|
||||
"type": "failure_analyzer_thought",
|
||||
"content": "Keyboard is closed, now re-executing search tap.",
|
||||
},
|
||||
{
|
||||
"type": "tool_call",
|
||||
"name": "_exec_click",
|
||||
"args": {"target": [500, 600]},
|
||||
"result": ["Tapped search button successfully", "<Bytes>"],
|
||||
},
|
||||
{
|
||||
"type": "tool_call",
|
||||
"name": "report_failure_analysis",
|
||||
"args": {
|
||||
"status": "fixed",
|
||||
"analysis": "Closed keyboard and tapped search.",
|
||||
},
|
||||
"result": "Success",
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
output = build_plan_and_history(
|
||||
plan, steps, "default", last_n_detailed=1, chronological_last_step=True
|
||||
)
|
||||
|
||||
assert "- **Step 1 (Most Recent Step, Start: 5.0s)**" in output
|
||||
assert "* [Operator Decision Loop]:" in output
|
||||
assert " - Let's click search." in output
|
||||
assert (
|
||||
"* [Planned Action]: Tapped 'Search' at None (Intercepted by"
|
||||
" Pre-Execution Safety Net)" in output
|
||||
)
|
||||
assert "* [Pre-Execution Safety Net]:" in output
|
||||
assert "- [Safety Net Check]: (Action not executed: Target button is not visible)" in output
|
||||
assert "* [Failure Analyzer Recovery Loop]:" in output
|
||||
assert " - [Tool Call]: Pressed key 'BACK'" in output
|
||||
assert " - [Tool Call]: Tapped element at [500, 600]" in output
|
||||
|
||||
|
||||
def test_build_plan_and_history_repaired_result():
|
||||
plan = "- [ ] Subgoal 1"
|
||||
steps = [
|
||||
{
|
||||
"step_id": "step_1",
|
||||
"step_number": 1,
|
||||
"relative_time": "2.5s",
|
||||
"summary": "Tapped button with repair",
|
||||
"action_taken": [{"action": "click"}],
|
||||
"last_execution_result": {
|
||||
"execution": [
|
||||
{
|
||||
"action": "click",
|
||||
"attempts": ["pre-validation failed"],
|
||||
"repair": ("Tapped center to wake controls and then tapped target"),
|
||||
}
|
||||
],
|
||||
"status": "success",
|
||||
"repair_status": "fixed",
|
||||
},
|
||||
}
|
||||
]
|
||||
output = build_plan_and_history(plan, steps, "default", last_n_detailed=1)
|
||||
assert " * [Validator Execution Result]: Action executed." not in output
|
||||
assert " * [Result]: Repaired: Tapped center to wake controls and then tapped target" in output
|
||||
|
||||
|
||||
def test_get_recent_subgoal_hashes_robust_milestone_exclusion(tmp_path):
|
||||
# Scenario:
|
||||
# 1. Milestone 1 is completed (hash_completed_1).
|
||||
# 2. We had a failed attempt 'hash_failed_old' during Milestone 1.
|
||||
# 3. Milestone 2 is completed (hash_completed_2).
|
||||
# 4. We are now working on Milestone C (hash_active_c).
|
||||
# 5. Milestone C had two previous failed renamed versions: Failed Attempt A and Failed Attempt B.
|
||||
# 6. subgoal_hash_chain.json maps: A -> B -> C.
|
||||
|
||||
base_dir = tmp_path
|
||||
notes_dir = base_dir / "notes"
|
||||
notes_dir.mkdir(parents=True, exist_ok=True)
|
||||
chain_path = notes_dir / "subgoal_hash_chain.json"
|
||||
|
||||
hash_completed_1 = hashlib.md5(b"Completed Milestone 1").hexdigest()
|
||||
hash_completed_2 = hashlib.md5(b"Completed Milestone 2").hexdigest()
|
||||
hash_active_c = hashlib.md5(b"Active Milestone C").hexdigest()
|
||||
|
||||
hash_failed_old = hashlib.md5(b"Failed Attempt Old").hexdigest()
|
||||
hash_failed_a = hashlib.md5(b"Failed Attempt A").hexdigest()
|
||||
hash_failed_b = hashlib.md5(b"Failed Attempt B").hexdigest()
|
||||
|
||||
# Write the hash chain representing: A -> B -> C
|
||||
import json
|
||||
|
||||
chain_data = {hash_failed_a: hash_failed_b, hash_failed_b: hash_active_c}
|
||||
chain_path.write_text(json.dumps(chain_data), encoding="utf-8")
|
||||
|
||||
steps = [
|
||||
# Steps from Completed Milestone 1
|
||||
{"step_id": "s1", "extra_metadata": {"subgoal_hash": hash_completed_1}},
|
||||
# Old failed attempt during Milestone 1
|
||||
{"step_id": "s2", "extra_metadata": {"subgoal_hash": hash_failed_old}},
|
||||
# Steps from Completed Milestone 2 (Most recent completed milestone)
|
||||
{"step_id": "s3", "extra_metadata": {"subgoal_hash": hash_completed_2}},
|
||||
{"step_id": "s4", "extra_metadata": {"subgoal_hash": hash_completed_2}},
|
||||
# Steps from Failed Attempt A (Part of the active task slot's past renames)
|
||||
{"step_id": "s5", "extra_metadata": {"subgoal_hash": hash_failed_a}},
|
||||
{"step_id": "s6", "extra_metadata": {"subgoal_hash": hash_failed_a}},
|
||||
# Steps from Failed Attempt B (Part of the active task slot's past renames)
|
||||
{"step_id": "s7", "extra_metadata": {"subgoal_hash": hash_failed_b}},
|
||||
{"step_id": "s8", "extra_metadata": {"subgoal_hash": hash_failed_b}},
|
||||
# Steps from Active Milestone C
|
||||
{"step_id": "s9", "extra_metadata": {"subgoal_hash": hash_active_c}},
|
||||
]
|
||||
|
||||
# Test raw transitive alias resolver
|
||||
from artemis.utils.task_tree import get_all_subgoal_aliases
|
||||
|
||||
aliases = get_all_subgoal_aliases(hash_active_c, base_dir)
|
||||
assert aliases == {hash_active_c, hash_failed_b, hash_failed_a}
|
||||
|
||||
keep_hashes = get_recent_subgoal_hashes(steps, hash_active_c, base_dir)
|
||||
|
||||
# Should keep:
|
||||
# 1. The current active subgoal (hash_active_c)
|
||||
assert hash_active_c in keep_hashes
|
||||
|
||||
# 2. The failed/renamed subgoals during active slot (hash_failed_b, hash_failed_a) - resolved transitively via chain
|
||||
assert hash_failed_b in keep_hashes
|
||||
assert hash_failed_a in keep_hashes
|
||||
|
||||
# 3. The single most recent completed subgoal (hash_completed_2) for continuity context
|
||||
assert hash_completed_2 in keep_hashes
|
||||
|
||||
# Should prune:
|
||||
# 4. Older completed subgoals (hash_completed_1)
|
||||
assert hash_completed_1 not in keep_hashes
|
||||
|
||||
# 5. Older failed subgoals from previously completed milestones (hash_failed_old)
|
||||
assert hash_failed_old not in keep_hashes
|
||||
|
||||
|
||||
def test_build_plan_and_history_for_failure_analyzer():
|
||||
plan_with_subgoals = """- [/] Goal 1
|
||||
- [x] Subtask 1.1
|
||||
- [/] Subtask 1.2"""
|
||||
|
||||
steps = [
|
||||
{
|
||||
"step_id": "step_1",
|
||||
"step_number": 1,
|
||||
"relative_time": "12.3s",
|
||||
"summary": "Tapped full screen",
|
||||
"action_taken": [{"action": "click", "target_text": "Fullscreen"}],
|
||||
"operator_raw_thinking": "Let's tap it",
|
||||
"operator_native_thinking": "Thinking to tap",
|
||||
"last_execution_result": {"status": "failed"},
|
||||
"interleaved_events": [
|
||||
{"type": "thought", "content": "I should find fullscreen icon."},
|
||||
{
|
||||
"type": "tool_call",
|
||||
"name": "safety_net_validation",
|
||||
"args": {"target": "Fullscreen"},
|
||||
"result": [
|
||||
False,
|
||||
"TARGET_DISAPPEARED",
|
||||
"Target button is not visible",
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
# Test with subgoals and for_failure_analyzer=True
|
||||
output_analyzer_with_plan = build_plan_and_history(
|
||||
plan_with_subgoals,
|
||||
steps,
|
||||
"default",
|
||||
last_n_detailed=1,
|
||||
for_failure_analyzer=True,
|
||||
)
|
||||
assert "--- Task Plan ---" in output_analyzer_with_plan
|
||||
assert (
|
||||
"*(Note: Provided for context only. Do not execute the remaining"
|
||||
" plan.)*" in output_analyzer_with_plan
|
||||
)
|
||||
assert "--- Execution History ---" in output_analyzer_with_plan
|
||||
assert (
|
||||
"- **Step 1 (Most Recent Step (Failed to execute, this is the step you"
|
||||
" need to focus on and repair), Start: 12.3s)**" in output_analyzer_with_plan
|
||||
)
|
||||
assert "[Operator Decision Loop]" in output_analyzer_with_plan
|
||||
assert "I should find fullscreen icon." in output_analyzer_with_plan
|
||||
assert (
|
||||
"* [Planned Action]: Tapped 'Fullscreen' at None (Intercepted by"
|
||||
" Pre-Execution Safety Net)" in output_analyzer_with_plan
|
||||
)
|
||||
|
||||
# Test with empty plan and for_failure_analyzer=True (should completely omit plan block)
|
||||
output_analyzer_empty_plan = build_plan_and_history(
|
||||
"", steps, "default", last_n_detailed=1, for_failure_analyzer=True
|
||||
)
|
||||
assert "--- Task Plan ---" not in output_analyzer_empty_plan
|
||||
assert (
|
||||
"*(Note: Provided for context only. Do not execute the remaining"
|
||||
" plan.)*" not in output_analyzer_empty_plan
|
||||
)
|
||||
assert "--- Execution History ---" in output_analyzer_empty_plan
|
||||
@@ -0,0 +1,349 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from artemis.utils.ui_filter import (
|
||||
_clip_bounds,
|
||||
_is_semantic_empty,
|
||||
filter_node,
|
||||
filter_ui_hierarchy,
|
||||
)
|
||||
|
||||
|
||||
def test_is_semantic_empty():
|
||||
# Interactive or textual nodes are NOT empty
|
||||
assert not _is_semantic_empty({"text": "Hello"})
|
||||
assert not _is_semantic_empty({"content-desc": "Button"})
|
||||
assert not _is_semantic_empty({"clickable": True})
|
||||
assert not _is_semantic_empty({"clickable": "true"})
|
||||
assert not _is_semantic_empty({"focusable": True})
|
||||
assert not _is_semantic_empty({"className": "android.widget.ImageView"})
|
||||
|
||||
# Empty containers ARE semantic empty
|
||||
assert _is_semantic_empty(
|
||||
{
|
||||
"text": "",
|
||||
"clickable": False,
|
||||
"className": "android.widget.FrameLayout",
|
||||
}
|
||||
)
|
||||
assert _is_semantic_empty({})
|
||||
|
||||
|
||||
def test_clip_bounds():
|
||||
node = {"bounds": "[10,-10][1200,100]"}
|
||||
bounds = {"left": 10, "top": -10, "right": 1200, "bottom": 100}
|
||||
ancestor_bounds = {"left": 0, "top": 0, "right": 1080, "bottom": 2400}
|
||||
clipped = _clip_bounds(node, bounds, ancestor_bounds)
|
||||
|
||||
assert clipped["left"] == 10
|
||||
assert clipped["top"] == 0
|
||||
assert clipped["right"] == 1080
|
||||
assert clipped["bottom"] == 100
|
||||
assert node["bounds"] == "[10,0][1080,100]"
|
||||
assert "is_clipped" not in node
|
||||
|
||||
|
||||
def test_clip_bounds_fully_clipped():
|
||||
node = {"bounds": "[1200,2500][1300,2600]"}
|
||||
bounds = {"left": 1200, "top": 2500, "right": 1300, "bottom": 2600}
|
||||
ancestor_bounds = {"left": 0, "top": 0, "right": 1080, "bottom": 2400}
|
||||
clipped = _clip_bounds(node, bounds, ancestor_bounds)
|
||||
|
||||
assert clipped["left"] == 1080
|
||||
assert clipped["top"] == 2400
|
||||
assert clipped["right"] == 1080
|
||||
assert clipped["bottom"] == 2400
|
||||
assert node["is_clipped"] is True
|
||||
assert node["bounds"] == "[1080,2400][1080,2400]"
|
||||
|
||||
|
||||
def test_semantic_hoisting():
|
||||
hierarchy = [
|
||||
{
|
||||
"bounds": "[0,0][100,100]",
|
||||
"text": "",
|
||||
"clickable": False,
|
||||
"children": [
|
||||
{
|
||||
"bounds": "[10,10][90,90]",
|
||||
"text": "Child Button",
|
||||
"clickable": True,
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
# Mode: hoist
|
||||
res_hoist = filter_ui_hierarchy(hierarchy, semantic_pruning="hoist")
|
||||
assert len(res_hoist) == 1
|
||||
assert res_hoist[0]["text"] == "Child Button"
|
||||
|
||||
# Mode: safe
|
||||
res_safe = filter_ui_hierarchy(hierarchy, semantic_pruning="safe")
|
||||
assert len(res_safe) == 1
|
||||
assert res_safe[0]["text"] == ""
|
||||
assert len(res_safe[0]["children"]) == 1
|
||||
|
||||
|
||||
def test_dynamic_min_size():
|
||||
# Short side is 1000. 0.5% of 1000 is 5px.
|
||||
# Element size is 4x4 -> filtered out.
|
||||
node = {"bounds": "[0,0][4,4]", "text": "Tiny", "clickable": True}
|
||||
res = filter_node(node, 1000, 2000, min_element_size=2, use_dynamic_min_size=True)
|
||||
assert res is None
|
||||
|
||||
# Element size is 6x6 -> kept.
|
||||
node = {"bounds": "[0,0][6,6]", "text": "Good", "clickable": True}
|
||||
res = filter_node(node, 1000, 2000, min_element_size=2, use_dynamic_min_size=True)
|
||||
assert res is not None
|
||||
|
||||
|
||||
def test_ancestor_bounds_propagation():
|
||||
hierarchy = {
|
||||
"bounds": "[0,0][100,100]",
|
||||
"text": "Parent",
|
||||
"clickable": True,
|
||||
"children": [{"bounds": "[50,50][150,150]", "text": "Child", "clickable": True}],
|
||||
}
|
||||
|
||||
# The child is at [50,50][150,150], but parent is [0,0][100,100].
|
||||
# So child should be clipped to [50,50][100,100].
|
||||
res = filter_node(hierarchy, 1000, 1000, clip_bounds=True)
|
||||
assert res is not None
|
||||
assert res["children"][0]["bounds"] == "[50,50][100,100]"
|
||||
|
||||
|
||||
def test_sibling_merging():
|
||||
hierarchy = {
|
||||
"clickable": True,
|
||||
"bounds": "[0,0][200,200]",
|
||||
"children": [
|
||||
{
|
||||
"className": "android.widget.ImageView",
|
||||
"bounds": "[10,10][50,50]",
|
||||
"clickable": False,
|
||||
"focusable": False,
|
||||
},
|
||||
{
|
||||
"className": "android.widget.TextView",
|
||||
"text": "Settings",
|
||||
"bounds": "[60,10][150,50]",
|
||||
"clickable": False,
|
||||
"focusable": False,
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
# The parent is clickable, has 1 text child and 1 image child without text.
|
||||
# They should be merged!
|
||||
res = filter_node(hierarchy, 1000, 1000)
|
||||
assert res is not None
|
||||
assert "children" in res
|
||||
assert len(res["children"]) == 1
|
||||
assert res["children"][0]["text"] == "Settings"
|
||||
assert res["children"][0]["content-desc"] == "[Icon]"
|
||||
assert res["children"][0]["bounds"] == "[10,10][150,50]"
|
||||
|
||||
|
||||
def test_sibling_merging_not_triggered_if_child_clickable():
|
||||
hierarchy = {
|
||||
"clickable": True,
|
||||
"bounds": "[0,0][200,200]",
|
||||
"children": [
|
||||
{
|
||||
"className": "android.widget.ImageView",
|
||||
"bounds": "[10,10][50,50]",
|
||||
"clickable": True, # Child is clickable
|
||||
"focusable": False,
|
||||
},
|
||||
{
|
||||
"className": "android.widget.TextView",
|
||||
"text": "Settings",
|
||||
"bounds": "[60,10][150,50]",
|
||||
"clickable": False,
|
||||
"focusable": False,
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
res = filter_node(hierarchy, 1000, 1000)
|
||||
assert res is not None
|
||||
assert "children" in res
|
||||
assert len(res["children"]) == 2 # No merge
|
||||
|
||||
|
||||
def test_sibling_merging_not_triggered_if_form_control():
|
||||
hierarchy = {
|
||||
"clickable": True,
|
||||
"bounds": "[0,0][200,200]",
|
||||
"children": [
|
||||
{
|
||||
"className": "android.widget.CheckBox", # Form control
|
||||
"bounds": "[10,10][50,50]",
|
||||
"clickable": True, # Make it clickable so it's not pruned
|
||||
"focusable": False,
|
||||
},
|
||||
{
|
||||
"className": "android.widget.TextView",
|
||||
"text": "Enable",
|
||||
"bounds": "[60,10][150,50]",
|
||||
"clickable": False,
|
||||
"focusable": False,
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
res = filter_node(hierarchy, 1000, 1000)
|
||||
assert res is not None
|
||||
assert "children" in res
|
||||
assert len(res["children"]) == 2 # No merge
|
||||
|
||||
|
||||
def test_parent_child_redundancy():
|
||||
# Case 1: Redundant TextView removed
|
||||
hierarchy = {
|
||||
"text": "Confirm",
|
||||
"clickable": True,
|
||||
"bounds": "[0,0][200,200]",
|
||||
"children": [
|
||||
{
|
||||
"className": "android.widget.TextView",
|
||||
"text": "Confirm",
|
||||
"bounds": "[10,10][190,190]",
|
||||
"clickable": False,
|
||||
"focusable": False,
|
||||
}
|
||||
],
|
||||
}
|
||||
res = filter_node(hierarchy, 1000, 1000)
|
||||
assert res is not None
|
||||
assert "children" in res
|
||||
assert len(res["children"]) == 0 # Child removed
|
||||
|
||||
# Case 2: Redundant CheckBox text cleared
|
||||
hierarchy2 = {
|
||||
"text": "Confirm",
|
||||
"clickable": True,
|
||||
"bounds": "[0,0][200,200]",
|
||||
"children": [
|
||||
{
|
||||
"className": "android.widget.CheckBox",
|
||||
"text": "Confirm",
|
||||
"bounds": "[10,10][190,190]",
|
||||
"clickable": False,
|
||||
"focusable": False,
|
||||
}
|
||||
],
|
||||
}
|
||||
res2 = filter_node(hierarchy2, 1000, 1000)
|
||||
assert res2 is not None
|
||||
assert "children" in res2
|
||||
assert len(res2["children"]) == 1
|
||||
assert res2["children"][0]["text"] == ""
|
||||
|
||||
# Case 3: Redundant interactive node text cleared
|
||||
hierarchy3 = {
|
||||
"text": "Confirm",
|
||||
"clickable": True,
|
||||
"bounds": "[0,0][200,200]",
|
||||
"children": [
|
||||
{
|
||||
"className": "android.widget.TextView",
|
||||
"text": "Confirm",
|
||||
"bounds": "[10,10][190,190]",
|
||||
"clickable": True,
|
||||
"focusable": False,
|
||||
}
|
||||
],
|
||||
}
|
||||
res3 = filter_node(hierarchy3, 1000, 1000)
|
||||
assert res3 is not None
|
||||
assert "children" in res3
|
||||
assert len(res3["children"]) == 1
|
||||
assert res3["children"][0]["text"] == ""
|
||||
|
||||
# Case 4: No match
|
||||
hierarchy4 = {
|
||||
"text": "Confirm",
|
||||
"clickable": True,
|
||||
"bounds": "[0,0][200,200]",
|
||||
"children": [
|
||||
{
|
||||
"className": "android.widget.TextView",
|
||||
"text": "Cancel",
|
||||
"bounds": "[10,10][190,190]",
|
||||
"clickable": False,
|
||||
"focusable": False,
|
||||
}
|
||||
],
|
||||
}
|
||||
res4 = filter_node(hierarchy4, 1000, 1000)
|
||||
assert res4 is not None
|
||||
assert "children" in res4
|
||||
assert len(res4["children"]) == 1
|
||||
assert res4["children"][0]["text"] == "Cancel"
|
||||
|
||||
|
||||
def test_fixed_system_bars_clamping_and_discarding():
|
||||
# Simulate a full-width fixed bottom navigation bar at [0, 2250][1080, 2400]
|
||||
# and two candidates: one slightly occluded (should be safely clamped) and one mostly covered (should be discarded).
|
||||
hierarchy = [
|
||||
{
|
||||
"resource-id": "com.google.android.youtube:id/bottom_navigation",
|
||||
"bounds": "[0,2250][1080,2400]",
|
||||
"scrollable": False,
|
||||
"clickable": False,
|
||||
},
|
||||
{
|
||||
"text": "Partially occluded video item",
|
||||
"bounds": "[0,2100][1080,2300]",
|
||||
"scrollable": False,
|
||||
"clickable": True,
|
||||
},
|
||||
{
|
||||
"text": "Almost entirely covered sliver item",
|
||||
"bounds": "[0,2245][1080,2300]",
|
||||
"scrollable": False,
|
||||
"clickable": True,
|
||||
},
|
||||
]
|
||||
|
||||
filtered = filter_ui_hierarchy(
|
||||
hierarchy, screen_width=1080, screen_height=2400, min_element_size=20
|
||||
)
|
||||
# The bottom_navigation bar itself is kept (or filtered depending on semantics, here kept if not empty / checked).
|
||||
# Specifically check the clamped item and discarded item:
|
||||
texts = [el.get("text") for el in filtered]
|
||||
assert "Partially occluded video item" in texts
|
||||
assert "Almost entirely covered sliver item" not in texts
|
||||
|
||||
clamped_item = next(el for el in filtered if el.get("text") == "Partially occluded video item")
|
||||
# Its original bottom was 2300, clamped safely to the bar's top at 2250
|
||||
assert clamped_item["bounds"] == "[0,2100][1080,2250]"
|
||||
|
||||
|
||||
def test_mutual_occlusion_warning_injection():
|
||||
from artemis.utils.visualization import format_minimal_list_with_elements
|
||||
|
||||
# Simulate two overlapping non-ancestor interactive items with >= 50% overlap
|
||||
fused_xml = [
|
||||
{"text": "Search Video Result", "bounds": "[50,1800][1000,2000]"},
|
||||
{"text": "Create Floating Button", "bounds": "[750,1850][1000,2000]"},
|
||||
]
|
||||
|
||||
minimal_list, elements, labels = format_minimal_list_with_elements(
|
||||
fused_xml, width=1080, height=2400
|
||||
)
|
||||
assert "WARNING: may overlap with [2], possible occlusion" in minimal_list
|
||||
assert "WARNING: may overlap with [1], possible occlusion" in minimal_list
|
||||
@@ -0,0 +1,243 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
from artemis.utils.ui_hierarchy import (
|
||||
ElementBounds,
|
||||
Point,
|
||||
find_element_by_resource_id,
|
||||
get_bounds_for_element,
|
||||
get_element_text,
|
||||
is_element_focused,
|
||||
text_input_is_empty,
|
||||
)
|
||||
|
||||
|
||||
def test_text_input_is_empty():
|
||||
assert text_input_is_empty(text=None, hint_text=None)
|
||||
assert text_input_is_empty(text="", hint_text=None)
|
||||
assert text_input_is_empty(text="", hint_text="")
|
||||
assert text_input_is_empty(text="text", hint_text="text")
|
||||
|
||||
assert not text_input_is_empty(text="text", hint_text=None)
|
||||
assert not text_input_is_empty(text="text", hint_text="")
|
||||
|
||||
|
||||
def test_find_element_by_resource_id():
|
||||
ui_hierarchy = [
|
||||
{
|
||||
"resourceId": "com.example:id/button1",
|
||||
"text": "Button 1",
|
||||
"children": [],
|
||||
},
|
||||
{
|
||||
"resourceId": "com.example:id/container",
|
||||
"children": [
|
||||
{
|
||||
"resourceId": "com.example:id/nested_button",
|
||||
"text": "Nested Button",
|
||||
"children": [],
|
||||
}
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button1")
|
||||
assert result is not None
|
||||
assert result["resourceId"] == "com.example:id/button1"
|
||||
assert result["text"] == "Button 1"
|
||||
|
||||
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/nested_button")
|
||||
assert result is not None
|
||||
assert result["resourceId"] == "com.example:id/nested_button"
|
||||
assert result["text"] == "Nested Button"
|
||||
|
||||
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/nonexistent")
|
||||
assert result is None
|
||||
|
||||
result = find_element_by_resource_id([], "com.example:id/button1")
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_find_element_by_resource_id_with_index():
|
||||
ui_hierarchy = [
|
||||
{
|
||||
"resourceId": "com.example:id/button",
|
||||
"text": "Button 1",
|
||||
"children": [],
|
||||
},
|
||||
{
|
||||
"resourceId": "com.example:id/button",
|
||||
"text": "Button 2",
|
||||
"children": [],
|
||||
},
|
||||
{
|
||||
"resourceId": "com.example:id/container",
|
||||
"children": [
|
||||
{
|
||||
"resourceId": "com.example:id/button",
|
||||
"text": "Button 3",
|
||||
"children": [],
|
||||
}
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button", index=0)
|
||||
assert result is not None
|
||||
assert result["text"] == "Button 1"
|
||||
|
||||
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button", index=1)
|
||||
assert result is not None
|
||||
assert result["text"] == "Button 2"
|
||||
|
||||
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button", index=2)
|
||||
assert result is not None
|
||||
assert result["text"] == "Button 3"
|
||||
|
||||
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button", index=3)
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_find_element_by_resource_id_rich_hierarchy():
|
||||
rich_hierarchy = [
|
||||
{
|
||||
"attributes": {"resource-id": "com.example:id/button1"},
|
||||
"children": [],
|
||||
},
|
||||
{
|
||||
"attributes": {"resource-id": "com.example:id/container"},
|
||||
"children": [
|
||||
{
|
||||
"attributes": {"resource-id": "com.example:id/nested_button"},
|
||||
"children": [],
|
||||
}
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
result = find_element_by_resource_id(
|
||||
rich_hierarchy, "com.example:id/button1", is_rich_hierarchy=True
|
||||
)
|
||||
assert result is not None
|
||||
assert result["resource-id"] == "com.example:id/button1"
|
||||
|
||||
result = find_element_by_resource_id(
|
||||
rich_hierarchy, "com.example:id/nested_button", is_rich_hierarchy=True
|
||||
)
|
||||
assert result is not None
|
||||
assert result["resource-id"] == "com.example:id/nested_button"
|
||||
|
||||
result = find_element_by_resource_id(
|
||||
rich_hierarchy, "com.example:id/nonexistent", is_rich_hierarchy=True
|
||||
)
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_is_element_focused():
|
||||
focused_element = {"focused": "true"}
|
||||
assert is_element_focused(focused_element)
|
||||
|
||||
non_focused_element = {"focused": "false"}
|
||||
assert not is_element_focused(non_focused_element)
|
||||
|
||||
no_focused_element = {"text": "some text"}
|
||||
assert not is_element_focused(no_focused_element)
|
||||
|
||||
none_focused_element = {"focused": None}
|
||||
assert not is_element_focused(none_focused_element)
|
||||
|
||||
|
||||
def test_get_element_text():
|
||||
element = {"text": "Button Text", "hintText": "Hint Text"}
|
||||
assert get_element_text(element) == "Button Text"
|
||||
assert get_element_text(element, hint_text=False) == "Button Text"
|
||||
assert get_element_text(element, hint_text=True) == "Hint Text"
|
||||
|
||||
element_no_text = {"hintText": "Hint Text"}
|
||||
assert get_element_text(element_no_text) is None
|
||||
assert get_element_text(element_no_text, hint_text=True) == "Hint Text"
|
||||
element_no_hint = {"text": "Button Text"}
|
||||
assert get_element_text(element_no_hint) == "Button Text"
|
||||
assert get_element_text(element_no_hint, hint_text=True) is None
|
||||
|
||||
empty_element = {}
|
||||
assert get_element_text(empty_element) is None
|
||||
assert get_element_text(empty_element, hint_text=True) is None
|
||||
|
||||
|
||||
def test_get_bounds_for_element():
|
||||
element_with_bounds = {"bounds": {"x": 10, "y": 20, "width": 100, "height": 50}}
|
||||
bounds = get_bounds_for_element(element_with_bounds)
|
||||
assert bounds is not None
|
||||
assert isinstance(bounds, ElementBounds)
|
||||
assert bounds.x == 10
|
||||
assert bounds.y == 20
|
||||
assert bounds.width == 100
|
||||
assert bounds.height == 50
|
||||
|
||||
element_no_bounds = {"text": "Button"}
|
||||
bounds = get_bounds_for_element(element_no_bounds)
|
||||
assert bounds is None
|
||||
|
||||
# Suppress logger output for the invalid bounds test case
|
||||
with patch("artemis.utils.ui_hierarchy.logger.error"):
|
||||
element_invalid_bounds = {
|
||||
"bounds": {
|
||||
"x": "invalid", # Should be int
|
||||
"y": 20,
|
||||
"width": 100,
|
||||
"height": 50,
|
||||
}
|
||||
}
|
||||
bounds = get_bounds_for_element(element_invalid_bounds)
|
||||
assert bounds is None
|
||||
|
||||
|
||||
def test_element_bounds():
|
||||
bounds = ElementBounds(x=10, y=20, width=100, height=50)
|
||||
|
||||
center = bounds.get_center()
|
||||
assert isinstance(center, Point)
|
||||
assert center.x == 60
|
||||
assert center.y == 45
|
||||
|
||||
center_point = bounds.get_relative_point(0.5, 0.5)
|
||||
assert isinstance(center_point, Point)
|
||||
assert center_point.x == 60
|
||||
assert center_point.y == 45
|
||||
|
||||
top_left = bounds.get_relative_point(0.0, 0.0)
|
||||
assert top_left.x == 10
|
||||
assert top_left.y == 20
|
||||
|
||||
bottom_right = bounds.get_relative_point(1.0, 1.0)
|
||||
assert bottom_right.x == 110
|
||||
assert bottom_right.y == 70
|
||||
custom_point = bounds.get_relative_point(0.95, 0.95)
|
||||
assert custom_point.x == 105
|
||||
assert custom_point.y == 67
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_text_input_is_empty()
|
||||
test_find_element_by_resource_id()
|
||||
test_find_element_by_resource_id_with_index()
|
||||
test_find_element_by_resource_id_rich_hierarchy()
|
||||
test_is_element_focused()
|
||||
test_get_element_text()
|
||||
test_get_bounds_for_element()
|
||||
test_element_bounds()
|
||||
print("All tests passed")
|
||||
@@ -0,0 +1,77 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import json
|
||||
from artemis.utils.verification import (
|
||||
append_verification_chat,
|
||||
get_verification_chat_path,
|
||||
get_verification_chat_rounds,
|
||||
read_verification_chat,
|
||||
)
|
||||
|
||||
|
||||
def test_get_verification_chat_path(tmp_path):
|
||||
subgoal_hash = "test_hash"
|
||||
expected_path = tmp_path / "notes" / f"verification_chat_{subgoal_hash}.json"
|
||||
path = get_verification_chat_path(tmp_path, subgoal_hash)
|
||||
assert path == expected_path
|
||||
|
||||
|
||||
def test_read_verification_chat_not_exists(tmp_path):
|
||||
path = tmp_path / "non_existent.json"
|
||||
turns = read_verification_chat(path)
|
||||
assert turns == []
|
||||
|
||||
|
||||
def test_read_verification_chat_exists(tmp_path):
|
||||
path = tmp_path / "chat.json"
|
||||
expected_turns = [{"role": "operator", "round": 1, "content": "hello"}]
|
||||
path.write_text(json.dumps(expected_turns), encoding="utf-8")
|
||||
turns = read_verification_chat(path)
|
||||
assert turns == expected_turns
|
||||
|
||||
|
||||
def test_get_verification_chat_rounds():
|
||||
turns = [
|
||||
{"role": "operator", "round": 1, "content": "op1"},
|
||||
{"role": "checker", "round": 1, "content": "chk1"},
|
||||
{"role": "operator", "round": 2, "content": "op2"},
|
||||
]
|
||||
max_op, max_chk = get_verification_chat_rounds(turns)
|
||||
assert max_op == 2
|
||||
assert max_chk == 1
|
||||
|
||||
|
||||
def test_get_verification_chat_rounds_empty():
|
||||
max_op, max_chk = get_verification_chat_rounds([])
|
||||
assert max_op == 0
|
||||
assert max_chk == 0
|
||||
|
||||
|
||||
def test_append_verification_chat(tmp_path):
|
||||
path = tmp_path / "chat.json"
|
||||
|
||||
# Append to empty/non-existent
|
||||
success = append_verification_chat(path, "operator", "hello", 1)
|
||||
assert success
|
||||
assert path.exists()
|
||||
turns = json.loads(path.read_text(encoding="utf-8"))
|
||||
assert turns == [{"role": "operator", "round": 1, "content": "hello"}]
|
||||
|
||||
# Append again
|
||||
success = append_verification_chat(path, "checker", "hi", 2)
|
||||
assert success
|
||||
turns = json.loads(path.read_text(encoding="utf-8"))
|
||||
assert len(turns) == 2
|
||||
assert turns[1] == {"role": "checker", "round": 2, "content": "hi"}
|
||||
Reference in new issue
Block a user