somew1nd
a45daf23f8
Report device actions as dispatched and preserve full failure details. Route Flash through McpActionExecutor and use ToolFailure across helper tools and agent loops instead of inferring failure from response text. Require model-provided descriptions for coordinate targets. Keep observed element metadata separate from those descriptions in safety checks, visual summaries, history, and frontend labels. Add Gemini Interactions API video analysis with configurable processing mode, chunk size, and timeouts. Centralize Google provider capabilities and token usage normalization, and update google-genai to 2.22.0. Preserve system instructions on follow-up interactions, replan pending chunks when falling back to static processing, and reject unanswered native video and audio segments instead of caching placeholder results. Keep helper-tool failure status intact through history tools and committee calls. Validation: 1,762 Python unit tests passed (3 skipped), 40 frontend formatter tests passed, and the production UI build completed. Ruff lint, formatting, the quality ratchet, and git diff checks passed.
345 lines
11 KiB
Python
345 lines
11 KiB
Python
# Copyright 2026 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
from unittest.mock import AsyncMock, Mock, patch
|
|
from adbutils import AdbClient
|
|
from langchain_core.messages import AIMessage, ToolMessage
|
|
from artemis.agents.diagnoser.diagnoser import Diagnoser
|
|
from artemis.context import ArtemisContext
|
|
from artemis.core.tool_failure import ToolFailure
|
|
from artemis.graph.state import State
|
|
import pytest
|
|
|
|
|
|
@pytest.fixture(scope="session", autouse=True)
|
|
def ensure_device_connected():
|
|
try:
|
|
client = AdbClient()
|
|
if not client.device_list():
|
|
pytest.skip("No active Android device/emulator found. Skipping diagnostic tool tests.")
|
|
except Exception as e:
|
|
pytest.skip(f"ADB Client connection failed: {e}. Skipping tests.")
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_context():
|
|
ctx = Mock(spec=ArtemisContext)
|
|
ctx.llm_config = Mock()
|
|
ctx.data_engine = Mock()
|
|
ctx.data_engine.base_dir = "/tmp"
|
|
ctx.data_engine.get_agent_friendly_steps.return_value = []
|
|
|
|
# Mock execution_setup
|
|
mock_setup = Mock()
|
|
mock_setup.video_recording_tools_enabled = True
|
|
ctx.execution_setup = mock_setup
|
|
|
|
return ctx
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_state():
|
|
state = Mock(spec=State)
|
|
state.initial_goal = "Test goal"
|
|
state.latest_screenshot = None
|
|
state.latest_ui_hierarchy = []
|
|
return state
|
|
|
|
|
|
@patch("artemis.agents.diagnoser.diagnoser.get_llm")
|
|
@patch("artemis.tools.video_tool.get_video_analyzer_tool")
|
|
@patch("artemis.tools.log_tool.get_analyze_logs_tool")
|
|
@patch("artemis.agents.diagnoser.diagnoser.get_ui_hierarchy_tool")
|
|
@pytest.mark.asyncio
|
|
async def test_maestro_uninstall_timeout(
|
|
mock_get_ui_tool,
|
|
mock_get_log_tool,
|
|
mock_get_video_tool,
|
|
mock_get_llm,
|
|
mock_context,
|
|
mock_state,
|
|
):
|
|
"""Test Maestro Uninstall Timeout / Blocker Failure."""
|
|
# Mock LLM calls
|
|
mock_llm = Mock()
|
|
mock_llm.bind_tools.return_value = mock_llm
|
|
|
|
mock_response_1 = AIMessage(
|
|
content="I need to check the UI hierarchy.",
|
|
tool_calls=[
|
|
{
|
|
"name": "get_ui_hierarchy",
|
|
"args": {},
|
|
"id": "call_hierarchy",
|
|
}
|
|
],
|
|
)
|
|
mock_response_2 = AIMessage(
|
|
content=(
|
|
"Diagnoser identifies the blocker failure and recommends restarting"
|
|
" ADB/uninstalling conflicting tools manually."
|
|
)
|
|
)
|
|
|
|
responses = [mock_response_1, mock_response_2]
|
|
call_count = 0
|
|
|
|
async def mock_astream(*args, **kwargs):
|
|
nonlocal call_count
|
|
if call_count < len(responses):
|
|
yield responses[call_count]
|
|
call_count += 1
|
|
|
|
mock_llm.astream.side_effect = mock_astream
|
|
mock_get_llm.return_value = mock_llm
|
|
|
|
# Mock tool
|
|
mock_ui_tool = AsyncMock()
|
|
mock_ui_tool.name = "get_ui_hierarchy"
|
|
mock_ui_tool.args = {}
|
|
mock_ui_tool.ainvoke.return_value = ToolFailure("Error: pm list packages timed out")
|
|
mock_get_ui_tool.return_value = mock_ui_tool
|
|
|
|
# Run agent
|
|
agent = Diagnoser(mock_context)
|
|
result = await agent.run("Diagnose why the app crashed", mock_state)
|
|
|
|
assert "recommends restarting ADB" in result
|
|
assert "uninstalling conflicting tools" in result
|
|
|
|
# Assert get_ui_hierarchy was called
|
|
mock_ui_tool.ainvoke.assert_called_once()
|
|
|
|
# Assert that the ToolMessage was sent back to LLM
|
|
assert len(mock_llm.astream.call_args_list) == 2
|
|
sent_messages = mock_llm.astream.call_args_list[1][0][0]
|
|
tool_message = sent_messages[-2]
|
|
assert isinstance(tool_message, ToolMessage)
|
|
assert tool_message.tool_call_id == "call_hierarchy"
|
|
assert tool_message.content == "Error: pm list packages timed out"
|
|
assert tool_message.status == "error"
|
|
|
|
|
|
@patch("artemis.agents.diagnoser.diagnoser.get_llm")
|
|
@patch("artemis.tools.video_tool.get_video_analyzer_tool")
|
|
@patch("artemis.tools.log_tool.get_analyze_logs_tool")
|
|
@patch("artemis.agents.diagnoser.diagnoser.get_ui_hierarchy_tool")
|
|
@pytest.mark.asyncio
|
|
async def test_uiautomator2_connection_lost(
|
|
mock_get_ui_tool,
|
|
mock_get_log_tool,
|
|
mock_get_video_tool,
|
|
mock_get_llm,
|
|
mock_context,
|
|
mock_state,
|
|
):
|
|
"""Test UIAutomator2 Daemon Connection Lost."""
|
|
# Mock LLM
|
|
mock_llm = Mock()
|
|
mock_llm.bind_tools.return_value = mock_llm
|
|
|
|
mock_response_1 = AIMessage(
|
|
content="Let me check the UI hierarchy.",
|
|
tool_calls=[
|
|
{
|
|
"name": "get_ui_hierarchy",
|
|
"args": {},
|
|
"id": "call_hierarchy_lost",
|
|
}
|
|
],
|
|
)
|
|
mock_response_2 = AIMessage(content="Diagnoser reports connection loss with UIAutomator2.")
|
|
|
|
responses = [mock_response_1, mock_response_2]
|
|
call_count = 0
|
|
|
|
async def mock_astream(*args, **kwargs):
|
|
nonlocal call_count
|
|
if call_count < len(responses):
|
|
yield responses[call_count]
|
|
call_count += 1
|
|
|
|
mock_llm.astream.side_effect = mock_astream
|
|
mock_get_llm.return_value = mock_llm
|
|
|
|
# Mock tool to throw ConnectionError
|
|
mock_ui_tool = AsyncMock()
|
|
mock_ui_tool.name = "get_ui_hierarchy"
|
|
mock_ui_tool.args = {}
|
|
mock_ui_tool.ainvoke.side_effect = ConnectionError(
|
|
"UIAutomator2 connection lost, reconnecting..."
|
|
)
|
|
mock_get_ui_tool.return_value = mock_ui_tool
|
|
|
|
# Run agent
|
|
agent = Diagnoser(mock_context)
|
|
result = await agent.run("Diagnose crash", mock_state)
|
|
|
|
assert "reports connection loss" in result
|
|
|
|
# Assert that tool error is wrapped and returned in the ToolMessage
|
|
assert len(mock_llm.astream.call_args_list) == 2
|
|
sent_messages = mock_llm.astream.call_args_list[1][0][0]
|
|
tool_message = sent_messages[-2]
|
|
assert isinstance(tool_message, ToolMessage)
|
|
assert tool_message.tool_call_id == "call_hierarchy_lost"
|
|
assert "UIAutomator2 connection lost, reconnecting..." in tool_message.content
|
|
assert tool_message.status == "error"
|
|
|
|
# Verify that the trace span recorded the failure
|
|
record_trace_calls = mock_context.data_engine.record_trace.call_args_list
|
|
tool_trace_calls = [c for c in record_trace_calls if c[1].get("name") == "get_ui_hierarchy"]
|
|
failed_trace = next((c for c in tool_trace_calls if c[1].get("status") == "failed"), None)
|
|
assert failed_trace is not None
|
|
assert "UIAutomator2 connection lost, reconnecting..." in failed_trace[1]["payload"]["error"]
|
|
|
|
|
|
@patch("artemis.agents.diagnoser.diagnoser.get_llm")
|
|
@patch("artemis.tools.video_tool.get_video_analyzer_tool")
|
|
@patch("artemis.tools.log_tool.get_analyze_logs_tool")
|
|
@patch("artemis.agents.diagnoser.diagnoser.get_ui_hierarchy_tool")
|
|
@pytest.mark.asyncio
|
|
async def test_ui_hierarchy_dump_stall(
|
|
mock_get_ui_tool,
|
|
mock_get_log_tool,
|
|
mock_get_video_tool,
|
|
mock_get_llm,
|
|
mock_context,
|
|
mock_state,
|
|
):
|
|
"""Test UI Hierarchy Dump Stall (UI Thread Frozen)."""
|
|
# Mock LLM
|
|
mock_llm = Mock()
|
|
mock_llm.bind_tools.return_value = mock_llm
|
|
|
|
mock_response_1 = AIMessage(
|
|
content="Let me fetch the UI hierarchy to see what's on screen.",
|
|
tool_calls=[
|
|
{
|
|
"name": "get_ui_hierarchy",
|
|
"args": {},
|
|
"id": "call_hierarchy_stall",
|
|
}
|
|
],
|
|
)
|
|
mock_response_2 = AIMessage(
|
|
content=(
|
|
"Diagnoser identifies frozen UI and suggests rendering-safe"
|
|
" recovery actions (like ADB wake commands or a wait/retry cycle)."
|
|
)
|
|
)
|
|
|
|
responses = [mock_response_1, mock_response_2]
|
|
call_count = 0
|
|
|
|
async def mock_astream(*args, **kwargs):
|
|
nonlocal call_count
|
|
if call_count < len(responses):
|
|
yield responses[call_count]
|
|
call_count += 1
|
|
|
|
mock_llm.astream.side_effect = mock_astream
|
|
mock_get_llm.return_value = mock_llm
|
|
|
|
# Mock tool
|
|
mock_ui_tool = AsyncMock()
|
|
mock_ui_tool.name = "get_ui_hierarchy"
|
|
mock_ui_tool.args = {}
|
|
mock_ui_tool.ainvoke.return_value = ToolFailure(
|
|
"Error retrieving UI hierarchy: dump_hierarchy timed out after 3 seconds"
|
|
)
|
|
mock_get_ui_tool.return_value = mock_ui_tool
|
|
|
|
# Run agent
|
|
agent = Diagnoser(mock_context)
|
|
result = await agent.run("Diagnose stall", mock_state)
|
|
|
|
assert "rendering-safe recovery actions" in result
|
|
|
|
# Assert ToolMessage was passed back with correct error details
|
|
assert len(mock_llm.astream.call_args_list) == 2
|
|
sent_messages = mock_llm.astream.call_args_list[1][0][0]
|
|
tool_message = sent_messages[-2]
|
|
assert isinstance(tool_message, ToolMessage)
|
|
assert tool_message.tool_call_id == "call_hierarchy_stall"
|
|
assert "dump_hierarchy timed out after 3 seconds" in tool_message.content
|
|
assert tool_message.status == "error"
|
|
|
|
|
|
@patch("artemis.agents.diagnoser.diagnoser.get_llm")
|
|
@patch("artemis.tools.video_tool.get_video_analyzer_tool")
|
|
@patch("artemis.tools.log_tool.get_analyze_logs_tool")
|
|
@patch("artemis.agents.diagnoser.diagnoser.get_ui_hierarchy_tool")
|
|
@pytest.mark.asyncio
|
|
async def test_device_offline(
|
|
mock_get_ui_tool,
|
|
mock_get_log_tool,
|
|
mock_get_video_tool,
|
|
mock_get_llm,
|
|
mock_context,
|
|
mock_state,
|
|
):
|
|
"""Test Device Offline (ADB Transport Lost)."""
|
|
# Mock LLM
|
|
mock_llm = Mock()
|
|
mock_llm.bind_tools.return_value = mock_llm
|
|
|
|
mock_response_1 = AIMessage(
|
|
content="Let's grab the UI hierarchy.",
|
|
tool_calls=[
|
|
{
|
|
"name": "get_ui_hierarchy",
|
|
"args": {},
|
|
"id": "call_hierarchy_offline",
|
|
}
|
|
],
|
|
)
|
|
mock_response_2 = AIMessage(
|
|
content=("The device is offline. Diagnoser flags the offline device state.")
|
|
)
|
|
|
|
responses = [mock_response_1, mock_response_2]
|
|
call_count = 0
|
|
|
|
async def mock_astream(*args, **kwargs):
|
|
nonlocal call_count
|
|
if call_count < len(responses):
|
|
yield responses[call_count]
|
|
call_count += 1
|
|
|
|
mock_llm.astream.side_effect = mock_astream
|
|
mock_get_llm.return_value = mock_llm
|
|
|
|
# Mock tool
|
|
mock_ui_tool = AsyncMock()
|
|
mock_ui_tool.name = "get_ui_hierarchy"
|
|
mock_ui_tool.args = {}
|
|
mock_ui_tool.ainvoke.side_effect = RuntimeError("device offline")
|
|
mock_get_ui_tool.return_value = mock_ui_tool
|
|
|
|
# Run agent
|
|
agent = Diagnoser(mock_context)
|
|
result = await agent.run("Diagnose offline", mock_state)
|
|
|
|
assert "flags the offline device state" in result
|
|
|
|
# Assert ToolMessage was passed back with correct error details
|
|
assert len(mock_llm.astream.call_args_list) == 2
|
|
sent_messages = mock_llm.astream.call_args_list[1][0][0]
|
|
tool_message = sent_messages[-2]
|
|
assert isinstance(tool_message, ToolMessage)
|
|
assert tool_message.tool_call_id == "call_hierarchy_offline"
|
|
assert "device offline" in tool_message.content
|
|
assert tool_message.status == "error"
|