somew1nd
5d92939d91
Preserve user guidance through history compression, stabilize Flash index targets, and retire withdrawn checks without applying stale verdicts. Improve runtime diagnostics, tool responses, and history rendering. Simplify documentation and comments. Validation: 2076 tests passed in the full suite; 381 affected tests and 26 frontend tests passed after cleanup. Ruff and the quality ratchet passed.
371 lines
12 KiB
Python
371 lines
12 KiB
Python
# Copyright 2026 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
from unittest.mock import AsyncMock, Mock, patch
|
|
from langchain_core.messages import AIMessage, ToolMessage
|
|
from langchain_core.tools import StructuredTool
|
|
from artemis.agents.diagnoser.diagnoser import Diagnoser
|
|
from artemis.context import ArtemisContext
|
|
from artemis.core.tool_failure import ToolFailure
|
|
from artemis.graph.state import State
|
|
import pytest
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_context():
|
|
ctx = Mock(spec=ArtemisContext)
|
|
ctx.llm_config = Mock()
|
|
ctx.data_engine = Mock()
|
|
ctx.data_engine.base_dir = "/tmp"
|
|
ctx.data_engine.get_agent_friendly_steps.return_value = []
|
|
|
|
# Mock execution_setup
|
|
mock_setup = Mock()
|
|
mock_setup.video_recording_tools_enabled = True
|
|
ctx.execution_setup = mock_setup
|
|
|
|
return ctx
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_state():
|
|
state = Mock(spec=State)
|
|
state.initial_goal = "Test goal"
|
|
state.latest_screenshot = None
|
|
state.latest_ui_hierarchy = []
|
|
return state
|
|
|
|
|
|
@patch("artemis.agents.diagnoser.diagnoser.get_llm")
|
|
@patch("artemis.agents.diagnoser.diagnoser.get_video_analyzer_tool")
|
|
@patch("artemis.agents.diagnoser.diagnoser.get_analyze_logs_tool")
|
|
@pytest.mark.asyncio
|
|
async def test_diagnostic_agent_simple_call(
|
|
mock_get_log_tool,
|
|
mock_get_video_tool,
|
|
mock_get_llm,
|
|
mock_context,
|
|
mock_state,
|
|
):
|
|
"""Test that DiagnosticAgent calls LLM and returns result."""
|
|
mock_llm = Mock()
|
|
mock_llm.bind_tools.return_value = mock_llm
|
|
|
|
mock_response = AIMessage(content="I diagnosed the issue: it is a network error.")
|
|
|
|
async def mock_astream(*args, **kwargs):
|
|
yield mock_response
|
|
|
|
mock_llm.astream.side_effect = mock_astream
|
|
mock_get_llm.return_value = mock_llm
|
|
|
|
# Mock tools
|
|
mock_video_tool = Mock()
|
|
mock_video_tool.name = "video_analyzer"
|
|
mock_get_video_tool.return_value = mock_video_tool
|
|
|
|
mock_log_tool = Mock()
|
|
mock_log_tool.name = "analyze_logs"
|
|
mock_get_log_tool.return_value = mock_log_tool
|
|
|
|
agent = Diagnoser(mock_context)
|
|
result = await agent.run("Diagnose why the app crashed", mock_state)
|
|
|
|
assert result == "I diagnosed the issue: it is a network error."
|
|
|
|
|
|
@patch("artemis.agents.diagnoser.diagnoser.get_llm")
|
|
@patch("artemis.agents.diagnoser.diagnoser.get_video_analyzer_tool")
|
|
@patch("artemis.agents.diagnoser.diagnoser.get_analyze_logs_tool")
|
|
@pytest.mark.asyncio
|
|
async def test_diagnostic_agent_tool_calls(
|
|
mock_get_log_tool,
|
|
mock_get_video_tool,
|
|
mock_get_llm,
|
|
mock_context,
|
|
mock_state,
|
|
):
|
|
"""Test that DiagnosticAgent handles tool calls."""
|
|
mock_llm = Mock()
|
|
mock_llm.bind_tools.return_value = mock_llm
|
|
|
|
# Simulate response with tool calls
|
|
mock_response = AIMessage(
|
|
content="I need to check logs and video.",
|
|
tool_calls=[
|
|
{
|
|
"name": "analyze_logs",
|
|
"args": {"specific_query": "Check for crashes"},
|
|
"id": "call_1",
|
|
},
|
|
{
|
|
"name": "video_analyzer",
|
|
"args": {
|
|
"time_description": "around 10s",
|
|
"purpose": "Check UI",
|
|
},
|
|
"id": "call_2",
|
|
},
|
|
],
|
|
)
|
|
# Second call returns final answer
|
|
mock_final_response = AIMessage(content="Diagnosis complete.")
|
|
|
|
responses = [mock_response, mock_final_response]
|
|
call_count = 0
|
|
|
|
async def mock_astream(*args, **kwargs):
|
|
nonlocal call_count
|
|
if call_count < len(responses):
|
|
yield responses[call_count]
|
|
call_count += 1
|
|
|
|
mock_llm.astream.side_effect = mock_astream
|
|
mock_get_llm.return_value = mock_llm
|
|
|
|
# Mock tools
|
|
mock_log_tool = AsyncMock()
|
|
mock_log_tool.name = "analyze_logs"
|
|
mock_log_tool.args = {"specific_query": None, "state": None}
|
|
mock_log_tool.ainvoke.return_value = "Logs show no crash."
|
|
mock_get_log_tool.return_value = mock_log_tool
|
|
|
|
mock_video_tool = AsyncMock()
|
|
mock_video_tool.name = "video_analyzer"
|
|
mock_video_tool.args = {
|
|
"time_description": None,
|
|
"purpose": None,
|
|
"state": None,
|
|
}
|
|
mock_video_tool.ainvoke.return_value = ToolMessage(
|
|
content="Video shows UI loading.", tool_call_id="call_2"
|
|
)
|
|
mock_get_video_tool.return_value = mock_video_tool
|
|
|
|
agent = Diagnoser(mock_context)
|
|
result = await agent.run("Diagnose issue", mock_state)
|
|
|
|
assert result == "Diagnosis complete."
|
|
mock_log_tool.ainvoke.assert_called_once_with(
|
|
{"specific_query": "Check for crashes", "state": mock_state}
|
|
)
|
|
mock_video_tool.ainvoke.assert_called_once()
|
|
|
|
|
|
@patch("artemis.agents.diagnoser.diagnoser.get_llm")
|
|
@patch("artemis.agents.diagnoser.diagnoser.get_video_analyzer_tool")
|
|
@patch("artemis.agents.diagnoser.diagnoser.get_analyze_logs_tool")
|
|
@pytest.mark.asyncio
|
|
async def test_diagnostic_agent_no_video_tool_when_disabled(
|
|
mock_get_log_tool,
|
|
mock_get_video_tool,
|
|
mock_get_llm,
|
|
mock_context,
|
|
mock_state,
|
|
):
|
|
"""Test that Diagnoser does not bind video tool when disabled."""
|
|
# Set video recording to False
|
|
mock_context.execution_setup.video_recording_tools_enabled = False
|
|
|
|
mock_llm = Mock()
|
|
mock_llm.bind_tools.return_value = mock_llm
|
|
|
|
mock_response = AIMessage(content="Diagnosis complete.")
|
|
|
|
async def mock_astream(*args, **kwargs):
|
|
yield mock_response
|
|
|
|
mock_llm.astream.side_effect = mock_astream
|
|
mock_get_llm.return_value = mock_llm
|
|
|
|
# Mock tools
|
|
mock_video_tool = Mock()
|
|
mock_video_tool.name = "video_analyzer"
|
|
mock_get_video_tool.return_value = mock_video_tool
|
|
|
|
mock_log_tool = Mock()
|
|
mock_log_tool.name = "analyze_logs"
|
|
mock_get_log_tool.return_value = mock_log_tool
|
|
|
|
agent = Diagnoser(mock_context)
|
|
await agent.run("Diagnose issue", mock_state)
|
|
|
|
# Verify bind_tools was called
|
|
mock_llm.bind_tools.assert_called_once()
|
|
bound_tools = mock_llm.bind_tools.call_args[1]["tools"]
|
|
|
|
# Verify video_analyzer is NOT in bound tools
|
|
tool_names = [t.name for t in bound_tools if hasattr(t, "name")]
|
|
assert "video_analyzer" not in tool_names
|
|
# Verify log tool IS in bound tools (should still be there)
|
|
assert "analyze_logs" in tool_names
|
|
|
|
|
|
@patch("artemis.agents.diagnoser.diagnoser.get_llm")
|
|
@patch("artemis.tools.video_tool.VideoAnalyzer")
|
|
@pytest.mark.asyncio
|
|
async def test_diagnoser_background_job_lifecycle(
|
|
mock_video_analyzer_class, mock_get_llm, mock_state
|
|
):
|
|
# Setup context
|
|
ctx = ArtemisContext.model_construct(
|
|
device=Mock(),
|
|
llm_config=Mock(),
|
|
execution_setup=Mock(video_recording_tools_enabled=True),
|
|
)
|
|
ctx.data_engine = None
|
|
|
|
# The Mock VideoAnalyzer agent running inside the background job
|
|
mock_agent_instance = AsyncMock()
|
|
|
|
async def mock_agent_run(time_desc, purpose):
|
|
return "Video shows step successfully completed.", "success"
|
|
|
|
mock_agent_instance.run.side_effect = mock_agent_run
|
|
mock_video_analyzer_class.return_value = mock_agent_instance
|
|
|
|
# Prepare LLM responses
|
|
mock_llm = Mock()
|
|
mock_llm.bind_tools.return_value = mock_llm
|
|
|
|
# Turn 1: LLM requests tool call
|
|
mock_response_1 = AIMessage(
|
|
content="Checking the video first.",
|
|
tool_calls=[
|
|
{
|
|
"name": "video_analyzer",
|
|
"args": {
|
|
"time_description": "from 0s to 5s",
|
|
"purpose": "check UI",
|
|
},
|
|
"id": "call_video_1",
|
|
}
|
|
],
|
|
)
|
|
# Turn 2: LLM gets the initial job start message, continues, then background job finishes, and result is injected.
|
|
mock_response_2 = AIMessage(content="Final complete diagnosis.")
|
|
|
|
responses = [mock_response_1, mock_response_2]
|
|
call_count = 0
|
|
|
|
async def mock_astream(*args, **kwargs):
|
|
nonlocal call_count
|
|
if call_count < len(responses):
|
|
yield responses[call_count]
|
|
call_count += 1
|
|
|
|
mock_llm.astream.side_effect = mock_astream
|
|
mock_get_llm.return_value = mock_llm
|
|
|
|
# Instantiate the Diagnoser
|
|
agent = Diagnoser(ctx)
|
|
|
|
# We want to use the actual video analyzer tool so it triggers the background task
|
|
from artemis.tools.video_tool import get_video_analyzer_tool
|
|
|
|
real_video_tool = get_video_analyzer_tool(ctx, role="diagnoser")
|
|
|
|
with patch(
|
|
"artemis.tools.log_tool.get_analyze_logs_tool",
|
|
return_value=Mock(name="analyze_logs"),
|
|
):
|
|
with patch(
|
|
"artemis.tools.video_tool.get_video_analyzer_tool",
|
|
return_value=real_video_tool,
|
|
):
|
|
result = await agent.run("Diagnose video issue", mock_state)
|
|
|
|
assert result == "Final complete diagnosis."
|
|
# Verify job registry
|
|
assert len(ctx.background_jobs) == 1
|
|
job_id = list(ctx.background_jobs.keys())[0]
|
|
job = ctx.background_jobs[job_id]
|
|
assert job["tool_call_id"] == "call_video_1"
|
|
assert job["status"] == "completed"
|
|
assert job["result"] == "Video shows step successfully completed."
|
|
assert job["consumed"] is True
|
|
|
|
|
|
# --- tool results: status is structural, never sniffed from the words -----------------
|
|
|
|
|
|
# A helper tool reports failure structurally (``ToolFailure``); free-form text
|
|
# that merely starts with "Error" is an ordinary answer.
|
|
_STATUS_CASES = [
|
|
pytest.param(ToolFailure("Error: note 'progress' not found"), "error", id="tool_failure"),
|
|
pytest.param("Error 404 was typed into the search box", "success", id="plain_error_text"),
|
|
]
|
|
|
|
|
|
def _text_tool(name: str, result):
|
|
async def _run(key: str) -> str:
|
|
return result
|
|
|
|
return StructuredTool.from_function(coroutine=_run, name=name, description=name)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("result, expected_status", _STATUS_CASES)
|
|
async def test_run_tool_message_status_is_structural(
|
|
result, expected_status, mock_context, mock_state
|
|
):
|
|
agent = Diagnoser(mock_context)
|
|
tool = _text_tool("read_note", result)
|
|
|
|
messages = await agent._run_tool(
|
|
{"name": "read_note", "args": {"key": "progress"}, "id": "c1"}, [tool], mock_state
|
|
)
|
|
|
|
assert len(messages) == 1
|
|
assert isinstance(messages[0], ToolMessage)
|
|
assert messages[0].tool_call_id == "c1"
|
|
assert messages[0].status == expected_status
|
|
assert messages[0].content == str(result)
|
|
|
|
|
|
# --- submit_answer rendering ------------------------------------------------------------
|
|
|
|
|
|
def test_submit_answer_renders_numbered_steps(mock_context):
|
|
agent = Diagnoser(mock_context)
|
|
call = {
|
|
"name": "submit_answer",
|
|
"args": {
|
|
"analysis": "Airplane mode blocks the toggle.",
|
|
"actionable_steps": ["Turn off airplane mode", "Retry the toggle"],
|
|
},
|
|
}
|
|
|
|
outcome = agent._handle_submit_call(call, [])
|
|
|
|
assert outcome == (
|
|
"Analysis: Airplane mode blocks the toggle.\n"
|
|
"Actionable steps:\n"
|
|
"1. Turn off airplane mode\n"
|
|
"2. Retry the toggle"
|
|
)
|
|
|
|
|
|
def test_submit_answer_without_steps_omits_section(mock_context):
|
|
agent = Diagnoser(mock_context)
|
|
call = {
|
|
"name": "submit_answer",
|
|
"args": {"analysis": "Nothing to fix.", "actionable_steps": []},
|
|
}
|
|
|
|
outcome = agent._handle_submit_call(call, [])
|
|
|
|
assert outcome == "Analysis: Nothing to fix."
|
|
assert "Actionable" not in outcome
|