test: add comprehensive unit, integration, and e2e test suites
Introduce complete test coverage for the ARTEMIS platform across multiple layers: Unit tests: Add tests for agent nodes (Planner, Operator, Validator, Diagnoser, FlashRunner, Explorer), tools, utils (OCR/XML fusion, UI hierarchy), and data engine. Integration tests: Add live tool verification, reactive runner, IME, and standalone workflow tests. E2E tests: Add end-to-end scenario tests for media playback and failure recovery flows. Fixtures & mocks: Add mock inputs (audio, video, screenshot) and test utilities.
This commit is contained in:
1 parent
c1803fcb86
commit
5d2283deaa
115 files changed
+22813
No files matched your search
@@ -0,0 +1,21 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Tests for checker tools.
|
||||
|
||||
The Checker agent does not define its own specific exposed tools for testing.
|
||||
It primarily reuses common tools (e.g. from scratchpad) and its behavior is
|
||||
tested at the agent level. Therefore, there are no specific checker tools to
|
||||
test here.
|
||||
"""
|
||||
@@ -0,0 +1,95 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.data_engine.engine import DataEngine
|
||||
import pytest
|
||||
|
||||
INPUTS_DIR = Path(__file__).parent / "inputs"
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def inputs_dir():
|
||||
return INPUTS_DIR
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_adb_client():
|
||||
client = MagicMock()
|
||||
# Dummy outputs for commands and physical interactions that tools might use
|
||||
client.shell.return_value = "dummy adb output"
|
||||
# Basic UI hierarchy for mobile read_hierarchy tool
|
||||
client.get_hierarchy.return_value = (
|
||||
"<?xml version='1.0' encoding='UTF-8'?><hierarchy><node index='0'"
|
||||
" text='dummy'/></hierarchy>"
|
||||
)
|
||||
return client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def artemis_context(inputs_dir, mock_adb_client):
|
||||
"""Provides a ArtemisContext with loaded database and actual side-effects,
|
||||
|
||||
except the ADB connection is customized to read from fixtures instead of a
|
||||
real device.
|
||||
"""
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
|
||||
ctx.execution_setup = MagicMock()
|
||||
ctx.execution_setup.traces_path = str(inputs_dir)
|
||||
|
||||
# Initialize real DataEngine if the fixture database exists
|
||||
db_path = inputs_dir / "data_engine.db"
|
||||
if db_path.exists():
|
||||
engine = DataEngine(ctx)
|
||||
engine.start_session(goal="Test goal")
|
||||
import uuid
|
||||
|
||||
engine.current_step_id = uuid.uuid4()
|
||||
ctx.data_engine = engine
|
||||
else:
|
||||
ctx.data_engine = None
|
||||
|
||||
ctx.ui_adb_client = mock_adb_client
|
||||
|
||||
# Fake device representation
|
||||
ctx.device = MagicMock()
|
||||
ctx.device.device_width = 1080
|
||||
ctx.device.device_height = 2400
|
||||
|
||||
# Trace output directory setup to point inside inputs or a dedicated test folder
|
||||
trace_out = inputs_dir / "trace_output"
|
||||
trace_out.mkdir(exist_ok=True)
|
||||
ctx.trace_dir = trace_out
|
||||
|
||||
ctx.agent_config = MagicMock()
|
||||
ctx.agent_config.blacklisted_tools = {}
|
||||
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_state(inputs_dir):
|
||||
"""Provides a dummy graph State with realistic artifact paths for tools that need state injection."""
|
||||
from artemis.graph.state import State
|
||||
|
||||
state = MagicMock(spec=State)
|
||||
state.latest_screenshot = str(inputs_dir / "screenshot.jpg")
|
||||
state.video_path = str(inputs_dir / "recording.mp4")
|
||||
state.indexed_points = []
|
||||
state.indexed_elements = []
|
||||
return state
|
||||
@@ -0,0 +1,128 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from artemis.tools.diagnoser_submit_answer_tool import get_submit_answer_tool
|
||||
|
||||
|
||||
def test_diagnoser_submit_answer_tool(artemis_context):
|
||||
tool = get_submit_answer_tool(artemis_context)
|
||||
|
||||
# Execute the tool with some typical payload
|
||||
result = tool.invoke(
|
||||
{
|
||||
"analysis": "The problem is caused by a network timeout.",
|
||||
"actionable_steps": ["Check network connection", "Restart the server"],
|
||||
}
|
||||
)
|
||||
|
||||
# Assert that the tool executes without raising errors and produces expected output structure
|
||||
assert result == "Answer submitted successfully."
|
||||
assert "error" not in result.lower()
|
||||
|
||||
|
||||
def test_submit_answer_tool_subclass_and_registry():
|
||||
"""Verify SubmitAnswerTool is an ArtemisTool subclass and registered in ToolRegistry."""
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.diagnoser_submit_answer_tool import (
|
||||
DiagnoserSubmitAnswerTool,
|
||||
SubmitAnswer,
|
||||
SubmitAnswerArgs,
|
||||
SubmitAnswerTool,
|
||||
submit_answer,
|
||||
)
|
||||
|
||||
assert issubclass(SubmitAnswerTool, ArtemisTool)
|
||||
assert issubclass(SubmitAnswer, ArtemisTool)
|
||||
assert issubclass(DiagnoserSubmitAnswerTool, ArtemisTool)
|
||||
assert isinstance(submit_answer, ArtemisTool)
|
||||
assert isinstance(submit_answer, SubmitAnswerTool)
|
||||
|
||||
assert submit_answer.name == "submit_answer"
|
||||
assert submit_answer.category == "custom"
|
||||
assert submit_answer.args_schema == SubmitAnswerArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("submit_answer")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, SubmitAnswerTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = submit_answer.to_genai_declaration()
|
||||
assert declaration.name == "submit_answer"
|
||||
assert "analysis" in declaration.parameters.properties
|
||||
assert "actionable_steps" in declaration.parameters.properties
|
||||
|
||||
|
||||
def test_ask_diagnoser_tool_subclass_and_registry():
|
||||
"""Verify AskDiagnoserTool is an ArtemisTool subclass and registered in ToolRegistry."""
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.diagnostic_tool import (
|
||||
AskDiagnoser,
|
||||
AskDiagnoserArgs,
|
||||
AskDiagnoserTool,
|
||||
DiagnosticTool,
|
||||
ask_diagnoser,
|
||||
)
|
||||
|
||||
assert issubclass(AskDiagnoserTool, ArtemisTool)
|
||||
assert issubclass(AskDiagnoser, ArtemisTool)
|
||||
assert issubclass(DiagnosticTool, ArtemisTool)
|
||||
assert isinstance(ask_diagnoser, ArtemisTool)
|
||||
assert isinstance(ask_diagnoser, AskDiagnoserTool)
|
||||
|
||||
assert ask_diagnoser.name == "ask_diagnoser"
|
||||
assert ask_diagnoser.category == "custom"
|
||||
assert ask_diagnoser.args_schema == AskDiagnoserArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("ask_diagnoser")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, AskDiagnoserTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = ask_diagnoser.to_genai_declaration()
|
||||
assert declaration.name == "ask_diagnoser"
|
||||
assert "query" in declaration.parameters.properties
|
||||
|
||||
|
||||
def test_analyze_logs_tool_subclass_and_registry():
|
||||
"""Verify AnalyzeLogsTool is an ArtemisTool subclass and registered in ToolRegistry."""
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.log_tool import (
|
||||
AnalyzeLogs,
|
||||
AnalyzeLogsArgs,
|
||||
AnalyzeLogsTool,
|
||||
LogTool,
|
||||
analyze_logs,
|
||||
)
|
||||
|
||||
assert issubclass(AnalyzeLogsTool, ArtemisTool)
|
||||
assert issubclass(AnalyzeLogs, ArtemisTool)
|
||||
assert issubclass(LogTool, ArtemisTool)
|
||||
assert isinstance(analyze_logs, ArtemisTool)
|
||||
assert isinstance(analyze_logs, AnalyzeLogsTool)
|
||||
|
||||
assert analyze_logs.name == "analyze_logs"
|
||||
assert analyze_logs.category == "custom"
|
||||
assert analyze_logs.args_schema == AnalyzeLogsArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("analyze_logs")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, AnalyzeLogsTool)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = analyze_logs.to_genai_declaration()
|
||||
assert declaration.name == "analyze_logs"
|
||||
assert "specific_query" in declaration.parameters.properties
|
||||
@@ -0,0 +1,83 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from artemis.agents.explorer.explorer import Explorer
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def explorer_instance(artemis_context, mock_state):
|
||||
explorer = Explorer(artemis_context)
|
||||
explorer.image_name = "test_img"
|
||||
explorer.screenshot_path = mock_state.latest_screenshot
|
||||
|
||||
# Initialize the image pool with img_0 required by processing tools
|
||||
explorer.image_pool = {
|
||||
"img_0": {
|
||||
"path": mock_state.latest_screenshot,
|
||||
"transform": {
|
||||
"offset_x": 0.0,
|
||||
"offset_y": 0.0,
|
||||
"scale_x": 1.0,
|
||||
"scale_y": 1.0,
|
||||
},
|
||||
}
|
||||
}
|
||||
return explorer
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exec_detect_objects(explorer_instance):
|
||||
result = await explorer_instance.exec_detect_objects(
|
||||
["button", "text"], target_image_id="img_0"
|
||||
)
|
||||
assert isinstance(result, dict)
|
||||
assert "text" in result
|
||||
assert "image_path" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exec_ask_perception_tool(explorer_instance):
|
||||
result = await explorer_instance.exec_ask_perception_tool("login", 500, 500, ["button"])
|
||||
assert isinstance(result, dict)
|
||||
assert "text" in result
|
||||
assert "image_paths" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exec_get_ocr_list(explorer_instance):
|
||||
result = await explorer_instance.exec_get_ocr_list()
|
||||
assert isinstance(result, dict)
|
||||
assert "text" in result
|
||||
assert "image_path" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exec_inspect_region(explorer_instance):
|
||||
# Depending on cv2 logic, it might return an error string if coordinates are out of bounds or image is missing
|
||||
# But it shouldn't raise unhandled exceptions
|
||||
result = await explorer_instance.exec_inspect_region(100, 100, 400, 400, 2.0)
|
||||
assert isinstance(result, dict)
|
||||
assert "text" in result
|
||||
assert "image_path" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exec_ask_image_processor(explorer_instance):
|
||||
result = await explorer_instance.exec_ask_image_processor(
|
||||
"Invert colors", target_image_id="img_0"
|
||||
)
|
||||
assert isinstance(result, dict)
|
||||
assert "text" in result
|
||||
assert "image_paths" in result
|
||||
@@ -0,0 +1,101 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import json
|
||||
from artemis.agents.history_analyzer.history_analyzer import HistoryAnalyzer
|
||||
from artemis.config import parse_llm_config
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exec_get_step_details(artemis_context):
|
||||
"""Test the exec_get_step_details method with manually configured steps."""
|
||||
analyzer = HistoryAnalyzer(artemis_context)
|
||||
analyzer.history_steps = [
|
||||
{
|
||||
"step_number": 0,
|
||||
"relative_time": 0.0,
|
||||
"summary": "Initial step",
|
||||
"action_taken": "none",
|
||||
"operator_raw_thinking": "Starting up",
|
||||
"last_execution_result": "Success",
|
||||
},
|
||||
{
|
||||
"step_number": 1,
|
||||
"relative_time": 5.0,
|
||||
"summary": "Second step",
|
||||
"action_taken": "click",
|
||||
"operator_raw_thinking": "Clicking button",
|
||||
"last_execution_result": "Success",
|
||||
},
|
||||
]
|
||||
|
||||
first_step_num = 0
|
||||
result = analyzer.exec_get_step_details(first_step_num, first_step_num)
|
||||
|
||||
assert "Error:" not in result
|
||||
assert "No steps found" not in result
|
||||
|
||||
parsed = json.loads(result)
|
||||
assert isinstance(parsed, list)
|
||||
assert len(parsed) == 1
|
||||
assert parsed[0]["step_number"] == first_step_num
|
||||
assert "summary" in parsed[0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_step_details_tool_with_fixture(artemis_context):
|
||||
"""Test _get_step_details_tool against real steps loaded from the inputs/data_engine.db fixture."""
|
||||
if not artemis_context.data_engine:
|
||||
pytest.skip("DataEngine fixture not initialized")
|
||||
|
||||
artemis_context.data_engine.current_session_id = "1a7fc344-ea69-4b1a-b066-d8a667502b8c"
|
||||
steps = artemis_context.data_engine.get_agent_friendly_steps()
|
||||
assert len(steps) > 0
|
||||
|
||||
analyzer = HistoryAnalyzer(artemis_context)
|
||||
tool = analyzer._get_step_details_tool(steps)
|
||||
|
||||
result = tool.invoke({"start_step": 1, "end_step": 3})
|
||||
assert isinstance(result, str)
|
||||
assert "Error:" not in result
|
||||
assert "No steps found" not in result
|
||||
|
||||
parsed = json.loads(result)
|
||||
assert isinstance(parsed, list)
|
||||
assert len(parsed) == 3
|
||||
assert parsed[0]["step_number"] == 1
|
||||
assert "YouTube" in parsed[0]["summary"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_history_analyzer_blackbox_run(artemis_context):
|
||||
"""End-to-end blackbox test of HistoryAnalyzer.run(query) using real steps from inputs/data_engine.db and real LLM calls."""
|
||||
if not artemis_context.data_engine:
|
||||
pytest.skip("DataEngine fixture not initialized")
|
||||
|
||||
artemis_context.llm_config = parse_llm_config()
|
||||
artemis_context.data_engine.current_session_id = "1a7fc344-ea69-4b1a-b066-d8a667502b8c"
|
||||
|
||||
analyzer = HistoryAnalyzer(artemis_context)
|
||||
query = (
|
||||
"What application did the operator open and what query did they search"
|
||||
" for? Briefly summarize the session steps."
|
||||
)
|
||||
result = await analyzer.run(query)
|
||||
|
||||
assert isinstance(result, str)
|
||||
assert len(result) > 0
|
||||
assert "Error:" not in result
|
||||
assert "No history recorded" not in result
|
||||
@@ -0,0 +1,19 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""This file is intentionally empty.
|
||||
|
||||
The hopper agent does not expose any tools directly. It is invoked
|
||||
programmatically by other tools (e.g., launch_app_tool) as an LLM utility.
|
||||
"""
|
||||
@@ -0,0 +1,43 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from artemis.tools.image_processor_tool import get_ask_image_processor_tool
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ask_image_processor_tool(artemis_context, mock_state):
|
||||
"""Tests the ask_image_processor tool as a blackbox.
|
||||
|
||||
Allows actual side effects and full lifecycle execution (including real LLM
|
||||
API calls). Uses shared pytest fixtures for realistic context.
|
||||
"""
|
||||
tool = get_ask_image_processor_tool(artemis_context)
|
||||
|
||||
# Passing a simple visual reasoning instruction.
|
||||
# The tool will use target_image_id="img_0" corresponding to the artifact in mock_state.
|
||||
result = await tool.ainvoke(
|
||||
{
|
||||
"instruction": "Identify the primary colors present in this image.",
|
||||
"target_image_id": "img_0",
|
||||
"state": mock_state,
|
||||
}
|
||||
)
|
||||
|
||||
# Assert execution without errors and produces expected output structure
|
||||
assert result is not None
|
||||
assert isinstance(result, dict)
|
||||
assert "outputs" in result
|
||||
assert "summary" in result
|
||||
assert isinstance(result["outputs"], list)
|
||||
Binary file not shown.
@@ -0,0 +1,61 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from artemis.agents.log_analyzer.log_analyzer import LogAnalyzerNode
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_spawn_log_reader(artemis_context, mock_state):
|
||||
"""Test the spawn_log_reader tool exposed by LogAnalyzerNode."""
|
||||
from artemis.sdk.utils import load_llm_config_override
|
||||
from pathlib import Path
|
||||
|
||||
node = LogAnalyzerNode(ctx=artemis_context)
|
||||
artemis_context.llm_config = load_llm_config_override(Path("llm-config.json"))
|
||||
if artemis_context.data_engine:
|
||||
artemis_context.data_engine.start_session("test_session")
|
||||
tool = node._get_spawn_log_reader_tool()
|
||||
|
||||
# Let the tool run its lifecycle. It will use the real LLM APIs.
|
||||
# Note: State injection is normally handled by LangGraph/tool invocation,
|
||||
# but since we are testing the coroutine directly, we can pass it manually.
|
||||
result = await tool.coroutine(
|
||||
specific_query=("Find any errors related to network timeout in the logs."),
|
||||
state=mock_state,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert isinstance(result, (str, list))
|
||||
assert len(result) > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_log_analyzer_run(artemis_context, mock_state):
|
||||
"""Test LogAnalyzerNode.run end-to-end execution with grounding configuration."""
|
||||
from artemis.config.llm import parse_llm_config
|
||||
|
||||
node = LogAnalyzerNode(ctx=artemis_context)
|
||||
artemis_context.llm_config = parse_llm_config()
|
||||
if artemis_context.data_engine:
|
||||
artemis_context.data_engine.start_session("test_log_analyzer_session")
|
||||
|
||||
result = await node.run(
|
||||
prompt="Analyze recent error logs for potential crash or exception tags.",
|
||||
state=mock_state,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert isinstance(result, (str, list))
|
||||
assert len(result) > 0
|
||||
@@ -0,0 +1,58 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Tests for object_detector universal tools."""
|
||||
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.object_detection_tool import (
|
||||
ObjectDetection,
|
||||
ObjectDetectionArgs,
|
||||
ObjectDetectionTool,
|
||||
ObjectDetectorTool,
|
||||
OperatorObjectDetection,
|
||||
OperatorObjectDetectionArgs,
|
||||
OperatorObjectDetectionTool,
|
||||
OperatorObjectDetectorTool,
|
||||
object_detection,
|
||||
operator_object_detection,
|
||||
)
|
||||
|
||||
|
||||
def test_object_detector_tool_subclass_and_registry():
|
||||
"""Verify ObjectDetectionTool is an ArtemisTool subclass and registered in ToolRegistry."""
|
||||
assert issubclass(ObjectDetectionTool, ArtemisTool)
|
||||
assert issubclass(ObjectDetection, ArtemisTool)
|
||||
assert issubclass(ObjectDetectorTool, ArtemisTool)
|
||||
assert issubclass(OperatorObjectDetectionTool, ArtemisTool)
|
||||
assert issubclass(OperatorObjectDetectionTool, ObjectDetectionTool)
|
||||
assert issubclass(OperatorObjectDetection, ArtemisTool)
|
||||
assert issubclass(OperatorObjectDetectorTool, ArtemisTool)
|
||||
|
||||
assert isinstance(object_detection, ArtemisTool)
|
||||
assert isinstance(object_detection, ObjectDetectionTool)
|
||||
assert isinstance(operator_object_detection, ArtemisTool)
|
||||
assert isinstance(operator_object_detection, OperatorObjectDetectionTool)
|
||||
|
||||
assert object_detection.name == "object_detection"
|
||||
assert object_detection.category == "perception"
|
||||
assert object_detection.args_schema == ObjectDetectionArgs
|
||||
|
||||
assert operator_object_detection.name == "object_detection"
|
||||
assert operator_object_detection.category == "perception"
|
||||
assert operator_object_detection.args_schema == OperatorObjectDetectionArgs
|
||||
|
||||
# Registry lookup
|
||||
reg_tool = ToolRegistry.get("object_detection")
|
||||
assert reg_tool is not None
|
||||
assert isinstance(reg_tool, ObjectDetectionTool)
|
||||
@@ -0,0 +1,68 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from langchain_core.tools import BaseTool
|
||||
from artemis.agents.operator.operator import OperatorNode
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_operator_tools(artemis_context, mock_state):
|
||||
"""Test all tools exposed by the OperatorNode."""
|
||||
node = OperatorNode(ctx=artemis_context)
|
||||
|
||||
click_tool = node._get_click_tool()
|
||||
assert isinstance(click_tool, BaseTool)
|
||||
res = click_tool.invoke({"target": 1, "times": 1, "delay_ms": 100})
|
||||
assert res == "Action Recorded"
|
||||
|
||||
input_text_tool = node._get_input_text_tool()
|
||||
assert isinstance(input_text_tool, BaseTool)
|
||||
res = input_text_tool.invoke({"text": "hello", "target": 1, "clear_exist": True})
|
||||
assert res == "Action Recorded"
|
||||
|
||||
swipe_tool = node._get_swipe_tool()
|
||||
assert isinstance(swipe_tool, BaseTool)
|
||||
res = swipe_tool.invoke({"gesture": "up"})
|
||||
assert res == "Action Recorded"
|
||||
|
||||
press_key_tool = node._get_press_key_tool()
|
||||
assert isinstance(press_key_tool, BaseTool)
|
||||
res = press_key_tool.invoke({"key": "ENTER"})
|
||||
assert res == "Action Recorded"
|
||||
|
||||
manage_app_tool = node._get_manage_app_tool()
|
||||
assert isinstance(manage_app_tool, BaseTool)
|
||||
res = manage_app_tool.invoke({"action": "launch", "app_name": "Settings"})
|
||||
assert res == "Action Recorded"
|
||||
|
||||
wait_for_delay_tool = node._get_wait_for_delay_tool()
|
||||
assert isinstance(wait_for_delay_tool, BaseTool)
|
||||
res = wait_for_delay_tool.invoke({"time_in_ms": 1000})
|
||||
assert res == "Action Recorded"
|
||||
|
||||
long_press_tool = node._get_long_press_tool()
|
||||
assert isinstance(long_press_tool, BaseTool)
|
||||
res = long_press_tool.invoke({"target": 1, "duration": 1000})
|
||||
assert res == "Action Recorded"
|
||||
|
||||
wait_for_text_tool = node._get_wait_for_text_tool()
|
||||
assert isinstance(wait_for_text_tool, BaseTool)
|
||||
res = wait_for_text_tool.invoke({"text": "Success", "state": "visible", "timeout_ms": 5000})
|
||||
assert res == "Action Recorded"
|
||||
|
||||
reply_tool = node._get_reply_to_checker_tool()
|
||||
assert isinstance(reply_tool, BaseTool)
|
||||
res = reply_tool.invoke({"reasoning": "This is a test reply."})
|
||||
assert isinstance(res, str)
|
||||
@@ -0,0 +1,61 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from artemis.agents.outputter.tools import (
|
||||
get_search_history_tool,
|
||||
get_step_details_tool,
|
||||
get_step_screenshot_tool,
|
||||
)
|
||||
from artemis.context import ArtemisContext
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def history_steps(artemis_context: ArtemisContext):
|
||||
if artemis_context.data_engine:
|
||||
return artemis_context.data_engine.get_agent_friendly_steps()
|
||||
return []
|
||||
|
||||
|
||||
def test_get_step_details_tool(history_steps):
|
||||
"""Test get_step_details_tool to ensure it runs without errors."""
|
||||
tool = get_step_details_tool(history_steps)
|
||||
result = tool.invoke({"start_step": 1, "end_step": 5})
|
||||
|
||||
assert isinstance(result, str)
|
||||
assert len(result) > 0
|
||||
|
||||
|
||||
def test_get_step_screenshot_tool(artemis_context: ArtemisContext, history_steps):
|
||||
"""Test get_step_screenshot_tool to ensure it handles requests gracefully."""
|
||||
tool = get_step_screenshot_tool(artemis_context, history_steps)
|
||||
|
||||
result = tool.invoke({"step_number": 1})
|
||||
|
||||
if isinstance(result, list):
|
||||
assert len(result) == 2
|
||||
assert result[0]["type"] == "text"
|
||||
assert result[1]["type"] == "image_url"
|
||||
else:
|
||||
assert isinstance(result, str)
|
||||
assert len(result) > 0
|
||||
|
||||
|
||||
def test_get_search_history_tool(artemis_context: ArtemisContext):
|
||||
"""Test get_search_history_tool to ensure it can search through history."""
|
||||
tool = get_search_history_tool(artemis_context)
|
||||
result = tool.invoke({"query": "test"})
|
||||
|
||||
assert isinstance(result, str)
|
||||
assert len(result) > 0
|
||||
@@ -0,0 +1,17 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
# The planner agent does not have exposed tools of its own to test here.
|
||||
# It uses shared tools from artemis.tools.scratchpad, which are tested in their respective modules.
|
||||
@@ -0,0 +1,16 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
# The summarizer agent does not have exposed tools of its own to test here.
|
||||
@@ -0,0 +1,75 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import base64
|
||||
from pathlib import Path
|
||||
|
||||
from artemis.agents.validator.failure_analyzer import ValidationErrorCategory
|
||||
from artemis.agents.validator.validator import ValidatorNode
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_safety_net_validation(artemis_context, mock_state):
|
||||
"""Test the safety_net_validation tool."""
|
||||
node = ValidatorNode(artemis_context)
|
||||
|
||||
action_item = {
|
||||
"action": "tap",
|
||||
"coordinates": [100, 200],
|
||||
"target_text": "Submit",
|
||||
}
|
||||
|
||||
# We pass session=None so it relies on fallback or fails gracefully
|
||||
# We assert it runs without unhandled exceptions.
|
||||
try:
|
||||
passed, category, reason = await node._validate_action_precondition(
|
||||
session=None, action_item=action_item, state=mock_state
|
||||
)
|
||||
except Exception as e:
|
||||
pytest.fail(f"Tool raised an exception: {e}")
|
||||
|
||||
assert isinstance(passed, bool)
|
||||
assert isinstance(category, ValidationErrorCategory)
|
||||
assert isinstance(reason, str)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_safety_net_pixel_validation(artemis_context, mock_state):
|
||||
"""Test the safety_net_pixel_validation tool."""
|
||||
node = ValidatorNode(artemis_context)
|
||||
|
||||
action_item = {"action": "tap", "coordinates": [100, 200]}
|
||||
|
||||
# Read the real screenshot from the mock_state to provide as pre_screenshot_b64
|
||||
screenshot_path = mock_state.latest_screenshot
|
||||
pre_screenshot_b64 = "dummy_base64"
|
||||
if screenshot_path and Path(screenshot_path).exists():
|
||||
with open(screenshot_path, "rb") as f:
|
||||
pre_screenshot_b64 = base64.b64encode(f.read()).decode("utf-8")
|
||||
|
||||
try:
|
||||
passed, category, reason = await node._validate_action_precondition_pixel(
|
||||
session=None,
|
||||
action_item=action_item,
|
||||
pre_screenshot_b64=pre_screenshot_b64,
|
||||
original_coords=[100, 200],
|
||||
state=mock_state,
|
||||
)
|
||||
except Exception as e:
|
||||
pytest.fail(f"Tool raised an exception: {e}")
|
||||
|
||||
assert isinstance(passed, bool)
|
||||
assert isinstance(category, ValidationErrorCategory)
|
||||
assert isinstance(reason, str)
|
||||
@@ -0,0 +1,103 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from artemis.agents.video_analyzer.video_analyzer import VideoAnalyzer
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_analyzer_tools(artemis_context, mock_state, inputs_dir):
|
||||
artemis_context.device.mobile_platform = "android"
|
||||
artemis_context.adb_client = artemis_context.ui_adb_client
|
||||
|
||||
agent = VideoAnalyzer(ctx=artemis_context)
|
||||
|
||||
agent.local_files_to_cleanup = set()
|
||||
agent.local_dirs_to_cleanup = set()
|
||||
agent.cloud_files_to_cleanup = set()
|
||||
agent.blackboard_entries = []
|
||||
agent.sub_system_prompt = "You are a sub agent"
|
||||
agent.audio_system_prompt = "You are an audio sub agent"
|
||||
agent.model_name = "gemini-3.6-flash"
|
||||
|
||||
# We must provide a valid FunctionDeclaration for the tool otherwise Pydantic will complain
|
||||
from google.genai import types
|
||||
|
||||
agent.submit_answer_tool = types.FunctionDeclaration(
|
||||
name="submit_answer",
|
||||
description=("Submit the final findings for the requested segment analysis."),
|
||||
parameters=types.Schema(
|
||||
type=types.Type.OBJECT,
|
||||
properties={
|
||||
"summary": types.Schema(type=types.Type.STRING, description="summary"),
|
||||
"analysis": types.Schema(type=types.Type.STRING, description="analysis"),
|
||||
"timeline_events": types.Schema(
|
||||
type=types.Type.ARRAY,
|
||||
items=types.Schema(
|
||||
type=types.Type.OBJECT,
|
||||
properties={
|
||||
"start_time": types.Schema(type=types.Type.NUMBER),
|
||||
"end_time": types.Schema(type=types.Type.NUMBER),
|
||||
"transcription": types.Schema(type=types.Type.STRING),
|
||||
"confidence_score": types.Schema(type=types.Type.NUMBER),
|
||||
"verification_timestamp_secs": types.Schema(type=types.Type.NUMBER),
|
||||
},
|
||||
),
|
||||
),
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
import google.genai as genai
|
||||
from artemis.config import settings
|
||||
|
||||
agent.client = genai.Client(
|
||||
api_key=settings.GOOGLE_API_KEY.get_secret_value() if settings.GOOGLE_API_KEY else None
|
||||
)
|
||||
artemis_context._genai_client = agent.client
|
||||
|
||||
mock_session = MagicMock()
|
||||
mock_session.local_video_path = Path(inputs_dir) / "recording.mp4"
|
||||
mock_session.start_time = 0.0
|
||||
mock_session.data_engine_start_time = None
|
||||
mock_session.process = MagicMock()
|
||||
mock_session.process.returncode = None
|
||||
mock_session.android_video_segments = []
|
||||
mock_session.android_segment_index = 0
|
||||
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
sample_video = Path(inputs_dir) / "recording.mp4"
|
||||
mock_driver = MagicMock()
|
||||
mock_driver.stop_video_recording = AsyncMock(return_value=sample_video)
|
||||
artemis_context._active_driver = mock_driver
|
||||
|
||||
with patch(
|
||||
"artemis.controllers.android_controller.get_active_session",
|
||||
return_value=mock_session,
|
||||
):
|
||||
result = await agent.exec_extract_segment_metadata(start_time=0.0, end_time=1.0)
|
||||
assert isinstance(result, str)
|
||||
assert len(result) > 0
|
||||
|
||||
# Test sub-agent with extremely short audio query to not waste resources and time
|
||||
audio_result = await agent.exec_analyze_audio_only(
|
||||
start_time=0.0,
|
||||
end_time=1.0,
|
||||
specific_query="Reply with a short summary only.",
|
||||
)
|
||||
assert isinstance(audio_result, str)
|
||||
Reference in new issue
Block a user