test: add comprehensive unit, integration, and e2e test suites

Introduce complete test coverage for the ARTEMIS platform across multiple layers:
Unit tests: Add tests for agent nodes (Planner, Operator, Validator, Diagnoser, FlashRunner, Explorer), tools, utils (OCR/XML fusion, UI hierarchy), and data engine.
Integration tests: Add live tool verification, reactive runner, IME, and standalone workflow tests.
E2E tests: Add end-to-end scenario tests for media playback and failure recovery flows.
Fixtures & mocks: Add mock inputs (audio, video, screenshot) and test utilities.
This commit is contained in:
chiwanko11 committed 2026-08-13 12:18:32 -07:00
1 parent c1803fcb86
commit 5d2283deaa
115 files changed
+22813

No files matched your search

+21
View File
@@ -0,0 +1,21 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for checker tools.
The Checker agent does not define its own specific exposed tools for testing.
It primarily reuses common tools (e.g. from scratchpad) and its behavior is
tested at the agent level. Therefore, there are no specific checker tools to
test here.
"""
+95
View File
@@ -0,0 +1,95 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pathlib import Path
from unittest.mock import MagicMock
from artemis.context import ArtemisContext
from artemis.data_engine.engine import DataEngine
import pytest
INPUTS_DIR = Path(__file__).parent / "inputs"
@pytest.fixture(scope="session")
def inputs_dir():
return INPUTS_DIR
@pytest.fixture
def mock_adb_client():
client = MagicMock()
# Dummy outputs for commands and physical interactions that tools might use
client.shell.return_value = "dummy adb output"
# Basic UI hierarchy for mobile read_hierarchy tool
client.get_hierarchy.return_value = (
"<?xml version='1.0' encoding='UTF-8'?><hierarchy><node index='0'"
" text='dummy'/></hierarchy>"
)
return client
@pytest.fixture
def artemis_context(inputs_dir, mock_adb_client):
"""Provides a ArtemisContext with loaded database and actual side-effects,
except the ADB connection is customized to read from fixtures instead of a
real device.
"""
ctx = MagicMock(spec=ArtemisContext)
ctx.execution_setup = MagicMock()
ctx.execution_setup.traces_path = str(inputs_dir)
# Initialize real DataEngine if the fixture database exists
db_path = inputs_dir / "data_engine.db"
if db_path.exists():
engine = DataEngine(ctx)
engine.start_session(goal="Test goal")
import uuid
engine.current_step_id = uuid.uuid4()
ctx.data_engine = engine
else:
ctx.data_engine = None
ctx.ui_adb_client = mock_adb_client
# Fake device representation
ctx.device = MagicMock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
# Trace output directory setup to point inside inputs or a dedicated test folder
trace_out = inputs_dir / "trace_output"
trace_out.mkdir(exist_ok=True)
ctx.trace_dir = trace_out
ctx.agent_config = MagicMock()
ctx.agent_config.blacklisted_tools = {}
return ctx
@pytest.fixture
def mock_state(inputs_dir):
"""Provides a dummy graph State with realistic artifact paths for tools that need state injection."""
from artemis.graph.state import State
state = MagicMock(spec=State)
state.latest_screenshot = str(inputs_dir / "screenshot.jpg")
state.video_path = str(inputs_dir / "recording.mp4")
state.indexed_points = []
state.indexed_elements = []
return state
@@ -0,0 +1,128 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from artemis.tools.diagnoser_submit_answer_tool import get_submit_answer_tool
def test_diagnoser_submit_answer_tool(artemis_context):
tool = get_submit_answer_tool(artemis_context)
# Execute the tool with some typical payload
result = tool.invoke(
{
"analysis": "The problem is caused by a network timeout.",
"actionable_steps": ["Check network connection", "Restart the server"],
}
)
# Assert that the tool executes without raising errors and produces expected output structure
assert result == "Answer submitted successfully."
assert "error" not in result.lower()
def test_submit_answer_tool_subclass_and_registry():
"""Verify SubmitAnswerTool is an ArtemisTool subclass and registered in ToolRegistry."""
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.diagnoser_submit_answer_tool import (
DiagnoserSubmitAnswerTool,
SubmitAnswer,
SubmitAnswerArgs,
SubmitAnswerTool,
submit_answer,
)
assert issubclass(SubmitAnswerTool, ArtemisTool)
assert issubclass(SubmitAnswer, ArtemisTool)
assert issubclass(DiagnoserSubmitAnswerTool, ArtemisTool)
assert isinstance(submit_answer, ArtemisTool)
assert isinstance(submit_answer, SubmitAnswerTool)
assert submit_answer.name == "submit_answer"
assert submit_answer.category == "custom"
assert submit_answer.args_schema == SubmitAnswerArgs
# Registry lookup
reg_tool = ToolRegistry.get("submit_answer")
assert reg_tool is not None
assert isinstance(reg_tool, SubmitAnswerTool)
# GenAI FunctionDeclaration export
declaration = submit_answer.to_genai_declaration()
assert declaration.name == "submit_answer"
assert "analysis" in declaration.parameters.properties
assert "actionable_steps" in declaration.parameters.properties
def test_ask_diagnoser_tool_subclass_and_registry():
"""Verify AskDiagnoserTool is an ArtemisTool subclass and registered in ToolRegistry."""
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.diagnostic_tool import (
AskDiagnoser,
AskDiagnoserArgs,
AskDiagnoserTool,
DiagnosticTool,
ask_diagnoser,
)
assert issubclass(AskDiagnoserTool, ArtemisTool)
assert issubclass(AskDiagnoser, ArtemisTool)
assert issubclass(DiagnosticTool, ArtemisTool)
assert isinstance(ask_diagnoser, ArtemisTool)
assert isinstance(ask_diagnoser, AskDiagnoserTool)
assert ask_diagnoser.name == "ask_diagnoser"
assert ask_diagnoser.category == "custom"
assert ask_diagnoser.args_schema == AskDiagnoserArgs
# Registry lookup
reg_tool = ToolRegistry.get("ask_diagnoser")
assert reg_tool is not None
assert isinstance(reg_tool, AskDiagnoserTool)
# GenAI FunctionDeclaration export
declaration = ask_diagnoser.to_genai_declaration()
assert declaration.name == "ask_diagnoser"
assert "query" in declaration.parameters.properties
def test_analyze_logs_tool_subclass_and_registry():
"""Verify AnalyzeLogsTool is an ArtemisTool subclass and registered in ToolRegistry."""
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.log_tool import (
AnalyzeLogs,
AnalyzeLogsArgs,
AnalyzeLogsTool,
LogTool,
analyze_logs,
)
assert issubclass(AnalyzeLogsTool, ArtemisTool)
assert issubclass(AnalyzeLogs, ArtemisTool)
assert issubclass(LogTool, ArtemisTool)
assert isinstance(analyze_logs, ArtemisTool)
assert isinstance(analyze_logs, AnalyzeLogsTool)
assert analyze_logs.name == "analyze_logs"
assert analyze_logs.category == "custom"
assert analyze_logs.args_schema == AnalyzeLogsArgs
# Registry lookup
reg_tool = ToolRegistry.get("analyze_logs")
assert reg_tool is not None
assert isinstance(reg_tool, AnalyzeLogsTool)
# GenAI FunctionDeclaration export
declaration = analyze_logs.to_genai_declaration()
assert declaration.name == "analyze_logs"
assert "specific_query" in declaration.parameters.properties
@@ -0,0 +1,83 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from artemis.agents.explorer.explorer import Explorer
import pytest
@pytest.fixture
def explorer_instance(artemis_context, mock_state):
explorer = Explorer(artemis_context)
explorer.image_name = "test_img"
explorer.screenshot_path = mock_state.latest_screenshot
# Initialize the image pool with img_0 required by processing tools
explorer.image_pool = {
"img_0": {
"path": mock_state.latest_screenshot,
"transform": {
"offset_x": 0.0,
"offset_y": 0.0,
"scale_x": 1.0,
"scale_y": 1.0,
},
}
}
return explorer
@pytest.mark.asyncio
async def test_exec_detect_objects(explorer_instance):
result = await explorer_instance.exec_detect_objects(
["button", "text"], target_image_id="img_0"
)
assert isinstance(result, dict)
assert "text" in result
assert "image_path" in result
@pytest.mark.asyncio
async def test_exec_ask_perception_tool(explorer_instance):
result = await explorer_instance.exec_ask_perception_tool("login", 500, 500, ["button"])
assert isinstance(result, dict)
assert "text" in result
assert "image_paths" in result
@pytest.mark.asyncio
async def test_exec_get_ocr_list(explorer_instance):
result = await explorer_instance.exec_get_ocr_list()
assert isinstance(result, dict)
assert "text" in result
assert "image_path" in result
@pytest.mark.asyncio
async def test_exec_inspect_region(explorer_instance):
# Depending on cv2 logic, it might return an error string if coordinates are out of bounds or image is missing
# But it shouldn't raise unhandled exceptions
result = await explorer_instance.exec_inspect_region(100, 100, 400, 400, 2.0)
assert isinstance(result, dict)
assert "text" in result
assert "image_path" in result
@pytest.mark.asyncio
async def test_exec_ask_image_processor(explorer_instance):
result = await explorer_instance.exec_ask_image_processor(
"Invert colors", target_image_id="img_0"
)
assert isinstance(result, dict)
assert "text" in result
assert "image_paths" in result
@@ -0,0 +1,101 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from artemis.agents.history_analyzer.history_analyzer import HistoryAnalyzer
from artemis.config import parse_llm_config
import pytest
@pytest.mark.asyncio
async def test_exec_get_step_details(artemis_context):
"""Test the exec_get_step_details method with manually configured steps."""
analyzer = HistoryAnalyzer(artemis_context)
analyzer.history_steps = [
{
"step_number": 0,
"relative_time": 0.0,
"summary": "Initial step",
"action_taken": "none",
"operator_raw_thinking": "Starting up",
"last_execution_result": "Success",
},
{
"step_number": 1,
"relative_time": 5.0,
"summary": "Second step",
"action_taken": "click",
"operator_raw_thinking": "Clicking button",
"last_execution_result": "Success",
},
]
first_step_num = 0
result = analyzer.exec_get_step_details(first_step_num, first_step_num)
assert "Error:" not in result
assert "No steps found" not in result
parsed = json.loads(result)
assert isinstance(parsed, list)
assert len(parsed) == 1
assert parsed[0]["step_number"] == first_step_num
assert "summary" in parsed[0]
@pytest.mark.asyncio
async def test_get_step_details_tool_with_fixture(artemis_context):
"""Test _get_step_details_tool against real steps loaded from the inputs/data_engine.db fixture."""
if not artemis_context.data_engine:
pytest.skip("DataEngine fixture not initialized")
artemis_context.data_engine.current_session_id = "1a7fc344-ea69-4b1a-b066-d8a667502b8c"
steps = artemis_context.data_engine.get_agent_friendly_steps()
assert len(steps) > 0
analyzer = HistoryAnalyzer(artemis_context)
tool = analyzer._get_step_details_tool(steps)
result = tool.invoke({"start_step": 1, "end_step": 3})
assert isinstance(result, str)
assert "Error:" not in result
assert "No steps found" not in result
parsed = json.loads(result)
assert isinstance(parsed, list)
assert len(parsed) == 3
assert parsed[0]["step_number"] == 1
assert "YouTube" in parsed[0]["summary"]
@pytest.mark.asyncio
async def test_history_analyzer_blackbox_run(artemis_context):
"""End-to-end blackbox test of HistoryAnalyzer.run(query) using real steps from inputs/data_engine.db and real LLM calls."""
if not artemis_context.data_engine:
pytest.skip("DataEngine fixture not initialized")
artemis_context.llm_config = parse_llm_config()
artemis_context.data_engine.current_session_id = "1a7fc344-ea69-4b1a-b066-d8a667502b8c"
analyzer = HistoryAnalyzer(artemis_context)
query = (
"What application did the operator open and what query did they search"
" for? Briefly summarize the session steps."
)
result = await analyzer.run(query)
assert isinstance(result, str)
assert len(result) > 0
assert "Error:" not in result
assert "No history recorded" not in result
+19
View File
@@ -0,0 +1,19 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""This file is intentionally empty.
The hopper agent does not expose any tools directly. It is invoked
programmatically by other tools (e.g., launch_app_tool) as an LLM utility.
"""
@@ -0,0 +1,43 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from artemis.tools.image_processor_tool import get_ask_image_processor_tool
import pytest
@pytest.mark.asyncio
async def test_ask_image_processor_tool(artemis_context, mock_state):
"""Tests the ask_image_processor tool as a blackbox.
Allows actual side effects and full lifecycle execution (including real LLM
API calls). Uses shared pytest fixtures for realistic context.
"""
tool = get_ask_image_processor_tool(artemis_context)
# Passing a simple visual reasoning instruction.
# The tool will use target_image_id="img_0" corresponding to the artifact in mock_state.
result = await tool.ainvoke(
{
"instruction": "Identify the primary colors present in this image.",
"target_image_id": "img_0",
"state": mock_state,
}
)
# Assert execution without errors and produces expected output structure
assert result is not None
assert isinstance(result, dict)
assert "outputs" in result
assert "summary" in result
assert isinstance(result["outputs"], list)
Binary file not shown.
@@ -0,0 +1,61 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from artemis.agents.log_analyzer.log_analyzer import LogAnalyzerNode
import pytest
@pytest.mark.asyncio
async def test_spawn_log_reader(artemis_context, mock_state):
"""Test the spawn_log_reader tool exposed by LogAnalyzerNode."""
from artemis.sdk.utils import load_llm_config_override
from pathlib import Path
node = LogAnalyzerNode(ctx=artemis_context)
artemis_context.llm_config = load_llm_config_override(Path("llm-config.json"))
if artemis_context.data_engine:
artemis_context.data_engine.start_session("test_session")
tool = node._get_spawn_log_reader_tool()
# Let the tool run its lifecycle. It will use the real LLM APIs.
# Note: State injection is normally handled by LangGraph/tool invocation,
# but since we are testing the coroutine directly, we can pass it manually.
result = await tool.coroutine(
specific_query=("Find any errors related to network timeout in the logs."),
state=mock_state,
)
assert result is not None
assert isinstance(result, (str, list))
assert len(result) > 0
@pytest.mark.asyncio
async def test_log_analyzer_run(artemis_context, mock_state):
"""Test LogAnalyzerNode.run end-to-end execution with grounding configuration."""
from artemis.config.llm import parse_llm_config
node = LogAnalyzerNode(ctx=artemis_context)
artemis_context.llm_config = parse_llm_config()
if artemis_context.data_engine:
artemis_context.data_engine.start_session("test_log_analyzer_session")
result = await node.run(
prompt="Analyze recent error logs for potential crash or exception tags.",
state=mock_state,
)
assert result is not None
assert isinstance(result, (str, list))
assert len(result) > 0
@@ -0,0 +1,58 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for object_detector universal tools."""
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.object_detection_tool import (
ObjectDetection,
ObjectDetectionArgs,
ObjectDetectionTool,
ObjectDetectorTool,
OperatorObjectDetection,
OperatorObjectDetectionArgs,
OperatorObjectDetectionTool,
OperatorObjectDetectorTool,
object_detection,
operator_object_detection,
)
def test_object_detector_tool_subclass_and_registry():
"""Verify ObjectDetectionTool is an ArtemisTool subclass and registered in ToolRegistry."""
assert issubclass(ObjectDetectionTool, ArtemisTool)
assert issubclass(ObjectDetection, ArtemisTool)
assert issubclass(ObjectDetectorTool, ArtemisTool)
assert issubclass(OperatorObjectDetectionTool, ArtemisTool)
assert issubclass(OperatorObjectDetectionTool, ObjectDetectionTool)
assert issubclass(OperatorObjectDetection, ArtemisTool)
assert issubclass(OperatorObjectDetectorTool, ArtemisTool)
assert isinstance(object_detection, ArtemisTool)
assert isinstance(object_detection, ObjectDetectionTool)
assert isinstance(operator_object_detection, ArtemisTool)
assert isinstance(operator_object_detection, OperatorObjectDetectionTool)
assert object_detection.name == "object_detection"
assert object_detection.category == "perception"
assert object_detection.args_schema == ObjectDetectionArgs
assert operator_object_detection.name == "object_detection"
assert operator_object_detection.category == "perception"
assert operator_object_detection.args_schema == OperatorObjectDetectionArgs
# Registry lookup
reg_tool = ToolRegistry.get("object_detection")
assert reg_tool is not None
assert isinstance(reg_tool, ObjectDetectionTool)
@@ -0,0 +1,68 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from langchain_core.tools import BaseTool
from artemis.agents.operator.operator import OperatorNode
import pytest
@pytest.mark.asyncio
async def test_operator_tools(artemis_context, mock_state):
"""Test all tools exposed by the OperatorNode."""
node = OperatorNode(ctx=artemis_context)
click_tool = node._get_click_tool()
assert isinstance(click_tool, BaseTool)
res = click_tool.invoke({"target": 1, "times": 1, "delay_ms": 100})
assert res == "Action Recorded"
input_text_tool = node._get_input_text_tool()
assert isinstance(input_text_tool, BaseTool)
res = input_text_tool.invoke({"text": "hello", "target": 1, "clear_exist": True})
assert res == "Action Recorded"
swipe_tool = node._get_swipe_tool()
assert isinstance(swipe_tool, BaseTool)
res = swipe_tool.invoke({"gesture": "up"})
assert res == "Action Recorded"
press_key_tool = node._get_press_key_tool()
assert isinstance(press_key_tool, BaseTool)
res = press_key_tool.invoke({"key": "ENTER"})
assert res == "Action Recorded"
manage_app_tool = node._get_manage_app_tool()
assert isinstance(manage_app_tool, BaseTool)
res = manage_app_tool.invoke({"action": "launch", "app_name": "Settings"})
assert res == "Action Recorded"
wait_for_delay_tool = node._get_wait_for_delay_tool()
assert isinstance(wait_for_delay_tool, BaseTool)
res = wait_for_delay_tool.invoke({"time_in_ms": 1000})
assert res == "Action Recorded"
long_press_tool = node._get_long_press_tool()
assert isinstance(long_press_tool, BaseTool)
res = long_press_tool.invoke({"target": 1, "duration": 1000})
assert res == "Action Recorded"
wait_for_text_tool = node._get_wait_for_text_tool()
assert isinstance(wait_for_text_tool, BaseTool)
res = wait_for_text_tool.invoke({"text": "Success", "state": "visible", "timeout_ms": 5000})
assert res == "Action Recorded"
reply_tool = node._get_reply_to_checker_tool()
assert isinstance(reply_tool, BaseTool)
res = reply_tool.invoke({"reasoning": "This is a test reply."})
assert isinstance(res, str)
@@ -0,0 +1,61 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from artemis.agents.outputter.tools import (
get_search_history_tool,
get_step_details_tool,
get_step_screenshot_tool,
)
from artemis.context import ArtemisContext
import pytest
@pytest.fixture
def history_steps(artemis_context: ArtemisContext):
if artemis_context.data_engine:
return artemis_context.data_engine.get_agent_friendly_steps()
return []
def test_get_step_details_tool(history_steps):
"""Test get_step_details_tool to ensure it runs without errors."""
tool = get_step_details_tool(history_steps)
result = tool.invoke({"start_step": 1, "end_step": 5})
assert isinstance(result, str)
assert len(result) > 0
def test_get_step_screenshot_tool(artemis_context: ArtemisContext, history_steps):
"""Test get_step_screenshot_tool to ensure it handles requests gracefully."""
tool = get_step_screenshot_tool(artemis_context, history_steps)
result = tool.invoke({"step_number": 1})
if isinstance(result, list):
assert len(result) == 2
assert result[0]["type"] == "text"
assert result[1]["type"] == "image_url"
else:
assert isinstance(result, str)
assert len(result) > 0
def test_get_search_history_tool(artemis_context: ArtemisContext):
"""Test get_search_history_tool to ensure it can search through history."""
tool = get_search_history_tool(artemis_context)
result = tool.invoke({"query": "test"})
assert isinstance(result, str)
assert len(result) > 0
+17
View File
@@ -0,0 +1,17 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# The planner agent does not have exposed tools of its own to test here.
# It uses shared tools from artemis.tools.scratchpad, which are tested in their respective modules.
@@ -0,0 +1,16 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# The summarizer agent does not have exposed tools of its own to test here.
@@ -0,0 +1,75 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import base64
from pathlib import Path
from artemis.agents.validator.failure_analyzer import ValidationErrorCategory
from artemis.agents.validator.validator import ValidatorNode
import pytest
@pytest.mark.asyncio
async def test_safety_net_validation(artemis_context, mock_state):
"""Test the safety_net_validation tool."""
node = ValidatorNode(artemis_context)
action_item = {
"action": "tap",
"coordinates": [100, 200],
"target_text": "Submit",
}
# We pass session=None so it relies on fallback or fails gracefully
# We assert it runs without unhandled exceptions.
try:
passed, category, reason = await node._validate_action_precondition(
session=None, action_item=action_item, state=mock_state
)
except Exception as e:
pytest.fail(f"Tool raised an exception: {e}")
assert isinstance(passed, bool)
assert isinstance(category, ValidationErrorCategory)
assert isinstance(reason, str)
@pytest.mark.asyncio
async def test_safety_net_pixel_validation(artemis_context, mock_state):
"""Test the safety_net_pixel_validation tool."""
node = ValidatorNode(artemis_context)
action_item = {"action": "tap", "coordinates": [100, 200]}
# Read the real screenshot from the mock_state to provide as pre_screenshot_b64
screenshot_path = mock_state.latest_screenshot
pre_screenshot_b64 = "dummy_base64"
if screenshot_path and Path(screenshot_path).exists():
with open(screenshot_path, "rb") as f:
pre_screenshot_b64 = base64.b64encode(f.read()).decode("utf-8")
try:
passed, category, reason = await node._validate_action_precondition_pixel(
session=None,
action_item=action_item,
pre_screenshot_b64=pre_screenshot_b64,
original_coords=[100, 200],
state=mock_state,
)
except Exception as e:
pytest.fail(f"Tool raised an exception: {e}")
assert isinstance(passed, bool)
assert isinstance(category, ValidationErrorCategory)
assert isinstance(reason, str)
@@ -0,0 +1,103 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
from artemis.agents.video_analyzer.video_analyzer import VideoAnalyzer
@pytest.mark.asyncio
async def test_video_analyzer_tools(artemis_context, mock_state, inputs_dir):
artemis_context.device.mobile_platform = "android"
artemis_context.adb_client = artemis_context.ui_adb_client
agent = VideoAnalyzer(ctx=artemis_context)
agent.local_files_to_cleanup = set()
agent.local_dirs_to_cleanup = set()
agent.cloud_files_to_cleanup = set()
agent.blackboard_entries = []
agent.sub_system_prompt = "You are a sub agent"
agent.audio_system_prompt = "You are an audio sub agent"
agent.model_name = "gemini-3.6-flash"
# We must provide a valid FunctionDeclaration for the tool otherwise Pydantic will complain
from google.genai import types
agent.submit_answer_tool = types.FunctionDeclaration(
name="submit_answer",
description=("Submit the final findings for the requested segment analysis."),
parameters=types.Schema(
type=types.Type.OBJECT,
properties={
"summary": types.Schema(type=types.Type.STRING, description="summary"),
"analysis": types.Schema(type=types.Type.STRING, description="analysis"),
"timeline_events": types.Schema(
type=types.Type.ARRAY,
items=types.Schema(
type=types.Type.OBJECT,
properties={
"start_time": types.Schema(type=types.Type.NUMBER),
"end_time": types.Schema(type=types.Type.NUMBER),
"transcription": types.Schema(type=types.Type.STRING),
"confidence_score": types.Schema(type=types.Type.NUMBER),
"verification_timestamp_secs": types.Schema(type=types.Type.NUMBER),
},
),
),
},
),
)
import google.genai as genai
from artemis.config import settings
agent.client = genai.Client(
api_key=settings.GOOGLE_API_KEY.get_secret_value() if settings.GOOGLE_API_KEY else None
)
artemis_context._genai_client = agent.client
mock_session = MagicMock()
mock_session.local_video_path = Path(inputs_dir) / "recording.mp4"
mock_session.start_time = 0.0
mock_session.data_engine_start_time = None
mock_session.process = MagicMock()
mock_session.process.returncode = None
mock_session.android_video_segments = []
mock_session.android_segment_index = 0
from unittest.mock import AsyncMock
sample_video = Path(inputs_dir) / "recording.mp4"
mock_driver = MagicMock()
mock_driver.stop_video_recording = AsyncMock(return_value=sample_video)
artemis_context._active_driver = mock_driver
with patch(
"artemis.controllers.android_controller.get_active_session",
return_value=mock_session,
):
result = await agent.exec_extract_segment_metadata(start_time=0.0, end_time=1.0)
assert isinstance(result, str)
assert len(result) > 0
# Test sub-agent with extremely short audio query to not waste resources and time
audio_result = await agent.exec_analyze_audio_only(
start_time=0.0,
end_time=1.0,
specific_query="Reply with a short summary only.",
)
assert isinstance(audio_result, str)