somew1nd
9a5aaf99b7
- Showcase UI Reorganization: * Move web UI dashboard from root directory into apps/showcase_ui/ for cleaner project structure * Add 'artemis ui' CLI command (artemis/interfaces/cli/commands/ui.py) and launcher scripts for the interactive showcase console - Standalone MCP Server & Notifiers (mcp_server/): * Implement dedicated MCP server package with mobile automation tools (device_state, inspect_trace, task_manager, task_runner) * Add multi-channel notifiers (AgentAPI, Desktop notifications, Webhooks, File output, and Composite routing) * Introduce background task runner and trace store for autonomous mobile execution monitoring - Cross-Platform Runtime & Toolchain Support: * Add artemis/platform/ with OS-specific implementations for Linux, macOS (Darwin), and Windows * Add artemis/runtime/supervisor.py for resilient process and agent runtime supervision * Add artemis/toolchain/ with dependency resolution and SDK descriptors * Add cross-platform launcher scripts (start.sh, start.bat) and installer scripts (scripts/install_deps.*) - Core Enhancements & Diagnostics: * Enhance coordinate normalization, OCR API fallback, and perception pipeline * Add credential validation (credentials_validator.py) and system diagnostics * Update bilingual documentation (README.md, README_CN.md) and architecture diagrams - Test Suite Expansion: * Add integration tests for cloud gateway, remote ADB, and Artemis login * Add unit tests covering core diagnostics, platform detection, toolchain, credentials validator, optional OCR, and MCP tools
457 lines
15 KiB
Python
457 lines
15 KiB
Python
# Copyright 2026 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import os
|
|
from pathlib import Path
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
from artemis.agents.video_analyzer.video_analyzer import VideoAnalyzer
|
|
from artemis.context import ArtemisContext
|
|
import pytest
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_video_analyzer_run():
|
|
# Mock context
|
|
mock_ctx = MagicMock(spec=ArtemisContext)
|
|
mock_ctx.data_engine = None
|
|
mock_ctx.llm_config = MagicMock()
|
|
mock_ctx.llm_config.utils.video_analyzer = MagicMock()
|
|
mock_ctx.llm_config.utils.video_analyzer.model = "gemini-3.7-flash"
|
|
|
|
# Mock Gemini Client
|
|
mock_client = MagicMock()
|
|
mock_client.aio = MagicMock()
|
|
mock_client.aio.models = MagicMock()
|
|
mock_chunk = MagicMock()
|
|
mock_chunk.text = "Final analysis: The action succeeded."
|
|
mock_chunk.function_calls = []
|
|
|
|
async def mock_stream():
|
|
yield mock_chunk
|
|
|
|
mock_client.aio.models.generate_content_stream = AsyncMock(return_value=mock_stream())
|
|
|
|
# Mock file reading for prompts
|
|
mock_prompt = "# Dynamic Video Analyzer..."
|
|
|
|
with (
|
|
patch(
|
|
"artemis.agents.video_analyzer.video_analyzer.genai.Client",
|
|
return_value=mock_client,
|
|
),
|
|
patch(
|
|
"pathlib.Path.exists",
|
|
autospec=True,
|
|
side_effect=lambda self: not str(self).endswith(".jpg"),
|
|
),
|
|
patch("pathlib.Path.read_text", return_value=mock_prompt),
|
|
patch(
|
|
"artemis.agents.video_analyzer.video_analyzer.cleanup_abandoned_gemini_files",
|
|
AsyncMock(),
|
|
),
|
|
):
|
|
analyzer = VideoAnalyzer(mock_ctx)
|
|
result, status = await analyzer.run(
|
|
time_description="from 10s to 15s", purpose="Verify action"
|
|
)
|
|
|
|
assert result == "Final analysis: The action succeeded."
|
|
assert status == "success"
|
|
assert mock_client.aio.models.generate_content_stream.called
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_video_analyzer_preserves_thought_signature():
|
|
# Mock context
|
|
mock_ctx = MagicMock(spec=ArtemisContext)
|
|
mock_ctx.data_engine = None
|
|
mock_ctx.llm_config = MagicMock()
|
|
mock_ctx.llm_config.utils.video_analyzer = MagicMock()
|
|
mock_ctx.llm_config.utils.video_analyzer.model = "gemini-3.7-flash"
|
|
|
|
# Mock MobileDeviceController
|
|
mock_controller = MagicMock()
|
|
mock_controller.extract_segment_metadata = AsyncMock(
|
|
return_value=MagicMock(
|
|
success=True,
|
|
video_path="/tmp/video.mp4",
|
|
actual_start_relative_time=10.0,
|
|
)
|
|
)
|
|
|
|
from google.genai import types
|
|
|
|
dummy_signature = b"dummy_thought_signature_bytes_56789"
|
|
|
|
# Turn 1: Model calls spawn_sub_agent
|
|
spawn_call = types.FunctionCall(
|
|
name="spawn_sub_agent",
|
|
args={
|
|
"start_time": 10.0,
|
|
"end_time": 15.0,
|
|
"specific_query": "Verify login screen",
|
|
},
|
|
)
|
|
spawn_part = types.Part(
|
|
function_call=spawn_call,
|
|
thought_signature=dummy_signature,
|
|
)
|
|
|
|
mock_chunk_1 = MagicMock()
|
|
mock_chunk_1.text = ""
|
|
mock_chunk_1.function_calls = [spawn_call]
|
|
mock_chunk_1.candidates = [
|
|
types.Candidate(content=types.Content(role="model", parts=[spawn_part]))
|
|
]
|
|
|
|
# Turn 2: Model returns final answer
|
|
mock_chunk_2 = MagicMock()
|
|
mock_chunk_2.text = "Final analysis: The spawn succeeded and login button is visible."
|
|
mock_chunk_2.function_calls = []
|
|
mock_chunk_2.candidates = [
|
|
types.Candidate(
|
|
content=types.Content(
|
|
role="model",
|
|
parts=[types.Part.from_text(text=mock_chunk_2.text)],
|
|
)
|
|
)
|
|
]
|
|
|
|
# We mock generate_content_stream
|
|
call_count = 0
|
|
captured_contents = []
|
|
|
|
async def mock_stream_1():
|
|
yield mock_chunk_1
|
|
|
|
async def mock_stream_2():
|
|
yield mock_chunk_2
|
|
|
|
async def mock_generate_content_stream(model, contents, config):
|
|
nonlocal call_count
|
|
has_submit_answer = False
|
|
if config.tools:
|
|
for t in config.tools:
|
|
fd_list = getattr(t, "function_declarations", None)
|
|
if fd_list:
|
|
for fd in fd_list:
|
|
if fd.name == "submit_answer":
|
|
has_submit_answer = True
|
|
break
|
|
if not has_submit_answer:
|
|
captured_contents.append(list(contents)) # copy history contents passed to call
|
|
call_count += 1
|
|
if call_count == 1:
|
|
return mock_stream_1()
|
|
else:
|
|
return mock_stream_2()
|
|
else:
|
|
submit_call = types.FunctionCall(
|
|
name="submit_answer",
|
|
args={
|
|
"confidence_score": 0.9,
|
|
"summary": "Sub-agent verification finished",
|
|
"analysis": "Analysis here",
|
|
"verification_timestamp_secs": 10.0,
|
|
},
|
|
)
|
|
mock_sub_chunk = MagicMock()
|
|
mock_sub_chunk.text = ""
|
|
mock_sub_chunk.function_calls = [submit_call]
|
|
|
|
async def sub_stream():
|
|
yield mock_sub_chunk
|
|
|
|
return sub_stream()
|
|
|
|
# Mock Gemini Client
|
|
mock_client = MagicMock()
|
|
mock_client.aio = MagicMock()
|
|
mock_client.aio.files = MagicMock()
|
|
|
|
mock_file = MagicMock()
|
|
mock_file.name = "dummy_cloud_file_123"
|
|
mock_file.uri = "https://gemini-file-api/dummy_cloud_file_123"
|
|
mock_file.mime_type = "video/mp4"
|
|
mock_client.aio.files.upload = AsyncMock(return_value=mock_file)
|
|
|
|
mock_f_state = MagicMock()
|
|
mock_f_state.state.name = "ACTIVE"
|
|
mock_client.aio.files.get = AsyncMock(return_value=mock_f_state)
|
|
mock_client.aio.files.delete = AsyncMock()
|
|
|
|
mock_client.aio.models = MagicMock()
|
|
mock_client.aio.models.generate_content_stream = AsyncMock(
|
|
side_effect=mock_generate_content_stream
|
|
)
|
|
|
|
# Mock file reading for prompts
|
|
mock_prompt = "# Dynamic Video Analyzer..."
|
|
|
|
with (
|
|
patch(
|
|
"artemis.agents.video_analyzer.video_analyzer.genai.Client",
|
|
return_value=mock_client,
|
|
),
|
|
patch(
|
|
"artemis.agents.video_analyzer.video_analyzer.get_controller",
|
|
return_value=mock_controller,
|
|
),
|
|
patch(
|
|
"pathlib.Path.exists",
|
|
autospec=True,
|
|
side_effect=lambda self: not str(self).endswith(".jpg"),
|
|
),
|
|
patch(
|
|
"pathlib.Path.stat",
|
|
return_value=os.stat_result((0o100644, 0, 0, 0, 0, 0, 1024 * 1024, 0, 0, 0)),
|
|
),
|
|
patch("pathlib.Path.read_text", return_value=mock_prompt),
|
|
patch(
|
|
"artemis.agents.video_analyzer.video_analyzer.cleanup_abandoned_gemini_files",
|
|
AsyncMock(),
|
|
),
|
|
patch(
|
|
"artemis.agents.video_analyzer.video_analyzer.compress_video_for_api",
|
|
AsyncMock(return_value=Path("/tmp/video.mp4")),
|
|
),
|
|
):
|
|
analyzer = VideoAnalyzer(mock_ctx)
|
|
result, status = await analyzer.run(
|
|
time_description="from 10s to 15s", purpose="Verify login"
|
|
)
|
|
|
|
assert status == "success"
|
|
assert "spawn succeeded" in result
|
|
assert call_count == 2
|
|
|
|
# Verify that the second model call's history contains the model turn with the thought_signature
|
|
second_call_history = captured_contents[1]
|
|
assert len(second_call_history) >= 4
|
|
|
|
model_turn = second_call_history[1]
|
|
assert model_turn.role == "model"
|
|
assert len(model_turn.parts) == 1
|
|
assert model_turn.parts[0].function_call.name == "spawn_sub_agent"
|
|
assert model_turn.parts[0].thought_signature == dummy_signature
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_video_analyzer_sub_agent_confidence_validation():
|
|
# Mock context
|
|
mock_ctx = MagicMock(spec=ArtemisContext)
|
|
mock_ctx.data_engine = None
|
|
mock_ctx.llm_config = MagicMock()
|
|
mock_ctx.llm_config.utils.video_analyzer = MagicMock()
|
|
mock_ctx.llm_config.utils.video_analyzer.model = "gemini-3.7-flash"
|
|
|
|
# Mock MobileDeviceController
|
|
mock_controller = MagicMock()
|
|
mock_controller.extract_segment_metadata = AsyncMock(
|
|
return_value=MagicMock(
|
|
success=True,
|
|
video_path="/tmp/video.mp4",
|
|
actual_start_relative_time=10.0,
|
|
)
|
|
)
|
|
|
|
from google.genai import types
|
|
|
|
# Main agent responses
|
|
# Turn 1: Calls spawn_sub_agent
|
|
spawn_call = types.FunctionCall(
|
|
name="spawn_sub_agent",
|
|
args={
|
|
"start_time": 10.0,
|
|
"end_time": 15.0,
|
|
"specific_query": "Verify login screen",
|
|
},
|
|
)
|
|
mock_main_chunk_1 = MagicMock()
|
|
mock_main_chunk_1.text = ""
|
|
mock_main_chunk_1.function_calls = [spawn_call]
|
|
|
|
# Turn 2: Returns final answer
|
|
mock_main_chunk_2 = MagicMock()
|
|
mock_main_chunk_2.text = "Final blackboard status checked."
|
|
mock_main_chunk_2.function_calls = []
|
|
|
|
# Sub-agent responses
|
|
# Turn 1: Calls submit_answer with invalid confidence_score (string)
|
|
invalid_submit_call = types.FunctionCall(
|
|
name="submit_answer",
|
|
args={
|
|
"timeline_events": [
|
|
{
|
|
"confidence_score": "HIGH",
|
|
"transcription": "Invalid confidence score call",
|
|
"verification_timestamp_secs": 10.0,
|
|
}
|
|
],
|
|
"summary": "Invalid confidence score call",
|
|
"analysis": "Some analysis",
|
|
},
|
|
)
|
|
sub_dummy_signature = b"sub_dummy_signature_9999"
|
|
sub_part_1 = types.Part(
|
|
function_call=invalid_submit_call,
|
|
thought_signature=sub_dummy_signature,
|
|
)
|
|
mock_sub_chunk_1 = MagicMock()
|
|
mock_sub_chunk_1.text = ""
|
|
mock_sub_chunk_1.function_calls = [invalid_submit_call]
|
|
mock_sub_chunk_1.candidates = [
|
|
types.Candidate(content=types.Content(role="model", parts=[sub_part_1]))
|
|
]
|
|
|
|
# Turn 2: Calls submit_answer with valid confidence_score
|
|
valid_submit_call = types.FunctionCall(
|
|
name="submit_answer",
|
|
args={
|
|
"timeline_events": [
|
|
{
|
|
"confidence_score": 0.8,
|
|
"transcription": "Valid confidence score call",
|
|
"verification_timestamp_secs": 10.0,
|
|
}
|
|
],
|
|
"summary": "Valid confidence score call",
|
|
"analysis": "Some analysis",
|
|
},
|
|
)
|
|
mock_sub_chunk_2 = MagicMock()
|
|
mock_sub_chunk_2.text = ""
|
|
mock_sub_chunk_2.function_calls = [valid_submit_call]
|
|
mock_sub_chunk_2.candidates = [
|
|
types.Candidate(
|
|
content=types.Content(role="model", parts=[types.Part(function_call=valid_submit_call)])
|
|
)
|
|
]
|
|
|
|
# Stream generator mocks
|
|
main_calls = 0
|
|
sub_calls = 0
|
|
captured_sub_contents = []
|
|
|
|
async def mock_main_stream():
|
|
nonlocal main_calls
|
|
if main_calls == 0:
|
|
main_calls += 1
|
|
yield mock_main_chunk_1
|
|
else:
|
|
yield mock_main_chunk_2
|
|
|
|
async def mock_sub_stream():
|
|
nonlocal sub_calls
|
|
if sub_calls == 0:
|
|
sub_calls += 1
|
|
yield mock_sub_chunk_1
|
|
else:
|
|
yield mock_sub_chunk_2
|
|
|
|
async def mock_generate_content_stream(model, contents, config):
|
|
system_instruction = config.system_instruction
|
|
if "Dynamic Video Analyzer" in str(system_instruction):
|
|
return mock_main_stream()
|
|
else:
|
|
captured_sub_contents.append(list(contents))
|
|
return mock_sub_stream()
|
|
|
|
# Mock Gemini Client
|
|
mock_client = MagicMock()
|
|
mock_client.aio = MagicMock()
|
|
mock_client.aio.files = MagicMock()
|
|
|
|
mock_file = MagicMock()
|
|
mock_file.name = "dummy_cloud_file_123"
|
|
mock_file.uri = "https://gemini-file-api/dummy_cloud_file_123"
|
|
mock_file.mime_type = "video/mp4"
|
|
mock_client.aio.files.upload = AsyncMock(return_value=mock_file)
|
|
|
|
mock_f_state = MagicMock()
|
|
mock_f_state.state.name = "ACTIVE"
|
|
mock_client.aio.files.get = AsyncMock(return_value=mock_f_state)
|
|
mock_client.aio.files.delete = AsyncMock()
|
|
|
|
mock_client.aio.models = MagicMock()
|
|
mock_client.aio.models.generate_content_stream = AsyncMock(
|
|
side_effect=mock_generate_content_stream
|
|
)
|
|
|
|
# Mock file reading for prompts
|
|
mock_main_prompt = "# Dynamic Video Analyzer..."
|
|
mock_sub_prompt = "# Sub Agent..."
|
|
|
|
def mock_read_text(self, encoding="utf-8"):
|
|
if "video_analyzer.md" in str(self):
|
|
return mock_main_prompt
|
|
elif "video_sub_agent.md" in str(self):
|
|
return mock_sub_prompt
|
|
return ""
|
|
|
|
with (
|
|
patch(
|
|
"artemis.agents.video_analyzer.video_analyzer.genai.Client",
|
|
return_value=mock_client,
|
|
),
|
|
patch(
|
|
"artemis.agents.video_analyzer.video_analyzer.get_controller",
|
|
return_value=mock_controller,
|
|
),
|
|
patch(
|
|
"pathlib.Path.exists",
|
|
autospec=True,
|
|
side_effect=lambda self: not str(self).endswith(".jpg"),
|
|
),
|
|
patch(
|
|
"pathlib.Path.stat",
|
|
return_value=os.stat_result((0o100644, 0, 0, 0, 0, 0, 1024 * 1024, 0, 0, 0)),
|
|
),
|
|
patch("pathlib.Path.read_text", mock_read_text),
|
|
patch("pathlib.Path.mkdir"),
|
|
patch("pathlib.Path.touch"),
|
|
patch("pathlib.Path.is_dir", return_value=True),
|
|
patch(
|
|
"artemis.agents.video_analyzer.video_analyzer.cleanup_abandoned_gemini_files",
|
|
AsyncMock(),
|
|
),
|
|
patch(
|
|
"artemis.agents.video_analyzer.video_analyzer.compress_video_for_api",
|
|
AsyncMock(return_value=Path("/tmp/video.mp4")),
|
|
),
|
|
patch(
|
|
"asyncio.create_subprocess_exec",
|
|
AsyncMock(return_value=MagicMock(communicate=AsyncMock(return_value=(b"", b"")))),
|
|
),
|
|
):
|
|
analyzer = VideoAnalyzer(mock_ctx)
|
|
result, status = await analyzer.run(
|
|
time_description="from 10s to 15s", purpose="Verify login"
|
|
)
|
|
|
|
assert status == "success"
|
|
assert sub_calls == 1
|
|
assert len(analyzer.blackboard_entries) == 1
|
|
entry = analyzer.blackboard_entries[0]
|
|
assert entry["confidence_score"] == 0.8
|
|
assert entry["summary"] == "Valid confidence score call"
|
|
|
|
# Verify that sub-agent turn 2 received the model turn with the thought_signature
|
|
assert len(captured_sub_contents) >= 2
|
|
sub_turn_2_contents = captured_sub_contents[1]
|
|
model_turns = [c for c in sub_turn_2_contents if c.role == "model"]
|
|
assert len(model_turns) == 1
|
|
assert model_turns[0].parts[0].thought_signature == sub_dummy_signature
|