feat: add Chinese UI, Responses models, and mirrored Windows setup
Localize the web console and persist custom model and capability settings. Use OpenAI Responses exclusively and move runtime limits into environment configuration. Bootstrap uv and the virtual environment on Windows, support optional China mirrors, and package source distributions without environments, build output, or credentials. Update regression tests and repository quality checks for the new behavior.
This commit is contained in:
chenmingxuan committed
1 parent
371aa6df56
commit
2ddec53954
183 files changed
+3427
-11857
No files matched your search
File diff suppressed because it is too large.
Load diff
@@ -301,34 +301,6 @@ def test_enrich_candidates_withholds_degenerate_bounds():
|
||||
assert explorer._enrich_candidates([cand]) == [cand]
|
||||
|
||||
|
||||
def test_native_submit_enriches_registered_candidates():
|
||||
explorer = _indexed_explorer()
|
||||
explorer._register_label("O2", explorer.screen_index.ocr_elements[0])
|
||||
call = MagicMock()
|
||||
call.name = "submit_answer"
|
||||
call.args = {
|
||||
"candidates": [
|
||||
{"label": "O2", "coords": [185, 54], "description": "header"},
|
||||
{"label": "D3", "coords": [900, 100], "description": "icon"},
|
||||
],
|
||||
"fallback_message": "",
|
||||
}
|
||||
turn_record = {"tool_calls": []}
|
||||
|
||||
raw = explorer._process_submit_answer([call], turn_record, [])
|
||||
|
||||
data = json.loads(raw)
|
||||
assert list(data) == ["candidates", "fallback_message"]
|
||||
assert data["candidates"][0] == {
|
||||
"label": "O2",
|
||||
"coords": [185, 54],
|
||||
"description": "header",
|
||||
"bounds": [92, 41, 277, 66],
|
||||
"source": "ocr",
|
||||
}
|
||||
assert data["candidates"][1] == call.args["candidates"][1]
|
||||
|
||||
|
||||
def test_universal_submit_enriches_registered_candidates():
|
||||
explorer = _indexed_explorer()
|
||||
explorer._register_label("1", explorer.screen_index.exact_matches("Settings")[0])
|
||||
|
||||
@@ -18,7 +18,6 @@ import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from google.genai import types
|
||||
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
|
||||
import pytest
|
||||
|
||||
@@ -61,36 +60,6 @@ def _state(screenshot: str) -> MagicMock:
|
||||
return state
|
||||
|
||||
|
||||
def _submit_response(coords=(500, 600)) -> MagicMock:
|
||||
call = MagicMock()
|
||||
call.name = "submit_answer"
|
||||
call.args = {
|
||||
"candidates": [{"label": "D1", "coords": list(coords), "description": "target"}],
|
||||
"fallback_message": "",
|
||||
}
|
||||
response = MagicMock()
|
||||
response.function_calls = [call]
|
||||
response.candidates = [MagicMock()]
|
||||
response.candidates[0].content = types.Content(
|
||||
role="model",
|
||||
parts=[types.Part.from_function_call(name="submit_answer", args=call.args)],
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def _native_client(responses) -> MagicMock:
|
||||
client = MagicMock()
|
||||
client.aio.models.generate_content = AsyncMock(side_effect=list(responses))
|
||||
cache = SimpleNamespace(name="cachedContents/abc")
|
||||
client.aio.caches.create = AsyncMock(return_value=cache)
|
||||
client.aio.caches.delete = AsyncMock()
|
||||
client.aio.files.upload = AsyncMock(
|
||||
return_value=SimpleNamespace(uri="files/1", mime_type="image/jpeg", name="files/1")
|
||||
)
|
||||
client.aio.files.delete = AsyncMock()
|
||||
return client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def screenshot(tmp_path):
|
||||
path = tmp_path / "screen.jpg"
|
||||
@@ -110,9 +79,7 @@ def _isolated_env(monkeypatch):
|
||||
|
||||
|
||||
def test_construction_creates_no_genai_client():
|
||||
with patch(f"{EXPLORER}.genai.Client") as client_cls:
|
||||
explorer = Explorer(_context())
|
||||
client_cls.assert_not_called()
|
||||
explorer = Explorer(_context())
|
||||
assert explorer.client is None
|
||||
assert explorer.tier is None
|
||||
|
||||
@@ -128,7 +95,7 @@ def test_tier_sets_turn_budget_and_exposed_tools(version, max_turns, expected_to
|
||||
explorer = Explorer(_context())
|
||||
with patch(f"{EXPLORER}.is_ocr_configured", return_value=True):
|
||||
explorer._apply_tier(get_tier(version))
|
||||
native_names = {t.name for t in explorer.get_exposed_tools()}
|
||||
native_names = {t["function"]["name"] for t in explorer.get_exposed_tools()}
|
||||
universal_names = {t["function"]["name"] for t in explorer._universal_exposed_tools()}
|
||||
|
||||
assert explorer.max_turns == max_turns
|
||||
@@ -141,7 +108,7 @@ def test_unconfigured_ocr_and_user_denylist_hide_tools_on_ultra():
|
||||
explorer = Explorer(_context(denylist=["inspect_region"]))
|
||||
with patch(f"{EXPLORER}.is_ocr_configured", return_value=False):
|
||||
explorer._apply_tier(get_tier("ultra"))
|
||||
names = {t.name for t in explorer.get_exposed_tools()}
|
||||
names = {t["function"]["name"] for t in explorer.get_exposed_tools()}
|
||||
assert "get_ocr_list" not in names
|
||||
assert "inspect_region" not in names
|
||||
assert {"detect_objects", "ask_image_processor", "ask_perception_tool"} <= names
|
||||
@@ -152,21 +119,6 @@ def test_unconfigured_ocr_and_user_denylist_hide_tools_on_ultra():
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_gemini_model_without_google_key_uses_universal_engine():
|
||||
with patch.object(settings, "GOOGLE_API_KEY", None):
|
||||
explorer = Explorer(_context(model="gemini-3.8-flash"))
|
||||
assert explorer.use_native_gemini is False
|
||||
|
||||
|
||||
def test_gemini_model_with_shared_client_uses_native_engine():
|
||||
ctx = _context(model="google/gemini-3.8-pro")
|
||||
ctx._genai_client = MagicMock()
|
||||
with patch.object(settings, "GOOGLE_API_KEY", None):
|
||||
explorer = Explorer(ctx)
|
||||
assert explorer.use_native_gemini is True
|
||||
assert explorer.model_name == "gemini-3.8-pro"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_universal_run_without_google_key_never_touches_genai(screenshot):
|
||||
ctx = _context(model="claude-sonnet-4")
|
||||
@@ -191,15 +143,13 @@ async def test_universal_run_without_google_key_never_touches_genai(screenshot):
|
||||
storage.get_image.return_value = None
|
||||
|
||||
with (
|
||||
patch.object(settings, "GOOGLE_API_KEY", None),
|
||||
patch(f"{EXPLORER}.genai.Client") as client_cls,
|
||||
patch.object(settings, "OPENAI_API_KEY", None),
|
||||
patch(f"{UNIVERSAL_RUNNER}.get_llm", return_value=llm),
|
||||
patch(f"{RUN_SETUP}.StorageManager", return_value=storage),
|
||||
):
|
||||
explorer = Explorer(ctx)
|
||||
raw = await explorer.run("find it", "", screenshot, _state(screenshot), version="pro")
|
||||
|
||||
client_cls.assert_not_called()
|
||||
outcome = json.loads(raw)
|
||||
assert outcome["candidates"][0]["coords"] == [100, 200]
|
||||
# The universal engine only sees the pro tier's tools.
|
||||
@@ -283,45 +233,6 @@ async def test_universal_dispatch_rejects_denylisted_tool():
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_caching_tier_defaults():
|
||||
explorer = Explorer(_context())
|
||||
with patch.object(settings, "EXPLORER_CACHING", None):
|
||||
assert explorer._resolve_caching(get_tier("pro"), None) is False
|
||||
assert explorer._resolve_caching(get_tier("ultra"), None) is True
|
||||
|
||||
|
||||
def test_caching_precedence(monkeypatch):
|
||||
ctx = _context()
|
||||
explorer = Explorer(ctx)
|
||||
pro = get_tier("pro")
|
||||
|
||||
with patch.object(settings, "EXPLORER_CACHING", None):
|
||||
ctx.agent_config.explorer = SimpleNamespace(caching=True)
|
||||
assert explorer._resolve_caching(pro, None) is True # agent config beats tier
|
||||
|
||||
monkeypatch.setenv("ARTEMIS_EXPLORER_CACHING", "false")
|
||||
assert explorer._resolve_caching(pro, None) is False # env beats agent config
|
||||
|
||||
assert explorer._resolve_caching(pro, True) is True # explicit beats env
|
||||
|
||||
with patch.object(settings, "EXPLORER_CACHING", True):
|
||||
monkeypatch.delenv("ARTEMIS_EXPLORER_CACHING")
|
||||
ctx.agent_config.explorer = SimpleNamespace(caching=None)
|
||||
ctx.execution_setup = SimpleNamespace(explorer=SimpleNamespace(caching=False))
|
||||
assert explorer._resolve_caching(pro, None) is False # execution setup beats settings
|
||||
|
||||
ctx.execution_setup = SimpleNamespace(explorer=SimpleNamespace(caching=None))
|
||||
assert explorer._resolve_caching(pro, None) is True # settings beat tier
|
||||
|
||||
|
||||
def test_caching_ignores_mock_config_values():
|
||||
"""A MagicMock context must fall through to the tier default, not force True."""
|
||||
ctx = MagicMock()
|
||||
explorer = Explorer(ctx)
|
||||
with patch.object(settings, "EXPLORER_CACHING", None):
|
||||
assert explorer._resolve_caching(get_tier("pro"), None) is False
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Tier-aware prompt
|
||||
# --------------------------------------------------------------------------- #
|
||||
@@ -381,7 +292,9 @@ def test_prompt_sections_render_shape():
|
||||
assert lines[0] == "# IDENTITY"
|
||||
assert any(line.startswith("# OPERATING PRINCIPLES") for line in lines)
|
||||
body = [line for line in lines if line and not line.startswith("#")]
|
||||
assert all(line.startswith("- ") or line.startswith("You are") for line in body)
|
||||
assert "输出语言要求" in prompt
|
||||
assert "简体中文" in prompt
|
||||
assert any(line.startswith("You are") for line in body)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
@@ -402,14 +315,12 @@ async def test_flash_run_returns_detected_candidates(screenshot):
|
||||
)
|
||||
with (
|
||||
patch(f"{RUN_SETUP}._run_object_detection", detection),
|
||||
patch(f"{EXPLORER}.genai.Client") as client_cls,
|
||||
):
|
||||
explorer = Explorer(_context())
|
||||
raw = await explorer.run(
|
||||
"gear icon | send button", "", screenshot, _state(screenshot), version="flash"
|
||||
)
|
||||
|
||||
client_cls.assert_not_called()
|
||||
outcome = json.loads(raw)
|
||||
assert [c["label"] for c in outcome["candidates"]] == ["D1", "D2"]
|
||||
assert outcome["candidates"][0] == {
|
||||
@@ -435,114 +346,6 @@ async def test_flash_run_reports_missing_detection(screenshot):
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_ultra_run_uses_async_cache_api(screenshot):
|
||||
ctx = _context()
|
||||
client = _native_client([_submit_response()])
|
||||
ctx._genai_client = client
|
||||
storage = MagicMock()
|
||||
storage.get_image.return_value = None
|
||||
|
||||
with (
|
||||
patch(f"{EXPLORER}.genai.Client") as client_cls,
|
||||
patch(f"{RUN_SETUP}.StorageManager", return_value=storage),
|
||||
patch.object(settings, "EXPLORER_CACHING", None),
|
||||
):
|
||||
explorer = Explorer(ctx)
|
||||
raw = await explorer.run("target", "", screenshot, _state(screenshot), version="ultra")
|
||||
|
||||
client_cls.assert_not_called()
|
||||
assert json.loads(raw)["candidates"][0]["coords"] == [500, 600]
|
||||
client.aio.caches.create.assert_awaited_once()
|
||||
client.aio.caches.delete.assert_awaited_once_with(name="cachedContents/abc")
|
||||
config = client.aio.models.generate_content.call_args.kwargs["config"]
|
||||
assert config.cached_content == "cachedContents/abc"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_pro_run_skips_cache_by_default(screenshot):
|
||||
ctx = _context()
|
||||
client = _native_client([_submit_response()])
|
||||
ctx._genai_client = client
|
||||
storage = MagicMock()
|
||||
storage.get_image.return_value = None
|
||||
|
||||
with (
|
||||
patch(f"{RUN_SETUP}.StorageManager", return_value=storage),
|
||||
patch.object(settings, "EXPLORER_CACHING", None),
|
||||
):
|
||||
await Explorer(ctx).run("target", "", screenshot, _state(screenshot), version="pro")
|
||||
|
||||
client.aio.caches.create.assert_not_awaited()
|
||||
client.aio.caches.delete.assert_not_awaited()
|
||||
config = client.aio.models.generate_content.call_args.kwargs["config"]
|
||||
assert {t.name for t in config.tools[0].function_declarations} == {
|
||||
"ask_perception_tool",
|
||||
"submit_answer",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_file_api_uploads_and_deletes_asynchronously(screenshot, monkeypatch):
|
||||
monkeypatch.setenv("ARTEMIS_USE_FILE_API", "true")
|
||||
ctx = _context()
|
||||
client = _native_client([_submit_response()])
|
||||
ctx._genai_client = client
|
||||
storage = MagicMock()
|
||||
storage.get_image.return_value = None
|
||||
|
||||
with patch(f"{RUN_SETUP}.StorageManager", return_value=storage):
|
||||
await Explorer(ctx).run("target", "", screenshot, _state(screenshot), version="pro")
|
||||
|
||||
client.aio.files.upload.assert_awaited_once_with(file=screenshot)
|
||||
client.aio.files.delete.assert_awaited_once_with(name="files/1")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_closes_http_client_on_both_engines(screenshot):
|
||||
storage = MagicMock()
|
||||
storage.get_image.return_value = None
|
||||
|
||||
# Native engine
|
||||
ctx = _context()
|
||||
client = _native_client([_submit_response()])
|
||||
ctx._genai_client = client
|
||||
native = Explorer(ctx)
|
||||
native.http_client = SimpleNamespace(aclose=AsyncMock())
|
||||
with patch(f"{RUN_SETUP}.StorageManager", return_value=storage):
|
||||
await native.run("t", "", screenshot, _state(screenshot), version="pro")
|
||||
native_close = native.http_client # already reset
|
||||
assert native_close is None
|
||||
|
||||
# Universal engine (model call fails -> finally still closes)
|
||||
llm = MagicMock()
|
||||
llm.bind_tools.return_value.ainvoke = AsyncMock(side_effect=RuntimeError("boom"))
|
||||
universal = Explorer(_context(model="claude-sonnet-4"))
|
||||
http_client = SimpleNamespace(aclose=AsyncMock())
|
||||
universal.http_client = http_client
|
||||
with (
|
||||
patch(f"{RUN_SETUP}.StorageManager", return_value=storage),
|
||||
patch(f"{UNIVERSAL_RUNNER}.get_llm", return_value=llm),
|
||||
pytest.raises(RuntimeError),
|
||||
):
|
||||
await universal.run("t", "", screenshot, _state(screenshot), version="pro")
|
||||
http_client.aclose.assert_awaited_once()
|
||||
assert universal.http_client is None
|
||||
|
||||
|
||||
def test_missing_tool_calls_without_candidates_does_not_crash():
|
||||
explorer = Explorer(_context())
|
||||
response = MagicMock()
|
||||
response.candidates = None
|
||||
contents = []
|
||||
turn_record = {"tool_calls": []}
|
||||
explorer._handle_missing_tool_calls(contents, response, turn_record, ["hi"], 3, 1)
|
||||
assert contents[0].role == "model"
|
||||
assert contents[0].parts[0].text == "hi"
|
||||
assert "2 more" in contents[1].parts[0].text
|
||||
assert turn_record["tool_calls"][0]["name"] == "hallucinated_plain_text"
|
||||
|
||||
|
||||
def test_tier_table_is_single_source_for_versions():
|
||||
assert set(EXPLORER_TIERS) == {"flash", "pro", "ultra"}
|
||||
assert get_tier("bogus").name == "flash"
|
||||
|
||||
@@ -50,9 +50,9 @@ def mock_context(tmp_path):
|
||||
|
||||
def test_explorer_universal_engine_detection(mock_context):
|
||||
"""Test that Explorer routes to Universal Engine when non-Gemini model is configured."""
|
||||
with patch.object(settings, "GOOGLE_API_KEY", None):
|
||||
with patch.object(settings, "OPENAI_API_KEY", None):
|
||||
agent = Explorer(mock_context)
|
||||
assert agent.use_native_gemini is False
|
||||
assert not hasattr(agent, "_ensure_genai_client")
|
||||
assert agent.model_name == "claude-3-7-sonnet"
|
||||
|
||||
|
||||
|
||||
@@ -54,8 +54,8 @@ from artemis.memory.transcript import (
|
||||
|
||||
# SHA-256 snapshots of the legacy system message with the fixed inputs below.
|
||||
# Update these when an intentional template change alters the rendered prompt.
|
||||
GOLDEN_EMPTY_PLAN = "513e0341014b9a2342f608ad4dd173b7dcf5704ce53ee028ba302a9a1b696f96"
|
||||
GOLDEN_SENTINEL_PLAN = "012fe1b08645266d5a7e518023e20fc6fbfa65d4f2d985847ebb06e0184c67ee"
|
||||
GOLDEN_EMPTY_PLAN = "d715128625911aec46c83138e15aabfa967b385ab4c878509fe36ab4304fae2a"
|
||||
GOLDEN_SENTINEL_PLAN = "e2cf4ede75ffd9e283159f70dcd5c94a169c18e9f6dd4d1e7aacf460e956eaf3"
|
||||
|
||||
SCREENSHOT_B64 = base64.b64encode(b"fake-jpeg-bytes").decode("utf-8")
|
||||
|
||||
@@ -77,7 +77,7 @@ async def _render_legacy_system(plan_and_history: str) -> str:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_legacy_prompt_bytes_match_pre_split_golden():
|
||||
async def test_legacy_prompt_bytes_match_chinese_prompt_golden():
|
||||
empty = await _render_legacy_system("No plan or history yet.")
|
||||
sentinel = await _render_legacy_system("PLAN_AND_HISTORY_SENTINEL\nline2")
|
||||
assert hashlib.sha256(empty.encode("utf-8")).hexdigest() == GOLDEN_EMPTY_PLAN
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,616 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.agents.video_analyzer import chunk_conversation
|
||||
from artemis.agents.video_analyzer.chunk_conversation import (
|
||||
_MISSING_SUMMARY_ERROR,
|
||||
_NO_TOOL_CALL_ERROR,
|
||||
SubAgentAnswerExhausted,
|
||||
_ChunkMedia,
|
||||
_drive_sub_agent_loop,
|
||||
_run_native_chunk_conversation,
|
||||
)
|
||||
from artemis.agents.video_analyzer.video_analyzer import VideoAnalyzer
|
||||
from artemis.context import ArtemisContext
|
||||
from google.genai import types
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_analyzer_run():
|
||||
# Mock context
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = None
|
||||
mock_ctx.llm_config = MagicMock()
|
||||
mock_ctx.llm_config.utils.video_analyzer = MagicMock()
|
||||
mock_ctx.llm_config.utils.video_analyzer.model = "gemini-3.7-flash"
|
||||
mock_ctx.agent_config = SimpleNamespace(video_analyzer=SimpleNamespace(processing="static"))
|
||||
|
||||
# Mock Gemini Client
|
||||
mock_client = MagicMock()
|
||||
mock_client.aio = MagicMock()
|
||||
mock_client.aio.models = MagicMock()
|
||||
mock_chunk = MagicMock()
|
||||
mock_chunk.text = "Final analysis: The action succeeded."
|
||||
mock_chunk.function_calls = []
|
||||
|
||||
async def mock_stream():
|
||||
yield mock_chunk
|
||||
|
||||
mock_client.aio.models.generate_content_stream = AsyncMock(return_value=mock_stream())
|
||||
|
||||
# Mock file reading for prompts
|
||||
mock_prompt = "# Dynamic Video Analyzer..."
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.genai.Client",
|
||||
return_value=mock_client,
|
||||
),
|
||||
patch(
|
||||
"pathlib.Path.exists",
|
||||
autospec=True,
|
||||
side_effect=lambda self: not str(self).endswith(".jpg"),
|
||||
),
|
||||
patch("pathlib.Path.read_text", return_value=mock_prompt),
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.cleanup_abandoned_gemini_files",
|
||||
AsyncMock(),
|
||||
),
|
||||
):
|
||||
analyzer = VideoAnalyzer(mock_ctx)
|
||||
result, status = await analyzer.run(
|
||||
time_description="from 10s to 15s", purpose="Verify action"
|
||||
)
|
||||
|
||||
assert result == "Final analysis: The action succeeded."
|
||||
assert status == "success"
|
||||
assert mock_client.aio.models.generate_content_stream.called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_analyzer_preserves_thought_signature():
|
||||
# Mock context
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = None
|
||||
mock_ctx.llm_config = MagicMock()
|
||||
mock_ctx.llm_config.utils.video_analyzer = MagicMock()
|
||||
mock_ctx.llm_config.utils.video_analyzer.model = "gemini-3.7-flash"
|
||||
mock_ctx.agent_config = SimpleNamespace(video_analyzer=SimpleNamespace(processing="static"))
|
||||
|
||||
# Mock MobileDeviceController
|
||||
mock_controller = MagicMock()
|
||||
mock_controller.extract_segment_metadata = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
success=True,
|
||||
video_path="/tmp/video.mp4",
|
||||
actual_start_relative_time=10.0,
|
||||
)
|
||||
)
|
||||
|
||||
from google.genai import types
|
||||
|
||||
dummy_signature = b"dummy_thought_signature_bytes_56789"
|
||||
|
||||
# Turn 1: Model calls spawn_sub_agent
|
||||
spawn_call = types.FunctionCall(
|
||||
name="spawn_sub_agent",
|
||||
args={
|
||||
"start_time": 10.0,
|
||||
"end_time": 15.0,
|
||||
"specific_query": "Verify login screen",
|
||||
},
|
||||
)
|
||||
spawn_part = types.Part(
|
||||
function_call=spawn_call,
|
||||
thought_signature=dummy_signature,
|
||||
)
|
||||
|
||||
mock_chunk_1 = MagicMock()
|
||||
mock_chunk_1.text = ""
|
||||
mock_chunk_1.function_calls = [spawn_call]
|
||||
mock_chunk_1.candidates = [
|
||||
types.Candidate(content=types.Content(role="model", parts=[spawn_part]))
|
||||
]
|
||||
|
||||
# Turn 2: Model returns final answer
|
||||
mock_chunk_2 = MagicMock()
|
||||
mock_chunk_2.text = "Final analysis: The spawn succeeded and login button is visible."
|
||||
mock_chunk_2.function_calls = []
|
||||
mock_chunk_2.candidates = [
|
||||
types.Candidate(
|
||||
content=types.Content(
|
||||
role="model",
|
||||
parts=[types.Part.from_text(text=mock_chunk_2.text)],
|
||||
)
|
||||
)
|
||||
]
|
||||
|
||||
# We mock generate_content_stream
|
||||
call_count = 0
|
||||
captured_contents = []
|
||||
|
||||
async def mock_stream_1():
|
||||
yield mock_chunk_1
|
||||
|
||||
async def mock_stream_2():
|
||||
yield mock_chunk_2
|
||||
|
||||
async def mock_generate_content_stream(model, contents, config):
|
||||
nonlocal call_count
|
||||
has_submit_answer = False
|
||||
if config.tools:
|
||||
for t in config.tools:
|
||||
fd_list = getattr(t, "function_declarations", None)
|
||||
if fd_list:
|
||||
for fd in fd_list:
|
||||
if fd.name == "submit_answer":
|
||||
has_submit_answer = True
|
||||
break
|
||||
if not has_submit_answer:
|
||||
captured_contents.append(list(contents)) # copy history contents passed to call
|
||||
call_count += 1
|
||||
if call_count == 1:
|
||||
return mock_stream_1()
|
||||
else:
|
||||
return mock_stream_2()
|
||||
else:
|
||||
submit_call = types.FunctionCall(
|
||||
name="submit_answer",
|
||||
args={
|
||||
"confidence_score": 0.9,
|
||||
"summary": "Sub-agent verification finished",
|
||||
"analysis": "Analysis here",
|
||||
"verification_timestamp_secs": 10.0,
|
||||
},
|
||||
)
|
||||
mock_sub_chunk = MagicMock()
|
||||
mock_sub_chunk.text = ""
|
||||
mock_sub_chunk.function_calls = [submit_call]
|
||||
|
||||
async def sub_stream():
|
||||
yield mock_sub_chunk
|
||||
|
||||
return sub_stream()
|
||||
|
||||
# Mock Gemini Client
|
||||
mock_client = MagicMock()
|
||||
mock_client.aio = MagicMock()
|
||||
mock_client.aio.files = MagicMock()
|
||||
|
||||
mock_file = MagicMock()
|
||||
mock_file.name = "dummy_cloud_file_123"
|
||||
mock_file.uri = "https://gemini-file-api/dummy_cloud_file_123"
|
||||
mock_file.mime_type = "video/mp4"
|
||||
mock_client.aio.files.upload = AsyncMock(return_value=mock_file)
|
||||
|
||||
mock_f_state = MagicMock()
|
||||
mock_f_state.state.name = "ACTIVE"
|
||||
mock_client.aio.files.get = AsyncMock(return_value=mock_f_state)
|
||||
mock_client.aio.files.delete = AsyncMock()
|
||||
|
||||
mock_client.aio.models = MagicMock()
|
||||
mock_client.aio.models.generate_content_stream = AsyncMock(
|
||||
side_effect=mock_generate_content_stream
|
||||
)
|
||||
|
||||
# Mock file reading for prompts
|
||||
mock_prompt = "# Dynamic Video Analyzer..."
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.genai.Client",
|
||||
return_value=mock_client,
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.get_controller",
|
||||
return_value=mock_controller,
|
||||
),
|
||||
patch(
|
||||
"pathlib.Path.exists",
|
||||
autospec=True,
|
||||
side_effect=lambda self: not str(self).endswith(".jpg"),
|
||||
),
|
||||
patch(
|
||||
"pathlib.Path.stat",
|
||||
return_value=os.stat_result((0o100644, 0, 0, 0, 0, 0, 1024 * 1024, 0, 0, 0)),
|
||||
),
|
||||
patch("pathlib.Path.read_text", return_value=mock_prompt),
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.cleanup_abandoned_gemini_files",
|
||||
AsyncMock(),
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.compress_video_for_api",
|
||||
AsyncMock(return_value=Path("/tmp/video.mp4")),
|
||||
),
|
||||
):
|
||||
analyzer = VideoAnalyzer(mock_ctx)
|
||||
result, status = await analyzer.run(
|
||||
time_description="from 10s to 15s", purpose="Verify login"
|
||||
)
|
||||
|
||||
assert status == "success"
|
||||
assert "spawn succeeded" in result
|
||||
assert call_count == 2
|
||||
|
||||
# Verify that the second model call's history contains the model turn with the thought_signature
|
||||
second_call_history = captured_contents[1]
|
||||
assert len(second_call_history) >= 4
|
||||
|
||||
model_turn = second_call_history[1]
|
||||
assert model_turn.role == "model"
|
||||
assert len(model_turn.parts) == 1
|
||||
assert model_turn.parts[0].function_call.name == "spawn_sub_agent"
|
||||
assert model_turn.parts[0].thought_signature == dummy_signature
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_analyzer_sub_agent_confidence_validation():
|
||||
# Mock context
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = None
|
||||
mock_ctx.llm_config = MagicMock()
|
||||
mock_ctx.llm_config.utils.video_analyzer = MagicMock()
|
||||
mock_ctx.llm_config.utils.video_analyzer.model = "gemini-3.7-flash"
|
||||
mock_ctx.agent_config = SimpleNamespace(video_analyzer=SimpleNamespace(processing="static"))
|
||||
|
||||
# Mock MobileDeviceController
|
||||
mock_controller = MagicMock()
|
||||
mock_controller.extract_segment_metadata = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
success=True,
|
||||
video_path="/tmp/video.mp4",
|
||||
actual_start_relative_time=10.0,
|
||||
)
|
||||
)
|
||||
|
||||
from google.genai import types
|
||||
|
||||
# Main agent responses
|
||||
# Turn 1: Calls spawn_sub_agent
|
||||
spawn_call = types.FunctionCall(
|
||||
name="spawn_sub_agent",
|
||||
args={
|
||||
"start_time": 10.0,
|
||||
"end_time": 15.0,
|
||||
"specific_query": "Verify login screen",
|
||||
},
|
||||
)
|
||||
mock_main_chunk_1 = MagicMock()
|
||||
mock_main_chunk_1.text = ""
|
||||
mock_main_chunk_1.function_calls = [spawn_call]
|
||||
|
||||
# Turn 2: Returns final answer
|
||||
mock_main_chunk_2 = MagicMock()
|
||||
mock_main_chunk_2.text = "Final blackboard status checked."
|
||||
mock_main_chunk_2.function_calls = []
|
||||
|
||||
# Sub-agent responses
|
||||
# Turn 1: Calls submit_answer with invalid confidence_score (string)
|
||||
invalid_submit_call = types.FunctionCall(
|
||||
name="submit_answer",
|
||||
args={
|
||||
"timeline_events": [
|
||||
{
|
||||
"confidence_score": "HIGH",
|
||||
"transcription": "Invalid confidence score call",
|
||||
"verification_timestamp_secs": 10.0,
|
||||
}
|
||||
],
|
||||
"summary": "Invalid confidence score call",
|
||||
"analysis": "Some analysis",
|
||||
},
|
||||
)
|
||||
sub_dummy_signature = b"sub_dummy_signature_9999"
|
||||
sub_part_1 = types.Part(
|
||||
function_call=invalid_submit_call,
|
||||
thought_signature=sub_dummy_signature,
|
||||
)
|
||||
mock_sub_chunk_1 = MagicMock()
|
||||
mock_sub_chunk_1.text = ""
|
||||
mock_sub_chunk_1.function_calls = [invalid_submit_call]
|
||||
mock_sub_chunk_1.candidates = [
|
||||
types.Candidate(content=types.Content(role="model", parts=[sub_part_1]))
|
||||
]
|
||||
|
||||
# Turn 2: Calls submit_answer with valid confidence_score
|
||||
valid_submit_call = types.FunctionCall(
|
||||
name="submit_answer",
|
||||
args={
|
||||
"timeline_events": [
|
||||
{
|
||||
"confidence_score": 0.8,
|
||||
"transcription": "Valid confidence score call",
|
||||
"verification_timestamp_secs": 10.0,
|
||||
}
|
||||
],
|
||||
"summary": "Valid confidence score call",
|
||||
"analysis": "Some analysis",
|
||||
},
|
||||
)
|
||||
mock_sub_chunk_2 = MagicMock()
|
||||
mock_sub_chunk_2.text = ""
|
||||
mock_sub_chunk_2.function_calls = [valid_submit_call]
|
||||
mock_sub_chunk_2.candidates = [
|
||||
types.Candidate(
|
||||
content=types.Content(role="model", parts=[types.Part(function_call=valid_submit_call)])
|
||||
)
|
||||
]
|
||||
|
||||
# Stream generator mocks
|
||||
main_calls = 0
|
||||
sub_calls = 0
|
||||
captured_sub_contents = []
|
||||
|
||||
async def mock_main_stream():
|
||||
nonlocal main_calls
|
||||
if main_calls == 0:
|
||||
main_calls += 1
|
||||
yield mock_main_chunk_1
|
||||
else:
|
||||
yield mock_main_chunk_2
|
||||
|
||||
async def mock_sub_stream():
|
||||
nonlocal sub_calls
|
||||
if sub_calls == 0:
|
||||
sub_calls += 1
|
||||
yield mock_sub_chunk_1
|
||||
else:
|
||||
yield mock_sub_chunk_2
|
||||
|
||||
async def mock_generate_content_stream(model, contents, config):
|
||||
system_instruction = config.system_instruction
|
||||
if "Dynamic Video Analyzer" in str(system_instruction):
|
||||
return mock_main_stream()
|
||||
else:
|
||||
captured_sub_contents.append(list(contents))
|
||||
return mock_sub_stream()
|
||||
|
||||
# Mock Gemini Client
|
||||
mock_client = MagicMock()
|
||||
mock_client.aio = MagicMock()
|
||||
mock_client.aio.files = MagicMock()
|
||||
|
||||
mock_file = MagicMock()
|
||||
mock_file.name = "dummy_cloud_file_123"
|
||||
mock_file.uri = "https://gemini-file-api/dummy_cloud_file_123"
|
||||
mock_file.mime_type = "video/mp4"
|
||||
mock_client.aio.files.upload = AsyncMock(return_value=mock_file)
|
||||
|
||||
mock_f_state = MagicMock()
|
||||
mock_f_state.state.name = "ACTIVE"
|
||||
mock_client.aio.files.get = AsyncMock(return_value=mock_f_state)
|
||||
mock_client.aio.files.delete = AsyncMock()
|
||||
|
||||
mock_client.aio.models = MagicMock()
|
||||
mock_client.aio.models.generate_content_stream = AsyncMock(
|
||||
side_effect=mock_generate_content_stream
|
||||
)
|
||||
|
||||
# Mock file reading for prompts
|
||||
mock_main_prompt = "# Dynamic Video Analyzer..."
|
||||
mock_sub_prompt = "# Sub Agent..."
|
||||
|
||||
def mock_read_text(self, encoding="utf-8"):
|
||||
if "video_analyzer.md" in str(self):
|
||||
return mock_main_prompt
|
||||
elif "video_sub_agent.md" in str(self):
|
||||
return mock_sub_prompt
|
||||
return ""
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.genai.Client",
|
||||
return_value=mock_client,
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.get_controller",
|
||||
return_value=mock_controller,
|
||||
),
|
||||
patch(
|
||||
"pathlib.Path.exists",
|
||||
autospec=True,
|
||||
side_effect=lambda self: not str(self).endswith(".jpg"),
|
||||
),
|
||||
patch(
|
||||
"pathlib.Path.stat",
|
||||
return_value=os.stat_result((0o100644, 0, 0, 0, 0, 0, 1024 * 1024, 0, 0, 0)),
|
||||
),
|
||||
patch("pathlib.Path.read_text", mock_read_text),
|
||||
patch("pathlib.Path.mkdir"),
|
||||
patch("pathlib.Path.touch"),
|
||||
patch("pathlib.Path.is_dir", return_value=True),
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.cleanup_abandoned_gemini_files",
|
||||
AsyncMock(),
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.compress_video_for_api",
|
||||
AsyncMock(return_value=Path("/tmp/video.mp4")),
|
||||
),
|
||||
patch(
|
||||
"asyncio.create_subprocess_exec",
|
||||
AsyncMock(return_value=MagicMock(communicate=AsyncMock(return_value=(b"", b"")))),
|
||||
),
|
||||
):
|
||||
analyzer = VideoAnalyzer(mock_ctx)
|
||||
result, status = await analyzer.run(
|
||||
time_description="from 10s to 15s", purpose="Verify login"
|
||||
)
|
||||
|
||||
assert status == "success"
|
||||
assert sub_calls == 1
|
||||
assert len(analyzer.blackboard_entries) == 1
|
||||
entry = analyzer.blackboard_entries[0]
|
||||
assert entry["confidence_score"] == 0.8
|
||||
assert entry["summary"] == "Valid confidence score call"
|
||||
|
||||
# Verify that sub-agent turn 2 received the model turn with the thought_signature
|
||||
assert len(captured_sub_contents) >= 2
|
||||
sub_turn_2_contents = captured_sub_contents[1]
|
||||
model_turns = [c for c in sub_turn_2_contents if c.role == "model"]
|
||||
assert len(model_turns) == 1
|
||||
assert model_turns[0].parts[0].thought_signature == sub_dummy_signature
|
||||
|
||||
|
||||
# ------------------------------------------------- static sub-agent loop
|
||||
|
||||
|
||||
def _static_media(tmp_path) -> _ChunkMedia:
|
||||
media = _ChunkMedia()
|
||||
media.path = tmp_path / "segment.mp4"
|
||||
media.path.write_bytes(b"video")
|
||||
media.compressed_path = media.path
|
||||
media.actual_start = 0.0
|
||||
media.prompt_with_context = "Watch the toggle."
|
||||
return media
|
||||
|
||||
|
||||
def _valid_static_event() -> dict:
|
||||
return {
|
||||
"start_time": 1.0,
|
||||
"end_time": 2.0,
|
||||
"transcription": "Battery Saver toggled on",
|
||||
"confidence_score": 0.9,
|
||||
"verification_timestamp_secs": 1.5,
|
||||
}
|
||||
|
||||
|
||||
def _scripted_static_turns(*turns):
|
||||
"""Fakes ``_stream_sub_agent_turn`` with ``(text, function_calls)`` per turn."""
|
||||
scripted = list(turns)
|
||||
|
||||
async def fake_turn(analyzer, sub_agent_contents, trace_id):
|
||||
text, function_calls = scripted.pop(0)
|
||||
return text, function_calls, []
|
||||
|
||||
return fake_turn
|
||||
|
||||
|
||||
def _static_analyzer() -> MagicMock:
|
||||
analyzer = MagicMock()
|
||||
analyzer.model_name = "gemini-3.7-flash"
|
||||
analyzer.sub_system_prompt = "system"
|
||||
analyzer.ctx.data_engine = None
|
||||
return analyzer
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_static_submit_answer_without_summary_is_sent_back_then_accepted(tmp_path):
|
||||
first = types.FunctionCall(
|
||||
name="submit_answer", args={"timeline_events": [_valid_static_event()]}
|
||||
)
|
||||
second = types.FunctionCall(
|
||||
name="submit_answer",
|
||||
args={"summary": "s2", "analysis": "a2", "timeline_events": []},
|
||||
)
|
||||
contents: list = []
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
chunk_conversation,
|
||||
"_stream_sub_agent_turn",
|
||||
new=_scripted_static_turns(("", [first]), ("", [second])),
|
||||
),
|
||||
patch.object(chunk_conversation, "_persist_timeline_events", new=AsyncMock()) as persist,
|
||||
):
|
||||
answer = await _drive_sub_agent_loop(
|
||||
_static_analyzer(), contents, None, _static_media(tmp_path), 0.0, 5.0, "verify"
|
||||
)
|
||||
|
||||
assert answer == ("s2", "a2")
|
||||
# Only the accepted answer reaches persistence.
|
||||
persist.assert_awaited_once()
|
||||
assert persist.await_args.args[1] == []
|
||||
rejection = contents[-1]
|
||||
assert rejection.role == "user"
|
||||
assert rejection.parts[0].function_response.response["error"] == _MISSING_SUMMARY_ERROR
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_static_loop_raises_when_no_turn_calls_submit_answer(tmp_path):
|
||||
with (
|
||||
patch.object(
|
||||
chunk_conversation,
|
||||
"_stream_sub_agent_turn",
|
||||
new=_scripted_static_turns(("text only", []), ("still text", [])),
|
||||
),
|
||||
patch.object(chunk_conversation, "_persist_timeline_events", new=AsyncMock()) as persist,
|
||||
pytest.raises(SubAgentAnswerExhausted) as raised,
|
||||
):
|
||||
await _drive_sub_agent_loop(
|
||||
_static_analyzer(), [], None, _static_media(tmp_path), 0.0, 5.0, "verify"
|
||||
)
|
||||
|
||||
assert raised.value.reason == _NO_TOOL_CALL_ERROR
|
||||
persist.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_static_last_turn_timeline_failure_raises_and_persists_nothing(tmp_path):
|
||||
bad = dict(_valid_static_event(), confidence_score="HIGH")
|
||||
out_of_range = dict(_valid_static_event(), verification_timestamp_secs=42.0)
|
||||
first = types.FunctionCall(
|
||||
name="submit_answer", args={"summary": "s1", "timeline_events": [bad]}
|
||||
)
|
||||
second = types.FunctionCall(
|
||||
name="submit_answer", args={"summary": "s2", "timeline_events": [out_of_range]}
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
chunk_conversation,
|
||||
"_stream_sub_agent_turn",
|
||||
new=_scripted_static_turns(("", [first]), ("", [second])),
|
||||
),
|
||||
patch.object(chunk_conversation, "_persist_timeline_events", new=AsyncMock()) as persist,
|
||||
pytest.raises(SubAgentAnswerExhausted) as raised,
|
||||
):
|
||||
await _drive_sub_agent_loop(
|
||||
_static_analyzer(), [], None, _static_media(tmp_path), 0.0, 5.0, "verify"
|
||||
)
|
||||
|
||||
assert "verification_timestamp_secs" in raised.value.reason
|
||||
assert "42.0" in str(raised.value)
|
||||
persist.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exhausted_static_chunk_is_not_committed_to_the_blackboard(tmp_path):
|
||||
analyzer = _static_analyzer()
|
||||
uploaded = SimpleNamespace(name="files/abc", uri="https://files/abc", mime_type="video/mp4")
|
||||
analyzer.upload_and_poll_file = AsyncMock(return_value=uploaded)
|
||||
analyzer.client.aio.files.delete = AsyncMock()
|
||||
analyzer.cloud_files_to_cleanup = set()
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
chunk_conversation,
|
||||
"_stream_sub_agent_turn",
|
||||
new=_scripted_static_turns(("text only", []), ("still text", [])),
|
||||
),
|
||||
pytest.raises(SubAgentAnswerExhausted),
|
||||
):
|
||||
await _run_native_chunk_conversation(
|
||||
analyzer, _static_media(tmp_path), 0.0, 5.0, 0.0, 5.0, "verify", "lease-1"
|
||||
)
|
||||
|
||||
analyzer.blackboard.complete_segment.assert_not_called()
|
||||
analyzer.client.aio.files.delete.assert_awaited_once_with(name="files/abc")
|
||||
@@ -50,9 +50,9 @@ def test_video_analyzer_universal_engine_detection(mock_context):
|
||||
"""Test that VideoAnalyzer routes to Universal Engine when non-Gemini model is set."""
|
||||
from artemis.config import settings
|
||||
|
||||
with patch.object(settings, "GOOGLE_API_KEY", None):
|
||||
with patch.object(settings, "OPENAI_API_KEY", None):
|
||||
agent = VideoAnalyzer(mock_context)
|
||||
assert agent.use_native_gemini is False
|
||||
assert not hasattr(agent, "_ensure_genai_client")
|
||||
assert agent.model_name == "claude-3-7-sonnet"
|
||||
|
||||
|
||||
|
||||
@@ -1,157 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Native Gemini audio agent loop: only accepted answers reach the blackboard."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
from artemis.agents.video_analyzer.audio_native import (
|
||||
_BAD_CONFIDENCE_ERROR,
|
||||
_AudioMedia,
|
||||
_drive_audio_agent_loop,
|
||||
_run_native_audio_conversation,
|
||||
)
|
||||
from artemis.agents.video_analyzer.chunk_conversation import (
|
||||
_MISSING_SUMMARY_ERROR,
|
||||
_NO_TOOL_CALL_ERROR,
|
||||
SUB_AGENT_MAX_TURNS,
|
||||
)
|
||||
from artemis.agents.video_analyzer.reliability import (
|
||||
SubAgentAnswerExhausted,
|
||||
VideoFailureCategory,
|
||||
classify_video_failure,
|
||||
)
|
||||
from artemis.agents.video_analyzer.universal_tools import build_submit_answer_declaration
|
||||
from google.genai import types
|
||||
import pytest
|
||||
|
||||
|
||||
def _audio_response(text: str, function_calls: list) -> SimpleNamespace:
|
||||
"""Fakes a non-streamed ``generate_content`` response for one audio turn."""
|
||||
parts = [types.Part(function_call=fc) for fc in function_calls]
|
||||
if text:
|
||||
parts.append(types.Part.from_text(text=text))
|
||||
return SimpleNamespace(
|
||||
text=text,
|
||||
function_calls=function_calls,
|
||||
candidates=[SimpleNamespace(content=SimpleNamespace(parts=parts))],
|
||||
)
|
||||
|
||||
|
||||
def _audio_analyzer(*turns) -> MagicMock:
|
||||
"""Analyzer stub whose Gemini client answers the scripted ``(text, calls)`` turns."""
|
||||
analyzer = MagicMock()
|
||||
analyzer.model_name = "gemini-3.7-flash"
|
||||
analyzer.audio_system_prompt = "system"
|
||||
analyzer.submit_answer_tool = build_submit_answer_declaration()
|
||||
analyzer.model_call_timeout_seconds = 5.0
|
||||
analyzer.ctx.data_engine = None
|
||||
analyzer.client.aio.models.generate_content = AsyncMock(
|
||||
side_effect=[_audio_response(text, calls) for text, calls in turns]
|
||||
)
|
||||
return analyzer
|
||||
|
||||
|
||||
def _audio_media(tmp_path) -> _AudioMedia:
|
||||
media = _AudioMedia()
|
||||
media.audio_path = tmp_path / "segment.wav"
|
||||
media.audio_path.write_bytes(b"audio")
|
||||
media.actual_start = 0.0
|
||||
media.prompt_with_context = "Listen for the chime."
|
||||
return media
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_audio_submit_answer_without_summary_is_sent_back_then_accepted():
|
||||
first = types.FunctionCall(
|
||||
name="submit_answer", args={"summary": " ", "confidence_score": 0.9}
|
||||
)
|
||||
second = types.FunctionCall(
|
||||
name="submit_answer",
|
||||
args={"summary": "chime\nheard", "analysis": "a2", "confidence_score": 0.7},
|
||||
)
|
||||
analyzer = _audio_analyzer(("", [first]), ("", [second]))
|
||||
contents: list = []
|
||||
|
||||
answer = await _drive_audio_agent_loop(analyzer, contents)
|
||||
|
||||
assert answer == ("chime heard", "a2", 0.7)
|
||||
assert analyzer.client.aio.models.generate_content.await_count == 2
|
||||
# The blank summary went back to the model as a function_response error.
|
||||
rejections = [
|
||||
c
|
||||
for c in contents
|
||||
if c.role == "user" and c.parts and c.parts[0].function_response is not None
|
||||
]
|
||||
assert len(rejections) == 1
|
||||
assert rejections[0].parts[0].function_response.response["error"] == _MISSING_SUMMARY_ERROR
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_audio_loop_raises_when_no_turn_calls_submit_answer():
|
||||
analyzer = _audio_analyzer(("text only", []), ("still text", []))
|
||||
|
||||
with pytest.raises(SubAgentAnswerExhausted) as raised:
|
||||
await _drive_audio_agent_loop(analyzer, [])
|
||||
|
||||
assert raised.value.turns == SUB_AGENT_MAX_TURNS
|
||||
assert raised.value.reason == _NO_TOOL_CALL_ERROR
|
||||
assert "No summary provided" not in str(raised.value)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_audio_loop_raises_when_confidence_score_is_never_valid():
|
||||
first = types.FunctionCall(
|
||||
name="submit_answer", args={"summary": "s1", "confidence_score": "HIGH"}
|
||||
)
|
||||
second = types.FunctionCall(
|
||||
name="submit_answer", args={"summary": "s2", "confidence_score": 1.5}
|
||||
)
|
||||
analyzer = _audio_analyzer(("", [first]), ("", [second]))
|
||||
contents: list = []
|
||||
|
||||
with pytest.raises(SubAgentAnswerExhausted) as raised:
|
||||
await _drive_audio_agent_loop(analyzer, contents)
|
||||
|
||||
assert raised.value.reason == _BAD_CONFIDENCE_ERROR
|
||||
assert contents[-1].role == "user"
|
||||
assert contents[-1].parts[0].function_response.response["error"] == _BAD_CONFIDENCE_ERROR
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exhausted_audio_segment_is_not_committed_and_cloud_file_is_deleted(tmp_path):
|
||||
analyzer = _audio_analyzer(("text only", []), ("still text", []))
|
||||
uploaded = SimpleNamespace(name="files/abc", uri="https://files/abc", mime_type="audio/wav")
|
||||
analyzer.upload_and_poll_file = AsyncMock(return_value=uploaded)
|
||||
analyzer.client.aio.files.delete = AsyncMock()
|
||||
analyzer.cloud_files_to_cleanup = {"files/abc"}
|
||||
analyzer._record_blackboard_entry = MagicMock()
|
||||
analyzer.blackboard = MagicMock()
|
||||
|
||||
with pytest.raises(SubAgentAnswerExhausted):
|
||||
await _run_native_audio_conversation(
|
||||
analyzer, _audio_media(tmp_path), 0.0, 5.0, 0.0, 5.0, "chime", "lease-1"
|
||||
)
|
||||
|
||||
analyzer.blackboard.complete_segment.assert_not_called()
|
||||
analyzer._record_blackboard_entry.assert_not_called()
|
||||
analyzer.client.aio.files.delete.assert_awaited_once_with(name="files/abc")
|
||||
assert analyzer.cloud_files_to_cleanup == set()
|
||||
|
||||
|
||||
def test_sub_agent_exhaustion_is_classified_unknown():
|
||||
failure = classify_video_failure(SubAgentAnswerExhausted(2, "x"))
|
||||
|
||||
assert failure.category is VideoFailureCategory.UNKNOWN
|
||||
@@ -143,7 +143,7 @@ def analyzer_context():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_spawn_sub_agent_reuses_complete_persistent_coverage(analyzer_context):
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.OPENAI_API_KEY", None):
|
||||
analyzer = VideoAnalyzer(analyzer_context)
|
||||
claim = analyzer.blackboard.claim_segment(10.0, 20.0, "verify login")
|
||||
analyzer.blackboard.complete_segment(
|
||||
@@ -165,7 +165,7 @@ async def test_spawn_sub_agent_reuses_complete_persistent_coverage(analyzer_cont
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_spawn_sub_agent_returns_partial_and_keeps_prior_success(analyzer_context):
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.OPENAI_API_KEY", None):
|
||||
analyzer = VideoAnalyzer(analyzer_context)
|
||||
claim = analyzer.blackboard.claim_segment(0.0, 60.0, "summarize flow")
|
||||
analyzer.blackboard.complete_segment(
|
||||
@@ -207,7 +207,7 @@ async def test_video_coordinator_retries_transient_failures():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_audio_analysis_reuses_exact_completed_coverage(analyzer_context):
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.OPENAI_API_KEY", None):
|
||||
analyzer = VideoAnalyzer(analyzer_context)
|
||||
claim = analyzer.blackboard.claim_segment(2.0, 6.0, "hear notification", modality="audio")
|
||||
analyzer.blackboard.complete_segment(
|
||||
|
||||
@@ -80,7 +80,7 @@ def test_circuit_breaker_opens_and_half_opens():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_timeout_chunk_is_bisected_and_successful_leaves_survive():
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.OPENAI_API_KEY", None):
|
||||
analyzer = VideoAnalyzer(_context())
|
||||
|
||||
async def analyze(start: float, end: float, query: str) -> str:
|
||||
@@ -98,7 +98,7 @@ async def test_timeout_chunk_is_bisected_and_successful_leaves_survive():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_universal_primary_failure_uses_configured_fallback():
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.OPENAI_API_KEY", None):
|
||||
analyzer = VideoAnalyzer(_context())
|
||||
|
||||
primary = MagicMock()
|
||||
@@ -126,57 +126,8 @@ async def test_universal_primary_failure_uses_configured_fallback():
|
||||
assert get_model.call_args_list[1].kwargs["use_fallback"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_chunk_exhaustion_commits_universal_fallback(tmp_path):
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
|
||||
analyzer = VideoAnalyzer(_context())
|
||||
analyzer.use_native_gemini = True
|
||||
raw_video = tmp_path / "segment.mp4"
|
||||
raw_video.write_bytes(b"video")
|
||||
extracted = SimpleNamespace(
|
||||
success=True,
|
||||
video_path=raw_video,
|
||||
actual_start_relative_time=0.0,
|
||||
duration_seconds=5.0,
|
||||
warning=None,
|
||||
)
|
||||
controller = SimpleNamespace(extract_segment_metadata=AsyncMock(return_value=extracted))
|
||||
fallback_text = "[from 0.0s to 5.0s] Summary: fallback recovered Analysis: grounded"
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.get_controller",
|
||||
return_value=controller,
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.compress_video_for_api",
|
||||
new=AsyncMock(return_value=raw_video),
|
||||
),
|
||||
patch.object(
|
||||
analyzer,
|
||||
"upload_and_poll_file",
|
||||
new=AsyncMock(side_effect=RuntimeError("503 provider unavailable")),
|
||||
) as upload,
|
||||
patch.object(
|
||||
analyzer,
|
||||
"_exec_single_chunk_universal",
|
||||
new=AsyncMock(return_value=fallback_text),
|
||||
) as fallback,
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.asyncio.sleep",
|
||||
new=AsyncMock(),
|
||||
),
|
||||
):
|
||||
result = await analyzer._exec_single_chunk(0.0, 5.0, "verify transition")
|
||||
|
||||
assert result == fallback_text
|
||||
assert upload.await_count == 2
|
||||
fallback.assert_awaited_once()
|
||||
assert analyzer.blackboard.missing_intervals(0.0, 5.0, "verify transition") == []
|
||||
|
||||
|
||||
def test_action_timestamps_drive_dense_sampling():
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.OPENAI_API_KEY", None):
|
||||
analyzer = VideoAnalyzer(_context())
|
||||
step = SimpleNamespace(timestamp=104.0, action_taken={"name": "tap"})
|
||||
storage = SimpleNamespace(get_steps=lambda _session: [step])
|
||||
@@ -304,7 +255,7 @@ class _AgenticRejected(Exception):
|
||||
@pytest.mark.asyncio
|
||||
async def test_pending_chunk_sized_for_agentic_ceiling_is_not_run_statically(tmp_path):
|
||||
"""A chunk planned at 600s whose run degraded before it started is handed back."""
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.OPENAI_API_KEY", None):
|
||||
analyzer = VideoAnalyzer(_context())
|
||||
analyzer.use_native_gemini = True
|
||||
# A sibling chunk already flipped the run to static (60s ceiling).
|
||||
@@ -318,7 +269,7 @@ async def test_pending_chunk_sized_for_agentic_ceiling_is_not_run_statically(tmp
|
||||
return_value=controller,
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.chunk_native._run_native_chunk_conversation",
|
||||
"artemis.agents.video_analyzer.chunk_runner._prepare_chunk_media",
|
||||
new=AsyncMock(),
|
||||
) as static_path,
|
||||
patch.object(analyzer, "_exec_single_chunk_universal", new=AsyncMock()) as universal,
|
||||
@@ -335,83 +286,10 @@ async def test_pending_chunk_sized_for_agentic_ceiling_is_not_run_statically(tmp
|
||||
assert analyzer.blackboard.missing_intervals(0.0, 150.0, "verify toggle") == [(0.0, 150.0)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sibling_chunk_degraded_mid_run_is_replanned_into_static_pieces(tmp_path):
|
||||
"""Two 600s chunks: the first is rejected while the second is still in media prep."""
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
|
||||
analyzer = VideoAnalyzer(_context())
|
||||
analyzer.use_native_gemini = True
|
||||
analyzer.video_processing = "agentic"
|
||||
analyzer.chunk_size_seconds = analyzer.agentic_chunk_size_seconds
|
||||
assert analyzer.chunk_size_seconds == 600.0
|
||||
|
||||
raw_video = tmp_path / "segment.mp4"
|
||||
raw_video.write_bytes(b"video")
|
||||
first_rejected = asyncio.Event()
|
||||
|
||||
async def extract(start: float, end: float):
|
||||
if start >= 600.0 and analyzer.video_processing == "agentic":
|
||||
# The second 600s chunk is mid-prep when the first one degrades.
|
||||
await first_rejected.wait()
|
||||
return SimpleNamespace(
|
||||
success=True,
|
||||
video_path=raw_video,
|
||||
actual_start_relative_time=start,
|
||||
duration_seconds=end - start,
|
||||
warning=None,
|
||||
)
|
||||
|
||||
controller = SimpleNamespace(extract_segment_metadata=AsyncMock(side_effect=extract))
|
||||
agentic_calls: list[tuple[float, float]] = []
|
||||
|
||||
async def agentic(analyzer_, media, cs, ce, *rest):
|
||||
agentic_calls.append((cs, ce))
|
||||
first_rejected.set()
|
||||
raise _AgenticRejected("400 INVALID_ARGUMENT: processing 'agentic' is not supported")
|
||||
|
||||
async def static_answer(analyzer_, media, cs, ce, *rest):
|
||||
return f"[from {cs:.1f}s to {ce:.1f}s] Summary: static Analysis: ok"
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.get_controller",
|
||||
return_value=controller,
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.compress_video_for_api",
|
||||
new=AsyncMock(return_value=raw_video),
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.chunk_native.run_agentic_chunk_conversation",
|
||||
new=AsyncMock(side_effect=agentic),
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.chunk_native._run_native_chunk_conversation",
|
||||
new=AsyncMock(side_effect=static_answer),
|
||||
) as static_path,
|
||||
patch.object(analyzer, "_exec_single_chunk_universal", new=AsyncMock()) as universal,
|
||||
patch("artemis.agents.video_analyzer.video_analyzer._record_llm_event"),
|
||||
):
|
||||
result = await analyzer.exec_spawn_sub_agent(0.0, 1200.0, "verify toggle")
|
||||
|
||||
# Only the first chunk ever reached the agentic API; the second never ran
|
||||
# its 600s clip statically and neither went to the universal fallback.
|
||||
assert agentic_calls == [(0.0, 600.0)]
|
||||
universal.assert_not_awaited()
|
||||
segments = sorted((call.args[2], call.args[3]) for call in static_path.await_args_list)
|
||||
assert len(segments) == 20
|
||||
assert segments[0] == (0.0, 60.0)
|
||||
assert segments[-1] == (1140.0, 1200.0)
|
||||
assert all(ce - cs <= 60.0 for cs, ce in segments)
|
||||
assert "PARTIAL" not in result
|
||||
assert result.count("Summary: static") == 20
|
||||
assert analyzer.video_processing == "static"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_short_degraded_chunk_is_retried_statically_once():
|
||||
"""A 5s chunk (below min_chunk*2) is re-run as-is instead of failing."""
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.OPENAI_API_KEY", None):
|
||||
analyzer = VideoAnalyzer(_context())
|
||||
calls: list[tuple[float, float]] = []
|
||||
|
||||
@@ -436,7 +314,7 @@ async def test_short_degraded_chunk_is_retried_statically_once():
|
||||
@pytest.mark.asyncio
|
||||
async def test_static_retry_of_degraded_chunk_is_bounded():
|
||||
"""A second degrade on the same interval is terminal: no loop, no bisect."""
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.OPENAI_API_KEY", None):
|
||||
analyzer = VideoAnalyzer(_context())
|
||||
calls: list[tuple[float, float]] = []
|
||||
|
||||
|
||||
Reference in new issue
Block a user