feat: add agentic video analysis and clarify action results

Report device actions as dispatched and preserve full failure details. Route
Flash through McpActionExecutor and use ToolFailure across helper tools and
agent loops instead of inferring failure from response text.

Require model-provided descriptions for coordinate targets. Keep observed
element metadata separate from those descriptions in safety checks, visual
summaries, history, and frontend labels.

Add Gemini Interactions API video analysis with configurable processing mode,
chunk size, and timeouts. Centralize Google provider capabilities and token
usage normalization, and update google-genai to 2.22.0.

Preserve system instructions on follow-up interactions, replan pending chunks
when falling back to static processing, and reject unanswered native video
and audio segments instead of caching placeholder results. Keep helper-tool
failure status intact through history tools and committee calls.

Validation: 1,762 Python unit tests passed (3 skipped), 40 frontend formatter
tests passed, and the production UI build completed. Ruff lint, formatting,
the quality ratchet, and git diff checks passed.
This commit is contained in:
somew1nd committed 2026-09-08 17:05:54 -07:00
1 parent 611d1a0947
commit a45daf23f8
120 files changed
+6301 -1710

No files matched your search

+162
View File
@@ -3,6 +3,7 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
import asyncio
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
@@ -10,6 +11,7 @@ from langchain_core.messages import AIMessage, HumanMessage
import pytest
from artemis.agents.video_analyzer.reliability import (
AgenticVideoDegraded,
VideoCircuitBreaker,
VideoFailureCategory,
classify_video_failure,
@@ -290,3 +292,163 @@ async def test_invoke_with_retry_caps_unknown_failures_by_policy():
await va._invoke_with_retry(op, "test-op", max_attempts=5)
# UNKNOWN failures follow the shared policy, not the caller's outer cap.
assert calls == retry_policy_for(FailureCategory.UNKNOWN).max_attempts
# ------------------------------------------------ mid-run agentic degrade
class _AgenticRejected(Exception):
status_code = 400
@pytest.mark.asyncio
async def test_pending_chunk_sized_for_agentic_ceiling_is_not_run_statically(tmp_path):
"""A chunk planned at 600s whose run degraded before it started is handed back."""
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
analyzer = VideoAnalyzer(_context())
analyzer.use_native_gemini = True
# A sibling chunk already flipped the run to static (60s ceiling).
analyzer.video_processing = "static"
analyzer.chunk_size_seconds = analyzer.static_chunk_size_seconds
controller = SimpleNamespace(extract_segment_metadata=AsyncMock())
with (
patch(
"artemis.agents.video_analyzer.video_analyzer.get_controller",
return_value=controller,
),
patch(
"artemis.agents.video_analyzer.chunk_native._run_native_chunk_conversation",
new=AsyncMock(),
) as static_path,
patch.object(analyzer, "_exec_single_chunk_universal", new=AsyncMock()) as universal,
pytest.raises(AgenticVideoDegraded) as raised,
):
await analyzer._exec_single_chunk(0.0, 150.0, "verify toggle")
# Refused before any extraction/upload; no static or fallback analysis.
controller.extract_segment_metadata.assert_not_awaited()
static_path.assert_not_awaited()
universal.assert_not_awaited()
assert classify_video_failure(raised.value).should_split
# The lease is released so the re-planned pieces can be claimed.
assert analyzer.blackboard.missing_intervals(0.0, 150.0, "verify toggle") == [(0.0, 150.0)]
@pytest.mark.asyncio
async def test_sibling_chunk_degraded_mid_run_is_replanned_into_static_pieces(tmp_path):
"""Two 600s chunks: the first is rejected while the second is still in media prep."""
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
analyzer = VideoAnalyzer(_context())
analyzer.use_native_gemini = True
analyzer.video_processing = "agentic"
analyzer.chunk_size_seconds = analyzer.agentic_chunk_size_seconds
assert analyzer.chunk_size_seconds == 600.0
raw_video = tmp_path / "segment.mp4"
raw_video.write_bytes(b"video")
first_rejected = asyncio.Event()
async def extract(start: float, end: float):
if start >= 600.0 and analyzer.video_processing == "agentic":
# The second 600s chunk is mid-prep when the first one degrades.
await first_rejected.wait()
return SimpleNamespace(
success=True,
video_path=raw_video,
actual_start_relative_time=start,
duration_seconds=end - start,
warning=None,
)
controller = SimpleNamespace(extract_segment_metadata=AsyncMock(side_effect=extract))
agentic_calls: list[tuple[float, float]] = []
async def agentic(analyzer_, media, cs, ce, *rest):
agentic_calls.append((cs, ce))
first_rejected.set()
raise _AgenticRejected("400 INVALID_ARGUMENT: processing 'agentic' is not supported")
async def static_answer(analyzer_, media, cs, ce, *rest):
return f"[from {cs:.1f}s to {ce:.1f}s] Summary: static Analysis: ok"
with (
patch(
"artemis.agents.video_analyzer.video_analyzer.get_controller",
return_value=controller,
),
patch(
"artemis.agents.video_analyzer.video_analyzer.compress_video_for_api",
new=AsyncMock(return_value=raw_video),
),
patch(
"artemis.agents.video_analyzer.chunk_native.run_agentic_chunk_conversation",
new=AsyncMock(side_effect=agentic),
),
patch(
"artemis.agents.video_analyzer.chunk_native._run_native_chunk_conversation",
new=AsyncMock(side_effect=static_answer),
) as static_path,
patch.object(analyzer, "_exec_single_chunk_universal", new=AsyncMock()) as universal,
patch("artemis.agents.video_analyzer.video_analyzer._record_llm_event"),
):
result = await analyzer.exec_spawn_sub_agent(0.0, 1200.0, "verify toggle")
# Only the first chunk ever reached the agentic API; the second never ran
# its 600s clip statically and neither went to the universal fallback.
assert agentic_calls == [(0.0, 600.0)]
universal.assert_not_awaited()
segments = sorted((call.args[2], call.args[3]) for call in static_path.await_args_list)
assert len(segments) == 20
assert segments[0] == (0.0, 60.0)
assert segments[-1] == (1140.0, 1200.0)
assert all(ce - cs <= 60.0 for cs, ce in segments)
assert "PARTIAL" not in result
assert result.count("Summary: static") == 20
assert analyzer.video_processing == "static"
@pytest.mark.asyncio
async def test_short_degraded_chunk_is_retried_statically_once():
"""A 5s chunk (below min_chunk*2) is re-run as-is instead of failing."""
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
analyzer = VideoAnalyzer(_context())
calls: list[tuple[float, float]] = []
async def analyze(start: float, end: float, query: str) -> str:
calls.append((start, end))
if len(calls) == 1:
analyzer.video_processing = "static"
analyzer.chunk_size_seconds = analyzer.static_chunk_size_seconds
raise AgenticVideoDegraded("agentic rejected")
return f"[from {start:.1f}s to {end:.1f}s] Summary: static {query}"
with patch.object(analyzer, "_exec_single_chunk", side_effect=analyze) as child:
result = await analyzer.exec_spawn_sub_agent(0.0, 5.0, "find toggle")
assert child.await_count == 2
assert calls == [(0.0, 5.0), (0.0, 5.0)]
assert "PARTIAL" not in result
assert "failed" not in result.lower()
assert result.count("Summary: static find toggle") == 1
@pytest.mark.asyncio
async def test_static_retry_of_degraded_chunk_is_bounded():
"""A second degrade on the same interval is terminal: no loop, no bisect."""
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
analyzer = VideoAnalyzer(_context())
calls: list[tuple[float, float]] = []
async def analyze(start: float, end: float, query: str) -> str:
calls.append((start, end))
raise AgenticVideoDegraded("agentic rejected again")
with patch.object(analyzer, "_exec_single_chunk", side_effect=analyze) as child:
# 30s is above min_chunk*2, so a bisect would otherwise be legal.
result = await analyzer.exec_spawn_sub_agent(0.0, 30.0, "find toggle")
assert child.await_count == 2
assert calls == [(0.0, 30.0), (0.0, 30.0)]
assert result.startswith("All sub-agent chunks failed")
assert "0.0s-30.0s [bad_request]" in result