feat: harden video analysis and runtime resilience

2026-08-25 20:58:25 -07:00
1 parent 5ada543a09
commit bb16991f2c
@@ -0,0 +1,193 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock, patch
+
+from langchain_core.messages import AIMessage, HumanMessage
+import pytest
+
+from artemis.agents.video_analyzer.reliability import (
+    VideoCircuitBreaker,
+    VideoFailureCategory,
+    classify_video_failure,
+)
+from artemis.agents.video_analyzer.video_analyzer import VideoAnalyzer
+from artemis.context import ArtemisContext
+
+
+def _context() -> MagicMock:
+    ctx = MagicMock(spec=ArtemisContext)
+    ctx._video_blackboard = None
+    ctx._video_circuit_breaker = None
+    ctx._mobile_controller = None
+    ctx.execution_setup = None
+    ctx.data_engine = None
+    ctx.agent_config = SimpleNamespace(
+        video_analyzer=SimpleNamespace(
+            enable_ledger=True,
+            chunk_size_seconds=60.0,
+            min_chunk_seconds=4.0,
+            max_split_depth=4,
+            circuit_breaker_threshold=2,
+            circuit_breaker_cooldown_seconds=30.0,
+            action_window_seconds=2.0,
+            dense_action_fps=4.0,
+            max_dense_action_frames=24,
+        )
+    )
+    ctx.device = SimpleNamespace(device_id="reliability-device")
+    ctx.llm_config = MagicMock()
+    ctx.llm_config.utils.video_analyzer.model = "primary-video-model"
+    ctx.llm_config.utils.video_analyzer.temperature = 0.2
+    return ctx
+
+
+def test_failure_classifier_chooses_safe_recovery():
+    outage = classify_video_failure(RuntimeError("503 service unavailable"))
+    assert outage.category == VideoFailureCategory.PROVIDER_UNAVAILABLE
+    assert outage.retryable and outage.should_fallback
+    assert not outage.should_split
+
+    timeout = classify_video_failure(TimeoutError("clip timed out"))
+    assert timeout.category == VideoFailureCategory.TIMEOUT
+    assert timeout.should_split and timeout.should_fallback
+
+    invalid = SimpleNamespace(code=400)
+    invalid_error = RuntimeError("bad request")
+    invalid_error.code = invalid.code
+    classified = classify_video_failure(invalid_error)
+    assert classified.category == VideoFailureCategory.BAD_REQUEST
+    assert not classified.retryable
+
+
+def test_circuit_breaker_opens_and_half_opens():
+    breaker = VideoCircuitBreaker(threshold=2, cooldown_seconds=10.0)
+    failure = classify_video_failure(TimeoutError("timeout"))
+    breaker.record_failure("model", failure, now=10.0)
+    assert breaker.allow("model", now=10.1)
+    breaker.record_failure("model", failure, now=11.0)
+    assert not breaker.allow("model", now=15.0)
+    assert breaker.allow("model", now=21.1)
+    assert not breaker.allow("model", now=21.2)
+    breaker.record_success("model")
+    assert breaker.snapshot() == {}
+
+
+@pytest.mark.asyncio
+async def test_timeout_chunk_is_bisected_and_successful_leaves_survive():
+    with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
+        analyzer = VideoAnalyzer(_context())
+
+    async def analyze(start: float, end: float, query: str) -> str:
+        if end - start > 4.0:
+            raise TimeoutError("large clip timed out")
+        return f"[from {start:.1f}s to {end:.1f}s] Summary: recovered {query}"
+
+    with patch.object(analyzer, "_exec_single_chunk", side_effect=analyze) as child:
+        result = await analyzer.exec_spawn_sub_agent(0.0, 16.0, "find transition")
+
+    assert child.await_count == 7
+    assert result.count("recovered find transition") == 4
+    assert "PARTIAL" not in result
+
+
+@pytest.mark.asyncio
+async def test_universal_primary_failure_uses_configured_fallback():
+    with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
+        analyzer = VideoAnalyzer(_context())
+
+    primary = MagicMock()
+    primary_bound = MagicMock()
+    primary_bound.ainvoke = AsyncMock(side_effect=TimeoutError("primary timeout"))
+    primary.bind_tools.return_value = primary_bound
+    fallback = MagicMock()
+    fallback_bound = MagicMock()
+    fallback_bound.ainvoke = AsyncMock(return_value=AIMessage(content="fallback answer"))
+    fallback.bind_tools.return_value = fallback_bound
+
+    with patch(
+        "artemis.agents.video_analyzer.video_analyzer.get_llm",
+        side_effect=[primary, fallback],
+    ) as get_model:
+        response = await analyzer._invoke_universal_model(
+            [HumanMessage(content="inspect")],
+            [],
+            timeout=1.0,
+            label="test",
+        )
+
+    assert response.content == "fallback answer"
+    assert get_model.call_args_list[0].kwargs["is_utils"] is True
+    assert get_model.call_args_list[1].kwargs["use_fallback"] is True
+
+
+@pytest.mark.asyncio
+async def test_native_chunk_exhaustion_commits_universal_fallback(tmp_path):
+    with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
+        analyzer = VideoAnalyzer(_context())
+    analyzer.use_native_gemini = True
+    raw_video = tmp_path / "segment.mp4"
+    raw_video.write_bytes(b"video")
+    extracted = SimpleNamespace(
+        success=True,
+        video_path=raw_video,
+        actual_start_relative_time=0.0,
+        duration_seconds=5.0,
+        warning=None,
+    )
+    controller = SimpleNamespace(
+        extract_segment_metadata=AsyncMock(return_value=extracted)
+    )
+    fallback_text = "[from 0.0s to 5.0s] Summary: fallback recovered Analysis: grounded"
+
+    with (
+        patch(
+            "artemis.agents.video_analyzer.video_analyzer.get_controller",
+            return_value=controller,
+        ),
+        patch(
+            "artemis.agents.video_analyzer.video_analyzer.compress_video_for_api",
+            new=AsyncMock(return_value=raw_video),
+        ),
+        patch.object(
+            analyzer,
+            "upload_and_poll_file",
+            new=AsyncMock(side_effect=RuntimeError("503 provider unavailable")),
+        ) as upload,
+        patch.object(
+            analyzer,
+            "_exec_single_chunk_universal",
+            new=AsyncMock(return_value=fallback_text),
+        ) as fallback,
+        patch(
+            "artemis.agents.video_analyzer.video_analyzer.asyncio.sleep",
+            new=AsyncMock(),
+        ),
+    ):
+        result = await analyzer._exec_single_chunk(0.0, 5.0, "verify transition")
+
+    assert result == fallback_text
+    assert upload.await_count == 2
+    fallback.assert_awaited_once()
+    assert analyzer.blackboard.missing_intervals(0.0, 5.0, "verify transition") == []
+
+
+def test_action_timestamps_drive_dense_sampling():
+    with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
+        analyzer = VideoAnalyzer(_context())
+    step = SimpleNamespace(timestamp=104.0, action_taken={"name": "tap"})
+    storage = SimpleNamespace(get_steps=lambda _session: [step])
+    analyzer.ctx.data_engine = SimpleNamespace(
+        current_session_id="session-1",
+        session_start_time=100.0,
+        storage=storage,
+    )
+
+    offsets = analyzer._dense_action_offsets(0.0, 10.0, actual_start=0.0)
+
+    assert offsets[0] == 2.0
+    assert 4.0 in offsets
+    assert offsets[-1] == 6.0