# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import time from unittest.mock import AsyncMock, MagicMock, patch import pytest from artemis.agents.video_analyzer.blackboard import ( VideoBlackboard, get_video_blackboard, ) from artemis.agents.video_analyzer.video_analyzer import VideoAnalyzer, _invoke_with_retry from artemis.context import ArtemisContext def test_blackboard_persists_segments_observations_and_evidence(tmp_path): db_path = tmp_path / "trace.db" evidence_dir = tmp_path / "video_blackboard" / "evidence" source_image = tmp_path / "temporary.jpg" source_image.write_bytes(b"jpeg evidence") board = VideoBlackboard("video:recording-1", db_path=db_path, evidence_dir=evidence_dir) stored = board.add_observation( { "start": 1.0, "end": 1.5, "target": "verify login", "summary": "Login screen became visible.", "confidence_score": 0.95, "screenshot": str(source_image), } ) # A duplicate commit is idempotent. board.add_observation(stored) claim = board.claim_segment(0.0, 5.0, "verify login") assert claim.state == "claimed" board.complete_segment( 0.0, 5.0, "verify login", claim.lease_owner, "Login succeeded.", "The authenticated home screen appeared.", ) reopened = VideoBlackboard("video:recording-1", db_path=db_path, evidence_dir=evidence_dir) observations = reopened.list_observations() assert len(observations) == 1 persisted_evidence = observations[0]["screenshot"] assert persisted_evidence != str(source_image) assert (evidence_dir / f"{stored['observation_id']}.jpg").read_bytes() == b"jpeg evidence" assert reopened.missing_intervals(0.0, 5.0, "verify login") == [] cached = reopened.claim_segment(0.0, 5.0, "verify login") assert cached.state == "cached" assert cached.summary == "Login succeeded." assert any(entry.get("kind") == "segment_result" for entry in reopened.list_ledger_entries()) def test_blackboard_calculates_only_uncovered_intervals(tmp_path): board = VideoBlackboard("video:coverage", db_path=tmp_path / "coverage.db") for start, end in ((0.0, 10.0), (8.0, 20.0), (30.0, 40.0)): claim = board.claim_segment(start, end, "find spinner") board.complete_segment( start, end, "find spinner", claim.lease_owner, f"Analyzed {start}-{end}" ) assert board.missing_intervals(0.0, 50.0, "find spinner") == [ (20.0, 30.0), (40.0, 50.0), ] # Different questions can use the observations as context but do not share # query-specific completion coverage. assert board.missing_intervals(0.0, 10.0, "count spinner frames") == [(0.0, 10.0)] def test_blackboard_lease_suppresses_duplicates_and_is_reclaimable(): board = VideoBlackboard("video:lease", lease_seconds=0.02) first = board.claim_segment(0.0, 5.0, "verify tap") assert first.state == "claimed" assert board.claim_segment(0.0, 5.0, "verify tap").state == "in_progress" time.sleep(0.03) reclaimed = board.claim_segment(0.0, 5.0, "verify tap") assert reclaimed.state == "claimed" assert reclaimed.lease_owner != first.lease_owner board.fail_segment(0.0, 5.0, "verify tap", reclaimed.lease_owner, "temporary provider outage") assert board.claim_segment(0.0, 5.0, "verify tap").state == "claimed" def test_get_video_blackboard_is_shared_by_context(): ctx = MagicMock(spec=ArtemisContext) ctx._video_blackboard = None ctx.data_engine = None ctx.execution_setup = None ctx.device = MagicMock() ctx.device.device_id = "device-1" first = get_video_blackboard(ctx) first.add_observation( { "start": 0.0, "end": 1.0, "target": "shared", "summary": "Persisted in context.", } ) second = get_video_blackboard(ctx) assert first is second assert second.list_observations()[0]["summary"] == "Persisted in context." @pytest.fixture def analyzer_context(): ctx = MagicMock(spec=ArtemisContext) ctx._video_blackboard = None ctx._mobile_controller = None ctx.data_engine = None ctx.execution_setup = None ctx.agent_config = None ctx.device = MagicMock() ctx.device.device_id = "device-analyzer" ctx.llm_config = MagicMock() ctx.llm_config.utils.video_analyzer.model = "universal-test-model" ctx.llm_config.utils.video_analyzer.temperature = 0.2 return ctx @pytest.mark.asyncio async def test_spawn_sub_agent_reuses_complete_persistent_coverage(analyzer_context): with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None): analyzer = VideoAnalyzer(analyzer_context) claim = analyzer.blackboard.claim_segment(10.0, 20.0, "verify login") analyzer.blackboard.complete_segment( 10.0, 20.0, "verify login", claim.lease_owner, "Login succeeded.", "Home screen appeared.", ) with patch.object(analyzer, "_exec_single_chunk", new_callable=AsyncMock) as child: result = await analyzer.exec_spawn_sub_agent(10.0, 20.0, "verify login") child.assert_not_awaited() assert result.startswith("CACHED VIDEO ANALYSIS:") assert "Login succeeded" in result @pytest.mark.asyncio async def test_spawn_sub_agent_returns_partial_and_keeps_prior_success(analyzer_context): with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None): analyzer = VideoAnalyzer(analyzer_context) claim = analyzer.blackboard.claim_segment(0.0, 60.0, "summarize flow") analyzer.blackboard.complete_segment( 0.0, 60.0, "summarize flow", claim.lease_owner, "First minute completed.", ) with patch.object( analyzer, "_exec_single_chunk", new_callable=AsyncMock, side_effect=RuntimeError("provider unavailable"), ) as child: result = await analyzer.exec_spawn_sub_agent(0.0, 120.0, "summarize flow") child.assert_awaited_once_with(60.0, 120.0, "summarize flow") assert result.startswith("PARTIAL VIDEO ANALYSIS") assert "First minute completed" in result assert analyzer.blackboard.missing_intervals(0.0, 120.0, "summarize flow") == [(60.0, 120.0)] @pytest.mark.asyncio async def test_video_coordinator_retries_transient_failures(): operation = AsyncMock( side_effect=[TimeoutError("timeout one"), ConnectionError("reset"), "completed"] ) with patch( "artemis.agents.video_analyzer.video_analyzer.asyncio.sleep", new_callable=AsyncMock ) as sleep: result = await _invoke_with_retry(operation, "test coordinator") assert result == "completed" assert operation.await_count == 3 assert sleep.await_count == 2 @pytest.mark.asyncio async def test_audio_analysis_reuses_exact_completed_coverage(analyzer_context): with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None): analyzer = VideoAnalyzer(analyzer_context) claim = analyzer.blackboard.claim_segment( 2.0, 6.0, "hear notification", modality="audio" ) analyzer.blackboard.complete_segment( 2.0, 6.0, "hear notification", claim.lease_owner, "A notification sounded.", "One short bell was audible.", modality="audio", ) with patch( "artemis.agents.video_analyzer.video_analyzer.get_controller" ) as controller: result = await analyzer.exec_analyze_audio_only( 2.0, 6.0, "hear notification" ) controller.assert_not_called() assert result.startswith("CACHED AUDIO ANALYSIS:") assert "notification sounded" in result def test_blackboard_metrics_include_structured_failures(tmp_path): board = VideoBlackboard("video:metrics", db_path=tmp_path / "metrics.db") claim = board.claim_segment( 0.0, 5.0, "inspect", model_name="vlm-primary", source_generation=2, ) board.fail_segment( 0.0, 5.0, "inspect", claim.lease_owner, "timeout", error_category="timeout", ) assert board.metrics()["segments"] == {"retryable_failed": 1} assert board.metrics()["failure_categories"] == {"timeout": 1} def test_session_deletion_removes_video_blackboard_rows(tmp_path): from uuid import uuid4 from artemis.data_engine.engine import DataEngine ctx = MagicMock(spec=ArtemisContext) ctx.execution_setup = MagicMock(traces_path=str(tmp_path / "traces")) ctx.device = None engine = DataEngine(ctx) session_id = engine.start_session(goal="blackboard cleanup") video_id = uuid4() recording = tmp_path / "recording.mp4" recording.write_bytes(b"video") engine.record_video_start(video_id, "device", recording) board = VideoBlackboard(f"video:{video_id}", db_path=engine.storage.db_path) board.add_observation( {"start": 0.0, "end": 1.0, "target": "cleanup", "summary": "evidence"} ) assert board.list_observations() engine.storage.delete_session(session_id) reopened = VideoBlackboard(f"video:{video_id}", db_path=engine.storage.db_path) assert reopened.list_observations() == []