# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """VideoAnalyzer facade: entry point and shared runtime state. The execution engines live in sibling modules (``chunk_runner``, ``audio_runner``, ``universal_engine``, ``universal_tools``) and resolve their patchable collaborators through THIS module's namespace, so existing ``mock.patch("artemis.agents.video_analyzer.video_analyzer.")`` targets keep working. Keep the imports below in place even when the facade body no longer references them directly. """ import asyncio import base64 from datetime import datetime import glob import json import os from pathlib import Path import re import shutil import tempfile import time from typing import Any from langchain_core.messages import ( BaseMessage, HumanMessage, SystemMessage, ToolMessage, ) from artemis.agents.video_analyzer.conflict_resolution import ( ConflictResolutionService, ) from artemis.agents.video_analyzer.blackboard import get_video_blackboard from artemis.agents.video_analyzer.reliability import ( AgenticVideoDegraded, VideoCircuitBreaker, classify_video_failure, ) from artemis.agents.video_analyzer.universal_tools import ( UNIVERSAL_MAIN_TOOLS, UNIVERSAL_SUBMIT_ANSWER_TOOL, ) from artemis.config import get_temp_dir, settings from artemis.context import ArtemisContext from artemis.controllers.controller_factory import get_controller from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan, trace from artemis.llm.reliability import retry_policy_for from artemis.services.llm import _record_llm_event, _record_llm_retry, get_llm from artemis.utils.logger import get_logger from artemis.utils.video import ( compress_video_for_api, extract_audio_from_video, extract_frames_at_timestamps, extract_keyframes_from_video, get_active_session, ) try: from datetime import UTC except ImportError: UTC = UTC logger = get_logger(__name__) TRANSCODE_SEMAPHORE = asyncio.Semaphore(2) MAIN_AGENT_SEMAPHORE = asyncio.Semaphore(2) API_SEMAPHORE = asyncio.Semaphore(5) SLOWDOWN_THRESHOLD_SECONDS = 30.0 SLOWDOWN_FACTOR = 2.0 async def _invoke_with_retry(operation, label: str, max_attempts: int = 3): """Retry side-effect-free model coordination calls using shared classification. Retry decisions come from ``classify_video_failure`` plus the shared per-category policies in :mod:`artemis.llm.reliability` instead of ad-hoc status-code sets. ``operation`` must be restartable from scratch: each retry re-invokes it and any partially accumulated result (e.g. consumed stream chunks) from the failed attempt is discarded, never surfaced as a complete output. The original exception is re-raised on giving up so downstream ``classify_video_failure`` consumers keep seeing the provider's native error shape. """ attempt = 0 while True: attempt += 1 try: return await operation() except Exception as exc: failure = classify_video_failure(exc) if not failure.retryable: logger.warning( f"{label} non-retryable failure [{failure.category.value}] " f"on attempt {attempt}: {exc}" ) raise policy = retry_policy_for(failure.category) allowed_attempts = min(max_attempts, policy.max_attempts) if attempt >= allowed_attempts: logger.warning( f"{label} exhausted {attempt} attempt(s) [{failure.category.value}]: {exc}" ) raise delay = policy.delay_for(attempt) logger.warning( f"{label} retryable failure [{failure.category.value}] " f"({attempt}/{allowed_attempts}): {exc}; retrying from scratch " f"in {delay:.2f}s (partial output from the failed attempt is discarded)" ) _record_llm_retry( str(exc), delay, attempt=attempt, max_retries=allowed_attempts, source=label, ) await asyncio.sleep(delay) class VideoAnalyzer: def __init__(self, ctx: ArtemisContext): self.ctx = ctx self.blackboard = get_video_blackboard(ctx) self.blackboard_entries = self.blackboard.list_ledger_entries() agent_config = getattr(ctx, "agent_config", None) self.video_config = getattr(agent_config, "video_analyzer", None) self.enable_ledger = getattr( self.video_config, "enable_ledger", getattr(agent_config, "enable_video_ledger", True), ) self.static_chunk_size_seconds = float( getattr(self.video_config, "chunk_size_seconds", settings.ARTEMIS_VIDEO_CHUNK_SECONDS) ) self.chunk_size_seconds = self.static_chunk_size_seconds self.video_processing = "static" self.min_chunk_seconds = float(getattr(self.video_config, "min_chunk_seconds", 4.0)) self.max_split_depth = int(getattr(self.video_config, "max_split_depth", 4)) self.action_window_seconds = float(getattr(self.video_config, "action_window_seconds", 2.0)) self.dense_action_fps = float(getattr(self.video_config, "dense_action_fps", 4.0)) self.max_dense_action_frames = int( getattr(self.video_config, "max_dense_action_frames", 24) ) self.max_retries = int( getattr(self.video_config, "max_retries", settings.ARTEMIS_VIDEO_MAX_RETRIES) ) self.model_call_timeout_seconds = float( getattr( self.video_config, "model_call_timeout_seconds", settings.ARTEMIS_VIDEO_TIMEOUT_SECONDS, ) ) breaker = getattr(ctx, "_video_circuit_breaker", None) if breaker is None: breaker = VideoCircuitBreaker( threshold=int(getattr(self.video_config, "circuit_breaker_threshold", 3)), cooldown_seconds=float( getattr( self.video_config, "circuit_breaker_cooldown_seconds", 60.0, ) ), ) ctx._video_circuit_breaker = breaker self.circuit_breaker = breaker self.local_files_to_cleanup = set() self.local_dirs_to_cleanup = set() self.cloud_files_to_cleanup = set() sub_prompt_path = Path(__file__).parent / "video_sub_agent.md" self.sub_system_prompt = ( sub_prompt_path.read_text(encoding="utf-8") if sub_prompt_path.exists() else "" ) audio_prompt_path = Path(__file__).parent / "video_sub_agent_audio.md" self.audio_system_prompt = ( audio_prompt_path.read_text(encoding="utf-8") if audio_prompt_path.exists() else "" ) self._init_engine() def _get_universal_llm(self, *, use_fallback: bool = False): """Resolve the utility-scoped video model, including its configured fallback.""" return get_llm( self.ctx, name="video_analyzer", is_utils=True, use_fallback=use_fallback, ) async def _invoke_universal_model( self, messages: list[BaseMessage], tools: list[dict[str, Any]], *, timeout: float, label: str, force_fallback: bool = False, ): """Invoke primary then fallback model under a context-scoped circuit breaker.""" primary_key = f"video_analyzer:{self.model_name}:primary" attempts = [True] if force_fallback else [False, True] last_error: BaseException | None = None for use_fallback in attempts: if not use_fallback and not self.circuit_breaker.allow(primary_key): logger.warning(f"{label}: primary video-model circuit is open; using fallback") continue try: llm = self._get_universal_llm(use_fallback=use_fallback) except (ValueError, AttributeError) as config_error: if use_fallback: if last_error is not None: raise last_error raise config_error raise try: bound_llm = llm.bind_tools(tools) response = await asyncio.wait_for(bound_llm.ainvoke(messages), timeout=timeout) if not use_fallback: self.circuit_breaker.record_success(primary_key) return response except BaseException as error: failure = classify_video_failure(error) if not use_fallback: self.circuit_breaker.record_failure(primary_key, failure) last_error = error if use_fallback or not failure.should_fallback: raise logger.warning( f"{label}: {failure.category.value} from primary; trying configured fallback" ) if last_error is not None: raise last_error raise RuntimeError(f"{label}: no video model is available") def _record_reliability_metrics(self) -> None: """Persist a compact blackboard/circuit snapshot into the active trace.""" trace_id = CURRENT_TRACE_ID.get() if not self.ctx.data_engine or not trace_id: return try: self.ctx.data_engine.record_trace( type="video_analysis", name="video_analysis_reliability", payload={ "blackboard": self.blackboard.metrics(), "circuits": self.circuit_breaker.snapshot(), }, parent_trace_id=trace_id, ) except Exception as error: logger.warning(f"Failed to record video reliability metrics: {error}") async def _cleanup_universal_resources(self) -> None: """Release universal-path temporary media without touching durable evidence.""" if os.environ.get("KEEP_VIDEOS") or os.environ.get("ARTEMIS_DEBUG"): return for path in list(self.local_files_to_cleanup): try: if path.exists(): path.unlink() except OSError as error: logger.warning(f"Failed to delete temporary video artifact {path}: {error}") protected = { Path("/").resolve(), Path("/tmp").resolve(), Path("/var/tmp").resolve(), Path.home().resolve(), Path(tempfile.gettempdir()).resolve(), } for directory in sorted( self.local_dirs_to_cleanup, key=lambda value: len(value.parts), reverse=True, ): try: resolved = directory.resolve() if resolved not in protected and directory.exists(): shutil.rmtree(directory, ignore_errors=True) except OSError as error: logger.warning(f"Failed to delete temporary video directory {directory}: {error}") def _action_timestamps(self, start: float, end: float) -> list[float]: """Return test-relative timestamps for recorded actions in an interval.""" engine = self.ctx.data_engine session_id = getattr(engine, "current_session_id", None) if engine else None storage = getattr(engine, "storage", None) if engine else None session_start = getattr(engine, "session_start_time", None) if engine else None if not storage or not session_id or not isinstance(session_start, (int, float)): return [] try: return sorted( { round(float(step.timestamp) - float(session_start), 3) for step in storage.get_steps(session_id) if step.action_taken and start <= float(step.timestamp) - float(session_start) <= end } ) except Exception as error: logger.warning(f"Failed to load action timestamps for video sampling: {error}") return [] def _dense_action_offsets(self, start: float, end: float, actual_start: float) -> list[float]: """Build bounded, video-relative sampling offsets around mobile actions.""" if self.max_dense_action_frames <= 0: return [] interval = 1.0 / max(1.0, self.dense_action_fps) offsets: list[float] = [] for action_time in self._action_timestamps(start, end): window_start = max(start, action_time - self.action_window_seconds) window_end = min(end, action_time + self.action_window_seconds) cursor = window_start while cursor <= window_end + 1e-6: offsets.append(max(0.0, cursor - actual_start)) if len(offsets) >= self.max_dense_action_frames: return offsets cursor += interval return offsets @staticmethod def _audio_content_block(audio_path: Path) -> dict[str, Any]: suffix = audio_path.suffix.lower() mime_type = { ".aac": "audio/aac", ".m4a": "audio/mp4", ".mp3": "audio/mpeg", ".wav": "audio/wav", ".ogg": "audio/ogg", }.get(suffix, "application/octet-stream") return { "type": "audio", "base64": base64.b64encode(audio_path.read_bytes()).decode("utf-8"), "mime_type": mime_type, } def _refresh_blackboard(self) -> None: """Refresh the presentation ledger from persistent video memory.""" self.blackboard_entries = self.blackboard.list_ledger_entries() def _clean_blackboard(self) -> None: """Clean the local view without deleting another agent's durable records.""" self.blackboard_entries = ConflictResolutionService.clean(self.blackboard_entries) def _record_blackboard_entry( self, entry: dict[str, Any], *, modality: str = "video" ) -> dict[str, Any]: """Persist an observation immediately, then update this agent's ledger.""" stored = self.blackboard.add_observation(entry, modality=modality) observation_id = stored.get("observation_id") if not any( existing.get("observation_id") == observation_id for existing in self.blackboard_entries if isinstance(existing, dict) ): self.blackboard_entries.append(stored) return stored @staticmethod def _parse_chunk_result(result: str) -> tuple[str, str | None]: """Extract summary/analysis from the stable child response envelope.""" match = re.search(r"Summary:\s*(.*?)(?:\s+Analysis:\s*(.*))?$", result, re.DOTALL) if not match: return result.strip(), None summary = match.group(1).strip() analysis = match.group(2).strip() if match.group(2) else None return summary, analysis def _init_engine(self) -> None: llm_config = getattr(self.ctx, "llm_config", None) utils_cfg = getattr(llm_config, "utils", None) if llm_config else None llm_cfg = getattr(utils_cfg, "video_analyzer", None) if utils_cfg else None self.model_name = getattr(llm_cfg, "model", None) or settings.OPENAI_MODEL self.video_processing = "static" self.chunk_size_seconds = self.static_chunk_size_seconds @property def blackboard_ledger(self) -> str: self._clean_blackboard() if not self.blackboard_entries: return "No video segments analyzed yet." lines = [] for e in self.blackboard_entries: lines.append(f"{e['start']}s - {e['end']}s: {e['summary']}") return " | ".join(lines) def get_overlapping_warnings(self, start_time: float, end_time: float | None) -> list[dict]: warnings = [] if not isinstance(start_time, (int, float)) or ( end_time is not None and not isinstance(end_time, (int, float)) ): return warnings if end_time is None or (end_time - start_time) <= 0: return warnings req_duration = end_time - start_time for entry in self.blackboard_entries: if isinstance(entry, dict) and entry.get("end") != "unknown": try: if not isinstance(entry.get("start"), (int, float, str)) or not isinstance( entry.get("end"), (int, float, str) ): continue overlap_start = max(start_time, float(entry["start"])) overlap_end = min(end_time, float(entry["end"])) overlap_duration = max(0.0, overlap_end - overlap_start) if (overlap_duration / req_duration) > 0.8: warnings.append( { "target": entry.get("target", ""), "summary": entry.get("summary", ""), } ) except (ValueError, TypeError): pass return warnings @trace(type="tool", name="extract_segment_metadata") async def exec_extract_segment_metadata( self, start_time: float, end_time: float | None = None ) -> str: controller = get_controller(self.ctx) async with TRANSCODE_SEMAPHORE: result = await controller.extract_segment_metadata(start_time, end_time) if result.success and result.video_path: path = Path(result.video_path) self.local_files_to_cleanup.add(path) self.local_dirs_to_cleanup.add(path.parent) size_mb = ( result.file_size_mb if result.file_size_mb is not None else (path.stat().st_size / (1024 * 1024) if path.exists() else 0.0) ) if result.duration_seconds is not None: duration = result.duration_seconds elif end_time is not None: duration = end_time - start_time else: try: duration_cmd = [ "ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "default=noprint_wrappers=1:nokey=1", str(path), ] proc = await asyncio.create_subprocess_exec( *duration_cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) stdout, _ = await proc.communicate() duration = float(stdout.decode().strip()) except Exception as e: logger.warning(f"Failed to resolve duration via ffprobe for {path}: {e}") duration = 0.0 if isinstance(duration, (int, float)): duration = round(duration, 1) return json.dumps( { "duration_seconds": duration, "file_size_mb": ( round(size_mb, 2) if isinstance(size_mb, (int, float)) else size_mb ), } ) raise RuntimeError(f"Failed to get video segment: {result.message}") async def _resolve_open_ended_interval( self, requested_start: float ) -> tuple[float, float | None, str | None]: """Resolve an open-ended request to a stable numeric interval (or an error).""" controller = get_controller(self.ctx) async with TRANSCODE_SEMAPHORE: metadata_result = await controller.extract_segment_metadata(requested_start, None) if not metadata_result.success: return ( requested_start, None, f"Error fetching segment metadata: {metadata_result.message}", ) duration = getattr(metadata_result, "duration_seconds", None) actual_start = getattr(metadata_result, "actual_start_relative_time", requested_start) if not isinstance(duration, (int, float)) or duration <= 0: return ( requested_start, None, "Error fetching segment metadata: resolved duration is unavailable.", ) if not isinstance(actual_start, (int, float)): actual_start = requested_start requested_start = round(float(actual_start), 3) requested_end = round(requested_start + float(duration), 3) if metadata_result.video_path: metadata_path = Path(metadata_result.video_path) self.local_files_to_cleanup.add(metadata_path) self.local_dirs_to_cleanup.add(metadata_path.parent) return requested_start, requested_end, None def _plan_chunks(self, intervals: list[tuple[float, float]]) -> list[tuple[float, float]]: """Cuts intervals into consecutive chunks of at most ``chunk_size_seconds``.""" target_chunk_size = self.chunk_size_seconds chunks: list[tuple[float, float]] = [] for gap_start, gap_end in intervals: cursor = gap_start while cursor < gap_end: chunk_end = min(gap_end, cursor + target_chunk_size) chunks.append((round(cursor, 3), round(chunk_end, 3))) cursor = chunk_end return chunks @trace(type="tool", name="spawn_sub_agent") async def exec_spawn_sub_agent( self, start_time: float, end_time: float | None, specific_query: str ) -> str: if not isinstance(start_time, (int, float)): return "Error: start_time must be a numeric recording-relative timestamp." if not str(specific_query or "").strip(): return "Error: specific_query must describe the evidence to find." requested_start = round(float(start_time), 3) requested_end = round(float(end_time), 3) if isinstance(end_time, (int, float)) else None if requested_end is not None and requested_end <= requested_start: return "Error: end_time must be greater than start_time." # A closed interval does not need a throw-away full-range trim merely # to discover its duration. Open-ended requests are resolved once to a # stable numeric end before they enter persistent coverage. if requested_end is None: requested_start, requested_end, error = await self._resolve_open_ended_interval( requested_start ) if error is not None: return error cached_results = self.blackboard.format_cached_segments( requested_start, requested_end, specific_query ) missing = self.blackboard.missing_intervals(requested_start, requested_end, specific_query) if not missing: logger.info( "Reusing complete video-blackboard coverage for " f"{requested_start:.1f}s-{requested_end:.1f}s" ) return "CACHED VIDEO ANALYSIS: " + " ".join(cached_results) chunks = self._plan_chunks(missing) logger.info( f"Video blackboard planned {len(chunks)} uncovered chunk(s) for " f"{requested_start:.1f}s-{requested_end:.1f}s" ) async def analyze_with_recovery( cs: float, ce: float, depth: int = 0, *, retried_static: bool = False ) -> tuple[list[str], list[str]]: try: async with API_SEMAPHORE: value = await self._exec_single_chunk(cs, ce, specific_query) return [value], [] except Exception as error: failure = classify_video_failure(error) duration = ce - cs fits_ceiling = duration <= self.chunk_size_seconds + 1e-3 degraded = isinstance(error, AgenticVideoDegraded) if degraded and fits_ceiling and not retried_static: # Retry short intervals once in static mode without splitting. logger.warning( f"Retrying {cs:.1f}s-{ce:.1f}s in static mode: it already fits" f" the {self.chunk_size_seconds:.0f}s chunk ceiling" ) return await analyze_with_recovery(cs, ce, depth, retried_static=True) pieces = ( self._plan_chunks([(cs, ce)]) if failure.should_split and not fits_ceiling else [] ) if len(pieces) > 1: # Split clips planned before a downgrade at the new ceiling. logger.warning( f"Re-planning failed {cs:.1f}s-{ce:.1f}s chunk into {len(pieces)}" f" piece(s) of at most {self.chunk_size_seconds:.0f}s" f" after {failure.category.value}" ) parts = await asyncio.gather( *(analyze_with_recovery(ps, pe, depth) for ps, pe in pieces) ) return ( [item for values, _ in parts for item in values], [item for _, failures in parts for item in failures], ) # Bisecting only helps genuine size/timeout failures; a # degraded interval that fits the ceiling has had its one # static retry and is terminal. can_split = ( failure.should_split and not degraded and depth < self.max_split_depth and duration >= self.min_chunk_seconds * 2 ) if can_split: midpoint = round(cs + duration / 2.0, 3) logger.warning( f"Splitting failed {cs:.1f}s-{ce:.1f}s chunk at {midpoint:.1f}s " f"after {failure.category.value}" ) left, right = await asyncio.gather( analyze_with_recovery(cs, midpoint, depth + 1), analyze_with_recovery(midpoint, ce, depth + 1), ) return left[0] + right[0], left[1] + right[1] failure_text = f"{cs:.1f}s-{ce:.1f}s [{failure.category.value}]: {error}" logger.error(f"Video chunk terminal failure: {failure_text}") return [], [failure_text] recovered = await asyncio.gather(*(analyze_with_recovery(cs, ce) for cs, ce in chunks)) fresh_results = [item for values, _ in recovered for item in values] failed_chunks = [item for _, failures in recovered for item in failures] # Reload committed chunk summaries and event evidence from the shared # store before applying presentation-only conflict filtering. self._refresh_blackboard() self._clean_blackboard() valid_results = cached_results + fresh_results if not valid_results: return "All sub-agent chunks failed. Failed intervals: " + "; ".join(failed_chunks) combined = " ".join(valid_results) if failed_chunks: return ( "PARTIAL VIDEO ANALYSIS (successful chunks were persisted). " f"Failed intervals: {'; '.join(failed_chunks)}. Results: {combined}" ) self._record_reliability_metrics() return combined async def _exec_single_chunk( self, start_time: float, end_time: float | None, specific_query: str ) -> str: from artemis.agents.video_analyzer import chunk_runner return await chunk_runner.exec_single_chunk(self, start_time, end_time, specific_query) @trace(type="tool", name="analyze_audio_only") async def exec_analyze_audio_only( self, start_time: float, end_time: float | None, specific_query: str ) -> str: from artemis.agents.video_analyzer import audio_runner return await audio_runner.exec_analyze_audio_only( self, start_time, end_time, specific_query ) async def _exec_single_chunk_universal( self, compressed_path: Path, raw_path: Path, start_time: float, end_time: float | None, actual_start: float, prompt_with_context: str, specific_query: str, force_fallback: bool = False, ) -> str: from artemis.agents.video_analyzer import universal_engine return await universal_engine.exec_single_chunk_universal( self, compressed_path=compressed_path, raw_path=raw_path, start_time=start_time, end_time=end_time, actual_start=actual_start, prompt_with_context=prompt_with_context, specific_query=specific_query, force_fallback=force_fallback, ) async def _exec_analyze_audio_universal( self, audio_path: Path, start_time: float, end_time: float | None, actual_start: float, prompt_with_context: str, specific_query: str, force_fallback: bool = False, ) -> str: from artemis.agents.video_analyzer import universal_engine return await universal_engine.exec_analyze_audio_universal( self, audio_path=audio_path, start_time=start_time, end_time=end_time, actual_start=actual_start, prompt_with_context=prompt_with_context, specific_query=specific_query, force_fallback=force_fallback, ) async def _run_universal( self, time_description: str, purpose: str, system_prompt: str, *, force_fallback: bool = False, ) -> tuple[str, str]: from artemis.agents.video_analyzer import universal_engine return await universal_engine.run_universal( self, time_description, purpose, system_prompt, force_fallback=force_fallback, ) @trace(type="agent", name="video_analyzer") async def run(self, time_description: str, purpose: str) -> tuple[str, str]: ctx = self.ctx async with MAIN_AGENT_SEMAPHORE: logger.info( f"Video analyzer invoked with time: '{time_description}', purpose: '{purpose}'" ) self._refresh_blackboard() # Initialize engine & cleanup tracking self._init_engine() # Resolve model name and temperature from config llm_config = getattr(ctx, "llm_config", None) utils_cfg = getattr(llm_config, "utils", None) if llm_config else None llm_cfg = getattr(utils_cfg, "video_analyzer", None) if utils_cfg else None temperature = 0.2 if llm_cfg: self.model_name = llm_cfg.model if getattr(llm_cfg, "temperature", None) is not None: temperature = llm_cfg.temperature else: self.model_name = settings.OPENAI_MODEL # Track files for cleanup self.local_files_to_cleanup = set() self.local_dirs_to_cleanup = set() self.cloud_files_to_cleanup = set() sub_prompt_path = Path(__file__).parent / "video_sub_agent.md" self.sub_system_prompt = sub_prompt_path.read_text(encoding="utf-8") audio_prompt_path = Path(__file__).parent / "video_sub_agent_audio.md" self.audio_system_prompt = audio_prompt_path.read_text(encoding="utf-8") prompt_path = Path(__file__).parent / "video_analyzer.md" system_prompt = prompt_path.read_text(encoding="utf-8") if self.enable_ledger: system_prompt += ( "\n## Trust Protocol\nConsult Blackboard Ledger before" " searching. Evaluate if new search is necessary based on" " previous context. Explain reasoning before aborting or" " proceeding.\n" ) try: return await self._run_universal( time_description=time_description, purpose=purpose, system_prompt=system_prompt, ) finally: await self._cleanup_universal_resources()