# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Native Gemini execution path for audio-only segment analysis. Extracted from ``video_analyzer.py`` as a pure structural split. Patchable collaborators are looked up late through the facade module so ``mock.patch("artemis.agents.video_analyzer.video_analyzer.")`` targets keep working. """ import asyncio from pathlib import Path from google.genai import types from artemis.agents.video_analyzer import video_analyzer as _va from artemis.agents.video_analyzer.reliability import classify_video_failure from artemis.constants import SAFETY_SETTINGS_BLOCK_NONE from artemis.data_engine.trace import TraceSpan from artemis.llm.reliability import retry_policy_for from artemis.utils.logger import get_logger logger = get_logger(__name__) class _AudioMedia: """Mutable holder for per-attempt audio artifacts shared with failure handling.""" __slots__ = ("audio_path", "actual_start", "prompt_with_context") def __init__(self): self.audio_path: Path | None = None self.actual_start: float | None = None self.prompt_with_context: str | None = None async def exec_analyze_audio_only( analyzer, start_time: float, end_time: float | None, specific_query: str ) -> str: """Analyzes only the audio track of a segment, with claim/lease and retries.""" if not isinstance(start_time, (int, float)): return "Error: start_time must be a numeric recording-relative timestamp." if not str(specific_query or "").strip(): return "Error: specific_query must describe the audio evidence to find." max_retries = analyzer.native_max_retries current_start = round(start_time, 1) if isinstance(start_time, (int, float)) else start_time current_end = round(end_time, 1) if isinstance(end_time, (int, float)) else end_time if current_end is None: controller = _va.get_controller(analyzer.ctx) async with _va.TRANSCODE_SEMAPHORE: metadata = await controller.extract_segment_metadata(current_start, None) duration = getattr(metadata, "duration_seconds", None) actual_start = getattr(metadata, "actual_start_relative_time", current_start) if not metadata.success or not isinstance(duration, (int, float)) or duration <= 0: return f"Error fetching audio segment metadata: {metadata.message}" current_start = round(float(actual_start), 1) current_end = round(current_start + float(duration), 1) if metadata.video_path: metadata_path = Path(metadata.video_path) analyzer.local_files_to_cleanup.add(metadata_path) analyzer.local_dirs_to_cleanup.add(metadata_path.parent) if current_end <= current_start: return "Error: end_time must be greater than start_time." device_id = getattr(getattr(analyzer.ctx, "device", None), "device_id", None) active_session = _va.get_active_session(device_id) if device_id else None claim = analyzer.blackboard.claim_segment( current_start, current_end, specific_query, modality="audio", model_name=analyzer.model_name, source_generation=( active_session.generation if active_session is not None else None ), ) if claim.state == "cached": result = ( f"[from {current_start:.1f}s to {current_end:.1f}s] Audio Summary: " f"{claim.summary or 'No audio summary provided.'}" ) if claim.analysis: result += f" Analysis: {claim.analysis}" return "CACHED AUDIO ANALYSIS: " + result if claim.state == "in_progress": return ( f"[from {current_start:.1f}s to {current_end:.1f}s] " "Audio analysis is already in progress; do not duplicate it." ) lease_owner = claim.lease_owner if lease_owner is None: raise RuntimeError("Video blackboard returned an audio claim without a lease owner") media = _AudioMedia() for attempt in range(max_retries + 1): try: logger.info( f"Attempt {attempt + 1}/{max_retries + 1} for" f" analyze_audio_only: start={current_start}," f" end={current_end}" ) result = await _prepare_audio_media(analyzer, media, current_start, current_end) media.prompt_with_context = _build_audio_prompt( analyzer, result, media.actual_start, start_time, end_time, specific_query ) if not analyzer.use_native_gemini: audio_result = await analyzer._exec_analyze_audio_universal( audio_path=media.audio_path, start_time=current_start, end_time=current_end, actual_start=media.actual_start, prompt_with_context=media.prompt_with_context, specific_query=specific_query, ) summary, analysis = analyzer._parse_chunk_result(audio_result) analyzer.blackboard.complete_segment( current_start, current_end, specific_query, lease_owner, summary, analysis, modality="audio", ) return audio_result return await _run_native_audio_conversation( analyzer, media, current_start, current_end, start_time, end_time, specific_query, lease_owner, ) except Exception as e: # Same contract as the video sub-agent loop: classification # owns the retry decision and every retry restarts the attempt # from scratch. failure = classify_video_failure(e) attempts_so_far = attempt + 1 if not failure.retryable: logger.warning( f"Audio non-retryable failure [{failure.category.value}]" f" on attempt {attempts_so_far}: {e}" ) return await _finish_failed_audio( analyzer, media, current_start, current_end, specific_query, lease_owner, e, retryable=False, ) policy = retry_policy_for(failure.category) allowed_attempts = min(max_retries + 1, policy.max_attempts) if attempts_so_far >= allowed_attempts: logger.warning( f"Audio analysis exhausted {attempts_so_far} attempt(s)" f" [{failure.category.value}]: {e}" ) return await _finish_failed_audio( analyzer, media, current_start, current_end, specific_query, lease_owner, e, ) delay = policy.delay_for(attempts_so_far) logger.warning( f"Audio retryable failure [{failure.category.value}]" f" ({attempts_so_far}/{allowed_attempts}): {e}; retrying" f" from scratch in {delay:.2f}s" ) _va._record_llm_retry( str(e), delay, attempt=attempts_so_far, max_retries=allowed_attempts, source="video_audio_agent", ) await asyncio.sleep(delay) async def _finish_failed_audio( analyzer, media: _AudioMedia, current_start: float, current_end: float, specific_query: str, lease_owner: str, error: Exception, *, retryable: bool | None = None, ): """Records terminal audio failure, attempting the universal fallback first.""" failure = classify_video_failure(error) if ( analyzer.use_native_gemini and failure.should_fallback and media.audio_path is not None and media.actual_start is not None and media.prompt_with_context is not None ): try: logger.warning( "Native audio analysis failed" f" [{failure.category.value}]; using universal fallback" ) _va._record_llm_event( "llm_fallback", { "reason": "native_audio_analysis_failed", "category": failure.category.value, "error": str(error)[:500], }, ) fallback_result = await analyzer._exec_analyze_audio_universal( audio_path=media.audio_path, start_time=current_start, end_time=current_end, actual_start=media.actual_start, prompt_with_context=media.prompt_with_context, specific_query=specific_query, force_fallback=True, ) summary, analysis = analyzer._parse_chunk_result(fallback_result) analyzer.blackboard.complete_segment( current_start, current_end, specific_query, lease_owner, summary, analysis, modality="audio", ) return fallback_result except Exception as fallback_error: error = fallback_error failure = classify_video_failure(error) retryable = None analyzer.blackboard.fail_segment( current_start, current_end, specific_query, lease_owner, str(error), modality="audio", retryable=failure.retryable if retryable is None else retryable, error_category=failure.category.value, ) raise error async def _prepare_audio_media( analyzer, media: _AudioMedia, current_start: float, current_end: float ): """Extracts the segment and its audio track, tracking artifacts for cleanup.""" controller = _va.get_controller(analyzer.ctx) async with _va.TRANSCODE_SEMAPHORE: result = await controller.extract_segment_metadata(current_start, current_end) if not result.success or not result.video_path: raise Exception(f"Failed to get video segment: {result.message}") path = Path(result.video_path) analyzer.local_files_to_cleanup.add(path) analyzer.local_dirs_to_cleanup.add(path.parent) async with _va.TRANSCODE_SEMAPHORE: audio_path = await _va.extract_audio_from_video(path) analyzer.local_files_to_cleanup.add(audio_path) analyzer.local_dirs_to_cleanup.add(audio_path.parent) media.audio_path = audio_path actual_start = getattr(result, "actual_start_relative_time", current_start) if not isinstance(actual_start, (int, float)): actual_start = current_start media.actual_start = actual_start return result def _build_audio_prompt( analyzer, result, actual_start: float, start_time: float, end_time: float | None, specific_query: str, ) -> str: """Assembles the audio sub-agent prompt with timing context, ledger, and warnings.""" warnings = analyzer.get_overlapping_warnings(start_time, end_time) warning_block = "" if warnings: lines = [ "WARNING: The following queries were already tried in this timeframe:\n" ] for w in warnings: lines.append(f"Searched for: {w['target']} -> {w['summary']}") warning_block = "\n".join(lines) + "\n\n" duration_secs = getattr(result, "duration_seconds", None) actual_end = ( actual_start + duration_secs if isinstance(duration_secs, (int, float)) else None ) end_str = f" to {actual_end:.1f}s" if actual_end is not None else "" warning_val = getattr(result, "warning", None) truncation_note = ( f" (NOTE: {warning_val})" if isinstance(warning_val, str) and warning_val else "" ) prompt_with_context = ( "IMPORTANT CONTEXT: This audio segment corresponds to" " the test's relative time from" f" {actual_start:.1f}s{end_str}{truncation_note}. When" " you report timestamps, remember that your 00:00 is" f" exactly {actual_start:.1f}s in the test relative" " time. Please calculate and output all timestamps as" " the exact test relative time (e.g., if you hear" " something at 00:05, report" f" {actual_start + 5:.1f}s).\n\n" ) if analyzer.enable_ledger: prompt_with_context += ( "BLACKBOARD" f" LEDGER:\n{analyzer.blackboard_ledger}\nDirective:" " Use the Blackboard to understand what happened" " outside your assigned timeframe. Do not" " duplicate findings already on the board.\n\n" ) if warning_block: prompt_with_context += ( f"{warning_block}Directive: Do not repeat the" " failed searches listed above. Shift your" " attention to the current task.\n\n" ) prompt_with_context += f"{specific_query}" return prompt_with_context async def _run_native_audio_conversation( analyzer, media: _AudioMedia, current_start: float, current_end: float, start_time: float, end_time: float | None, specific_query: str, lease_owner: str, ) -> str: """Uploads the audio and drives the native Gemini audio agent to a committed answer.""" file = None try: with TraceSpan(name="upload_audio_to_gemini") as span: file = await analyzer.upload_and_poll_file(media.audio_path) span.result = f"Uploaded {file.name}" logger.info( f"Invoking Gemini for audio-only task with model {analyzer.model_name}..." ) sub_agent_contents = [ types.Content( role="user", parts=[ types.Part.from_uri(file_uri=file.uri, mime_type=file.mime_type) if hasattr(file, "uri") else file, types.Part.from_text(text=media.prompt_with_context), ], ) ] ( final_summary, final_analysis, final_confidence_score, ) = await _drive_audio_agent_loop(analyzer, sub_agent_contents) end_val = end_time if end_time is not None else "unknown" analyzer._record_blackboard_entry( { "start": start_time, "end": end_val, "target": specific_query, "summary": final_summary, "confidence_score": final_confidence_score, }, modality="audio", ) end_str = f"{end_time:.1f}s" if end_time is not None else "unknown" audio_result = ( f"[from {start_time:.1f}s to {end_str}] Summary:" f" {final_summary} Analysis: {final_analysis}" ) analyzer.blackboard.complete_segment( current_start, current_end, specific_query, lease_owner, final_summary, final_analysis, modality="audio", ) return audio_result finally: if file: logger.info(f"Cleaning up cloud file {file.name}...") try: await asyncio.wait_for( analyzer.client.aio.files.delete(name=file.name), timeout=30, ) analyzer.cloud_files_to_cleanup.discard(file.name) except Exception as ce: logger.error(f"Failed to delete cloud file {file.name}: {ce}") async def _drive_audio_agent_loop( analyzer, sub_agent_contents: list ) -> tuple[str, str, float]: """Iterates the audio agent conversation until submit_answer is accepted.""" sub_max_iterations = 2 sub_iterations = 0 final_confidence_score = 0.0 final_summary = "No summary provided." final_analysis = "No analysis provided." final_full_text = "" while sub_iterations < sub_max_iterations: sub_iterations += 1 if sub_iterations == sub_max_iterations - 1: sub_agent_contents.append( types.Content( role="user", parts=[ types.Part.from_text( text=( "[WARNING] You must call the" " submit_answer tool now, time" " is running out." ) ) ], ) ) with TraceSpan(name="gemini_generate_audio_content") as span: response = await asyncio.wait_for( analyzer.client.aio.models.generate_content( model=analyzer.model_name, contents=sub_agent_contents, config=types.GenerateContentConfig( system_instruction=analyzer.audio_system_prompt, tools=[ types.Tool( function_declarations=[analyzer.submit_answer_tool] ) ], safety_settings=SAFETY_SETTINGS_BLOCK_NONE, ), ), timeout=analyzer.model_call_timeout_seconds, ) full_text = response.text or "" final_full_text += full_text + "\n" function_calls = response.function_calls or [] if function_calls: ( answered, final_summary, final_analysis, final_confidence_score, ) = _handle_audio_function_calls( sub_agent_contents, response, function_calls, final_summary, final_analysis, final_confidence_score, ) if answered is True: break else: sub_agent_contents.append( types.Content( role="model", parts=[types.Part.from_text(text=full_text)], ) ) return final_summary, final_analysis, final_confidence_score def _handle_audio_function_calls( sub_agent_contents: list, response, function_calls: list, final_summary: str, final_analysis: str, final_confidence_score: float, ): """Processes audio-agent function calls; returns (answered, summary, analysis, score).""" answered = False for fc in function_calls: if ( fc.name.split(":")[-1] if ":" in fc.name else fc.name ) == "submit_answer": args = fc.args if ( "confidence_score" not in args or not isinstance( args["confidence_score"], (int, float), ) or not (0.0 <= float(args["confidence_score"]) <= 1.0) ): error_msg = ( "Error: 'confidence_score' is" " missing or out of bounds. It must" " be a float between 0.0 and 1.0" " inclusive. Fix the payload and" " call submit_answer again." ) sub_agent_contents.append( types.Content( role="model", parts=response.candidates[0].content.parts, ) ) sub_agent_contents.append( types.Content( role="user", parts=[ types.Part.from_function_response( name=fc.name, response={"error": error_msg}, ) ], ) ) answered = "error" break final_summary = ( args.get("summary", "No summary provided.") .strip() .replace("\n", " ") ) final_analysis = ( args.get("analysis", "No analysis provided.") .strip() .replace("\n", " ") ) final_confidence_score = float(args["confidence_score"]) answered = True break if answered is True or answered == "error": return answered, final_summary, final_analysis, final_confidence_score sub_agent_contents.append( types.Content( role="model", parts=response.candidates[0].content.parts, ) ) sub_agent_contents.append( types.Content( role="user", parts=[ types.Part.from_function_response( name=function_calls[0].name, response={ "error": ( "Tool not recognized." " Please use" " submit_answer." ) }, ) ], ) ) return answered, final_summary, final_analysis, final_confidence_score