302 lines
12 KiB
Python
302 lines
12 KiB
Python
# Copyright 2026 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""Responses execution path for audio-only segment analysis.
|
|
|
|
Extracted from ``video_analyzer.py`` as a pure structural split. Patchable
|
|
collaborators are looked up late through the facade module so
|
|
``mock.patch("artemis.agents.video_analyzer.video_analyzer.<name>")`` targets
|
|
keep working.
|
|
"""
|
|
|
|
import asyncio
|
|
from pathlib import Path
|
|
|
|
|
|
from artemis.agents.video_analyzer import video_analyzer as _va
|
|
from artemis.agents.video_analyzer.reliability import (
|
|
SubAgentAnswerExhausted,
|
|
classify_video_failure,
|
|
)
|
|
from artemis.data_engine.trace import TraceSpan
|
|
from artemis.llm.reliability import retry_policy_for
|
|
from artemis.utils.logger import get_logger
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
_BAD_CONFIDENCE_ERROR = (
|
|
"Error: 'confidence_score' is missing or out of bounds. It must be a float"
|
|
" between 0.0 and 1.0 inclusive. Fix the payload and call submit_answer again."
|
|
)
|
|
|
|
|
|
class _AudioMedia:
|
|
"""Mutable holder for per-attempt audio artifacts shared with failure handling."""
|
|
|
|
__slots__ = ("audio_path", "actual_start", "prompt_with_context")
|
|
|
|
def __init__(self):
|
|
self.audio_path: Path | None = None
|
|
self.actual_start: float | None = None
|
|
self.prompt_with_context: str | None = None
|
|
|
|
|
|
async def exec_analyze_audio_only(
|
|
analyzer, start_time: float, end_time: float | None, specific_query: str
|
|
) -> str:
|
|
"""Analyzes only the audio track of a segment, with claim/lease and retries."""
|
|
if not isinstance(start_time, (int, float)):
|
|
return "Error: start_time must be a numeric recording-relative timestamp."
|
|
if not str(specific_query or "").strip():
|
|
return "Error: specific_query must describe the audio evidence to find."
|
|
max_retries = analyzer.max_retries
|
|
current_start = round(start_time, 1) if isinstance(start_time, (int, float)) else start_time
|
|
current_end = round(end_time, 1) if isinstance(end_time, (int, float)) else end_time
|
|
|
|
if current_end is None:
|
|
controller = getattr(analyzer, "media_source", None) or _va.get_controller(analyzer.ctx)
|
|
async with _va.TRANSCODE_SEMAPHORE:
|
|
metadata = await controller.extract_segment_metadata(current_start, None)
|
|
duration = getattr(metadata, "duration_seconds", None)
|
|
actual_start = getattr(metadata, "actual_start_relative_time", current_start)
|
|
if not metadata.success or not isinstance(duration, (int, float)) or duration <= 0:
|
|
return f"Error fetching audio segment metadata: {metadata.message}"
|
|
current_start = round(float(actual_start), 1)
|
|
current_end = round(current_start + float(duration), 1)
|
|
if metadata.video_path:
|
|
metadata_path = Path(metadata.video_path)
|
|
analyzer.local_files_to_cleanup.add(metadata_path)
|
|
analyzer.local_dirs_to_cleanup.add(metadata_path.parent)
|
|
if current_end <= current_start:
|
|
return "Error: end_time must be greater than start_time."
|
|
|
|
device_id = getattr(getattr(analyzer.ctx, "device", None), "device_id", None)
|
|
active_session = _va.get_active_session(device_id) if device_id else None
|
|
claim = analyzer.blackboard.claim_segment(
|
|
current_start,
|
|
current_end,
|
|
specific_query,
|
|
modality="audio",
|
|
model_name=analyzer.model_name,
|
|
source_generation=(active_session.generation if active_session is not None else None),
|
|
)
|
|
if claim.state == "cached":
|
|
result = (
|
|
f"[from {current_start:.1f}s to {current_end:.1f}s] Audio Summary: "
|
|
f"{claim.summary or 'No audio summary provided.'}"
|
|
)
|
|
if claim.analysis:
|
|
result += f" Analysis: {claim.analysis}"
|
|
return "CACHED AUDIO ANALYSIS: " + result
|
|
if claim.state == "in_progress":
|
|
return (
|
|
f"[from {current_start:.1f}s to {current_end:.1f}s] "
|
|
"Audio analysis is already in progress; do not duplicate it."
|
|
)
|
|
lease_owner = claim.lease_owner
|
|
if lease_owner is None:
|
|
raise RuntimeError("Video blackboard returned an audio claim without a lease owner")
|
|
|
|
media = _AudioMedia()
|
|
|
|
for attempt in range(max_retries + 1):
|
|
try:
|
|
logger.info(
|
|
f"Attempt {attempt + 1}/{max_retries + 1} for"
|
|
f" analyze_audio_only: start={current_start},"
|
|
f" end={current_end}"
|
|
)
|
|
result = await _prepare_audio_media(analyzer, media, current_start, current_end)
|
|
media.prompt_with_context = _build_audio_prompt(
|
|
analyzer, result, media.actual_start, start_time, end_time, specific_query
|
|
)
|
|
|
|
audio_result = await analyzer._exec_analyze_audio_universal(
|
|
audio_path=media.audio_path,
|
|
start_time=current_start,
|
|
end_time=current_end,
|
|
actual_start=media.actual_start,
|
|
prompt_with_context=media.prompt_with_context,
|
|
specific_query=specific_query,
|
|
)
|
|
summary, analysis = analyzer._parse_chunk_result(audio_result)
|
|
analyzer.blackboard.complete_segment(
|
|
current_start,
|
|
current_end,
|
|
specific_query,
|
|
lease_owner,
|
|
summary,
|
|
analysis,
|
|
modality="audio",
|
|
)
|
|
return audio_result
|
|
|
|
except Exception as e:
|
|
# Same contract as the video sub-agent loop: classification
|
|
# owns the retry decision and every retry restarts the attempt
|
|
# from scratch.
|
|
failure = classify_video_failure(e)
|
|
attempts_so_far = attempt + 1
|
|
if not failure.retryable:
|
|
logger.warning(
|
|
f"Audio non-retryable failure [{failure.category.value}]"
|
|
f" on attempt {attempts_so_far}: {e}"
|
|
)
|
|
return await _finish_failed_audio(
|
|
analyzer,
|
|
media,
|
|
current_start,
|
|
current_end,
|
|
specific_query,
|
|
lease_owner,
|
|
e,
|
|
retryable=False,
|
|
)
|
|
policy = retry_policy_for(failure.category)
|
|
allowed_attempts = min(max_retries + 1, policy.max_attempts)
|
|
if attempts_so_far >= allowed_attempts:
|
|
logger.warning(
|
|
f"Audio analysis exhausted {attempts_so_far} attempt(s)"
|
|
f" [{failure.category.value}]: {e}"
|
|
)
|
|
return await _finish_failed_audio(
|
|
analyzer,
|
|
media,
|
|
current_start,
|
|
current_end,
|
|
specific_query,
|
|
lease_owner,
|
|
e,
|
|
)
|
|
delay = policy.delay_for(attempts_so_far)
|
|
logger.warning(
|
|
f"Audio retryable failure [{failure.category.value}]"
|
|
f" ({attempts_so_far}/{allowed_attempts}): {e}; retrying"
|
|
f" from scratch in {delay:.2f}s"
|
|
)
|
|
_va._record_llm_retry(
|
|
str(e),
|
|
delay,
|
|
attempt=attempts_so_far,
|
|
max_retries=allowed_attempts,
|
|
source="video_audio_agent",
|
|
)
|
|
await asyncio.sleep(delay)
|
|
|
|
|
|
async def _finish_failed_audio(
|
|
analyzer,
|
|
media: _AudioMedia,
|
|
current_start: float,
|
|
current_end: float,
|
|
specific_query: str,
|
|
lease_owner: str,
|
|
error: Exception,
|
|
*,
|
|
retryable: bool | None = None,
|
|
):
|
|
"""Records terminal audio failure, attempting the universal fallback first."""
|
|
failure = classify_video_failure(error)
|
|
analyzer.blackboard.fail_segment(
|
|
current_start,
|
|
current_end,
|
|
specific_query,
|
|
lease_owner,
|
|
str(error),
|
|
modality="audio",
|
|
retryable=failure.retryable if retryable is None else retryable,
|
|
error_category=failure.category.value,
|
|
)
|
|
raise error
|
|
|
|
|
|
async def _prepare_audio_media(
|
|
analyzer, media: _AudioMedia, current_start: float, current_end: float
|
|
):
|
|
"""Extracts the segment and its audio track, tracking artifacts for cleanup."""
|
|
controller = getattr(analyzer, "media_source", None) or _va.get_controller(analyzer.ctx)
|
|
async with _va.TRANSCODE_SEMAPHORE:
|
|
result = await controller.extract_segment_metadata(current_start, current_end)
|
|
if not result.success or not result.video_path:
|
|
raise Exception(f"Failed to get video segment: {result.message}")
|
|
|
|
path = Path(result.video_path)
|
|
analyzer.local_files_to_cleanup.add(path)
|
|
analyzer.local_dirs_to_cleanup.add(path.parent)
|
|
|
|
async with _va.TRANSCODE_SEMAPHORE:
|
|
audio_path = await _va.extract_audio_from_video(path)
|
|
analyzer.local_files_to_cleanup.add(audio_path)
|
|
analyzer.local_dirs_to_cleanup.add(audio_path.parent)
|
|
media.audio_path = audio_path
|
|
actual_start = getattr(result, "actual_start_relative_time", current_start)
|
|
if not isinstance(actual_start, (int, float)):
|
|
actual_start = current_start
|
|
media.actual_start = actual_start
|
|
return result
|
|
|
|
|
|
def _build_audio_prompt(
|
|
analyzer,
|
|
result,
|
|
actual_start: float,
|
|
start_time: float,
|
|
end_time: float | None,
|
|
specific_query: str,
|
|
) -> str:
|
|
"""Assembles the audio sub-agent prompt with timing context, ledger, and warnings."""
|
|
warnings = analyzer.get_overlapping_warnings(start_time, end_time)
|
|
warning_block = ""
|
|
if warnings:
|
|
lines = ["WARNING: The following queries were already tried in this timeframe:\n"]
|
|
for w in warnings:
|
|
lines.append(f"Searched for: {w['target']} -> {w['summary']}")
|
|
warning_block = "\n".join(lines) + "\n\n"
|
|
|
|
duration_secs = getattr(result, "duration_seconds", None)
|
|
actual_end = actual_start + duration_secs if isinstance(duration_secs, (int, float)) else None
|
|
end_str = f" to {actual_end:.1f}s" if actual_end is not None else ""
|
|
warning_val = getattr(result, "warning", None)
|
|
truncation_note = (
|
|
f" (NOTE: {warning_val})" if isinstance(warning_val, str) and warning_val else ""
|
|
)
|
|
|
|
prompt_with_context = (
|
|
"IMPORTANT CONTEXT: This audio segment corresponds to"
|
|
" the test's relative time from"
|
|
f" {actual_start:.1f}s{end_str}{truncation_note}. When"
|
|
" you report timestamps, remember that your 00:00 is"
|
|
f" exactly {actual_start:.1f}s in the test relative"
|
|
" time. Please calculate and output all timestamps as"
|
|
" the exact test relative time (e.g., if you hear"
|
|
" something at 00:05, report"
|
|
f" {actual_start + 5:.1f}s).\n\n"
|
|
)
|
|
if analyzer.enable_ledger:
|
|
prompt_with_context += (
|
|
"BLACKBOARD"
|
|
f" LEDGER:\n{analyzer.blackboard_ledger}\nDirective:"
|
|
" Use the Blackboard to understand what happened"
|
|
" outside your assigned timeframe. Do not"
|
|
" duplicate findings already on the board.\n\n"
|
|
)
|
|
if warning_block:
|
|
prompt_with_context += (
|
|
f"{warning_block}Directive: Do not repeat the"
|
|
" failed searches listed above. Shift your"
|
|
" attention to the current task.\n\n"
|
|
)
|
|
prompt_with_context += f"{specific_query}"
|
|
return prompt_with_context
|