chenmingxuan
2ddec53954
Localize the web console and persist custom model and capability settings. Use OpenAI Responses exclusively and move runtime limits into environment configuration. Bootstrap uv and the virtual environment on Windows, support optional China mirrors, and package source distributions without environments, build output, or credentials. Update regression tests and repository quality checks for the new behavior.
302 lines
12 KiB
Python
302 lines
12 KiB
Python
# Copyright 2026 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""Responses execution path for audio-only segment analysis.
|
|
|
|
Extracted from ``video_analyzer.py`` as a pure structural split. Patchable
|
|
collaborators are looked up late through the facade module so
|
|
``mock.patch("artemis.agents.video_analyzer.video_analyzer.<name>")`` targets
|
|
keep working.
|
|
"""
|
|
|
|
import asyncio
|
|
from pathlib import Path
|
|
|
|
|
|
from artemis.agents.video_analyzer import video_analyzer as _va
|
|
from artemis.agents.video_analyzer.reliability import (
|
|
SubAgentAnswerExhausted,
|
|
classify_video_failure,
|
|
)
|
|
from artemis.data_engine.trace import TraceSpan
|
|
from artemis.llm.reliability import retry_policy_for
|
|
from artemis.utils.logger import get_logger
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
_BAD_CONFIDENCE_ERROR = (
|
|
"Error: 'confidence_score' is missing or out of bounds. It must be a float"
|
|
" between 0.0 and 1.0 inclusive. Fix the payload and call submit_answer again."
|
|
)
|
|
|
|
|
|
class _AudioMedia:
|
|
"""Mutable holder for per-attempt audio artifacts shared with failure handling."""
|
|
|
|
__slots__ = ("audio_path", "actual_start", "prompt_with_context")
|
|
|
|
def __init__(self):
|
|
self.audio_path: Path | None = None
|
|
self.actual_start: float | None = None
|
|
self.prompt_with_context: str | None = None
|
|
|
|
|
|
async def exec_analyze_audio_only(
|
|
analyzer, start_time: float, end_time: float | None, specific_query: str
|
|
) -> str:
|
|
"""Analyzes only the audio track of a segment, with claim/lease and retries."""
|
|
if not isinstance(start_time, (int, float)):
|
|
return "Error: start_time must be a numeric recording-relative timestamp."
|
|
if not str(specific_query or "").strip():
|
|
return "Error: specific_query must describe the audio evidence to find."
|
|
max_retries = analyzer.max_retries
|
|
current_start = round(start_time, 1) if isinstance(start_time, (int, float)) else start_time
|
|
current_end = round(end_time, 1) if isinstance(end_time, (int, float)) else end_time
|
|
|
|
if current_end is None:
|
|
controller = _va.get_controller(analyzer.ctx)
|
|
async with _va.TRANSCODE_SEMAPHORE:
|
|
metadata = await controller.extract_segment_metadata(current_start, None)
|
|
duration = getattr(metadata, "duration_seconds", None)
|
|
actual_start = getattr(metadata, "actual_start_relative_time", current_start)
|
|
if not metadata.success or not isinstance(duration, (int, float)) or duration <= 0:
|
|
return f"Error fetching audio segment metadata: {metadata.message}"
|
|
current_start = round(float(actual_start), 1)
|
|
current_end = round(current_start + float(duration), 1)
|
|
if metadata.video_path:
|
|
metadata_path = Path(metadata.video_path)
|
|
analyzer.local_files_to_cleanup.add(metadata_path)
|
|
analyzer.local_dirs_to_cleanup.add(metadata_path.parent)
|
|
if current_end <= current_start:
|
|
return "Error: end_time must be greater than start_time."
|
|
|
|
device_id = getattr(getattr(analyzer.ctx, "device", None), "device_id", None)
|
|
active_session = _va.get_active_session(device_id) if device_id else None
|
|
claim = analyzer.blackboard.claim_segment(
|
|
current_start,
|
|
current_end,
|
|
specific_query,
|
|
modality="audio",
|
|
model_name=analyzer.model_name,
|
|
source_generation=(active_session.generation if active_session is not None else None),
|
|
)
|
|
if claim.state == "cached":
|
|
result = (
|
|
f"[from {current_start:.1f}s to {current_end:.1f}s] Audio Summary: "
|
|
f"{claim.summary or 'No audio summary provided.'}"
|
|
)
|
|
if claim.analysis:
|
|
result += f" Analysis: {claim.analysis}"
|
|
return "CACHED AUDIO ANALYSIS: " + result
|
|
if claim.state == "in_progress":
|
|
return (
|
|
f"[from {current_start:.1f}s to {current_end:.1f}s] "
|
|
"Audio analysis is already in progress; do not duplicate it."
|
|
)
|
|
lease_owner = claim.lease_owner
|
|
if lease_owner is None:
|
|
raise RuntimeError("Video blackboard returned an audio claim without a lease owner")
|
|
|
|
media = _AudioMedia()
|
|
|
|
for attempt in range(max_retries + 1):
|
|
try:
|
|
logger.info(
|
|
f"Attempt {attempt + 1}/{max_retries + 1} for"
|
|
f" analyze_audio_only: start={current_start},"
|
|
f" end={current_end}"
|
|
)
|
|
result = await _prepare_audio_media(analyzer, media, current_start, current_end)
|
|
media.prompt_with_context = _build_audio_prompt(
|
|
analyzer, result, media.actual_start, start_time, end_time, specific_query
|
|
)
|
|
|
|
audio_result = await analyzer._exec_analyze_audio_universal(
|
|
audio_path=media.audio_path,
|
|
start_time=current_start,
|
|
end_time=current_end,
|
|
actual_start=media.actual_start,
|
|
prompt_with_context=media.prompt_with_context,
|
|
specific_query=specific_query,
|
|
)
|
|
summary, analysis = analyzer._parse_chunk_result(audio_result)
|
|
analyzer.blackboard.complete_segment(
|
|
current_start,
|
|
current_end,
|
|
specific_query,
|
|
lease_owner,
|
|
summary,
|
|
analysis,
|
|
modality="audio",
|
|
)
|
|
return audio_result
|
|
|
|
except Exception as e:
|
|
# Same contract as the video sub-agent loop: classification
|
|
# owns the retry decision and every retry restarts the attempt
|
|
# from scratch.
|
|
failure = classify_video_failure(e)
|
|
attempts_so_far = attempt + 1
|
|
if not failure.retryable:
|
|
logger.warning(
|
|
f"Audio non-retryable failure [{failure.category.value}]"
|
|
f" on attempt {attempts_so_far}: {e}"
|
|
)
|
|
return await _finish_failed_audio(
|
|
analyzer,
|
|
media,
|
|
current_start,
|
|
current_end,
|
|
specific_query,
|
|
lease_owner,
|
|
e,
|
|
retryable=False,
|
|
)
|
|
policy = retry_policy_for(failure.category)
|
|
allowed_attempts = min(max_retries + 1, policy.max_attempts)
|
|
if attempts_so_far >= allowed_attempts:
|
|
logger.warning(
|
|
f"Audio analysis exhausted {attempts_so_far} attempt(s)"
|
|
f" [{failure.category.value}]: {e}"
|
|
)
|
|
return await _finish_failed_audio(
|
|
analyzer,
|
|
media,
|
|
current_start,
|
|
current_end,
|
|
specific_query,
|
|
lease_owner,
|
|
e,
|
|
)
|
|
delay = policy.delay_for(attempts_so_far)
|
|
logger.warning(
|
|
f"Audio retryable failure [{failure.category.value}]"
|
|
f" ({attempts_so_far}/{allowed_attempts}): {e}; retrying"
|
|
f" from scratch in {delay:.2f}s"
|
|
)
|
|
_va._record_llm_retry(
|
|
str(e),
|
|
delay,
|
|
attempt=attempts_so_far,
|
|
max_retries=allowed_attempts,
|
|
source="video_audio_agent",
|
|
)
|
|
await asyncio.sleep(delay)
|
|
|
|
|
|
async def _finish_failed_audio(
|
|
analyzer,
|
|
media: _AudioMedia,
|
|
current_start: float,
|
|
current_end: float,
|
|
specific_query: str,
|
|
lease_owner: str,
|
|
error: Exception,
|
|
*,
|
|
retryable: bool | None = None,
|
|
):
|
|
"""Records terminal audio failure, attempting the universal fallback first."""
|
|
failure = classify_video_failure(error)
|
|
analyzer.blackboard.fail_segment(
|
|
current_start,
|
|
current_end,
|
|
specific_query,
|
|
lease_owner,
|
|
str(error),
|
|
modality="audio",
|
|
retryable=failure.retryable if retryable is None else retryable,
|
|
error_category=failure.category.value,
|
|
)
|
|
raise error
|
|
|
|
|
|
async def _prepare_audio_media(
|
|
analyzer, media: _AudioMedia, current_start: float, current_end: float
|
|
):
|
|
"""Extracts the segment and its audio track, tracking artifacts for cleanup."""
|
|
controller = _va.get_controller(analyzer.ctx)
|
|
async with _va.TRANSCODE_SEMAPHORE:
|
|
result = await controller.extract_segment_metadata(current_start, current_end)
|
|
if not result.success or not result.video_path:
|
|
raise Exception(f"Failed to get video segment: {result.message}")
|
|
|
|
path = Path(result.video_path)
|
|
analyzer.local_files_to_cleanup.add(path)
|
|
analyzer.local_dirs_to_cleanup.add(path.parent)
|
|
|
|
async with _va.TRANSCODE_SEMAPHORE:
|
|
audio_path = await _va.extract_audio_from_video(path)
|
|
analyzer.local_files_to_cleanup.add(audio_path)
|
|
analyzer.local_dirs_to_cleanup.add(audio_path.parent)
|
|
media.audio_path = audio_path
|
|
actual_start = getattr(result, "actual_start_relative_time", current_start)
|
|
if not isinstance(actual_start, (int, float)):
|
|
actual_start = current_start
|
|
media.actual_start = actual_start
|
|
return result
|
|
|
|
|
|
def _build_audio_prompt(
|
|
analyzer,
|
|
result,
|
|
actual_start: float,
|
|
start_time: float,
|
|
end_time: float | None,
|
|
specific_query: str,
|
|
) -> str:
|
|
"""Assembles the audio sub-agent prompt with timing context, ledger, and warnings."""
|
|
warnings = analyzer.get_overlapping_warnings(start_time, end_time)
|
|
warning_block = ""
|
|
if warnings:
|
|
lines = ["WARNING: The following queries were already tried in this timeframe:\n"]
|
|
for w in warnings:
|
|
lines.append(f"Searched for: {w['target']} -> {w['summary']}")
|
|
warning_block = "\n".join(lines) + "\n\n"
|
|
|
|
duration_secs = getattr(result, "duration_seconds", None)
|
|
actual_end = actual_start + duration_secs if isinstance(duration_secs, (int, float)) else None
|
|
end_str = f" to {actual_end:.1f}s" if actual_end is not None else ""
|
|
warning_val = getattr(result, "warning", None)
|
|
truncation_note = (
|
|
f" (NOTE: {warning_val})" if isinstance(warning_val, str) and warning_val else ""
|
|
)
|
|
|
|
prompt_with_context = (
|
|
"IMPORTANT CONTEXT: This audio segment corresponds to"
|
|
" the test's relative time from"
|
|
f" {actual_start:.1f}s{end_str}{truncation_note}. When"
|
|
" you report timestamps, remember that your 00:00 is"
|
|
f" exactly {actual_start:.1f}s in the test relative"
|
|
" time. Please calculate and output all timestamps as"
|
|
" the exact test relative time (e.g., if you hear"
|
|
" something at 00:05, report"
|
|
f" {actual_start + 5:.1f}s).\n\n"
|
|
)
|
|
if analyzer.enable_ledger:
|
|
prompt_with_context += (
|
|
"BLACKBOARD"
|
|
f" LEDGER:\n{analyzer.blackboard_ledger}\nDirective:"
|
|
" Use the Blackboard to understand what happened"
|
|
" outside your assigned timeframe. Do not"
|
|
" duplicate findings already on the board.\n\n"
|
|
)
|
|
if warning_block:
|
|
prompt_with_context += (
|
|
f"{warning_block}Directive: Do not repeat the"
|
|
" failed searches listed above. Shift your"
|
|
" attention to the current task.\n\n"
|
|
)
|
|
prompt_with_context += f"{specific_query}"
|
|
return prompt_with_context
|