chenmingxuan/Artemis-Eapil
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 32 Wiki Activity

feat: add Chinese UI, Responses models, and mirrored Windows setup
CI / Python 3.12 / ubuntu-latest (push) Canceled after 0s
Details
CI / Python 3.12 / windows-latest (push) Canceled after 0s
Details
CI / Frontend tests and production build (push) Canceled after 0s
Details
CI / Build and smoke-test Python packages (push) Canceled after 0s
Details

Browse Source 
Localize the web console and persist custom model and capability settings.
Use OpenAI Responses exclusively and move runtime limits into environment configuration.
Bootstrap uv and the virtual environment on Windows, support optional China mirrors,
and package source distributions without environments, build output, or credentials.

Update regression tests and repository quality checks for the new behavior.
This commit is contained in:
chenmingxuan committed 2026-09-22 19:16:17 +08:00
1 parent 371aa6df56
commit 2ddec53954
183 files changed
+3427 -11857
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

artemis/agents/video_analyzer/__init__.py
+1 -1
View File
@@ -12,4 +12,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Video analyzer utility for analyzing video content with Gemini models."""
"""Video analyzer utility for analyzing video content with OpenAI Responses models."""
artemis/agents/video_analyzer/audio_native.py → artemis/agents/video_analyzer/audio_runner.py
+16 -258
View File
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Native Gemini execution path for audio-only segment analysis.
"""Responses execution path for audio-only segment analysis.
Extracted from ``video_analyzer.py`` as a pure structural split. Patchable
collaborators are looked up late through the facade module so
@@ -23,24 +23,12 @@ keep working.
import asyncio
from pathlib import Path
from google.genai import types
from artemis.agents.video_analyzer import video_analyzer as _va
from artemis.agents.video_analyzer.chunk_conversation import (
_MISSING_SUMMARY_ERROR,
_NO_TOOL_CALL_ERROR,
_SUBMIT_NOW_WARNING,
_WRONG_TOOL_ERROR,
SUB_AGENT_MAX_TURNS,
_append_rejection,
_clean_text,
_tool_name,
)
from artemis.agents.video_analyzer.reliability import (
SubAgentAnswerExhausted,
classify_video_failure,
)
from artemis.constants import SAFETY_SETTINGS_BLOCK_NONE
from artemis.data_engine.trace import TraceSpan
from artemis.llm.reliability import retry_policy_for
from artemis.utils.logger import get_logger
@@ -72,7 +60,7 @@ async def exec_analyze_audio_only(
return "Error: start_time must be a numeric recording-relative timestamp."
if not str(specific_query or "").strip():
return "Error: specific_query must describe the audio evidence to find."
max_retries = analyzer.native_max_retries
max_retries = analyzer.max_retries
current_start = round(start_time, 1) if isinstance(start_time, (int, float)) else start_time
current_end = round(end_time, 1) if isinstance(end_time, (int, float)) else end_time
@@ -134,37 +122,25 @@ async def exec_analyze_audio_only(
analyzer, result, media.actual_start, start_time, end_time, specific_query
)
if not analyzer.use_native_gemini:
audio_result = await analyzer._exec_analyze_audio_universal(
audio_path=media.audio_path,
start_time=current_start,
end_time=current_end,
actual_start=media.actual_start,
prompt_with_context=media.prompt_with_context,
specific_query=specific_query,
)
summary, analysis = analyzer._parse_chunk_result(audio_result)
analyzer.blackboard.complete_segment(
current_start,
current_end,
specific_query,
lease_owner,
summary,
analysis,
modality="audio",
)
return audio_result
return await _run_native_audio_conversation(
analyzer,
media,
audio_result = await analyzer._exec_analyze_audio_universal(
audio_path=media.audio_path,
start_time=current_start,
end_time=current_end,
actual_start=media.actual_start,
prompt_with_context=media.prompt_with_context,
specific_query=specific_query,
)
summary, analysis = analyzer._parse_chunk_result(audio_result)
analyzer.blackboard.complete_segment(
current_start,
current_end,
start_time,
end_time,
specific_query,
lease_owner,
summary,
analysis,
modality="audio",
)
return audio_result
except Exception as e:
# Same contract as the video sub-agent loop: classification
@@ -232,49 +208,6 @@ async def _finish_failed_audio(
):
"""Records terminal audio failure, attempting the universal fallback first."""
failure = classify_video_failure(error)
if (
analyzer.use_native_gemini
and failure.should_fallback
and media.audio_path is not None
and media.actual_start is not None
and media.prompt_with_context is not None
):
try:
logger.warning(
f"Native audio analysis failed [{failure.category.value}]; using universal fallback"
)
_va._record_llm_event(
"llm_fallback",
{
"reason": "native_audio_analysis_failed",
"category": failure.category.value,
"error": str(error)[:500],
},
)
fallback_result = await analyzer._exec_analyze_audio_universal(
audio_path=media.audio_path,
start_time=current_start,
end_time=current_end,
actual_start=media.actual_start,
prompt_with_context=media.prompt_with_context,
specific_query=specific_query,
force_fallback=True,
)
summary, analysis = analyzer._parse_chunk_result(fallback_result)
analyzer.blackboard.complete_segment(
current_start,
current_end,
specific_query,
lease_owner,
summary,
analysis,
modality="audio",
)
return fallback_result
except Exception as fallback_error:
error = fallback_error
failure = classify_video_failure(error)
retryable = None
analyzer.blackboard.fail_segment(
current_start,
current_end,
@@ -366,178 +299,3 @@ def _build_audio_prompt(
)
prompt_with_context += f"{specific_query}"
return prompt_with_context
async def _run_native_audio_conversation(
analyzer,
media: _AudioMedia,
current_start: float,
current_end: float,
start_time: float,
end_time: float | None,
specific_query: str,
lease_owner: str,
) -> str:
"""Uploads the audio and drives the native Gemini audio agent to a committed answer."""
file = None
try:
with TraceSpan(name="upload_audio_to_gemini") as span:
file = await analyzer.upload_and_poll_file(media.audio_path)
span.result = f"Uploaded {file.name}"
logger.info(f"Invoking Gemini for audio-only task with model {analyzer.model_name}...")
sub_agent_contents = [
types.Content(
role="user",
parts=[
types.Part.from_uri(file_uri=file.uri, mime_type=file.mime_type)
if hasattr(file, "uri")
else file,
types.Part.from_text(text=media.prompt_with_context),
],
)
]
(
final_summary,
final_analysis,
final_confidence_score,
) = await _drive_audio_agent_loop(analyzer, sub_agent_contents)
end_val = end_time if end_time is not None else "unknown"
analyzer._record_blackboard_entry(
{
"start": start_time,
"end": end_val,
"target": specific_query,
"summary": final_summary,
"confidence_score": final_confidence_score,
},
modality="audio",
)
end_str = f"{end_time:.1f}s" if end_time is not None else "unknown"
audio_result = (
f"[from {start_time:.1f}s to {end_str}] Summary:"
f" {final_summary} Analysis: {final_analysis}"
)
analyzer.blackboard.complete_segment(
current_start,
current_end,
specific_query,
lease_owner,
final_summary,
final_analysis,
modality="audio",
)
return audio_result
finally:
if file:
logger.info(f"Cleaning up cloud file {file.name}...")
try:
await asyncio.wait_for(
analyzer.client.aio.files.delete(name=file.name),
timeout=30,
)
analyzer.cloud_files_to_cleanup.discard(file.name)
except Exception as ce:
logger.error(f"Failed to delete cloud file {file.name}: {ce}")
async def _drive_audio_agent_loop(analyzer, sub_agent_contents: list) -> tuple[str, str, float]:
"""Return a validated (summary, analysis, confidence_score).
Raise ``SubAgentAnswerExhausted`` if no answer is accepted within the turn limit.
"""
rejection = _NO_TOOL_CALL_ERROR
for turn in range(1, SUB_AGENT_MAX_TURNS + 1):
if turn == SUB_AGENT_MAX_TURNS - 1:
sub_agent_contents.append(
types.Content(
role="user",
parts=[types.Part.from_text(text=_SUBMIT_NOW_WARNING)],
)
)
with TraceSpan(name="gemini_generate_audio_content") as span:
response = await asyncio.wait_for(
analyzer.client.aio.models.generate_content(
model=analyzer.model_name,
contents=sub_agent_contents,
config=types.GenerateContentConfig(
system_instruction=analyzer.audio_system_prompt,
tools=[types.Tool(function_declarations=[analyzer.submit_answer_tool])],
safety_settings=SAFETY_SETTINGS_BLOCK_NONE,
),
),
timeout=analyzer.model_call_timeout_seconds,
)
full_text = response.text or ""
function_calls = response.function_calls or []
if function_calls:
answer, rejection = _handle_audio_function_calls(
sub_agent_contents, response, function_calls, full_text
)
if answer is not None:
return answer
else:
rejection = _NO_TOOL_CALL_ERROR
sub_agent_contents.append(
types.Content(
role="model",
parts=[types.Part.from_text(text=full_text)],
)
)
raise SubAgentAnswerExhausted(SUB_AGENT_MAX_TURNS, rejection)
def _validate_audio_answer(summary: str, confidence_score) -> str | None:
"""Return a validation error for the audio answer, or None if valid."""
if not summary:
return _MISSING_SUMMARY_ERROR
if (
not isinstance(confidence_score, (int, float))
or isinstance(confidence_score, bool)
or not (0.0 <= float(confidence_score) <= 1.0)
):
return _BAD_CONFIDENCE_ERROR
return None
def _handle_audio_function_calls(
sub_agent_contents: list,
response,
function_calls: list,
full_text: str,
) -> tuple[tuple[str, str, float] | None, str | None]:
"""Return (answer, rejection), appending any rejection to the conversation."""
original_parts = response.candidates[0].content.parts
for fc in function_calls:
if _tool_name(fc.name) != "submit_answer":
continue
args = dict(fc.args or {})
summary = _clean_text(args.get("summary"))
analysis = _clean_text(args.get("analysis"))
rejection = _validate_audio_answer(summary, args.get("confidence_score"))
if rejection is not None:
_append_rejection(
sub_agent_contents, original_parts, function_calls, full_text, fc.name, rejection
)
return None, rejection
return (summary, analysis or "No analysis provided.", float(args["confidence_score"])), None
_append_rejection(
sub_agent_contents,
original_parts,
function_calls,
full_text,
function_calls[0].name,
_WRONG_TOOL_ERROR,
)
return None, _WRONG_TOOL_ERROR
artemis/agents/video_analyzer/chunk_agentic.py
-455
View File
@@ -1,455 +0,0 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Analyze video chunks through the Gemini Interactions API.
``processing: "agentic"`` lets the model search and revisit an uploaded clip.
Validation errors are returned as function results, with video context retained
through ``previous_interaction_id``. Only accepted answers reach the blackboard.
The stream consumer reconstructs function arguments from step deltas. Completion
events may omit steps, so it fetches the interaction when reconstruction fails.
"""
from __future__ import annotations
import asyncio
import copy
from dataclasses import dataclass, field
import json
from types import SimpleNamespace
from typing import Any
from artemis.agents.video_analyzer import gemini_files
from artemis.agents.video_analyzer import video_analyzer as _va
from artemis.agents.video_analyzer.chunk_conversation import (
_NO_TOOL_CALL_ERROR,
_SUBMIT_NOW_WARNING,
_WRONG_TOOL_ERROR,
SUB_AGENT_MAX_TURNS,
SubAgentAnswerExhausted,
_ChunkMedia,
_clean_text,
_persist_timeline_events,
_tool_name,
_validate_answer,
)
from artemis.agents.video_analyzer.universal_tools import UNIVERSAL_SUBMIT_ANSWER_TOOL
from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan
from artemis.llm.google import normalize_usage
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
STREAM_START_TIMEOUT_SECONDS = 60.0
INTERACTION_READ_TIMEOUT_SECONDS = 30.0
# Interaction statuses after which no follow-up turn can be sent
# (``google.genai`` ``InteractionStatus``).
_TERMINAL_FAILURE_STATUSES = frozenset({"failed", "cancelled", "incomplete", "budget_exceeded"})
def build_interactions_submit_tool() -> dict[str, Any]:
"""The ``submit_answer`` declaration in Interactions API (flat JSON schema) form.
Derived from the universal (OpenAI-style) declaration so the schema is
written once; ``timeline_events`` is required to match the native path.
"""
function = UNIVERSAL_SUBMIT_ANSWER_TOOL["function"]
parameters = copy.deepcopy(function["parameters"])
required = list(parameters.get("required", []))
if "timeline_events" not in required:
required.append("timeline_events")
parameters["required"] = required
return {
"type": "function",
"name": function["name"],
"description": function["description"],
"parameters": parameters,
}
def _steps(interaction: Any) -> list:
return list(getattr(interaction, "steps", None) or [])
def _find_function_call(interaction: Any):
"""The last ``function_call`` step of an interaction, if any."""
for step in reversed(_steps(interaction)):
if getattr(step, "type", None) == "function_call":
return step
return None
def _raise_if_failed(interaction: Any) -> None:
"""Raises when an interaction ended in a terminal failure state.
A follow-up with ``previous_interaction_id`` onto a failed interaction is
rejected by the API, so the failure surfaces here as an exception and
flows through the shared retry / classification layer instead.
"""
status = str(getattr(interaction, "status", None) or "").lower()
errors = list(getattr(interaction, "errors", None) or [])
if status not in _TERMINAL_FAILURE_STATUSES and not errors:
return
details = "; ".join(
f"[{getattr(error, 'code', None)}] {getattr(error, 'message', error)}" for error in errors
)
raise RuntimeError(
f"Gemini Interactions interaction {getattr(interaction, 'id', None)} ended with"
f" status {status or 'unknown'}" + (f": {details}" if details else "")
)
async def run_agentic_chunk_conversation(
analyzer,
media: _ChunkMedia,
current_start: float,
current_end: float,
start_time: float,
end_time: float | None,
specific_query: str,
lease_owner: str,
) -> str:
"""Uploads the chunk and drives the agentic sub-agent to a committed answer."""
file = None
try:
with TraceSpan(name="upload_video_to_gemini") as span:
file = await analyzer.upload_and_poll_file(media.compressed_path)
span.result = f"Uploaded {file.name}"
logger.info(
f"Invoking Gemini agentic video understanding for sub-agent task with model"
f" {analyzer.model_name} (streaming)..."
)
trace_id = CURRENT_TRACE_ID.get()
if analyzer.ctx.data_engine and trace_id:
analyzer.ctx.data_engine.record_trace(
type="llm_call",
name="video_sub_agent",
payload={
"contents": [
f"file://{file.name}",
media.prompt_with_context,
],
"system_instruction": analyzer.sub_system_prompt,
"video_processing": "agentic",
},
parent_trace_id=trace_id,
)
final_summary, final_analysis = await _drive_agentic_loop(
analyzer,
file,
trace_id,
media,
start_time,
end_time,
specific_query,
)
end_str = f"{current_end:.1f}s" if current_end is not None else "unknown"
chunk_result = (
f"[from {current_start:.1f}s to {end_str}] Summary:"
f" {final_summary} Analysis: {final_analysis}"
)
analyzer.blackboard.complete_segment(
current_start,
current_end,
specific_query,
lease_owner,
final_summary,
final_analysis,
)
return chunk_result
finally:
if file:
await gemini_files.delete_cloud_file(
analyzer.client, file.name, analyzer.cloud_files_to_cleanup
)
def _initial_request(analyzer, file, prompt: str, tool: dict[str, Any]) -> dict[str, Any]:
return {
"model": analyzer.model_name,
"input": [
{
"type": "video",
"uri": file.uri,
"mime_type": file.mime_type,
"processing": "agentic",
},
{"type": "text", "text": prompt},
],
"system_instruction": analyzer.sub_system_prompt,
"tools": [tool],
"stream": True,
}
def _follow_up_request(
analyzer, previous_id: str, steps: list[dict[str, Any]], tool: dict[str, Any]
) -> dict[str, Any]:
"""A stateful follow-up turn: the server keeps the video context.
Only the conversation state is inherited through
``previous_interaction_id``; ``system_instruction`` and ``tools`` are
interaction-scoped and must be re-sent on every turn.
"""
return {
"model": analyzer.model_name,
"previous_interaction_id": previous_id,
"input": steps,
"system_instruction": analyzer.sub_system_prompt,
"tools": [tool],
"stream": True,
}
def _function_result(call_id: str, error: str) -> dict[str, Any]:
return {
"type": "function_result",
"call_id": call_id,
"name": "submit_answer",
"result": {"error": error},
}
async def _drive_agentic_loop(
analyzer,
file,
trace_id,
media: _ChunkMedia,
start_time: float,
end_time: float | None,
specific_query: str,
) -> tuple[str, str]:
"""Return a validated answer and persist its timeline events.
Raise ``SubAgentAnswerExhausted`` if no answer is accepted within the turn limit.
"""
tool = build_interactions_submit_tool()
request = _initial_request(analyzer, file, media.prompt_with_context, tool)
rejection = _NO_TOOL_CALL_ERROR
for turn in range(1, SUB_AGENT_MAX_TURNS + 1):
interaction = await _stream_agentic_turn(analyzer, request, trace_id)
interaction_id = getattr(interaction, "id", None) or ""
call = _find_function_call(interaction)
if call is None:
rejection = _NO_TOOL_CALL_ERROR
steps = [{"type": "text", "text": _SUBMIT_NOW_WARNING}]
elif _tool_name(getattr(call, "name", None)) != "submit_answer":
rejection = _WRONG_TOOL_ERROR
steps = [_function_result(call.id, rejection)]
else:
args = dict(getattr(call, "arguments", None) or {})
timeline_events = args.get("timeline_events", [])
if not isinstance(timeline_events, list):
timeline_events = []
summary = _clean_text(args.get("summary"))
analysis = _clean_text(args.get("analysis"))
rejection = _validate_answer(summary, timeline_events, start_time, end_time)
if rejection is None:
await _persist_timeline_events(
analyzer, timeline_events, media, start_time, end_time, specific_query
)
return summary, analysis or "No analysis provided."
steps = [_function_result(call.id, rejection)]
if turn < SUB_AGENT_MAX_TURNS:
request = _follow_up_request(analyzer, interaction_id, steps, tool)
raise SubAgentAnswerExhausted(SUB_AGENT_MAX_TURNS, rejection)
async def _stream_agentic_turn(analyzer, request: dict[str, Any], trace_id):
"""Runs one traced streaming Interactions turn and returns the completed interaction."""
with TraceSpan(name="gemini_agentic_video_call") as span:
span.payload = {
"model": request.get("model"),
"video_processing": "agentic",
"previous_interaction_id": request.get("previous_interaction_id"),
}
stream = await asyncio.wait_for(
analyzer.client.aio.interactions.create(**request),
timeout=STREAM_START_TIMEOUT_SECONDS,
)
interaction = await asyncio.wait_for(
_consume_agentic_stream(analyzer, span, stream, trace_id),
timeout=analyzer.agentic_call_timeout_seconds,
)
call = _find_function_call(interaction)
span.result = (
f"Function call: {_tool_name(getattr(call, 'name', None))}"
if call is not None
else f"Status: {getattr(interaction, 'status', 'unknown')}"
)
return interaction
@dataclass
class _PendingStep:
"""A step under construction while its start/delta/stop events stream in."""
kind: str
call_id: str | None = None
name: str | None = None
seed_arguments: dict[str, Any] = field(default_factory=dict)
arguments_json: list[str] = field(default_factory=list)
text: list[str] = field(default_factory=list)
def finish(self):
"""Materializes the step in the shape the SDK's typed steps expose.
Returns ``None`` for a function call whose streamed arguments are not
valid JSON: a half-built call must not stand in for the real one, so
the step is dropped and the interaction is re-read by id instead.
"""
if self.kind == "function_call":
arguments: dict[str, Any] = dict(self.seed_arguments)
raw = "".join(self.arguments_json)
if raw.strip():
try:
parsed = json.loads(raw)
except ValueError:
logger.warning(
f"Could not parse streamed arguments for {self.name}: {raw[:200]!r};"
" dropping the step"
)
return None
if isinstance(parsed, dict):
arguments = parsed
return SimpleNamespace(
type="function_call", id=self.call_id, name=self.name, arguments=arguments
)
if self.kind == "model_output":
return SimpleNamespace(
type="model_output",
content=[SimpleNamespace(type="text", text="".join(self.text))],
)
return SimpleNamespace(type=self.kind, id=self.call_id)
async def _consume_agentic_stream(analyzer, span, stream, trace_id):
"""Consumes Interactions SSE events, rebuilding steps and mirroring progress.
Returns an interaction-shaped object (``id`` / ``status`` / ``steps`` /
``usage``). The interaction is re-read by id when the stream ended
without a completion event, or when it stopped at ``requires_action``
without a parsable function call.
"""
data_engine = analyzer.ctx.data_engine if trace_id else None
interaction_id: str | None = None
status: str | None = None
completed_steps: list | None = None
usage: dict[str, int] | None = None
pending: dict[int, _PendingStep] = {}
steps: list = []
processing_calls = 0
def _emit(text: str, *, thinking: bool) -> None:
if data_engine and text:
data_engine.stream_output(trace_id, text, is_thinking=thinking)
async for event in stream:
data = getattr(event, "data", event)
event_type = getattr(data, "event_type", None)
if event_type == "interaction.created":
created = getattr(data, "interaction", None)
interaction_id = getattr(created, "id", None) or interaction_id
elif event_type == "step.start":
step = getattr(data, "step", None)
index = getattr(data, "index", None)
kind = str(getattr(step, "type", None) or "")
pending[index] = _PendingStep(
kind=kind,
call_id=getattr(step, "id", None) or getattr(step, "call_id", None),
name=getattr(step, "name", None),
seed_arguments=dict(getattr(step, "arguments", None) or {}),
)
if kind == "processing_call":
processing_calls += 1
_emit(
f"Inspecting the recording (video lookup {processing_calls})\n",
thinking=True,
)
elif event_type == "step.delta":
delta = getattr(data, "delta", None)
delta_type = getattr(delta, "type", None)
current = pending.get(getattr(data, "index", None))
if delta_type == "arguments_delta":
fragment = getattr(delta, "arguments", None)
if current is not None and isinstance(fragment, str):
current.arguments_json.append(fragment)
elif delta_type == "text":
text = getattr(delta, "text", None)
if current is not None and isinstance(text, str):
current.text.append(text)
_emit(text or "", thinking=False)
elif delta_type == "thought_summary":
content = getattr(delta, "content", None)
_emit(getattr(content, "text", None) or "", thinking=True)
elif event_type == "step.stop":
current = pending.pop(getattr(data, "index", None), None)
finished = current.finish() if current is not None else None
if finished is not None:
steps.append(finished)
usage = normalize_usage(getattr(data, "usage", None)) or usage
elif event_type == "interaction.completed":
completed = getattr(data, "interaction", None)
interaction_id = getattr(completed, "id", None) or interaction_id
status = getattr(completed, "status", None) or status
completed_steps = getattr(completed, "steps", None) or None
usage = normalize_usage(getattr(completed, "usage", None)) or usage
_raise_if_failed(completed)
elif event_type == "interaction.status_update":
status = getattr(data, "status", None) or status
elif event_type == "error":
error = getattr(data, "error", None)
raise RuntimeError(
"Gemini Interactions stream error"
f" [{getattr(error, 'code', None)}]: {getattr(error, 'message', None)}"
)
for current in pending.values():
finished = current.finish()
if finished is not None:
steps.append(finished)
span.payload["processing_calls"] = processing_calls
if completed_steps:
steps = list(completed_steps)
needs_reread = status is None or (
status == "requires_action" and _find_function_call(SimpleNamespace(steps=steps)) is None
)
if needs_reread:
if not interaction_id:
raise RuntimeError("Gemini Interactions stream ended without an interaction")
logger.info(f"Re-reading interaction {interaction_id} (status={status})")
fetched = await asyncio.wait_for(
analyzer.client.aio.interactions.get(interaction_id),
timeout=INTERACTION_READ_TIMEOUT_SECONDS,
)
_raise_if_failed(fetched)
status = getattr(fetched, "status", None) or status
steps = _steps(fetched) or steps
usage = normalize_usage(getattr(fetched, "usage", None)) or usage
if usage:
span.payload["usage_metadata"] = usage
return SimpleNamespace(id=interaction_id, status=status, steps=steps, usage=usage)
artemis/agents/video_analyzer/chunk_conversation.py
-491
View File
@@ -1,491 +0,0 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Native Gemini sub-agent conversation and submit_answer handling for chunks.
Extracted from ``video_analyzer.py`` as a pure structural split. Patchable
collaborators are looked up late through the facade module so
``mock.patch("artemis.agents.video_analyzer.video_analyzer.<name>")`` targets
keep working.
"""
import asyncio
import glob
from pathlib import Path
import re
from google.genai import types
from artemis.agents.video_analyzer import gemini_files
from artemis.agents.video_analyzer import video_analyzer as _va
from artemis.agents.video_analyzer.reliability import SubAgentAnswerExhausted
from artemis.constants import SAFETY_SETTINGS_BLOCK_NONE
from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan
from artemis.llm.google import normalize_usage
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
#: Turns the chunk sub-agent gets to produce an accepted ``submit_answer``
#: (shared by the static and agentic paths).
SUB_AGENT_MAX_TURNS = 2
_SUBMIT_NOW_WARNING = "[WARNING] You must call the submit_answer tool now, time is running out."
_WRONG_TOOL_ERROR = "Tool not recognized. Please use submit_answer."
_NO_TOOL_CALL_ERROR = "The model replied with text only and never called submit_answer."
_MISSING_SUMMARY_ERROR = (
"submit_answer was called without a 'summary'. Call it again with a non-empty"
" 'summary' (and 'analysis') describing what you observed."
)
def _tool_name(name: str | None) -> str:
name = name or ""
return name.split(":")[-1] if ":" in name else name
def _clean_text(value) -> str:
return str(value or "").strip().replace("\n", " ")
def _validate_answer(
summary: str, timeline_events: list, start_time: float, end_time: float | None
) -> str | None:
"""Return a validation error for submit_answer, or None if valid."""
if not summary:
return _MISSING_SUMMARY_ERROR
return _validate_timeline_events(timeline_events, start_time, end_time)
class _ChunkMedia:
"""Mutable holder for per-attempt media artifacts shared with failure handling."""
__slots__ = ("path", "compressed_path", "actual_start", "prompt_with_context")
def __init__(self):
self.path: Path | None = None
self.compressed_path: Path | None = None
self.actual_start: float | None = None
self.prompt_with_context: str | None = None
async def _run_native_chunk_conversation(
analyzer,
media: _ChunkMedia,
current_start: float,
current_end: float,
start_time: float,
end_time: float | None,
specific_query: str,
lease_owner: str,
) -> str:
"""Uploads the chunk and drives the native Gemini sub-agent to a committed answer."""
file = None
try:
with TraceSpan(name="upload_video_to_gemini") as span:
file = await analyzer.upload_and_poll_file(media.compressed_path)
span.result = f"Uploaded {file.name}"
logger.info(
f"Invoking Gemini for sub-agent task with model {analyzer.model_name} (streaming)..."
)
trace_id = CURRENT_TRACE_ID.get()
if analyzer.ctx.data_engine and trace_id:
analyzer.ctx.data_engine.record_trace(
type="llm_call",
name="video_sub_agent",
payload={
"contents": [
f"file://{file.name}",
media.prompt_with_context,
],
"system_instruction": analyzer.sub_system_prompt,
},
parent_trace_id=trace_id,
)
sub_agent_contents = [
types.Content(
role="user",
parts=[
types.Part.from_uri(file_uri=file.uri, mime_type=file.mime_type)
if hasattr(file, "uri")
else file,
types.Part.from_text(text=media.prompt_with_context),
],
)
]
final_summary, final_analysis = await _drive_sub_agent_loop(
analyzer,
sub_agent_contents,
trace_id,
media,
start_time,
end_time,
specific_query,
)
end_str = f"{current_end:.1f}s" if current_end is not None else "unknown"
chunk_result = (
f"[from {current_start:.1f}s to {end_str}] Summary:"
f" {final_summary} Analysis: {final_analysis}"
)
analyzer.blackboard.complete_segment(
current_start,
current_end,
specific_query,
lease_owner,
final_summary,
final_analysis,
)
return chunk_result
finally:
if file:
await gemini_files.delete_cloud_file(
analyzer.client, file.name, analyzer.cloud_files_to_cleanup
)
async def _drive_sub_agent_loop(
analyzer,
sub_agent_contents: list,
trace_id,
media: _ChunkMedia,
start_time: float,
end_time: float | None,
specific_query: str,
) -> tuple[str, str]:
"""Return a validated answer and persist its timeline events.
Raise ``SubAgentAnswerExhausted`` if no answer is accepted within the turn limit.
"""
rejection = _NO_TOOL_CALL_ERROR
for turn in range(1, SUB_AGENT_MAX_TURNS + 1):
if turn == SUB_AGENT_MAX_TURNS - 1:
sub_agent_contents.append(
types.Content(
role="user",
parts=[types.Part.from_text(text=_SUBMIT_NOW_WARNING)],
)
)
full_text, function_calls, original_parts = await _stream_sub_agent_turn(
analyzer, sub_agent_contents, trace_id
)
if function_calls:
answer, rejection = await _handle_sub_agent_calls(
analyzer,
sub_agent_contents,
function_calls,
original_parts,
full_text,
media,
start_time,
end_time,
specific_query,
)
if answer is not None:
return answer
else:
rejection = _NO_TOOL_CALL_ERROR
sub_agent_contents.append(
types.Content(
role="model",
parts=[types.Part.from_text(text=full_text)],
)
)
raise SubAgentAnswerExhausted(SUB_AGENT_MAX_TURNS, rejection)
async def _stream_sub_agent_turn(
analyzer, sub_agent_contents: list, trace_id
) -> tuple[str, list, list]:
"""Starts one streamed Gemini turn and consumes it fully."""
with TraceSpan(name="gemini_stream_content_main") as span:
stream = await asyncio.wait_for(
analyzer.client.aio.models.generate_content_stream(
model=analyzer.model_name,
contents=sub_agent_contents,
config=types.GenerateContentConfig(
system_instruction=analyzer.sub_system_prompt,
tools=[types.Tool(function_declarations=[analyzer.submit_answer_tool])],
safety_settings=SAFETY_SETTINGS_BLOCK_NONE,
),
),
timeout=60,
)
full_text, function_calls, original_parts = await asyncio.wait_for(
_read_sub_agent_stream(analyzer, span, stream, trace_id),
timeout=analyzer.model_call_timeout_seconds,
)
return full_text, function_calls, original_parts
async def _read_sub_agent_stream(analyzer, span, target_stream, current_trace_id):
"""Consumes a streamed response, mirroring text/thoughts into the trace."""
text = ""
function_calls = []
accumulated_parts = []
async for chunk in target_stream:
usage = normalize_usage(getattr(chunk, "usage_metadata", None))
if usage:
span.payload["usage_metadata"] = usage
chunk_text = chunk.text or ""
if chunk_text:
text += chunk_text
if analyzer.ctx.data_engine and current_trace_id:
analyzer.ctx.data_engine.stream_output(current_trace_id, chunk_text)
if chunk.function_calls:
function_calls.extend(chunk.function_calls)
candidates = getattr(chunk, "candidates", None)
if candidates and isinstance(candidates, list) and len(candidates) > 0:
content = getattr(candidates[0], "content", None)
parts = getattr(content, "parts", None) if content else None
if parts:
for part in parts:
accumulated_parts.append(part)
if getattr(part, "thought", False) and part.text:
if analyzer.ctx.data_engine and current_trace_id:
analyzer.ctx.data_engine.stream_output(
current_trace_id,
part.text,
is_thinking=True,
)
else:
if chunk.function_calls:
for fc in chunk.function_calls:
accumulated_parts.append(types.Part(function_call=fc))
return text, function_calls, accumulated_parts
async def _handle_sub_agent_calls(
analyzer,
sub_agent_contents: list,
function_calls: list,
original_parts: list,
full_text: str,
media: _ChunkMedia,
start_time: float,
end_time: float | None,
specific_query: str,
) -> tuple[tuple[str, str] | None, str | None]:
"""Return ``(answer, rejection)`` after processing one turn's tool calls.
Persist events for accepted answers; append rejected calls to the conversation
with an error for the model to correct.
"""
for fc in function_calls:
if _tool_name(fc.name) != "submit_answer":
continue
args = dict(fc.args or {})
timeline_events = args.get("timeline_events", [])
if not isinstance(timeline_events, list):
timeline_events = []
summary = _clean_text(args.get("summary"))
analysis = _clean_text(args.get("analysis"))
rejection = _validate_answer(summary, timeline_events, start_time, end_time)
if rejection is not None:
_append_rejection(
sub_agent_contents, original_parts, function_calls, full_text, fc.name, rejection
)
return None, rejection
await _persist_timeline_events(
analyzer, timeline_events, media, start_time, end_time, specific_query
)
return (summary, analysis or "No analysis provided."), None
_append_rejection(
sub_agent_contents,
original_parts,
function_calls,
full_text,
function_calls[0].name,
_WRONG_TOOL_ERROR,
)
return None, _WRONG_TOOL_ERROR
def _validate_timeline_events(
timeline_events: list, start_time: float, end_time: float | None
) -> str | None:
"""Returns the first validation error message for the submitted events, if any."""
for event in timeline_events:
if (
"confidence_score" not in event
or not isinstance(
event["confidence_score"],
(int, float),
)
or isinstance(event["confidence_score"], bool)
or not (0.0 <= float(event["confidence_score"]) <= 1.0)
):
return (
"Error: 'confidence_score' is"
" missing or out of bounds in"
" one of the timeline_events."
" It must be a float between"
" 0.0 and 1.0 inclusive. Fix"
" the payload and call"
" submit_answer again."
)
if (
"verification_timestamp_secs" not in event
or not isinstance(
event["verification_timestamp_secs"],
(int, float),
)
or isinstance(event["verification_timestamp_secs"], bool)
):
return (
"Error:"
" 'verification_timestamp_secs'"
" is missing or invalid in one"
" of the timeline_events. Fix"
" the payload and call"
" submit_answer again."
)
verification_ts = float(
event.get(
"verification_timestamp_secs",
start_time,
)
)
confidence = float(event.get("confidence_score", 0.0)) # noqa: F841
if verification_ts < start_time or (end_time is not None and verification_ts > end_time):
return (
"Error:"
" 'verification_timestamp_secs'"
f" {verification_ts} must fall"
" within the start_time"
f" ({start_time}) and end_time"
f" ({end_time}) boundaries. Fix"
" the payload and call"
" submit_answer again."
)
return None
def _append_rejection(
sub_agent_contents: list,
original_parts: list,
function_calls: list,
full_text: str,
response_name: str,
error_msg: str,
) -> None:
"""Appends the model turn plus an error function response to the conversation."""
sub_agent_contents.append(
types.Content(
role="model",
parts=original_parts
if original_parts
else [types.Part(function_call=fc) for fc in function_calls]
+ ([types.Part.from_text(text=full_text)] if full_text else []),
)
)
sub_agent_contents.append(
types.Content(
role="user",
parts=[
types.Part.from_function_response(
name=response_name,
response={"error": (error_msg)},
)
],
)
)
async def _persist_timeline_events(
analyzer,
timeline_events: list,
media: _ChunkMedia,
start_time: float,
end_time: float | None,
specific_query: str,
) -> None:
"""Captures verification screenshots and records validated events on the blackboard."""
for event in timeline_events:
verification_ts = float(
event.get(
"verification_timestamp_secs",
start_time,
)
)
confidence = float(event.get("confidence_score", 0.0))
relative_offset = max(0.0, verification_ts - media.actual_start)
out_dir = _va.get_temp_dir("video_analyzer")
out_dir.mkdir(parents=True, exist_ok=True)
existing_imgs = glob.glob(f"{out_dir}/img_*.jpg")
max_idx = -1
for img in existing_imgs:
match = re.search(r"img_(\d+)\.jpg$", img)
if match:
max_idx = max(max_idx, int(match.group(1)))
next_idx = max_idx + 1
screenshot_output_path = out_dir / f"img_{next_idx}.jpg"
while screenshot_output_path.exists():
next_idx += 1
screenshot_output_path = out_dir / f"img_{next_idx}.jpg"
screenshot_output_path.touch()
analyzer.local_files_to_cleanup.add(screenshot_output_path)
ffmpeg_cmd = [
"ffmpeg",
"-ss",
str(relative_offset),
"-i",
str(media.path),
"-vframes",
"1",
"-q:v",
"2",
"-y",
str(screenshot_output_path),
]
try:
proc = await asyncio.create_subprocess_exec(
*ffmpeg_cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
await proc.communicate()
except Exception as e:
logger.warning(f"Failed to extract frame: {e}")
screenshot_path_str = str(screenshot_output_path)
entry = {
"start": event.get("start_time", start_time),
"end": event.get("end_time", end_time),
"target": specific_query,
"summary": event.get("transcription", ""),
"confidence_score": confidence,
}
if Path(screenshot_output_path).exists():
entry["screenshot"] = str(screenshot_path_str)
analyzer._record_blackboard_entry(entry)
artemis/agents/video_analyzer/chunk_native.py → artemis/agents/video_analyzer/chunk_runner.py
+29 -126
View File
@@ -12,29 +12,23 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Native Gemini execution path for a single video chunk sub-agent.
"""Responses execution path for a single video chunk sub-agent.
Extracted from ``video_analyzer.py`` as a pure structural split. Patchable
collaborators (``get_controller``, ``compress_video_for_api``, semaphores,
LLM event recorders, ...) are looked up late through the facade module so
``mock.patch("artemis.agents.video_analyzer.video_analyzer.<name>")`` targets
keep working. The sub-agent conversation itself lives in
``chunk_conversation``.
``universal_engine``.
"""
import asyncio
from pathlib import Path
from artemis.agents.video_analyzer import video_analyzer as _va
from artemis.agents.video_analyzer.chunk_agentic import run_agentic_chunk_conversation
from artemis.agents.video_analyzer.chunk_conversation import (
_ChunkMedia,
_run_native_chunk_conversation,
)
from artemis.agents.video_analyzer.reliability import (
AgenticVideoDegraded,
classify_video_failure,
is_agentic_rejection,
)
from artemis.llm.reliability import retry_policy_for
from artemis.utils.logger import get_logger
@@ -42,11 +36,23 @@ from artemis.utils.logger import get_logger
logger = get_logger(__name__)
class _ChunkMedia:
"""Mutable holder for per-attempt media artifacts shared with failure handling."""
__slots__ = ("path", "compressed_path", "actual_start", "prompt_with_context")
def __init__(self):
self.path: Path | None = None
self.compressed_path: Path | None = None
self.actual_start: float | None = None
self.prompt_with_context: str | None = None
async def exec_single_chunk(
analyzer, start_time: float, end_time: float | None, specific_query: str
) -> str:
"""Analyzes one closed chunk interval, with claim/lease and retry handling."""
max_retries = analyzer.native_max_retries
max_retries = analyzer.max_retries
current_start = round(start_time, 1) if isinstance(start_time, (int, float)) else start_time
current_end = round(end_time, 1) if isinstance(end_time, (int, float)) else end_time
@@ -85,7 +91,6 @@ async def exec_single_chunk(
while attempt <= max_retries:
# The mode this attempt was launched in: a parallel chunk may degrade
# the run while this one is still in flight.
attempt_mode = analyzer.video_processing
try:
logger.info(
f"Attempt {attempt + 1}/{max_retries + 1} for sub-agent:"
@@ -108,49 +113,25 @@ async def exec_single_chunk(
)
_ensure_fits_static_ceiling(analyzer, current_start, current_end)
if not analyzer.use_native_gemini:
chunk_result = await analyzer._exec_single_chunk_universal(
compressed_path=media.compressed_path,
raw_path=media.path,
start_time=current_start,
end_time=current_end,
actual_start=media.actual_start,
prompt_with_context=media.prompt_with_context,
specific_query=specific_query,
)
summary, analysis = analyzer._parse_chunk_result(chunk_result)
analyzer.blackboard.complete_segment(
current_start,
current_end,
specific_query,
lease_owner,
summary,
analysis,
)
return chunk_result
if analyzer.video_processing == "agentic":
return await run_agentic_chunk_conversation(
analyzer,
media,
current_start,
current_end,
start_time,
end_time,
specific_query,
lease_owner,
)
return await _run_native_chunk_conversation(
analyzer,
media,
chunk_result = await analyzer._exec_single_chunk_universal(
compressed_path=media.compressed_path,
raw_path=media.path,
start_time=current_start,
end_time=current_end,
actual_start=media.actual_start,
prompt_with_context=media.prompt_with_context,
specific_query=specific_query,
)
summary, analysis = analyzer._parse_chunk_result(chunk_result)
analyzer.blackboard.complete_segment(
current_start,
current_end,
start_time,
end_time,
specific_query,
lease_owner,
summary,
analysis,
)
return chunk_result
except Exception as e:
# Retry decisions are owned by the shared classification layer.
@@ -159,22 +140,6 @@ async def exec_single_chunk(
# stream output from the failed attempt is discarded, never
# surfaced as a complete answer.
failure = classify_video_failure(e)
if attempt_mode == "agentic" and is_agentic_rejection(failure, e):
# Switch to static and let the coordinator resize the interval.
_degrade_agentic_to_static(analyzer, e, failure)
return await _finish_failed_chunk(
analyzer,
media,
current_start,
current_end,
specific_query,
lease_owner,
AgenticVideoDegraded(
f"Agentic video processing rejected ({e}); re-plan"
f" {current_start:.1f}s-{current_end:.1f}s at the static chunk size"
),
retryable=False,
)
attempts_so_far = attempt + 1
if not failure.retryable:
logger.warning(
@@ -229,26 +194,6 @@ async def exec_single_chunk(
)
def _degrade_agentic_to_static(analyzer, error: Exception, failure) -> None:
"""Switches the rest of this run to static video understanding (idempotent)."""
if analyzer.video_processing != "agentic":
return
logger.warning(
f"Agentic video processing rejected ({error}); the rest of this run"
" analyzes video in static mode"
)
_va._record_llm_event(
"llm_fallback",
{
"reason": "agentic_video_rejected",
"category": failure.category.value,
"error": str(error)[:500],
},
)
analyzer.video_processing = "static"
analyzer.chunk_size_seconds = analyzer.static_chunk_size_seconds
def _ensure_fits_static_ceiling(analyzer, current_start: float, current_end: float) -> None:
"""Refuses to analyze a clip sized for the agentic ceiling once the run is static.
@@ -280,48 +225,6 @@ async def _finish_failed_chunk(
):
"""Records terminal failure, attempting the universal fallback first when eligible."""
failure = classify_video_failure(error)
if (
analyzer.use_native_gemini
and failure.should_fallback
and media.path is not None
and media.compressed_path is not None
and media.actual_start is not None
and media.prompt_with_context is not None
):
try:
logger.warning("Native video chunk exhausted retries; using universal fallback")
_va._record_llm_event(
"llm_fallback",
{
"reason": "native_video_chunk_failed",
"category": failure.category.value,
"error": str(error)[:500],
},
)
fallback_result = await analyzer._exec_single_chunk_universal(
compressed_path=media.compressed_path,
raw_path=media.path,
start_time=current_start,
end_time=current_end,
actual_start=media.actual_start,
prompt_with_context=media.prompt_with_context,
specific_query=specific_query,
force_fallback=True,
)
summary, analysis = analyzer._parse_chunk_result(fallback_result)
analyzer.blackboard.complete_segment(
current_start,
current_end,
specific_query,
lease_owner,
summary,
analysis,
)
return fallback_result
except Exception as fallback_error:
error = fallback_error
failure = classify_video_failure(error)
retryable = None
analyzer.blackboard.fail_segment(
current_start,
current_end,
artemis/agents/video_analyzer/gemini_files.py
-162
View File
@@ -1,162 +0,0 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Gemini File API helpers: upload/poll lifecycle and abandoned-file cleanup."""
import asyncio
from datetime import datetime
from pathlib import Path
import time
from artemis.utils.logger import get_logger
try:
from datetime import UTC
except ImportError:
UTC = UTC
logger = get_logger(__name__)
_LAST_CLEANUP_TIME = 0.0
_CLEANUP_LOCK = asyncio.Lock()
async def cleanup_abandoned_gemini_files(client) -> None:
"""Scan and delete remaining cloud video files whose creation time has elapsed standard TTL."""
global _LAST_CLEANUP_TIME
if _CLEANUP_LOCK.locked():
return
async with _CLEANUP_LOCK:
now_ts = time.time()
if now_ts - _LAST_CLEANUP_TIME < 3600:
return
_LAST_CLEANUP_TIME = now_ts
try:
pager = await asyncio.wait_for(client.aio.files.list(), timeout=30)
files = []
async for f in pager:
files.append(f)
now = datetime.now(UTC)
files_to_delete = []
for f in files:
display_name = getattr(f, "display_name", "") or ""
if (
display_name.startswith("compressed_")
or display_name.startswith("audio_")
or "artemis" in display_name
):
create_time = getattr(f, "created_at", None) or getattr(f, "create_time", None)
if not create_time:
continue
if isinstance(create_time, str):
try:
parsed_time = datetime.fromisoformat(create_time.replace("Z", "+00:00"))
except Exception:
continue
elif isinstance(create_time, datetime):
parsed_time = create_time
else:
continue
if parsed_time.tzinfo is None:
parsed_time = parsed_time.replace(tzinfo=UTC)
age_seconds = (now - parsed_time).total_seconds()
if age_seconds > 7200:
files_to_delete.append(f)
logger.info(
f"Marked expired cloud asset for deletion: {f.name}"
f" ({display_name}), Age: {age_seconds / 3600:.1f}h"
)
if files_to_delete:
logger.info(f"Purging {len(files_to_delete)} expired cloud assets in parallel...")
tasks = [
asyncio.wait_for(client.aio.files.delete(name=f.name), timeout=30)
for f in files_to_delete
]
await asyncio.gather(*tasks, return_exceptions=True)
except Exception as e:
logger.warning(f"Routine cloud maintenance skipped: {e}")
async def delete_cloud_file(
client, file_name: str, cloud_files_to_cleanup: set, *, timeout: float = 30.0
) -> None:
"""Best-effort deletion of an uploaded file; failures are logged, never raised."""
logger.info(f"Cleaning up cloud file {file_name}...")
try:
await asyncio.wait_for(client.aio.files.delete(name=file_name), timeout=timeout)
cloud_files_to_cleanup.discard(file_name)
except Exception as ce:
logger.error(f"Failed to delete cloud file {file_name}: {ce}")
async def upload_and_poll_file(client, compressed_path: Path, cloud_files_to_cleanup: set) -> any:
"""Upload a media file to the Gemini File API and poll until it is ACTIVE."""
logger.info(f"Uploading {compressed_path} to Gemini File API...")
file_size_mb = compressed_path.stat().st_size / (1024 * 1024)
upload_timeout = max(30.0, min(120.0, file_size_mb * 2.0))
file = await asyncio.wait_for(
client.aio.files.upload(file=compressed_path),
timeout=upload_timeout,
)
cloud_files_to_cleanup.add(file.name)
max_wait = max(60, min(180, 60 + int(file_size_mb * 2)))
wait_interval = 0.5
start_wait = time.time()
retry_count = 0
max_retries = 3
while True:
try:
f_state = await asyncio.wait_for(client.aio.files.get(name=file.name), timeout=20)
retry_count = 0
except Exception as poll_error:
retry_count += 1
logger.warning(
f"Temporary issue polling file {file.name} (attempt"
f" {retry_count}/{max_retries}): {poll_error}"
)
if retry_count > max_retries:
raise RuntimeError(
f"Failed to poll Gemini File API after {max_retries} attempts: {poll_error}"
)
await asyncio.sleep(2.0)
continue
if f_state.state.name == "ACTIVE":
break
elif f_state.state.name == "FAILED":
raise RuntimeError(f"Gemini File API processing failed for {file.name}")
if time.time() - start_wait > max_wait:
raise TimeoutError(
f"Gemini File API processing timeout for {file.name}"
f" (Waited {max_wait}s for size {file_size_mb:.1f}MB)"
)
logger.info(f"File {file.name} is {f_state.state.name}, waiting {wait_interval}s...")
await asyncio.sleep(wait_interval)
wait_interval = min(3.0, wait_interval * 1.5)
return file
artemis/agents/video_analyzer/native_coordinator.py
-541
View File
@@ -1,541 +0,0 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Native Gemini coordinator loop for the video analyzer main agent.
Extracted from ``video_analyzer.py`` as a pure structural split. Patchable
collaborators are looked up late through the facade module so
``mock.patch("artemis.agents.video_analyzer.video_analyzer.<name>")`` targets
keep working.
"""
import asyncio
import os
from pathlib import Path
import shutil
import tempfile
from google.genai import types
from artemis.agents.video_analyzer import video_analyzer as _va
from artemis.agents.video_analyzer.reliability import classify_video_failure
from artemis.constants import SAFETY_SETTINGS_BLOCK_NONE
from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan
from artemis.llm.google import normalize_usage
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
class _DummyResponse:
"""Aggregated view over a consumed stream, shaped like a full response."""
def __init__(self, text, function_calls, parts):
self.text = text
self.function_calls = function_calls
self.candidates = [
types.Candidate(
content=types.Content(
role="model",
parts=parts if parts else ([types.Part.from_text(text=text)] if text else []),
)
)
]
async def run_native(
analyzer,
ctx,
time_description: str,
purpose: str,
system_prompt: str,
temperature: float,
thinking_level,
) -> tuple[str, str]:
"""Drives the native Gemini coordinator loop to a final answer."""
tools_declaration = analyzer.tools_declaration
contents = [
types.Content(
role="user",
parts=[
types.Part.from_text(
text=(f"Time description: {time_description}\nPurpose: {purpose}")
)
],
)
]
agent_outcome = ""
status = "success"
max_iterations = 10
iterations = 0
last_ledger_index = 0
try:
while iterations < max_iterations:
iterations += 1
logger.info(f"Iteration {iterations}: Invoking Native Gemini SDK...")
analyzer._clean_blackboard()
last_ledger_index = _append_ledger_diff(analyzer, contents, last_ledger_index)
is_final_turn = iterations == max_iterations
_append_turn_guidance(contents, iterations, max_iterations, is_final_turn)
response = await _stream_main_turn(
analyzer,
ctx,
contents,
system_prompt,
temperature,
thinking_level,
tools_declaration,
is_final_turn,
)
function_calls = response.function_calls
if not function_calls:
agent_outcome = response.text
break
if response.candidates and response.candidates[0].content:
contents.append(response.candidates[0].content)
else:
contents.append(
types.Content(
role="model",
parts=[types.Part(function_call=fc) for fc in function_calls],
)
)
tool_response_parts = await _execute_native_tool_calls(analyzer, function_calls)
if tool_response_parts:
contents.append(types.Content(role="user", parts=tool_response_parts))
if iterations >= max_iterations and not agent_outcome:
agent_outcome = (
"Error: Video analyzer agent reached maximum iterations"
" without settling on a final answer."
)
status = "error"
except Exception as e:
logger.error(f"Video analyzer failed: {e}")
failure = classify_video_failure(e)
if failure.should_fallback:
logger.warning(
f"Native coordinator failed with {failure.category.value}; "
"continuing through universal fallback"
)
agent_outcome, status = await analyzer._run_universal(
time_description,
purpose,
system_prompt,
force_fallback=True,
)
else:
agent_outcome = f"Video analysis failed: {e}"
status = "error"
finally:
analyzer._record_reliability_metrics()
await _cleanup_native_run(analyzer)
return agent_outcome, status
def _append_ledger_diff(analyzer, contents: list, last_ledger_index: int) -> int:
"""Appends new blackboard entries (and proofs) since the last coordinator turn."""
new_entries = analyzer.blackboard_entries[last_ledger_index:]
if analyzer.enable_ledger and new_entries:
lines = []
image_parts = []
for e in new_entries:
line = f"[{e['start']}s - {e['end']}s] SUMMARY: {e['summary']}"
screenshot_file = e.get("screenshot") or e.get("screenshot_path")
if screenshot_file and Path(screenshot_file).exists():
line += f" PROOF: {Path(screenshot_file).name}"
image_parts.append(
types.Part.from_text(text=f"PROOF: {Path(screenshot_file).name}")
)
image_parts.append(
types.Part.from_bytes(
data=Path(screenshot_file).read_bytes(),
mime_type="image/jpeg",
)
)
lines.append(line)
diff_text = "NEW BLACKBOARD ENTRIES ADDED: " + " | ".join(lines)
contents.append(
types.Content(
role="user",
parts=[types.Part.from_text(text=diff_text)],
)
)
if image_parts:
contents.append(types.Content(role="user", parts=image_parts))
last_ledger_index = len(analyzer.blackboard_entries)
return last_ledger_index
def _append_turn_guidance(
contents: list, iterations: int, max_iterations: int, is_final_turn: bool
) -> None:
"""Appends per-turn steering messages (final-turn notice or progress nudge)."""
if is_final_turn:
contents.append(
types.Content(
role="user",
parts=[
types.Part.from_text(
text=(
"This is your final iteration; all"
" tools are stripped, and you must"
" provide your final answer directly."
)
)
],
)
)
else:
if iterations > 1:
contents.append(
types.Content(
role="user",
parts=[
types.Part.from_text(
text=(
"You have not completed the video"
" analysis yet (iteration"
f" {iterations} of"
f" {max_iterations})."
)
)
],
)
)
def _record_main_llm_trace(
ctx, trace_id, contents: list, system_prompt: str, tools_declaration, is_final_turn: bool
) -> None:
"""Records the sanitized coordinator LLM call into the trace store."""
serialized_contents = []
for c in contents:
if hasattr(c, "model_dump"):
dumped = c.model_dump()
if isinstance(dumped, dict) and "parts" in dumped and dumped["parts"]:
for p in dumped["parts"]:
if (
isinstance(p, dict)
and "inline_data" in p
and p["inline_data"]
and "data" in p["inline_data"]
):
p["inline_data"]["data"] = "<image_bytes_sanitized>"
serialized_contents.append(dumped)
else:
serialized_contents.append(str(c))
ctx.data_engine.record_trace(
type="llm_call",
name="video_analyzer_main",
payload={
"contents": serialized_contents,
"system_instruction": system_prompt,
"tools": (
[t.name for t in tools_declaration]
if (tools_declaration and not is_final_turn)
else []
),
},
parent_trace_id=trace_id,
)
async def _stream_main_turn(
analyzer,
ctx,
contents: list,
system_prompt: str,
temperature: float,
thinking_level,
tools_declaration,
is_final_turn: bool,
):
"""Runs one traced, retry-wrapped streaming turn of the main coordinator."""
with TraceSpan(name="gemini_main_agent_call") as span:
trace_id = CURRENT_TRACE_ID.get()
if ctx.data_engine and trace_id:
_record_main_llm_trace(
ctx, trace_id, contents, system_prompt, tools_declaration, is_final_turn
)
async def run_stream():
return await _consume_main_stream(
analyzer,
ctx,
span,
trace_id,
contents,
system_prompt,
temperature,
thinking_level,
tools_declaration,
is_final_turn,
)
response = await _va._invoke_with_retry(
lambda: asyncio.wait_for(run_stream(), timeout=300),
"Native Gemini video coordinator",
)
span.result = (
f"Function calls: {len(response.function_calls)}"
if response.function_calls
else "Final answer"
)
return response
async def _consume_main_stream(
analyzer,
ctx,
span,
trace_id,
contents: list,
system_prompt: str,
temperature: float,
thinking_level,
tools_declaration,
is_final_turn: bool,
) -> _DummyResponse:
"""Starts the coordinator stream and aggregates it into a response object."""
stream = await asyncio.wait_for(
analyzer.client.aio.models.generate_content_stream(
model=analyzer.model_name,
contents=contents,
config=types.GenerateContentConfig(
system_instruction=system_prompt,
temperature=temperature,
tools=[]
if is_final_turn
else [types.Tool(function_declarations=tools_declaration)],
safety_settings=SAFETY_SETTINGS_BLOCK_NONE,
**(
{"thinking_config": types.ThinkingConfig(thinking_level=thinking_level)}
if thinking_level
else {}
),
),
),
timeout=60,
)
text = ""
accumulated_parts = []
function_calls = []
async for chunk in stream:
usage = normalize_usage(getattr(chunk, "usage_metadata", None))
if usage:
span.payload["usage_metadata"] = usage
chunk_text = chunk.text or ""
text += chunk_text
if ctx.data_engine and trace_id and chunk_text:
ctx.data_engine.stream_output(trace_id, chunk_text, is_thinking=False)
if chunk.function_calls:
function_calls.extend(chunk.function_calls)
candidates = getattr(chunk, "candidates", None)
if candidates and isinstance(candidates, list) and len(candidates) > 0:
content = getattr(candidates[0], "content", None)
parts = getattr(content, "parts", None) if content else None
if parts:
for part in parts:
accumulated_parts.append(part)
if getattr(part, "thought", False) and part.text:
if ctx.data_engine and trace_id:
ctx.data_engine.stream_output(
trace_id,
part.text,
is_thinking=True,
)
else:
if chunk.function_calls:
for fc in chunk.function_calls:
accumulated_parts.append(types.Part(function_call=fc))
return _DummyResponse(text, function_calls, accumulated_parts)
async def _execute_native_tool_calls(analyzer, function_calls: list) -> list:
"""Executes coordinator tool calls (parallel where safe) and collects responses."""
sub_agent_calls = [
fc
for fc in function_calls
if (fc.name.split(":")[-1] if ":" in fc.name else fc.name) == "spawn_sub_agent"
]
audio_only_calls = [
fc
for fc in function_calls
if (fc.name.split(":")[-1] if ":" in fc.name else fc.name) == "analyze_audio_only"
]
other_calls = [
fc
for fc in function_calls
if (fc.name.split(":")[-1] if ":" in fc.name else fc.name)
not in ["spawn_sub_agent", "analyze_audio_only"]
]
tool_response_parts = []
if sub_agent_calls:
await _run_spawn_calls(analyzer, sub_agent_calls, tool_response_parts)
if audio_only_calls:
await _run_audio_calls(analyzer, audio_only_calls, tool_response_parts)
await _run_other_calls(analyzer, other_calls, tool_response_parts)
return tool_response_parts
async def _run_spawn_calls(analyzer, sub_agent_calls: list, tool_response_parts: list) -> None:
"""Runs spawn_sub_agent calls in parallel and appends their responses."""
logger.info(f"Executing {len(sub_agent_calls)} sub-agent calls in parallel...")
async def bounded_spawn(fc):
# exec_spawn_sub_agent limits each actual model leaf.
# Holding the same semaphore around the coordinator
# would deadlock when all slots wait for child slots.
return await analyzer.exec_spawn_sub_agent(
fc.args.get("start_time"),
fc.args.get("end_time"),
fc.args.get("specific_query"),
)
tasks = [bounded_spawn(fc) for fc in sub_agent_calls]
results = await asyncio.gather(*tasks, return_exceptions=True)
for fc, result in zip(sub_agent_calls, results):
if isinstance(result, Exception):
error_msg = f"Sub-agent failed. Error: {result}"
logger.error(error_msg)
tool_response_parts.append(
types.Part.from_function_response(name=fc.name, response={"error": error_msg})
)
else:
tool_response_parts.append(
types.Part.from_function_response(name=fc.name, response={"result": result})
)
async def _run_audio_calls(analyzer, audio_only_calls: list, tool_response_parts: list) -> None:
"""Runs analyze_audio_only calls in parallel and appends their responses."""
logger.info(f"Executing {len(audio_only_calls)} analyze_audio_only calls in parallel...")
async def bounded_audio_spawn(fc):
async with _va.API_SEMAPHORE:
return await analyzer.exec_analyze_audio_only(
fc.args.get("start_time"),
fc.args.get("end_time"),
fc.args.get("specific_query"),
)
tasks = [bounded_audio_spawn(fc) for fc in audio_only_calls]
results = await asyncio.gather(*tasks, return_exceptions=True)
for fc, result in zip(audio_only_calls, results):
if isinstance(result, Exception):
error_msg = f"Audio analysis failed. Error: {result}"
logger.error(error_msg)
tool_response_parts.append(
types.Part.from_function_response(name=fc.name, response={"error": error_msg})
)
else:
tool_response_parts.append(
types.Part.from_function_response(name=fc.name, response={"result": result})
)
async def _run_other_calls(analyzer, other_calls: list, tool_response_parts: list) -> None:
"""Runs remaining coordinator tool calls sequentially."""
for fc in other_calls:
name = fc.name.split(":")[-1] if ":" in fc.name else fc.name
args = fc.args
logger.info(f"Executing tool '{name}' sequentially...")
if name == "extract_segment_metadata":
try:
res = await analyzer.exec_extract_segment_metadata(
args.get("start_time"), args.get("end_time")
)
tool_response_parts.append(
types.Part.from_function_response(name=name, response={"result": res})
)
except Exception as e:
tool_response_parts.append(
types.Part.from_function_response(name=name, response={"error": str(e)})
)
async def _cleanup_native_run(analyzer) -> None:
"""Releases remaining cloud files and temporary local media after a run."""
if analyzer.cloud_files_to_cleanup:
logger.info(
f"Cleaning up {len(analyzer.cloud_files_to_cleanup)} remaining"
" cloud files in parallel..."
)
tasks = [
asyncio.wait_for(analyzer.client.aio.files.delete(name=name), timeout=30)
for name in list(analyzer.cloud_files_to_cleanup)
]
await asyncio.gather(*tasks, return_exceptions=True)
analyzer.cloud_files_to_cleanup.clear()
if os.environ.get("KEEP_VIDEOS") or os.environ.get("ARTEMIS_DEBUG"):
logger.info("Skipping cleanup of local video files and directories due to debug mode.")
else:
for path in list(analyzer.local_files_to_cleanup):
try:
if path.exists():
logger.info(f"Cleaning up local file {path}...")
path.unlink()
except Exception as e:
logger.error(f"Failed to delete local file {path}: {e}")
for dir_path in list(analyzer.local_dirs_to_cleanup):
try:
resolved = dir_path.resolve()
system_dirs = {
Path("/").resolve(),
Path("/tmp").resolve(),
Path("/var/tmp").resolve(),
Path.home().resolve(),
Path(tempfile.gettempdir()).resolve(),
}
if resolved in system_dirs:
logger.debug(f"Skipping deletion of root/system directory: {dir_path}")
continue
if dir_path.exists():
logger.info(f"Cleaning up local directory {dir_path}...")
shutil.rmtree(dir_path, ignore_errors=True)
except Exception as e:
logger.error(f"Failed to delete local directory {dir_path}: {e}")
artemis/agents/video_analyzer/universal_engine.py
+4 -2
View File
@@ -35,6 +35,8 @@ from langchain_core.messages import (
ToolMessage,
)
from artemis.llm.structured import content_to_text
from artemis.agents.video_analyzer import video_analyzer as _va
from artemis.agents.video_analyzer.reliability import (
SubAgentAnswerExhausted,
@@ -241,7 +243,7 @@ def _extract_submit_payload(response) -> tuple[str, str, list]:
return final_summary, final_analysis, list(args.get("timeline_events") or [])
raw_content = getattr(response, "content", None)
text_content = str(raw_content).strip() if raw_content else ""
text_content = content_to_text(raw_content).strip()
if text_content:
return text_content[:200].replace("\n", " "), text_content, []
@@ -435,7 +437,7 @@ async def run_universal(
messages.append(response)
if not getattr(response, "tool_calls", None):
agent_outcome = str(response.content)
agent_outcome = content_to_text(response.content)
break
for tc in response.tool_calls:
artemis/agents/video_analyzer/universal_tools.py
+1 -153
View File
@@ -14,14 +14,11 @@
"""Tool schema declarations for the video analyzer.
Holds both the OpenAI-style function schemas used by the universal
(LangChain) engine and the builders for the native Gemini
``FunctionDeclaration`` equivalents. Pure data/declarations only.
OpenAI function schemas for the Responses video analysis engine.
"""
from typing import Any
from google.genai import types
UNIVERSAL_SUBMIT_ANSWER_TOOL: dict[str, Any] = {
"type": "function",
@@ -172,152 +169,3 @@ UNIVERSAL_MAIN_TOOLS: list[dict[str, Any]] = [
},
},
]
def build_native_tools_declaration() -> list[types.FunctionDeclaration]:
"""Builds the native Gemini declarations for the coordinator tools."""
return [
types.FunctionDeclaration(
name="extract_segment_metadata",
description=(
"Retrieves metadata (such as duration_seconds and"
" file_size_mb) for a specific video segment. Does not"
" return the video content or perform content analysis."
),
parameters=types.Schema(
type=types.Type.OBJECT,
properties={
"start_time": types.Schema(
type=types.Type.NUMBER,
description="Start time in seconds",
),
"end_time": types.Schema(
type=types.Type.NUMBER,
description=(
"End time in seconds (optional, defaults to latest available time)"
),
),
},
required=["start_time"],
),
),
types.FunctionDeclaration(
name="spawn_sub_agent",
description=(
"Spawns a sub-agent to analyze both visual and audio"
" content of a video segment. Parallelizes chunk analysis"
" across workers. Returns combined summary and analysis."
),
parameters=types.Schema(
type=types.Type.OBJECT,
properties={
"start_time": types.Schema(
type=types.Type.NUMBER,
description="Start time in seconds",
),
"end_time": types.Schema(
type=types.Type.NUMBER,
description=(
"End time in seconds. Use the maximum available time if not specified."
),
),
"specific_query": types.Schema(
type=types.Type.STRING,
description=("Specific query or intent for the sub-agent"),
),
},
required=["start_time", "specific_query"],
),
),
types.FunctionDeclaration(
name="analyze_audio_only",
description=(
"Extracts and analyzes only the audio track from a video"
" segment. More efficient for non-visual tasks."
),
parameters=types.Schema(
type=types.Type.OBJECT,
properties={
"start_time": types.Schema(
type=types.Type.NUMBER,
description="Start time in seconds",
),
"end_time": types.Schema(
type=types.Type.NUMBER,
description="End time in seconds (optional)",
),
"specific_query": types.Schema(
type=types.Type.STRING,
description=("Specific query or intent for the audio content"),
),
},
required=["start_time", "specific_query"],
),
),
]
def build_submit_answer_declaration() -> types.FunctionDeclaration:
"""Builds the native Gemini declaration for the sub-agent submit_answer tool."""
return types.FunctionDeclaration(
name="submit_answer",
description=("Submit the final findings for the requested segment analysis."),
parameters=types.Schema(
type=types.Type.OBJECT,
properties={
"summary": types.Schema(
type=types.Type.STRING,
description=("A short paragraph summarizing of your findings."),
),
"analysis": types.Schema(
type=types.Type.STRING,
description="A short paragraph of detailed analysis.",
),
"timeline_events": types.Schema(
type=types.Type.ARRAY,
description=(
"List of detected events matching the query. If no"
" relevant events occurred in the segment, return"
" an empty array."
),
items=types.Schema(
type=types.Type.OBJECT,
properties={
"start_time": types.Schema(
type=types.Type.NUMBER,
description="Event start time (seconds)",
),
"end_time": types.Schema(
type=types.Type.NUMBER,
description="Event end time (seconds)",
),
"transcription": types.Schema(
type=types.Type.STRING,
description=(
"A concise transcription of what happened in the event. "
),
),
"confidence_score": types.Schema(
type=types.Type.NUMBER,
description=("Confidence score between 0 and 1"),
),
"verification_timestamp_secs": types.Schema(
type=types.Type.NUMBER,
description=(
"Exact timestamp (seconds) for verification screenshot."
),
),
},
required=[
"start_time",
"end_time",
"transcription",
"confidence_score",
"verification_timestamp_secs",
],
),
),
},
required=["summary", "analysis", "timeline_events"],
),
)
artemis/agents/video_analyzer/video_analyzer.py
+29 -121
View File
@@ -14,9 +14,8 @@
"""VideoAnalyzer facade: entry point and shared runtime state.
The execution engines live in sibling modules (``chunk_native``,
``audio_native``, ``universal_engine``, ``native_coordinator``,
``gemini_files``, ``universal_tools``) and resolve their patchable
The execution engines live in sibling modules (``chunk_runner``,
``audio_runner``, ``universal_engine``, ``universal_tools``) and resolve their patchable
collaborators through THIS module's namespace, so existing
``mock.patch("artemis.agents.video_analyzer.video_analyzer.<name>")``
targets keep working. Keep the imports below in place even when the facade
@@ -36,8 +35,6 @@ import tempfile
import time
from typing import Any
from google import genai
from google.genai import types
from langchain_core.messages import (
BaseMessage,
HumanMessage,
@@ -49,9 +46,6 @@ from artemis.agents.video_analyzer.conflict_resolution import (
ConflictResolutionService,
)
from artemis.agents.video_analyzer.blackboard import get_video_blackboard
from artemis.agents.video_analyzer.gemini_files import (
cleanup_abandoned_gemini_files,
)
from artemis.agents.video_analyzer.reliability import (
AgenticVideoDegraded,
VideoCircuitBreaker,
@@ -60,19 +54,11 @@ from artemis.agents.video_analyzer.reliability import (
from artemis.agents.video_analyzer.universal_tools import (
UNIVERSAL_MAIN_TOOLS,
UNIVERSAL_SUBMIT_ANSWER_TOOL,
build_native_tools_declaration,
build_submit_answer_declaration,
)
from artemis.config import get_temp_dir, settings
from artemis.constants import SAFETY_SETTINGS_BLOCK_NONE
from artemis.context import ArtemisContext
from artemis.controllers.controller_factory import get_controller
from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan, trace
from artemis.llm.google import (
is_gemini_model,
resolve_video_processing,
strip_provider_prefix,
)
from artemis.llm.reliability import retry_policy_for
from artemis.services.llm import _record_llm_event, _record_llm_retry, get_llm
from artemis.utils.logger import get_logger
@@ -159,16 +145,8 @@ class VideoAnalyzer:
getattr(agent_config, "enable_video_ledger", True),
)
self.static_chunk_size_seconds = float(
getattr(self.video_config, "chunk_size_seconds", 60.0)
getattr(self.video_config, "chunk_size_seconds", settings.ARTEMIS_VIDEO_CHUNK_SECONDS)
)
self.agentic_chunk_size_seconds = float(
getattr(self.video_config, "agentic_chunk_size_seconds", 600.0)
)
self.agentic_call_timeout_seconds = float(
getattr(self.video_config, "agentic_call_timeout_seconds", 300.0)
)
# Effective chunk ceiling; re-derived once the engine picks a
# video-processing mode (agentic sub-agents take long clips).
self.chunk_size_seconds = self.static_chunk_size_seconds
self.video_processing = "static"
self.min_chunk_seconds = float(getattr(self.video_config, "min_chunk_seconds", 4.0))
@@ -178,9 +156,15 @@ class VideoAnalyzer:
self.max_dense_action_frames = int(
getattr(self.video_config, "max_dense_action_frames", 24)
)
self.native_max_retries = int(getattr(self.video_config, "native_max_retries", 1))
self.max_retries = int(
getattr(self.video_config, "max_retries", settings.ARTEMIS_VIDEO_MAX_RETRIES)
)
self.model_call_timeout_seconds = float(
getattr(self.video_config, "model_call_timeout_seconds", 120.0)
getattr(
self.video_config,
"model_call_timeout_seconds",
settings.ARTEMIS_VIDEO_TIMEOUT_SECONDS,
)
)
breaker = getattr(ctx, "_video_circuit_breaker", None)
if breaker is None:
@@ -210,7 +194,6 @@ class VideoAnalyzer:
audio_prompt_path.read_text(encoding="utf-8") if audio_prompt_path.exists() else ""
)
self._init_tools()
self._init_engine()
def _get_universal_llm(self, *, use_fallback: bool = False):
@@ -406,65 +389,13 @@ class VideoAnalyzer:
analysis = match.group(2).strip() if match.group(2) else None
return summary, analysis
def _init_tools(self) -> None:
"""Initializes native and sub-agent tool declarations."""
self.tools_declaration = build_native_tools_declaration()
self.submit_answer_tool = build_submit_answer_declaration()
def _init_engine(self) -> None:
"""Initializes model engine and decides whether to use native Gemini or Universal path."""
ctx = self.ctx
llm_config = getattr(ctx, "llm_config", None)
llm_config = getattr(self.ctx, "llm_config", None)
utils_cfg = getattr(llm_config, "utils", None) if llm_config else None
llm_cfg = getattr(utils_cfg, "video_analyzer", None) if utils_cfg else None
self.model_name = strip_provider_prefix(
llm_cfg.model if (llm_cfg and hasattr(llm_cfg, "model")) else "gemini-3.8-flash"
)
has_google_key = bool(
settings.GOOGLE_API_KEY and settings.GOOGLE_API_KEY.get_secret_value()
)
# If client is already set on ctx or explicit Gemini configuration
self.client = getattr(ctx, "_genai_client", None)
if self.client is not None:
self.use_native_gemini = True
elif has_google_key and is_gemini_model(self.model_name):
try:
self.client = genai.Client(api_key=settings.GOOGLE_API_KEY.get_secret_value())
ctx._genai_client = self.client
self.use_native_gemini = True
asyncio.create_task(cleanup_abandoned_gemini_files(self.client))
except Exception as e:
logger.warning(
f"Failed to initialize native Gemini client: {e}. Using universal engine."
)
self.use_native_gemini = False
else:
self.use_native_gemini = False
self._resolve_video_processing()
def _resolve_video_processing(self) -> None:
"""Picks static vs agentic video understanding for the sub-agents.
Agentic processing (Gemini Interactions API) needs the native client
and a supporting model; the configured knob (``auto`` / ``agentic`` /
``static``) decides the rest. The chunk ceiling follows the mode
because agentic sub-agents are built to take long clips.
"""
configured = getattr(self.video_config, "processing", "auto")
mode = "static"
if self.use_native_gemini:
mode = resolve_video_processing(configured, self.model_name)
self.video_processing = mode
self.chunk_size_seconds = (
self.agentic_chunk_size_seconds if mode == "agentic" else self.static_chunk_size_seconds
)
if mode == "agentic":
logger.info(
f"Video sub-agents use agentic video understanding on {self.model_name}"
f" (chunk ceiling {self.chunk_size_seconds:.0f}s)"
)
self.model_name = getattr(llm_cfg, "model", None) or settings.OPENAI_MODEL
self.video_processing = "static"
self.chunk_size_seconds = self.static_chunk_size_seconds
@property
def blackboard_ledger(self) -> str:
@@ -476,13 +407,6 @@ class VideoAnalyzer:
lines.append(f"{e['start']}s - {e['end']}s: {e['summary']}")
return " | ".join(lines)
async def upload_and_poll_file(self, compressed_path: Path) -> any:
from artemis.agents.video_analyzer import gemini_files
return await gemini_files.upload_and_poll_file(
self.client, compressed_path, self.cloud_files_to_cleanup
)
def get_overlapping_warnings(self, start_time: float, end_time: float | None) -> list[dict]:
warnings = []
if not isinstance(start_time, (int, float)) or (
@@ -739,17 +663,17 @@ class VideoAnalyzer:
async def _exec_single_chunk(
self, start_time: float, end_time: float | None, specific_query: str
) -> str:
from artemis.agents.video_analyzer import chunk_native
from artemis.agents.video_analyzer import chunk_runner
return await chunk_native.exec_single_chunk(self, start_time, end_time, specific_query)
return await chunk_runner.exec_single_chunk(self, start_time, end_time, specific_query)
@trace(type="tool", name="analyze_audio_only")
async def exec_analyze_audio_only(
self, start_time: float, end_time: float | None, specific_query: str
) -> str:
from artemis.agents.video_analyzer import audio_native
from artemis.agents.video_analyzer import audio_runner
return await audio_native.exec_analyze_audio_only(
return await audio_runner.exec_analyze_audio_only(
self, start_time, end_time, specific_query
)
@@ -837,15 +761,12 @@ class VideoAnalyzer:
utils_cfg = getattr(llm_config, "utils", None) if llm_config else None
llm_cfg = getattr(utils_cfg, "video_analyzer", None) if utils_cfg else None
temperature = 0.2
thinking_level = None
if llm_cfg:
self.model_name = strip_provider_prefix(llm_cfg.model)
self.model_name = llm_cfg.model
if getattr(llm_cfg, "temperature", None) is not None:
temperature = llm_cfg.temperature
if getattr(llm_cfg, "thinking_level", None) is not None:
thinking_level = llm_cfg.thinking_level
else:
self.model_name = "gemini-3.8-flash"
self.model_name = settings.OPENAI_MODEL
# Track files for cleanup
self.local_files_to_cleanup = set()
@@ -868,24 +789,11 @@ class VideoAnalyzer:
" proceeding.\n"
)
if not self.use_native_gemini:
try:
return await self._run_universal(
time_description=time_description,
purpose=purpose,
system_prompt=system_prompt,
)
finally:
await self._cleanup_universal_resources()
from artemis.agents.video_analyzer import native_coordinator
return await native_coordinator.run_native(
self,
ctx,
time_description,
purpose,
system_prompt,
temperature,
thinking_level,
)
try:
return await self._run_universal(
time_description=time_description,
purpose=purpose,
system_prompt=system_prompt,
)
finally:
await self._cleanup_universal_resources()