chenmingxuan/Artemis-Eapil
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 32 Wiki Activity

feat: add agentic video analysis and clarify action results

Browse Source 
Report device actions as dispatched and preserve full failure details. Route
Flash through McpActionExecutor and use ToolFailure across helper tools and
agent loops instead of inferring failure from response text.

Require model-provided descriptions for coordinate targets. Keep observed
element metadata separate from those descriptions in safety checks, visual
summaries, history, and frontend labels.

Add Gemini Interactions API video analysis with configurable processing mode,
chunk size, and timeouts. Centralize Google provider capabilities and token
usage normalization, and update google-genai to 2.22.0.

Preserve system instructions on follow-up interactions, replan pending chunks
when falling back to static processing, and reject unanswered native video
and audio segments instead of caching placeholder results. Keep helper-tool
failure status intact through history tools and committee calls.

Validation: 1,762 Python unit tests passed (3 skipped), 40 frontend formatter
tests passed, and the production UI build completed. Ruff lint, formatting,
the quality ratchet, and git diff checks passed.
This commit is contained in:
somew1nd committed 2026-09-08 17:05:54 -07:00
1 parent 611d1a0947
commit a45daf23f8
120 files changed
+6301 -1710
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

artemis/agents/video_analyzer/audio_native.py
+69 -103
View File
@@ -26,7 +26,20 @@ from pathlib import Path
from google.genai import types
from artemis.agents.video_analyzer import video_analyzer as _va
from artemis.agents.video_analyzer.reliability import classify_video_failure
from artemis.agents.video_analyzer.chunk_conversation import (
_MISSING_SUMMARY_ERROR,
_NO_TOOL_CALL_ERROR,
_SUBMIT_NOW_WARNING,
_WRONG_TOOL_ERROR,
SUB_AGENT_MAX_TURNS,
_append_rejection,
_clean_text,
_tool_name,
)
from artemis.agents.video_analyzer.reliability import (
SubAgentAnswerExhausted,
classify_video_failure,
)
from artemis.constants import SAFETY_SETTINGS_BLOCK_NONE
from artemis.data_engine.trace import TraceSpan
from artemis.llm.reliability import retry_policy_for
@@ -34,6 +47,11 @@ from artemis.utils.logger import get_logger
logger = get_logger(__name__)
_BAD_CONFIDENCE_ERROR = (
"Error: 'confidence_score' is missing or out of bounds. It must be a float"
" between 0.0 and 1.0 inclusive. Fix the payload and call submit_answer again."
)
class _AudioMedia:
"""Mutable holder for per-attempt audio artifacts shared with failure handling."""
@@ -427,29 +445,18 @@ async def _run_native_audio_conversation(
async def _drive_audio_agent_loop(analyzer, sub_agent_contents: list) -> tuple[str, str, float]:
"""Iterates the audio agent conversation until submit_answer is accepted."""
sub_max_iterations = 2
sub_iterations = 0
final_confidence_score = 0.0
final_summary = "No summary provided."
final_analysis = "No analysis provided."
final_full_text = ""
"""Return a validated (summary, analysis, confidence_score).
while sub_iterations < sub_max_iterations:
sub_iterations += 1
if sub_iterations == sub_max_iterations - 1:
Raise ``SubAgentAnswerExhausted`` if no answer is accepted within the turn limit.
"""
rejection = _NO_TOOL_CALL_ERROR
for turn in range(1, SUB_AGENT_MAX_TURNS + 1):
if turn == SUB_AGENT_MAX_TURNS - 1:
sub_agent_contents.append(
types.Content(
role="user",
parts=[
types.Part.from_text(
text=(
"[WARNING] You must call the"
" submit_answer tool now, time"
" is running out."
)
)
],
parts=[types.Part.from_text(text=_SUBMIT_NOW_WARNING)],
)
)
@@ -468,26 +475,16 @@ async def _drive_audio_agent_loop(analyzer, sub_agent_contents: list) -> tuple[s
)
full_text = response.text or ""
final_full_text += full_text + "\n"
function_calls = response.function_calls or []
if function_calls:
(
answered,
final_summary,
final_analysis,
final_confidence_score,
) = _handle_audio_function_calls(
sub_agent_contents,
response,
function_calls,
final_summary,
final_analysis,
final_confidence_score,
answer, rejection = _handle_audio_function_calls(
sub_agent_contents, response, function_calls, full_text
)
if answered is True:
break
if answer is not None:
return answer
else:
rejection = _NO_TOOL_CALL_ERROR
sub_agent_contents.append(
types.Content(
role="model",
@@ -495,83 +492,52 @@ async def _drive_audio_agent_loop(analyzer, sub_agent_contents: list) -> tuple[s
)
)
return final_summary, final_analysis, final_confidence_score
raise SubAgentAnswerExhausted(SUB_AGENT_MAX_TURNS, rejection)
def _validate_audio_answer(summary: str, confidence_score) -> str | None:
"""Return a validation error for the audio answer, or None if valid."""
if not summary:
return _MISSING_SUMMARY_ERROR
if (
not isinstance(confidence_score, (int, float))
or isinstance(confidence_score, bool)
or not (0.0 <= float(confidence_score) <= 1.0)
):
return _BAD_CONFIDENCE_ERROR
return None
def _handle_audio_function_calls(
sub_agent_contents: list,
response,
function_calls: list,
final_summary: str,
final_analysis: str,
final_confidence_score: float,
):
"""Processes audio-agent function calls; returns (answered, summary, analysis, score)."""
answered = False
full_text: str,
) -> tuple[tuple[str, str, float] | None, str | None]:
"""Return (answer, rejection), appending any rejection to the conversation."""
original_parts = response.candidates[0].content.parts
for fc in function_calls:
if (fc.name.split(":")[-1] if ":" in fc.name else fc.name) == "submit_answer":
args = fc.args
if (
"confidence_score" not in args
or not isinstance(
args["confidence_score"],
(int, float),
)
or not (0.0 <= float(args["confidence_score"]) <= 1.0)
):
error_msg = (
"Error: 'confidence_score' is"
" missing or out of bounds. It must"
" be a float between 0.0 and 1.0"
" inclusive. Fix the payload and"
" call submit_answer again."
)
sub_agent_contents.append(
types.Content(
role="model",
parts=response.candidates[0].content.parts,
)
)
sub_agent_contents.append(
types.Content(
role="user",
parts=[
types.Part.from_function_response(
name=fc.name,
response={"error": error_msg},
)
],
)
)
answered = "error"
break
if _tool_name(fc.name) != "submit_answer":
continue
args = dict(fc.args or {})
summary = _clean_text(args.get("summary"))
analysis = _clean_text(args.get("analysis"))
final_summary = args.get("summary", "No summary provided.").strip().replace("\n", " ")
final_analysis = (
args.get("analysis", "No analysis provided.").strip().replace("\n", " ")
rejection = _validate_audio_answer(summary, args.get("confidence_score"))
if rejection is not None:
_append_rejection(
sub_agent_contents, original_parts, function_calls, full_text, fc.name, rejection
)
final_confidence_score = float(args["confidence_score"])
answered = True
break
return None, rejection
if answered is True or answered == "error":
return answered, final_summary, final_analysis, final_confidence_score
return (summary, analysis or "No analysis provided.", float(args["confidence_score"])), None
sub_agent_contents.append(
types.Content(
role="model",
parts=response.candidates[0].content.parts,
)
_append_rejection(
sub_agent_contents,
original_parts,
function_calls,
full_text,
function_calls[0].name,
_WRONG_TOOL_ERROR,
)
sub_agent_contents.append(
types.Content(
role="user",
parts=[
types.Part.from_function_response(
name=function_calls[0].name,
response={"error": ("Tool not recognized. Please use submit_answer.")},
)
],
)
)
return answered, final_summary, final_analysis, final_confidence_score
return None, _WRONG_TOOL_ERROR
artemis/agents/video_analyzer/chunk_agentic.py
+455
View File
@@ -0,0 +1,455 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Analyze video chunks through the Gemini Interactions API.
``processing: "agentic"`` lets the model search and revisit an uploaded clip.
Validation errors are returned as function results, with video context retained
through ``previous_interaction_id``. Only accepted answers reach the blackboard.
The stream consumer reconstructs function arguments from step deltas. Completion
events may omit steps, so it fetches the interaction when reconstruction fails.
"""
from __future__ import annotations
import asyncio
import copy
from dataclasses import dataclass, field
import json
from types import SimpleNamespace
from typing import Any
from artemis.agents.video_analyzer import gemini_files
from artemis.agents.video_analyzer import video_analyzer as _va
from artemis.agents.video_analyzer.chunk_conversation import (
_NO_TOOL_CALL_ERROR,
_SUBMIT_NOW_WARNING,
_WRONG_TOOL_ERROR,
SUB_AGENT_MAX_TURNS,
SubAgentAnswerExhausted,
_ChunkMedia,
_clean_text,
_persist_timeline_events,
_tool_name,
_validate_answer,
)
from artemis.agents.video_analyzer.universal_tools import UNIVERSAL_SUBMIT_ANSWER_TOOL
from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan
from artemis.llm.google import normalize_usage
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
STREAM_START_TIMEOUT_SECONDS = 60.0
INTERACTION_READ_TIMEOUT_SECONDS = 30.0
# Interaction statuses after which no follow-up turn can be sent
# (``google.genai`` ``InteractionStatus``).
_TERMINAL_FAILURE_STATUSES = frozenset({"failed", "cancelled", "incomplete", "budget_exceeded"})
def build_interactions_submit_tool() -> dict[str, Any]:
"""The ``submit_answer`` declaration in Interactions API (flat JSON schema) form.
Derived from the universal (OpenAI-style) declaration so the schema is
written once; ``timeline_events`` is required to match the native path.
"""
function = UNIVERSAL_SUBMIT_ANSWER_TOOL["function"]
parameters = copy.deepcopy(function["parameters"])
required = list(parameters.get("required", []))
if "timeline_events" not in required:
required.append("timeline_events")
parameters["required"] = required
return {
"type": "function",
"name": function["name"],
"description": function["description"],
"parameters": parameters,
}
def _steps(interaction: Any) -> list:
return list(getattr(interaction, "steps", None) or [])
def _find_function_call(interaction: Any):
"""The last ``function_call`` step of an interaction, if any."""
for step in reversed(_steps(interaction)):
if getattr(step, "type", None) == "function_call":
return step
return None
def _raise_if_failed(interaction: Any) -> None:
"""Raises when an interaction ended in a terminal failure state.
A follow-up with ``previous_interaction_id`` onto a failed interaction is
rejected by the API, so the failure surfaces here as an exception and
flows through the shared retry / classification layer instead.
"""
status = str(getattr(interaction, "status", None) or "").lower()
errors = list(getattr(interaction, "errors", None) or [])
if status not in _TERMINAL_FAILURE_STATUSES and not errors:
return
details = "; ".join(
f"[{getattr(error, 'code', None)}] {getattr(error, 'message', error)}" for error in errors
)
raise RuntimeError(
f"Gemini Interactions interaction {getattr(interaction, 'id', None)} ended with"
f" status {status or 'unknown'}" + (f": {details}" if details else "")
)
async def run_agentic_chunk_conversation(
analyzer,
media: _ChunkMedia,
current_start: float,
current_end: float,
start_time: float,
end_time: float | None,
specific_query: str,
lease_owner: str,
) -> str:
"""Uploads the chunk and drives the agentic sub-agent to a committed answer."""
file = None
try:
with TraceSpan(name="upload_video_to_gemini") as span:
file = await analyzer.upload_and_poll_file(media.compressed_path)
span.result = f"Uploaded {file.name}"
logger.info(
f"Invoking Gemini agentic video understanding for sub-agent task with model"
f" {analyzer.model_name} (streaming)..."
)
trace_id = CURRENT_TRACE_ID.get()
if analyzer.ctx.data_engine and trace_id:
analyzer.ctx.data_engine.record_trace(
type="llm_call",
name="video_sub_agent",
payload={
"contents": [
f"file://{file.name}",
media.prompt_with_context,
],
"system_instruction": analyzer.sub_system_prompt,
"video_processing": "agentic",
},
parent_trace_id=trace_id,
)
final_summary, final_analysis = await _drive_agentic_loop(
analyzer,
file,
trace_id,
media,
start_time,
end_time,
specific_query,
)
end_str = f"{current_end:.1f}s" if current_end is not None else "unknown"
chunk_result = (
f"[from {current_start:.1f}s to {end_str}] Summary:"
f" {final_summary} Analysis: {final_analysis}"
)
analyzer.blackboard.complete_segment(
current_start,
current_end,
specific_query,
lease_owner,
final_summary,
final_analysis,
)
return chunk_result
finally:
if file:
await gemini_files.delete_cloud_file(
analyzer.client, file.name, analyzer.cloud_files_to_cleanup
)
def _initial_request(analyzer, file, prompt: str, tool: dict[str, Any]) -> dict[str, Any]:
return {
"model": analyzer.model_name,
"input": [
{
"type": "video",
"uri": file.uri,
"mime_type": file.mime_type,
"processing": "agentic",
},
{"type": "text", "text": prompt},
],
"system_instruction": analyzer.sub_system_prompt,
"tools": [tool],
"stream": True,
}
def _follow_up_request(
analyzer, previous_id: str, steps: list[dict[str, Any]], tool: dict[str, Any]
) -> dict[str, Any]:
"""A stateful follow-up turn: the server keeps the video context.
Only the conversation state is inherited through
``previous_interaction_id``; ``system_instruction`` and ``tools`` are
interaction-scoped and must be re-sent on every turn.
"""
return {
"model": analyzer.model_name,
"previous_interaction_id": previous_id,
"input": steps,
"system_instruction": analyzer.sub_system_prompt,
"tools": [tool],
"stream": True,
}
def _function_result(call_id: str, error: str) -> dict[str, Any]:
return {
"type": "function_result",
"call_id": call_id,
"name": "submit_answer",
"result": {"error": error},
}
async def _drive_agentic_loop(
analyzer,
file,
trace_id,
media: _ChunkMedia,
start_time: float,
end_time: float | None,
specific_query: str,
) -> tuple[str, str]:
"""Return a validated answer and persist its timeline events.
Raise ``SubAgentAnswerExhausted`` if no answer is accepted within the turn limit.
"""
tool = build_interactions_submit_tool()
request = _initial_request(analyzer, file, media.prompt_with_context, tool)
rejection = _NO_TOOL_CALL_ERROR
for turn in range(1, SUB_AGENT_MAX_TURNS + 1):
interaction = await _stream_agentic_turn(analyzer, request, trace_id)
interaction_id = getattr(interaction, "id", None) or ""
call = _find_function_call(interaction)
if call is None:
rejection = _NO_TOOL_CALL_ERROR
steps = [{"type": "text", "text": _SUBMIT_NOW_WARNING}]
elif _tool_name(getattr(call, "name", None)) != "submit_answer":
rejection = _WRONG_TOOL_ERROR
steps = [_function_result(call.id, rejection)]
else:
args = dict(getattr(call, "arguments", None) or {})
timeline_events = args.get("timeline_events", [])
if not isinstance(timeline_events, list):
timeline_events = []
summary = _clean_text(args.get("summary"))
analysis = _clean_text(args.get("analysis"))
rejection = _validate_answer(summary, timeline_events, start_time, end_time)
if rejection is None:
await _persist_timeline_events(
analyzer, timeline_events, media, start_time, end_time, specific_query
)
return summary, analysis or "No analysis provided."
steps = [_function_result(call.id, rejection)]
if turn < SUB_AGENT_MAX_TURNS:
request = _follow_up_request(analyzer, interaction_id, steps, tool)
raise SubAgentAnswerExhausted(SUB_AGENT_MAX_TURNS, rejection)
async def _stream_agentic_turn(analyzer, request: dict[str, Any], trace_id):
"""Runs one traced streaming Interactions turn and returns the completed interaction."""
with TraceSpan(name="gemini_agentic_video_call") as span:
span.payload = {
"model": request.get("model"),
"video_processing": "agentic",
"previous_interaction_id": request.get("previous_interaction_id"),
}
stream = await asyncio.wait_for(
analyzer.client.aio.interactions.create(**request),
timeout=STREAM_START_TIMEOUT_SECONDS,
)
interaction = await asyncio.wait_for(
_consume_agentic_stream(analyzer, span, stream, trace_id),
timeout=analyzer.agentic_call_timeout_seconds,
)
call = _find_function_call(interaction)
span.result = (
f"Function call: {_tool_name(getattr(call, 'name', None))}"
if call is not None
else f"Status: {getattr(interaction, 'status', 'unknown')}"
)
return interaction
@dataclass
class _PendingStep:
"""A step under construction while its start/delta/stop events stream in."""
kind: str
call_id: str | None = None
name: str | None = None
seed_arguments: dict[str, Any] = field(default_factory=dict)
arguments_json: list[str] = field(default_factory=list)
text: list[str] = field(default_factory=list)
def finish(self):
"""Materializes the step in the shape the SDK's typed steps expose.
Returns ``None`` for a function call whose streamed arguments are not
valid JSON: a half-built call must not stand in for the real one, so
the step is dropped and the interaction is re-read by id instead.
"""
if self.kind == "function_call":
arguments: dict[str, Any] = dict(self.seed_arguments)
raw = "".join(self.arguments_json)
if raw.strip():
try:
parsed = json.loads(raw)
except ValueError:
logger.warning(
f"Could not parse streamed arguments for {self.name}: {raw[:200]!r};"
" dropping the step"
)
return None
if isinstance(parsed, dict):
arguments = parsed
return SimpleNamespace(
type="function_call", id=self.call_id, name=self.name, arguments=arguments
)
if self.kind == "model_output":
return SimpleNamespace(
type="model_output",
content=[SimpleNamespace(type="text", text="".join(self.text))],
)
return SimpleNamespace(type=self.kind, id=self.call_id)
async def _consume_agentic_stream(analyzer, span, stream, trace_id):
"""Consumes Interactions SSE events, rebuilding steps and mirroring progress.
Returns an interaction-shaped object (``id`` / ``status`` / ``steps`` /
``usage``). The interaction is re-read by id when the stream ended
without a completion event, or when it stopped at ``requires_action``
without a parsable function call.
"""
data_engine = analyzer.ctx.data_engine if trace_id else None
interaction_id: str | None = None
status: str | None = None
completed_steps: list | None = None
usage: dict[str, int] | None = None
pending: dict[int, _PendingStep] = {}
steps: list = []
processing_calls = 0
def _emit(text: str, *, thinking: bool) -> None:
if data_engine and text:
data_engine.stream_output(trace_id, text, is_thinking=thinking)
async for event in stream:
data = getattr(event, "data", event)
event_type = getattr(data, "event_type", None)
if event_type == "interaction.created":
created = getattr(data, "interaction", None)
interaction_id = getattr(created, "id", None) or interaction_id
elif event_type == "step.start":
step = getattr(data, "step", None)
index = getattr(data, "index", None)
kind = str(getattr(step, "type", None) or "")
pending[index] = _PendingStep(
kind=kind,
call_id=getattr(step, "id", None) or getattr(step, "call_id", None),
name=getattr(step, "name", None),
seed_arguments=dict(getattr(step, "arguments", None) or {}),
)
if kind == "processing_call":
processing_calls += 1
_emit(
f"Inspecting the recording (video lookup {processing_calls})\n",
thinking=True,
)
elif event_type == "step.delta":
delta = getattr(data, "delta", None)
delta_type = getattr(delta, "type", None)
current = pending.get(getattr(data, "index", None))
if delta_type == "arguments_delta":
fragment = getattr(delta, "arguments", None)
if current is not None and isinstance(fragment, str):
current.arguments_json.append(fragment)
elif delta_type == "text":
text = getattr(delta, "text", None)
if current is not None and isinstance(text, str):
current.text.append(text)
_emit(text or "", thinking=False)
elif delta_type == "thought_summary":
content = getattr(delta, "content", None)
_emit(getattr(content, "text", None) or "", thinking=True)
elif event_type == "step.stop":
current = pending.pop(getattr(data, "index", None), None)
finished = current.finish() if current is not None else None
if finished is not None:
steps.append(finished)
usage = normalize_usage(getattr(data, "usage", None)) or usage
elif event_type == "interaction.completed":
completed = getattr(data, "interaction", None)
interaction_id = getattr(completed, "id", None) or interaction_id
status = getattr(completed, "status", None) or status
completed_steps = getattr(completed, "steps", None) or None
usage = normalize_usage(getattr(completed, "usage", None)) or usage
_raise_if_failed(completed)
elif event_type == "interaction.status_update":
status = getattr(data, "status", None) or status
elif event_type == "error":
error = getattr(data, "error", None)
raise RuntimeError(
"Gemini Interactions stream error"
f" [{getattr(error, 'code', None)}]: {getattr(error, 'message', None)}"
)
for current in pending.values():
finished = current.finish()
if finished is not None:
steps.append(finished)
span.payload["processing_calls"] = processing_calls
if completed_steps:
steps = list(completed_steps)
needs_reread = status is None or (
status == "requires_action" and _find_function_call(SimpleNamespace(steps=steps)) is None
)
if needs_reread:
if not interaction_id:
raise RuntimeError("Gemini Interactions stream ended without an interaction")
logger.info(f"Re-reading interaction {interaction_id} (status={status})")
fetched = await asyncio.wait_for(
analyzer.client.aio.interactions.get(interaction_id),
timeout=INTERACTION_READ_TIMEOUT_SECONDS,
)
_raise_if_failed(fetched)
status = getattr(fetched, "status", None) or status
steps = _steps(fetched) or steps
usage = normalize_usage(getattr(fetched, "usage", None)) or usage
if usage:
span.payload["usage_metadata"] = usage
return SimpleNamespace(id=interaction_id, status=status, steps=steps, usage=usage)
artemis/agents/video_analyzer/chunk_conversation.py
+78 -87
View File
@@ -27,13 +27,46 @@ import re
from google.genai import types
from artemis.agents.video_analyzer import gemini_files
from artemis.agents.video_analyzer import video_analyzer as _va
from artemis.agents.video_analyzer.reliability import SubAgentAnswerExhausted
from artemis.constants import SAFETY_SETTINGS_BLOCK_NONE
from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan
from artemis.llm.google import normalize_usage
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
#: Turns the chunk sub-agent gets to produce an accepted ``submit_answer``
#: (shared by the static and agentic paths).
SUB_AGENT_MAX_TURNS = 2
_SUBMIT_NOW_WARNING = "[WARNING] You must call the submit_answer tool now, time is running out."
_WRONG_TOOL_ERROR = "Tool not recognized. Please use submit_answer."
_NO_TOOL_CALL_ERROR = "The model replied with text only and never called submit_answer."
_MISSING_SUMMARY_ERROR = (
"submit_answer was called without a 'summary'. Call it again with a non-empty"
" 'summary' (and 'analysis') describing what you observed."
)
def _tool_name(name: str | None) -> str:
name = name or ""
return name.split(":")[-1] if ":" in name else name
def _clean_text(value) -> str:
return str(value or "").strip().replace("\n", " ")
def _validate_answer(
summary: str, timeline_events: list, start_time: float, end_time: float | None
) -> str | None:
"""Return a validation error for submit_answer, or None if valid."""
if not summary:
return _MISSING_SUMMARY_ERROR
return _validate_timeline_events(timeline_events, start_time, end_time)
class _ChunkMedia:
"""Mutable holder for per-attempt media artifacts shared with failure handling."""
@@ -120,15 +153,9 @@ async def _run_native_chunk_conversation(
return chunk_result
finally:
if file:
logger.info(f"Cleaning up cloud file {file.name}...")
try:
await asyncio.wait_for(
analyzer.client.aio.files.delete(name=file.name),
timeout=30,
)
analyzer.cloud_files_to_cleanup.discard(file.name)
except Exception as ce:
logger.error(f"Failed to delete cloud file {file.name}: {ce}")
await gemini_files.delete_cloud_file(
analyzer.client, file.name, analyzer.cloud_files_to_cleanup
)
async def _drive_sub_agent_loop(
@@ -140,38 +167,27 @@ async def _drive_sub_agent_loop(
end_time: float | None,
specific_query: str,
) -> tuple[str, str]:
"""Iterates the sub-agent conversation until submit_answer is accepted."""
sub_max_iterations = 2
sub_iterations = 0
final_analysis = "No analysis provided."
final_summary = "No summary provided."
final_full_text = ""
"""Return a validated answer and persist its timeline events.
while sub_iterations < sub_max_iterations:
sub_iterations += 1
if sub_iterations == sub_max_iterations - 1:
Raise ``SubAgentAnswerExhausted`` if no answer is accepted within the turn limit.
"""
rejection = _NO_TOOL_CALL_ERROR
for turn in range(1, SUB_AGENT_MAX_TURNS + 1):
if turn == SUB_AGENT_MAX_TURNS - 1:
sub_agent_contents.append(
types.Content(
role="user",
parts=[
types.Part.from_text(
text=(
"[WARNING] You must call the"
" submit_answer tool now, time"
" is running out."
)
)
],
parts=[types.Part.from_text(text=_SUBMIT_NOW_WARNING)],
)
)
full_text, function_calls, original_parts = await _stream_sub_agent_turn(
analyzer, sub_agent_contents, trace_id
)
final_full_text += full_text + "\n"
if function_calls:
answered, final_summary, final_analysis = await _handle_sub_agent_calls(
answer, rejection = await _handle_sub_agent_calls(
analyzer,
sub_agent_contents,
function_calls,
@@ -181,12 +197,11 @@ async def _drive_sub_agent_loop(
start_time,
end_time,
specific_query,
final_summary,
final_analysis,
)
if answered is True:
break
if answer is not None:
return answer
else:
rejection = _NO_TOOL_CALL_ERROR
sub_agent_contents.append(
types.Content(
role="model",
@@ -194,7 +209,7 @@ async def _drive_sub_agent_loop(
)
)
return final_summary, final_analysis
raise SubAgentAnswerExhausted(SUB_AGENT_MAX_TURNS, rejection)
async def _stream_sub_agent_turn(
@@ -228,24 +243,9 @@ async def _read_sub_agent_stream(analyzer, span, target_stream, current_trace_id
function_calls = []
accumulated_parts = []
async for chunk in target_stream:
if hasattr(chunk, "usage_metadata") and chunk.usage_metadata:
span.payload["usage_metadata"] = {
"prompt_token_count": getattr(
chunk.usage_metadata,
"prompt_token_count",
0,
),
"candidates_token_count": getattr(
chunk.usage_metadata,
"candidates_token_count",
0,
),
"total_token_count": getattr(
chunk.usage_metadata,
"total_token_count",
0,
),
}
usage = normalize_usage(getattr(chunk, "usage_metadata", None))
if usage:
span.payload["usage_metadata"] = usage
chunk_text = chunk.text or ""
if chunk_text:
@@ -287,42 +287,33 @@ async def _handle_sub_agent_calls(
start_time: float,
end_time: float | None,
specific_query: str,
final_summary: str,
final_analysis: str,
):
"""Processes function calls from one turn; returns (answered, summary, analysis)."""
answered = False
) -> tuple[tuple[str, str] | None, str | None]:
"""Return ``(answer, rejection)`` after processing one turn's tool calls.
Persist events for accepted answers; append rejected calls to the conversation
with an error for the model to correct.
"""
for fc in function_calls:
if (fc.name.split(":")[-1] if ":" in fc.name else fc.name) == "submit_answer":
args = fc.args
timeline_events = args.get("timeline_events", [])
final_summary = args.get("summary", "No summary provided.").strip().replace("\n", " ")
final_analysis = (
args.get("analysis", "No analysis provided.").strip().replace("\n", " ")
if _tool_name(fc.name) != "submit_answer":
continue
args = dict(fc.args or {})
timeline_events = args.get("timeline_events", [])
if not isinstance(timeline_events, list):
timeline_events = []
summary = _clean_text(args.get("summary"))
analysis = _clean_text(args.get("analysis"))
rejection = _validate_answer(summary, timeline_events, start_time, end_time)
if rejection is not None:
_append_rejection(
sub_agent_contents, original_parts, function_calls, full_text, fc.name, rejection
)
return None, rejection
error_msg = _validate_timeline_events(timeline_events, start_time, end_time)
if error_msg is not None:
_append_rejection(
sub_agent_contents,
original_parts,
function_calls,
full_text,
fc.name,
error_msg,
)
answered = "error"
break
await _persist_timeline_events(
analyzer, timeline_events, media, start_time, end_time, specific_query
)
answered = True
break
if answered is True or answered == "error":
return answered, final_summary, final_analysis
await _persist_timeline_events(
analyzer, timeline_events, media, start_time, end_time, specific_query
)
return (summary, analysis or "No analysis provided."), None
_append_rejection(
sub_agent_contents,
@@ -330,9 +321,9 @@ async def _handle_sub_agent_calls(
function_calls,
full_text,
function_calls[0].name,
("Tool not recognized. Please use submit_answer."),
_WRONG_TOOL_ERROR,
)
return answered, final_summary, final_analysis
return None, _WRONG_TOOL_ERROR
def _validate_timeline_events(
artemis/agents/video_analyzer/chunk_native.py
+82 -2
View File
@@ -26,11 +26,16 @@ import asyncio
from pathlib import Path
from artemis.agents.video_analyzer import video_analyzer as _va
from artemis.agents.video_analyzer.chunk_agentic import run_agentic_chunk_conversation
from artemis.agents.video_analyzer.chunk_conversation import (
_ChunkMedia,
_run_native_chunk_conversation,
)
from artemis.agents.video_analyzer.reliability import classify_video_failure
from artemis.agents.video_analyzer.reliability import (
AgenticVideoDegraded,
classify_video_failure,
is_agentic_rejection,
)
from artemis.llm.reliability import retry_policy_for
from artemis.utils.logger import get_logger
@@ -76,12 +81,19 @@ async def exec_single_chunk(
media = _ChunkMedia()
for attempt in range(max_retries + 1):
attempt = 0
while attempt <= max_retries:
# The mode this attempt was launched in: a parallel chunk may degrade
# the run while this one is still in flight.
attempt_mode = analyzer.video_processing
try:
logger.info(
f"Attempt {attempt + 1}/{max_retries + 1} for sub-agent:"
f" start={current_start}, end={current_end}"
)
# A sibling may switch the run to static during queuing or media
# preparation. Check the size before preparation and before dispatch.
_ensure_fits_static_ceiling(analyzer, current_start, current_end)
result, slowdown_factor = await _prepare_chunk_media(
analyzer, media, current_start, current_end
)
@@ -94,6 +106,7 @@ async def exec_single_chunk(
end_time,
specific_query,
)
_ensure_fits_static_ceiling(analyzer, current_start, current_end)
if not analyzer.use_native_gemini:
chunk_result = await analyzer._exec_single_chunk_universal(
@@ -116,6 +129,18 @@ async def exec_single_chunk(
)
return chunk_result
if analyzer.video_processing == "agentic":
return await run_agentic_chunk_conversation(
analyzer,
media,
current_start,
current_end,
start_time,
end_time,
specific_query,
lease_owner,
)
return await _run_native_chunk_conversation(
analyzer,
media,
@@ -134,6 +159,22 @@ async def exec_single_chunk(
# stream output from the failed attempt is discarded, never
# surfaced as a complete answer.
failure = classify_video_failure(e)
if attempt_mode == "agentic" and is_agentic_rejection(failure, e):
# Switch to static and let the coordinator resize the interval.
_degrade_agentic_to_static(analyzer, e, failure)
return await _finish_failed_chunk(
analyzer,
media,
current_start,
current_end,
specific_query,
lease_owner,
AgenticVideoDegraded(
f"Agentic video processing rejected ({e}); re-plan"
f" {current_start:.1f}s-{current_end:.1f}s at the static chunk size"
),
retryable=False,
)
attempts_so_far = attempt + 1
if not failure.retryable:
logger.warning(
@@ -181,12 +222,51 @@ async def exec_single_chunk(
source="video_sub_agent",
)
await asyncio.sleep(delay)
attempt += 1
raise RuntimeError(
f"Video sub-agent exhausted retries for {current_start:.1f}s-{current_end:.1f}s"
)
def _degrade_agentic_to_static(analyzer, error: Exception, failure) -> None:
"""Switches the rest of this run to static video understanding (idempotent)."""
if analyzer.video_processing != "agentic":
return
logger.warning(
f"Agentic video processing rejected ({error}); the rest of this run"
" analyzes video in static mode"
)
_va._record_llm_event(
"llm_fallback",
{
"reason": "agentic_video_rejected",
"category": failure.category.value,
"error": str(error)[:500],
},
)
analyzer.video_processing = "static"
analyzer.chunk_size_seconds = analyzer.static_chunk_size_seconds
def _ensure_fits_static_ceiling(analyzer, current_start: float, current_end: float) -> None:
"""Refuses to analyze a clip sized for the agentic ceiling once the run is static.
Raises :class:`AgenticVideoDegraded`, which the attempt loop treats as a
non-retryable failure: the lease is released via ``fail_segment`` and the
coordinator re-plans the interval into static-sized pieces.
"""
if analyzer.video_processing == "agentic":
return
if current_end - current_start <= analyzer.chunk_size_seconds + 1e-3:
return
raise AgenticVideoDegraded(
f"Run degraded to static video processing while {current_start:.1f}s-{current_end:.1f}s"
f" was pending; re-plan it at the static chunk size"
f" ({analyzer.chunk_size_seconds:.0f}s)"
)
async def _finish_failed_chunk(
analyzer,
media: _ChunkMedia,
artemis/agents/video_analyzer/gemini_files.py
+12
View File
@@ -97,6 +97,18 @@ async def cleanup_abandoned_gemini_files(client) -> None:
logger.warning(f"Routine cloud maintenance skipped: {e}")
async def delete_cloud_file(
client, file_name: str, cloud_files_to_cleanup: set, *, timeout: float = 30.0
) -> None:
"""Best-effort deletion of an uploaded file; failures are logged, never raised."""
logger.info(f"Cleaning up cloud file {file_name}...")
try:
await asyncio.wait_for(client.aio.files.delete(name=file_name), timeout=timeout)
cloud_files_to_cleanup.discard(file_name)
except Exception as ce:
logger.error(f"Failed to delete cloud file {file_name}: {ce}")
async def upload_and_poll_file(client, compressed_path: Path, cloud_files_to_cleanup: set) -> any:
"""Upload a media file to the Gemini File API and poll until it is ACTIVE."""
logger.info(f"Uploading {compressed_path} to Gemini File API...")
artemis/agents/video_analyzer/native_coordinator.py
+4 -18
View File
@@ -32,6 +32,7 @@ from artemis.agents.video_analyzer import video_analyzer as _va
from artemis.agents.video_analyzer.reliability import classify_video_failure
from artemis.constants import SAFETY_SETTINGS_BLOCK_NONE
from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan
from artemis.llm.google import normalize_usage
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
@@ -349,24 +350,9 @@ async def _consume_main_stream(
function_calls = []
async for chunk in stream:
if hasattr(chunk, "usage_metadata") and chunk.usage_metadata:
span.payload["usage_metadata"] = {
"prompt_token_count": getattr(
chunk.usage_metadata,
"prompt_token_count",
0,
),
"candidates_token_count": getattr(
chunk.usage_metadata,
"candidates_token_count",
0,
),
"total_token_count": getattr(
chunk.usage_metadata,
"total_token_count",
0,
),
}
usage = normalize_usage(getattr(chunk, "usage_metadata", None))
if usage:
span.payload["usage_metadata"] = usage
chunk_text = chunk.text or ""
text += chunk_text
artemis/agents/video_analyzer/reliability.py
+40
View File
@@ -53,10 +53,50 @@ class VideoFailure:
_MEDIA_MARKERS = ("ffmpeg", "codec", "corrupt", "invalid video", "invalid audio", "media")
_AGENTIC_REJECTION_MARKERS = ("processing", "agentic")
class SubAgentAnswerExhausted(RuntimeError):
"""The sub-agent exhausted its turns without a valid submit_answer.
The caller records a failed segment so a later claim can retry it. The shared
classifier permits retries, splitting, and fallback for this failure.
"""
def __init__(self, turns: int, reason: str):
self.turns = turns
self.reason = reason
super().__init__(
f"submit_answer was not accepted within {turns} turn(s); last rejection: {reason}"
)
class AgenticVideoDegraded(RuntimeError):
"""Return an interval to the coordinator after switching to static mode.
The coordinator splits oversized clips and retries smaller clips as-is.
"""
def is_agentic_rejection(failure: VideoFailure, error: BaseException) -> bool:
"""True when a bad request is the API refusing agentic video processing.
Only an HTTP 400 whose message mentions ``processing`` / ``agentic``
qualifies; other bad requests (413 payload too large, 404, ...) keep the
shared handling.
"""
if failure.category is not VideoFailureCategory.BAD_REQUEST:
return False
if extract_status_code(error) != 400:
return False
message = str(error).lower()
return any(marker in message for marker in _AGENTIC_REJECTION_MARKERS)
def classify_video_failure(error: BaseException) -> VideoFailure:
"""Convert provider/transport/media exceptions into recovery decisions."""
if isinstance(error, AgenticVideoDegraded):
return VideoFailure(VideoFailureCategory.BAD_REQUEST, False, True, False)
generic = classify_failure(error)
message = str(error).lower()
has_media_marker = any(marker in message for marker in _MEDIA_MARKERS)
artemis/agents/video_analyzer/universal_engine.py
+32 -39
View File
@@ -36,7 +36,10 @@ from langchain_core.messages import (
)
from artemis.agents.video_analyzer import video_analyzer as _va
from artemis.agents.video_analyzer.reliability import classify_video_failure
from artemis.agents.video_analyzer.reliability import (
SubAgentAnswerExhausted,
classify_video_failure,
)
from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan
from artemis.utils.logger import get_logger
@@ -218,29 +221,33 @@ async def _invoke_universal_with_visual_retry(
def _extract_submit_payload(response) -> tuple[str, str, list]:
"""Pulls summary/analysis/timeline events from a universal model response."""
final_summary = "No summary provided."
final_analysis = "No analysis provided."
timeline_events = []
"""Extract a summary, analysis, and events from the model response.
if getattr(response, "tool_calls", None):
for tc in response.tool_calls:
if tc.get("name") == "submit_answer":
args = tc.get("args", {})
final_summary = (
str(args.get("summary", "")).strip().replace("\n", " ") or final_summary
)
final_analysis = (
str(args.get("analysis", "")).strip().replace("\n", " ") or final_analysis
)
timeline_events = args.get("timeline_events", [])
break
elif getattr(response, "content", None):
text_content = str(response.content).strip()
final_summary = text_content[:200].replace("\n", " ")
final_analysis = text_content
Use the first submit_answer call if its summary is non-empty. Otherwise,
fall back to response text, using its first 200 characters as the summary
and dropping any events from the rejected call. Raise SubAgentAnswerExhausted
if neither source provides an answer.
"""
for tc in getattr(response, "tool_calls", None) or []:
if tc.get("name") != "submit_answer":
continue
args = tc.get("args") or {}
final_summary = str(args.get("summary", "")).strip().replace("\n", " ")
if not final_summary:
break
final_analysis = (
str(args.get("analysis", "")).strip().replace("\n", " ") or "No analysis provided."
)
return final_summary, final_analysis, list(args.get("timeline_events") or [])
return final_summary, final_analysis, timeline_events
raw_content = getattr(response, "content", None)
text_content = str(raw_content).strip() if raw_content else ""
if text_content:
return text_content[:200].replace("\n", " "), text_content, []
raise SubAgentAnswerExhausted(
1, "the model neither called submit_answer with a summary nor replied with text"
)
async def _persist_universal_events(
@@ -361,23 +368,9 @@ async def exec_analyze_audio_universal(
label="Universal audio sub-agent",
force_fallback=force_fallback,
)
final_summary = "No audio summary provided."
final_analysis = "No audio analysis provided."
if getattr(response, "tool_calls", None):
for tc in response.tool_calls:
if tc.get("name") == "submit_answer":
args = tc.get("args", {})
final_summary = (
str(args.get("summary", "")).strip().replace("\n", " ") or final_summary
)
final_analysis = (
str(args.get("analysis", "")).strip().replace("\n", " ") or final_analysis
)
break
elif getattr(response, "content", None):
final_summary = str(response.content)[:200].replace("\n", " ")
final_analysis = str(response.content)
# Same resolution order as the video chunk (raises SubAgentAnswerExhausted
# when the model produced nothing usable); audio has no timeline events.
final_summary, final_analysis, _ = _extract_submit_payload(response)
end_val = end_time if end_time is not None else "unknown"
analyzer._record_blackboard_entry(
artemis/agents/video_analyzer/video_analyzer.py
+91 -21
View File
@@ -53,6 +53,7 @@ from artemis.agents.video_analyzer.gemini_files import (
cleanup_abandoned_gemini_files,
)
from artemis.agents.video_analyzer.reliability import (
AgenticVideoDegraded,
VideoCircuitBreaker,
classify_video_failure,
)
@@ -67,6 +68,11 @@ from artemis.constants import SAFETY_SETTINGS_BLOCK_NONE
from artemis.context import ArtemisContext
from artemis.controllers.controller_factory import get_controller
from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan, trace
from artemis.llm.google import (
is_gemini_model,
resolve_video_processing,
strip_provider_prefix,
)
from artemis.llm.reliability import retry_policy_for
from artemis.services.llm import _record_llm_event, _record_llm_retry, get_llm
from artemis.utils.logger import get_logger
@@ -152,7 +158,19 @@ class VideoAnalyzer:
"enable_ledger",
getattr(agent_config, "enable_video_ledger", True),
)
self.chunk_size_seconds = float(getattr(self.video_config, "chunk_size_seconds", 60.0))
self.static_chunk_size_seconds = float(
getattr(self.video_config, "chunk_size_seconds", 60.0)
)
self.agentic_chunk_size_seconds = float(
getattr(self.video_config, "agentic_chunk_size_seconds", 600.0)
)
self.agentic_call_timeout_seconds = float(
getattr(self.video_config, "agentic_call_timeout_seconds", 300.0)
)
# Effective chunk ceiling; re-derived once the engine picks a
# video-processing mode (agentic sub-agents take long clips).
self.chunk_size_seconds = self.static_chunk_size_seconds
self.video_processing = "static"
self.min_chunk_seconds = float(getattr(self.video_config, "min_chunk_seconds", 4.0))
self.max_split_depth = int(getattr(self.video_config, "max_split_depth", 4))
self.action_window_seconds = float(getattr(self.video_config, "action_window_seconds", 2.0))
@@ -399,25 +417,19 @@ class VideoAnalyzer:
llm_config = getattr(ctx, "llm_config", None)
utils_cfg = getattr(llm_config, "utils", None) if llm_config else None
llm_cfg = getattr(utils_cfg, "video_analyzer", None) if utils_cfg else None
model_str = (
llm_cfg.model if (llm_cfg and hasattr(llm_cfg, "model")) else "gemini-3.8-flash"
).lower()
self.model_name = (
self.model_name = strip_provider_prefix(
llm_cfg.model if (llm_cfg and hasattr(llm_cfg, "model")) else "gemini-3.8-flash"
)
if "/" in self.model_name:
self.model_name = self.model_name.split("/")[-1]
has_google_key = bool(
settings.GOOGLE_API_KEY and settings.GOOGLE_API_KEY.get_secret_value()
)
is_gemini_model = "gemini" in model_str
# If client is already set on ctx or explicit Gemini configuration
self.client = getattr(ctx, "_genai_client", None)
if self.client is not None:
self.use_native_gemini = True
elif has_google_key and is_gemini_model:
elif has_google_key and is_gemini_model(self.model_name):
try:
self.client = genai.Client(api_key=settings.GOOGLE_API_KEY.get_secret_value())
ctx._genai_client = self.client
@@ -430,6 +442,29 @@ class VideoAnalyzer:
self.use_native_gemini = False
else:
self.use_native_gemini = False
self._resolve_video_processing()
def _resolve_video_processing(self) -> None:
"""Picks static vs agentic video understanding for the sub-agents.
Agentic processing (Gemini Interactions API) needs the native client
and a supporting model; the configured knob (``auto`` / ``agentic`` /
``static``) decides the rest. The chunk ceiling follows the mode
because agentic sub-agents are built to take long clips.
"""
configured = getattr(self.video_config, "processing", "auto")
mode = "static"
if self.use_native_gemini:
mode = resolve_video_processing(configured, self.model_name)
self.video_processing = mode
self.chunk_size_seconds = (
self.agentic_chunk_size_seconds if mode == "agentic" else self.static_chunk_size_seconds
)
if mode == "agentic":
logger.info(
f"Video sub-agents use agentic video understanding on {self.model_name}"
f" (chunk ceiling {self.chunk_size_seconds:.0f}s)"
)
@property
def blackboard_ledger(self) -> str:
@@ -565,6 +600,18 @@ class VideoAnalyzer:
self.local_dirs_to_cleanup.add(metadata_path.parent)
return requested_start, requested_end, None
def _plan_chunks(self, intervals: list[tuple[float, float]]) -> list[tuple[float, float]]:
"""Cuts intervals into consecutive chunks of at most ``chunk_size_seconds``."""
target_chunk_size = self.chunk_size_seconds
chunks: list[tuple[float, float]] = []
for gap_start, gap_end in intervals:
cursor = gap_start
while cursor < gap_end:
chunk_end = min(gap_end, cursor + target_chunk_size)
chunks.append((round(cursor, 3), round(chunk_end, 3)))
cursor = chunk_end
return chunks
@trace(type="tool", name="spawn_sub_agent")
async def exec_spawn_sub_agent(
self, start_time: float, end_time: float | None, specific_query: str
@@ -600,21 +647,14 @@ class VideoAnalyzer:
)
return "CACHED VIDEO ANALYSIS: " + " ".join(cached_results)
target_chunk_size = self.chunk_size_seconds
chunks: list[tuple[float, float]] = []
for gap_start, gap_end in missing:
cursor = gap_start
while cursor < gap_end:
chunk_end = min(gap_end, cursor + target_chunk_size)
chunks.append((round(cursor, 3), round(chunk_end, 3)))
cursor = chunk_end
chunks = self._plan_chunks(missing)
logger.info(
f"Video blackboard planned {len(chunks)} uncovered chunk(s) for "
f"{requested_start:.1f}s-{requested_end:.1f}s"
)
async def analyze_with_recovery(
cs: float, ce: float, depth: int = 0
cs: float, ce: float, depth: int = 0, *, retried_static: bool = False
) -> tuple[list[str], list[str]]:
try:
async with API_SEMAPHORE:
@@ -623,8 +663,40 @@ class VideoAnalyzer:
except Exception as error:
failure = classify_video_failure(error)
duration = ce - cs
fits_ceiling = duration <= self.chunk_size_seconds + 1e-3
degraded = isinstance(error, AgenticVideoDegraded)
if degraded and fits_ceiling and not retried_static:
# Retry short intervals once in static mode without splitting.
logger.warning(
f"Retrying {cs:.1f}s-{ce:.1f}s in static mode: it already fits"
f" the {self.chunk_size_seconds:.0f}s chunk ceiling"
)
return await analyze_with_recovery(cs, ce, depth, retried_static=True)
pieces = (
self._plan_chunks([(cs, ce)])
if failure.should_split and not fits_ceiling
else []
)
if len(pieces) > 1:
# Split clips planned before a downgrade at the new ceiling.
logger.warning(
f"Re-planning failed {cs:.1f}s-{ce:.1f}s chunk into {len(pieces)}"
f" piece(s) of at most {self.chunk_size_seconds:.0f}s"
f" after {failure.category.value}"
)
parts = await asyncio.gather(
*(analyze_with_recovery(ps, pe, depth) for ps, pe in pieces)
)
return (
[item for values, _ in parts for item in values],
[item for _, failures in parts for item in failures],
)
# Bisecting only helps genuine size/timeout failures; a
# degraded interval that fits the ceiling has had its one
# static retry and is terminal.
can_split = (
failure.should_split
and not degraded
and depth < self.max_split_depth
and duration >= self.min_chunk_seconds * 2
)
@@ -767,9 +839,7 @@ class VideoAnalyzer:
temperature = 0.2
thinking_level = None
if llm_cfg:
self.model_name = llm_cfg.model
if "/" in self.model_name:
self.model_name = self.model_name.split("/")[-1]
self.model_name = strip_provider_prefix(llm_cfg.model)
if getattr(llm_cfg, "temperature", None) is not None:
temperature = llm_cfg.temperature
if getattr(llm_cfg, "thinking_level", None) is not None: