feat: add agentic video analysis and clarify action results

Report device actions as dispatched and preserve full failure details. Route
Flash through McpActionExecutor and use ToolFailure across helper tools and
agent loops instead of inferring failure from response text.

Require model-provided descriptions for coordinate targets. Keep observed
element metadata separate from those descriptions in safety checks, visual
summaries, history, and frontend labels.

Add Gemini Interactions API video analysis with configurable processing mode,
chunk size, and timeouts. Centralize Google provider capabilities and token
usage normalization, and update google-genai to 2.22.0.

Preserve system instructions on follow-up interactions, replan pending chunks
when falling back to static processing, and reject unanswered native video
and audio segments instead of caching placeholder results. Keep helper-tool
failure status intact through history tools and committee calls.

Validation: 1,762 Python unit tests passed (3 skipped), 40 frontend formatter
tests passed, and the production UI build completed. Ruff lint, formatting,
the quality ratchet, and git diff checks passed.

artemis/agents/checker/checker.py

@@ -44,6 +44,7 @@ from langchain_core.tools import BaseTool, tool
 
 from pydantic import BaseModel, Field, ValidationError
 
+from artemis.core.tool_failure import ToolFailure, is_tool_failure
 from artemis.constants import CHECKER_MAX_ITERATIONS
 from artemis.context import ArtemisContext
 from artemis.data_engine.context_vars import CURRENT_TRACE_ID
@@ -57,6 +58,7 @@ from artemis.tools.tool_wrapper import (
     tool_result_messages,
 )
 from artemis.utils.logger import get_logger
+from artemis.utils.task_tree import format_actions_clean
 from artemis.utils.ocr_api import is_ocr_configured, perform_ocr
 from artemis.utils.ocr_xml_fusion import (
     _crop_image_remove_status_bar,
@@ -204,7 +206,11 @@ def build_probe_argv(kind: str, params: dict[str, Any] | None = None) -> list[st
 
 
 async def _execute_probe(ctx: ArtemisContext, argv: list[str]) -> str:
-    """Executes a validated probe argv over the ADB device channel."""
+    """Executes a validated probe argv over the ADB device channel.
+
+    A failed probe answers with a :class:`ToolFailure` so the tool loop marks
+    the result as an error structurally rather than by its wording.
+    """
     try:
         adb_client = ctx.get_adb_client()
         device = adb_client.device(serial=ctx.device.device_id)
@@ -214,7 +220,7 @@ async def _execute_probe(ctx: ArtemisContext, argv: list[str]) -> str:
             text = text[:_PROBE_OUTPUT_LIMIT] + "\n... [output truncated]"
         return text
     except Exception as e:
-        return f"Error executing probe: {e}"
+        return ToolFailure(f"Error executing probe: {e}")
 
 
 def get_probe_tool(ctx: ArtemisContext) -> BaseTool:
@@ -230,7 +236,7 @@ def get_probe_tool(ctx: ArtemisContext) -> BaseTool:
         try:
             argv = build_probe_argv(kind, params)
         except ValueError as e:
-            return f"Error: {e}"
+            return ToolFailure(f"Error: {e}")
         return await _execute_probe(ctx, argv)
 
     return probe_device
@@ -330,8 +336,11 @@ def _format_history(ctx: ArtemisContext, limit: int = 60) -> str:
         action = s.get("action_taken")
         action_str = ""
         if action and not summary:
+            # The shared renderer keeps the target's provenance visible: a
+            # self-described coordinate target reads "'play button' (self-described)",
+            # an observed index target reads "'Login'" with no marker.
             try:
-                action_str = json.dumps(action, ensure_ascii=False)[:160]
+                action_str = format_actions_clean(action)[:160]
             except Exception:
                 action_str = str(action)[:160]
         lines.append(f"- Step {num} ({rel}): {summary or action_str}".rstrip())
@@ -456,7 +465,7 @@ async def _run_check_loop(
                     tool=tool_to_run, args=args, tool_call_id=tc["id"]
                 )
                 text, _ = split_multimodal_result(result_obj)
-                status = "error" if text.startswith("Error") else "success"
+                status = "error" if is_tool_failure(result_obj) else "success"
             except Exception as e:
                 result_obj = f"Error running tool {tool_name}: {e}"
                 status = "error"

artemis/agents/diagnoser/diagnoser.py

@@ -22,6 +22,7 @@ import uuid
 
 from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage, ToolMessage
 
+from artemis.core.tool_failure import is_tool_failure
 from artemis.context import ArtemisContext
 from artemis.data_engine.trace import (
     CURRENT_TRACE_ID,
@@ -30,6 +31,7 @@ from artemis.data_engine.trace import (
     trace_langchain_tool,
 )
 from artemis.graph.state import State
+from artemis.llm.google import usage_from_message
 from artemis.services.llm import acomplete, get_llm, invoke_llm_with_timeout_message
 from artemis.tools.command_tool import get_run_short_adb_command_tool
 from artemis.tools.diagnoser_submit_answer_tool import get_submit_answer_tool
@@ -304,24 +306,8 @@ class Diagnoser:
         return base_llm.bind_tools(tools=traced_tools)
 
     def _token_usage_payload(self, response) -> dict | None:
-        """Extracts token-usage counters from the response metadata, if present."""
-        if not hasattr(response, "usage_metadata") or response.usage_metadata is None:
-            return None
-        usage = response.usage_metadata
-        if isinstance(usage, dict):
-            prompt_token_count = usage.get("prompt_token_count")
-            candidates_token_count = usage.get("candidates_token_count")
-            cached_content_token_count = usage.get("cached_content_token_count")
-        else:
-            prompt_token_count = getattr(usage, "prompt_token_count", None)
-            candidates_token_count = getattr(usage, "candidates_token_count", None)
-            cached_content_token_count = getattr(usage, "cached_content_token_count", None)
-
-        return {
-            "prompt_token_count": prompt_token_count,
-            "candidates_token_count": candidates_token_count,
-            "cached_content_token_count": (cached_content_token_count),
-        }
+        """Extracts normalized token-usage counters from the response, if present."""
+        return usage_from_message(response)
 
     async def _invoke_model(
         self,
@@ -495,7 +481,7 @@ class Diagnoser:
                         result = get_tool_result_content(result_obj)
                         result_str, _ = split_multimodal_result(result)
                         span.result = result
-                        if result_str.startswith("Error"):
+                        if is_tool_failure(result_obj):
                             status = "error"
                             span.status = "failed"
                             span.error = result

artemis/agents/explorer/explorer.py

@@ -49,6 +49,7 @@ from artemis.config import settings
 from artemis.context import ArtemisContext
 from artemis.data_engine.trace import trace
 from artemis.graph.state import State
+from artemis.llm.google import is_gemini_model, strip_provider_prefix
 from artemis.utils.logger import get_logger
 from artemis.utils.ocr_api import is_ocr_configured
 
@@ -118,9 +119,7 @@ class Explorer(PerceptionToolsMixin, UniversalRunnerMixin, RunSetupMixin, Native
         llm_cfg = getattr(llm_config, "explorer", None) if llm_config else None
         model = getattr(llm_cfg, "model", None) if llm_cfg else None
         model_name = model if isinstance(model, str) and model else DEFAULT_EXPLORER_MODEL
-        if "/" in model_name:
-            model_name = model_name.split("/")[-1]
-        return model_name
+        return strip_provider_prefix(model_name)
 
     def _detect_native_engine(self) -> bool:
         """True when the native google-genai SDK should drive the reasoning loop.
@@ -129,7 +128,7 @@ class Explorer(PerceptionToolsMixin, UniversalRunnerMixin, RunSetupMixin, Native
         client already shared on the context) the SDK cannot be constructed,
         so such configurations run through the universal LangChain engine.
         """
-        if "gemini" not in self.model_name.lower():
+        if not is_gemini_model(self.model_name):
             return False
         if getattr(self.ctx, "_genai_client", None) is not None:
             return True

artemis/agents/flash/context_compressor.py

@@ -313,7 +313,7 @@ class ScrubEdgeCompressor:
 
         # Defensive fallback: never leave ToolMessage or HumanMessage content empty
         if not new_blocks:
-            new_blocks = [{"type": "text", "text": "Action completed." if rec["is_tool"] else ""}]
+            new_blocks = [{"type": "text", "text": "Action dispatched." if rec["is_tool"] else ""}]
 
         msg.content = new_blocks
         self._frozen.add(rec["idx"])

artemis/agents/flash/flash_runner.md

@@ -23,7 +23,7 @@ Keep in mind that the mobile device is an asynchronous, constantly evolving envi
 ---
 
 # 3. READING YOUR HISTORY & THE SESSION CLOCK
-- **Conversation = execution history**: The earlier turns of this conversation are your raw execution history. Each observation opens with `# CURRENT OBSERVATION [T+mm:ss]`, and each turn that executed a device action ends with an `--- Action Execution Result (T+mm:ss) ---` message (`Status: success` means the action was dispatched and observed; `Status: failed` carries the `Error:` line). Every timestamp is a session-relative `T+mm:ss` offset from the start of this session; it is the only clock you have, for pacing waits as well as for reasoning about elapsed time.
+- **Conversation = execution history**: The earlier turns of this conversation are your raw execution history. Each observation opens with `# CURRENT OBSERVATION [T+mm:ss]`, and each turn that executed a device action ends with an `--- Action Execution Result (T+mm:ss) ---` message (`Status: dispatched` means the device accepted the action; whether it had the intended effect is yours to judge from the observation that follows. `Status: failed` carries the `Error:` line). Every timestamp is a session-relative `T+mm:ss` offset from the start of this session; it is the only clock you have, for pacing waits as well as for reasoning about elapsed time.
 - **Compression**: Screenshots older than a few turns are replaced by short `--- Historical Visual Transition ---` summaries and old `--- Visible UI Elements ---` lists are dropped. Long spans render as `[Era N | Steps a–b | T+.. → T+..]` blocks (a merged synopsis, segment titles, and a per-step action ledger). Because of this, if you spot critical text that you will need later (e.g., verification codes, specific titles, search lists), **you must explicitly verbalize and write those details into your text reasoning** so they persist across turns.
 {% if "search_history" in available_tools %}- **History Lookup (`search_history`)**: Compressed history keeps the step numbers and `T+` offsets but not every detail. When you need exactly what you did or saw at an earlier step (a value you read, the result of an action, which screen you were on), call `search_history` with keywords and/or the `[start, end]` step range shown in a chunk block; it searches the full stored history (screen descriptions, actions, results, reasoning, tool calls, notes, on-screen text) and every hit carries its step number. It is a pre-decision helper: read its answer, then act.{% endif %}
 {% if "replay_steps" in available_tools %}- **Step Replay (`replay_steps`)**: Replays one earlier step (or a short range) exactly as your context showed it — screen description, reasoning, tool calls with their results, the action and its execution result. Use the step numbers shown in the compressed history{% if "search_history" in available_tools %} or returned by `search_history`{% endif %}.{% endif %}
@@ -39,7 +39,7 @@ Keep in mind that the mobile device is an asynchronous, constantly evolving envi
 ---
 
 # 5. ACTION RULES
-1. **Normalized Coordinates (`0-1000` Scale)**: Always use normalized `[x, y]` coordinates in `0-1000` scale for any location-based target (e.g., `target=[500, 600]` or `click_sequence=[[300, 400], [500, 600]]`).
+1. **Normalized Coordinates (`0-1000` Scale)**: Always use normalized `[x, y]` coordinates in `0-1000` scale for any location-based target (e.g., `target=[500, 600]` or `click_sequence=[[300, 400], [500, 600]]`). Every coordinate target must carry `target_description` (one entry per point for `click_sequence`): a short, meaningful name for what you are aiming at, as a user would call it (e.g. 'play button', 'video body'), never a generic 'element' or 'button'. Nothing is inferred on your behalf: it is recorded as your own statement, and your history shows it marked as self-described.
 2. **Do not determine element coordinates based on guesswork.** If the element is visible in the screenshot but its coordinates are missing from the provided `--- Visible UI Elements ---` list, please use the `ask_explorer` tool to locate the element first.
 {% if "manage_app" in available_tools %}3. **App Launching**: To start or reset an application, always use `manage_app(action="launch", app_name="...")` or `action="stop"`. Do not manually swipe across home screens.{% endif %}
 

artemis/agents/flash/flash_summarizer.md

@@ -5,7 +5,7 @@ You are the Step Summarizer for an Android UI automation agent. Your task is to
 
 # PERSPECTIVE & FORMAT CONSTRAINTS
 1. **First-Person Perspective**: Write strictly from the agent's first-person perspective using **"I"** (e.g., "In Step {{ step_number }}, I tapped...", "I swiped up on... and observed..."). NEVER use third-person terms like "The agent", "The operator", or "The system".
-2. **Single Continuous Paragraph**: Your output MUST be exactly **one compact, continuous paragraph** (1–3 sentences, 35–65 words).
+2. **Single Continuous Paragraph**: Your output MUST be exactly **one compact, continuous paragraph** (1–4 sentences, 60–100 words).
 3. **No Lists or Formatting**: Do NOT use bullet points, numbered lists, markdown headers, bold labels, or line breaks in your output. Your output must NEVER contain `---` separators or section-marker lines — those belong to the input, not the summary.
 
 ---
@@ -21,6 +21,15 @@ You are the Step Summarizer for an Android UI automation agent. Your task is to
 
 ---
 
+# OPERATOR FOCUS (ATTENTION GUIDANCE)
+The input may open with an `--- [0] OPERATOR FOCUS ... ---` block: the operator's own words about this step — the task goal, the active sub-goal, a user instruction, and the operator's reasoning naming the target it chose and the screen change it expected. The action line shows the operator's target as `'...' (self-described)` (its own statement) or as `'...'` (an element observed in the UI tree). Use all of this to direct your attention, never to supply facts:
+1. **Focused details are mandatory**: Every visible detail that relates to the named target, the stated expectation, the sub-goal or the goal — exact labels, values, amounts, counts, toggle/selection/enabled/loading states, positions — must be transcribed verbatim. Never paraphrase or skip them. If the target control is visible, name its exact label.
+2. **Absence is a fact**: If something the operator expected or looked for is NOT visible, say so plainly (e.g., "no confirmation dialog is visible", "the 'Pay' button is not on this screen"). Never describe an expected element or change merely because the operator expected it.
+3. **The rest stays complete**: Focus adds emphasis; it never narrows the description. Keep describing the whole screen and the whole transition at full detail.
+4. **Not evidence**: The focus block is intention, not observation. A self-described target is what the operator believed it aimed at; describe what the red marker actually landed on.
+
+---
+
 # EXAMPLES OF HIGH-QUALITY SUMMARIES
 
 - **Click / Navigation**:
@@ -35,6 +44,9 @@ You are the Step Summarizer for an Android UI automation agent. Your task is to
 - **Modal / Alert State**:
   "In Step 4, I tapped the 'Confirm Booking' button; an alert dialog titled 'Payment Method Required' appeared over the view with an 'Add Card' option."
 
+- **With operator focus** (action line `Tapped 'Wi-Fi toggle' (self-described) at [880, 410]`, reasoning expected the toggle to switch on):
+  "In Step 6, I tapped the switch marked in red on the 'Wi-Fi' row, which read 'Off' before the tap; afterwards the switch showed 'On' with 'Searching for networks…' beneath it, the network list below stayed empty, and the remaining rows ('Mobile network', 'Hotspot & tethering') were unchanged."
+
 ---
 
 # TASK

artemis/agents/flash/flash_summarizer_single.md

@@ -12,7 +12,7 @@ You are the Step Summarizer for an Android UI automation agent. Exactly ONE scre
 
 # PERSPECTIVE & FORMAT CONSTRAINTS
 1. **First-Person Perspective**: Write strictly from the agent's first-person perspective using **"I"** (e.g., "In Step {{ step_number }}, I tapped... on a screen showing..."). NEVER use third-person terms like "The agent", "The operator", or "The system".
-2. **Single Continuous Paragraph**: Your output MUST be exactly **one compact, continuous paragraph** (1–3 sentences, 35–65 words).
+2. **Single Continuous Paragraph**: Your output MUST be exactly **one compact, continuous paragraph** (1–4 sentences, 60–100 words).
 3. **No Lists or Formatting**: Do NOT use bullet points, numbered lists, markdown headers, bold labels, or line breaks. Your output must NEVER contain `---` separators or section-marker lines — those belong to the input, not the summary.
 
 ---
@@ -25,6 +25,15 @@ You are the Step Summarizer for an Android UI automation agent. Exactly ONE scre
 
 ---
 
+# OPERATOR FOCUS (ATTENTION GUIDANCE)
+The input may open with an `--- [0] OPERATOR FOCUS ... ---` block: the operator's own words about this step — the task goal, the active sub-goal, a user instruction, and the operator's reasoning naming the target it chose and the screen change it expected. The action line shows the operator's target as `'...' (self-described)` (its own statement) or as `'...'` (an element observed in the UI tree). Use all of this to direct your attention, never to supply facts:
+1. **Focused details are mandatory**: Every visible detail that relates to the named target, the stated expectation (as it stands on THIS frame only), the sub-goal or the goal — exact labels, values, amounts, counts, toggle/selection/enabled/loading states, positions — must be transcribed verbatim. Never paraphrase or skip them. If the target control is visible, name its exact label.
+2. **Absence is a fact**: If something the operator expected or looked for is NOT visible, say so plainly (e.g., "no confirmation dialog is visible", "the 'Pay' button is not on this screen"). Never describe an expected element or change merely because the operator expected it.
+3. **The rest stays complete**: Focus adds emphasis; it never narrows the description. Keep describing the whole screen and the whole transition at full detail.
+4. **Not evidence**: The focus block is intention, not observation. A self-described target is what the operator believed it aimed at; describe what the red marker actually landed on.
+
+---
+
 # EXAMPLES OF HIGH-QUALITY SINGLE-FRAME SUMMARIES
 
 - **Click on a list row**:
@@ -36,6 +45,9 @@ You are the Step Summarizer for an Android UI automation agent. Exactly ONE scre
 - **Swipe / Scroll**:
   "In Step 3, I swiped up starting from the red-marked point on the flight results list, which at that moment displayed 4 morning flights priced from '$310' under the header 'Best departing flights'."
 
+- **With operator focus** (action line `Tapped 'Wi-Fi toggle' (self-described) at [880, 410]`, sub-goal "Enable Wi-Fi"):
+  "In Step 6, I tapped the switch marked in red at the right end of the 'Wi-Fi' row, which read 'Off' at that moment; the Network & internet screen also listed 'Mobile network' with 'T-Mobile' beneath it and 'Hotspot & tethering' with 'Off', and no network list or dialog was visible."
+
 ---
 
 # TASK

artemis/agents/flash/runner.py

@@ -44,7 +44,7 @@ from langchain_core.messages import (
     ToolMessage,
 )
 
-from artemis.agents.flash.summarizer import VisualStepSummarizer
+from artemis.agents.flash.summarizer import VisualStepSummarizer, build_focus_context
 from artemis.agents.validator.tool_declarations import (
     ASK_EXPLORER_TOOL,
     CLICK_SEQUENCE_TOOL,
@@ -123,8 +123,10 @@ class _TurnRecord:
             return None
         for name, status, text in self.actions:
             if status != "success":
-                return {"status": "failed", "error": f"{name}: {text}"}
-        return {"status": "success"}
+                # Executor messages name the action themselves ("Error
+                # executing click: ..."); prefix only the ones that don't.
+                return {"status": "failed", "error": text if name in text else f"{name}: {text}"}
+        return {"status": "dispatched"}
 
 
 class FlashRunner:
@@ -284,10 +286,17 @@ class FlashRunner:
     # Per-turn helpers (observe / think)
     # ------------------------------------------------------------------
 
-    async def _read_injected_instruction(self) -> str | None:
-        """Returns the real-time injected instruction text for this turn, if any."""
+    async def _read_injected_instruction(self) -> tuple[str | None, str | None]:
+        """Consumes this turn's injected instruction: ``(instruction, notice)``.
+
+        ``instruction`` is the user's text verbatim — what the step record,
+        the chunk ledger and the visual-transition lens receive (the same
+        shape Pro stamps). ``notice`` wraps it with the operator-facing
+        directive and rides only on the observation tail. Both are None
+        when nothing was injected.
+        """
         if not (self.ctx.data_engine and self.ctx.data_engine.base_dir):
-            return None
+            return None, None
         try:
             injected_payload = await asyncio.to_thread(
                 _check_injected_instruction_file,
@@ -295,21 +304,22 @@ class FlashRunner:
             )
         except (OSError, ValueError, AttributeError) as e:
             logger.warning(f"Failed to check injected instruction in FlashRunner: {e}")
-            return None
+            return None, None
         if not (injected_payload and injected_payload.get("instruction")):
-            return None
-        injected_text = (
+            return None, None
+        instruction = str(injected_payload["instruction"])
+        notice = (
             "[REAL-TIME INJECTED INSTRUCTION from user]:"
-            f" {injected_payload['instruction']}\nYou MUST immediately"
+            f" {instruction}\nYou MUST immediately"
             " follow this instruction and adjust your plan/actions."
         )
         if injected_payload.get("release_loop"):
-            injected_text += (
+            notice += (
                 "\nThe user has explicitly authorized stopping any"
                 " ongoing monitoring loop; you may now wrap up and"
                 " complete the task."
             )
-        return injected_text
+        return instruction, notice
 
     def _build_tail(
         self,
@@ -597,16 +607,13 @@ class FlashRunner:
                 action_dict["normalized_start_coordinates"] = norm_start
                 action_dict["normalized_end_coordinates"] = norm_end
 
-            # Record-time enrichment computed by the executor from
-            # the pre-action frame (target_text / target_class /
-            # target_resource_id / target_label_source).
-            target_semantics = (exec_result.metadata or {}).get("target_semantics")
-            if isinstance(target_semantics, dict):
-                action_dict.update(target_semantics)
+            # The model's own statement of what it aimed at (target_description),
+            # validated and shaped by the executor into the fields Pro records.
+            action_dict.update((exec_result.metadata or {}).get("target_semantics") or {})
 
             succeeded = exec_result.status == "success"
             last_execution_result = {
-                "status": "success" if succeeded else "failed",
+                "status": "dispatched" if succeeded else "failed",
                 "result": exec_result.text_summary,
             }
             if not succeeded:
@@ -704,6 +711,13 @@ class FlashRunner:
                     exec_outcome=exec_result.text_summary,
                     action_key=str(tc_id),
                     data_engine_step_id=recorded_step_id,
+                    # The model's own reasoning for this turn steers the lens's
+                    # attention; the target it named rides on ``args``.
+                    focus=build_focus_context(
+                        intent=raw_text,
+                        goal=self.goal,
+                        injected_instruction=injected,
+                    ),
                 )
 
             if post_img_bytes:
@@ -719,7 +733,7 @@ class FlashRunner:
             content = (
                 raw_blocks
                 if isinstance(raw_blocks, list) and raw_blocks
-                else exec_result.text_summary or f"Action '{name}' completed."
+                else exec_result.text_summary or f"Action '{name}' dispatched."
             )
             messages.extend(
                 tool_result_messages(
@@ -868,8 +882,9 @@ class FlashRunner:
             self._commit_turn(ledger, previous_turn)
             previous_turn = None
 
-            # Check for real-time injected instructions
-            injected = await self._read_injected_instruction()
+            # Check for real-time injected instructions: the verbatim text
+            # is stamped on the step, the wrapped notice goes to the tail.
+            injected, injected_notice = await self._read_injected_instruction()
 
             # Tool restriction on the final turn (bounded loops only)
             is_final = limit is not None and turns == limit
@@ -878,7 +893,7 @@ class FlashRunner:
                 turns,
                 current_pre_screenshot_bytes,
                 current_xml_list,
-                injected=injected,
+                injected=injected_notice,
                 notices=pending_notices,
                 is_final=is_final,
             )

artemis/agents/flash/summarizer.py

@@ -39,6 +39,7 @@ from artemis.memory.step_memory import JobKey, StepMemoryService
 from artemis.services.llm import RobustChatModelWrapper, get_google_llm, get_llm
 from artemis.services.token_meter import record_llm_usage
 from artemis.utils.logger import get_logger
+from artemis.utils.task_tree import format_actions_clean
 from artemis.utils.visualization import draw_action_overlay_on_image
 
 logger = get_logger(__name__)
@@ -62,6 +63,79 @@ def degenerate_summary_reason(text: str) -> str | None:
     return None
 
 
+#: Header of the operator-focus block that leads the lens input. It shares
+#: the ``--- [`` shape of the frame headers so an echoed header trips the
+#: same degenerate-output guard.
+FOCUS_BLOCK_HEADER = "--- [0] OPERATOR FOCUS (context for attention, not evidence) ---"
+
+#: Cap on the operator reasoning excerpt carried in the focus block. The
+#: expectation statement closes the reasoning, so the tail is kept intact
+#: and the cut lands in the middle.
+FOCUS_INTENT_MAX_CHARS = 900
+FOCUS_INTENT_TAIL_CHARS = 600
+
+
+def _cap_intent(text: str) -> str:
+    if len(text) <= FOCUS_INTENT_MAX_CHARS:
+        return text
+    head_len = FOCUS_INTENT_MAX_CHARS - FOCUS_INTENT_TAIL_CHARS
+    head, tail = text[:head_len], text[-FOCUS_INTENT_TAIL_CHARS:]
+    cut = len(text) - len(head) - len(tail)
+    return f"{head}\n[… {cut} characters cut here …]\n{tail}"
+
+
+def build_focus_context(
+    *,
+    intent: str | None = None,
+    goal: str | None = None,
+    subgoal: str | None = None,
+    injected_instruction: str | None = None,
+) -> dict[str, str] | None:
+    """Collect operator context to guide the visual summary's focus.
+
+    Target descriptions remain on the action. Return None if all fields are empty.
+    """
+    focus: dict[str, str] = {}
+    if isinstance(goal, str) and goal.strip():
+        focus["goal"] = goal.strip()
+    if isinstance(subgoal, str) and subgoal.strip():
+        focus["subgoal"] = subgoal.strip()
+    if isinstance(injected_instruction, str) and injected_instruction.strip():
+        focus["injected_instruction"] = injected_instruction.strip()
+    if isinstance(intent, str) and intent.strip():
+        focus["intent"] = _cap_intent(intent.strip())
+    return focus or None
+
+
+def render_focus_block(focus: dict[str, str], *, dual: bool = True) -> str:
+    """Plain-text focus block (goal, sub-goal, instruction, then reasoning).
+
+    ``dual=False`` (a single decision frame, every Pro step) labels the
+    reasoning so its expectation is read as the operator's aim only: the
+    outcome is not in the frame and must not be described.
+    """
+    lines = [FOCUS_BLOCK_HEADER]
+    if focus.get("goal"):
+        lines.append(f"Task goal: {focus['goal']}")
+    if focus.get("subgoal"):
+        lines.append(f"Active sub-goal: {focus['subgoal']}")
+    if focus.get("injected_instruction"):
+        lines.append(f"User instruction at this step: {focus['injected_instruction']}")
+    if focus.get("intent"):
+        if dual:
+            lines.append(
+                "Operator reasoning for this step (why this target, what screen"
+                " change it expected):"
+            )
+        else:
+            lines.append(
+                "Operator reasoning for this step (why this target; the expected"
+                " outcome is NOT in this frame and must not be described):"
+            )
+        lines.append(focus["intent"])
+    return "\n".join(lines)
+
+
 class VisualStepSummarizer(StepMemoryService):
     """Visual-transition lens on top of the shared step-memory runtime.
 
@@ -148,6 +222,7 @@ class VisualStepSummarizer(StepMemoryService):
         *,
         action_key: str | None = None,
         data_engine_step_id: UUID | str | None = None,
+        focus: dict[str, str] | None = None,
     ) -> None:
         """Dispatches an asynchronous summarization task without blocking the caller.
 
@@ -155,6 +230,10 @@ class VisualStepSummarizer(StepMemoryService):
         tool_call_id (``action_key``) is retained as an alias so the message
         compressor can keep querying by it. Callers with neither provide the
         step ordinal, matching the legacy keying.
+
+        ``focus`` (:func:`build_focus_context`) is the operator's own context
+        for the step; the lens renders it ahead of the frames so the details
+        the operator was after are transcribed rather than summarized away.
         """
         key: JobKey
         aliases: tuple[JobKey, ...] = ()
@@ -173,9 +252,32 @@ class VisualStepSummarizer(StepMemoryService):
             "post_img_bytes": post_img_bytes,
             "exec_outcome": exec_outcome,
             "data_engine_step_id": data_engine_step_id,
+            "focus": focus,
         }
         self.submit(key, payload, aliases=aliases)
 
+    @staticmethod
+    def _action_phrase(action_name: str, action_args: dict[str, Any] | None) -> str:
+        """The action as every other history reader sees it (``format_actions_clean``).
+
+        A described coordinate target renders as ``'play button' (self-described)``,
+        an observed element as ``'Play'``; a Pro burst lists every member.
+        """
+        args = dict(action_args or {})
+        extra = args.pop("additional_actions", None)
+        # Flash's coordinate swipe names its points ``start``/``end``; the
+        # history renderer reads ``start_coordinates``/``end_coordinates``.
+        if action_name == "swipe" and "start" in args and "end" in args:
+            args.setdefault("start_coordinates", args["start"])
+            args.setdefault("end_coordinates", args["end"])
+        try:
+            first = {"action": action_name, **args}
+            actions = [first, *extra] if isinstance(extra, list) and extra else first
+            return format_actions_clean(actions)
+        except Exception as exc:
+            logger.debug(f"Action phrase rendering fell back to raw args: {exc}", exc_info=True)
+            return f"{action_name}({action_args})"
+
     def _meter_lens_call(self, response: Any) -> None:
         """Meter one raw-model lens call as an ``llm_usage`` trace, best-effort.
 
@@ -236,15 +338,15 @@ class VisualStepSummarizer(StepMemoryService):
             dual = bool(pre_bytes) and bool(post_bytes)
             template = self._prompt_template if dual else self._single_prompt_template
             rendered_prompt = Template(template).render(step_number=step_number)
-            content_blocks: list[dict[str, Any]] = [
-                {
-                    "type": "text",
-                    "text": (
-                        f"Step {step_number} Physical Action: {action_name}({action_args})\n"
-                        f"Controller Outcome: {exec_outcome}"
-                    ),
-                }
+            lead_lines = [
+                f"Step {step_number} Physical Action: {self._action_phrase(action_name, action_args)}",
+                f"Action arguments (verbatim): {action_name}({action_args})",
+                f"Controller Outcome: {exec_outcome}",
             ]
+            focus = input_data.get("focus")
+            if isinstance(focus, dict) and focus:
+                lead_lines.extend(["", render_focus_block(focus, dual=dual)])
+            content_blocks: list[dict[str, Any]] = [{"type": "text", "text": "\n".join(lead_lines)}]
 
             if pre_bytes:
                 # 🎨 Visually mark the exact action (tap ripple, sequence numbers, swipe arrow) on the BEFORE screenshot
@@ -318,10 +420,12 @@ class VisualStepSummarizer(StepMemoryService):
                     f"VisualStepSummarizer: Generated summary for Step {step_number}: {summary_text[:80]}..."
                 )
 
-                # Free binary image buffers from memory once summary is secured
+                # Free binary image buffers (and the focus text) once the
+                # summary is secured; a landed job is never re-rendered.
                 if key in self._step_inputs:
                     self._step_inputs[key]["pre_img_bytes"] = None
                     self._step_inputs[key]["post_img_bytes"] = None
+                    self._step_inputs[key]["focus"] = None
 
                 # Update DataEngine telemetry if active
                 if self.ctx.data_engine:

artemis/agents/history_analyzer/history_analyzer.py

@@ -25,6 +25,7 @@ from pathlib import Path
 from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage
 from langchain_core.tools import BaseTool
 
+from artemis.core.tool_failure import is_tool_failure
 from artemis.context import ArtemisContext
 from artemis.data_engine.trace import TraceSpan, trace
 from artemis.memory.context_policy import build_history_for
@@ -161,7 +162,7 @@ class HistoryAnalyzer:
                                 tool=tool, args=args, tool_call_id=tc["id"]
                             )
                             text, _ = split_multimodal_result(result)
-                            status = "error" if text.startswith("Error") else "success"
+                            status = "error" if is_tool_failure(result) else "success"
                             span.result = text
                             if status == "error":
                                 span.status = "failed"

artemis/agents/log_analyzer/log_analyzer.py

@@ -51,6 +51,7 @@ class SpawnLogReaderArgs(BaseModel):
 
 from langgraph.prebuilt import InjectedState
 
+from artemis.core.tool_failure import ToolFailure, is_tool_failure
 from artemis.context import ArtemisContext
 from artemis.data_engine.trace import (
     CURRENT_TRACE_ID,
@@ -102,7 +103,7 @@ class LogAnalyzerNode:
                     return self._raw_log_cache[cache_key]
 
                 result = await original_coroutine(**kwargs)
-                if isinstance(result, str) and not result.startswith("Failed to read logs"):
+                if isinstance(result, str) and not is_tool_failure(result):
                     self._raw_log_cache[cache_key] = result
                 return result
 
@@ -130,7 +131,7 @@ class LogAnalyzerNode:
                         )
                     except Exception as e:
                         logger.error(f"Failed to search logs locally: {e}")
-                        return f"Error searching logs: {e}"
+                        return ToolFailure(f"Error searching logs: {e}")
 
                 # If not cached, fetch them once via read_logs (or the original tool, which will fetch them)
                 # Note: search_logs itself fetches and filters. We can let the original_coroutine run,
@@ -161,7 +162,7 @@ class LogAnalyzerNode:
                     )
                 except Exception as e:
                     logger.error(f"Failed to fetch and search logs: {e}")
-                    return f"Error searching logs: {e}"
+                    return ToolFailure(f"Error searching logs: {e}")
 
             # Fallback to standard tool caching for other tools
             serialized_args = json.dumps(kwargs, sort_keys=True)
@@ -276,7 +277,7 @@ class LogAnalyzerNode:
                                 elif not isinstance(result, str):
                                     result = str(result)
                                 span.result = result
-                                if result.startswith("Error"):
+                                if is_tool_failure(result_obj):
                                     status = "error"
                                     span.status = "failed"
                                     span.error = result

artemis/agents/log_analyzer/output_analyzer.py

@@ -16,6 +16,7 @@ from pathlib import Path
 from typing import Annotated
 from langchain_core.messages import HumanMessage, SystemMessage, ToolMessage
 from langchain_core.tools import tool
+from artemis.core.tool_failure import ToolFailure, is_tool_failure
 from artemis.context import ArtemisContext
 from artemis.services.llm import get_llm, invoke_llm_with_timeout_message
 from artemis.tools.mobile.search_logs import search_and_merge_logs
@@ -147,15 +148,15 @@ class TaskOutputAnalyzerNode:
                     try:
                         result = tool_to_run.invoke(args)
                     except Exception as e:
-                        result = f"Failed to run tool {tool_name}: {e}"
+                        result = ToolFailure(f"Failed to run tool {tool_name}: {e}")
                 else:
-                    result = f"Error: Tool {tool_name} not found."
+                    result = ToolFailure(f"Error: Tool {tool_name} not found.")
 
                 current_messages.append(
                     ToolMessage(
                         tool_call_id=tc["id"],
                         content=str(result),
-                        status="success" if not str(result).startswith("Error") else "error",
+                        status="error" if is_tool_failure(result) else "success",
                     )
                 )
 

artemis/agents/operator/operator.json

@@ -1,3 +1,3 @@
 {
-  "main_template": "# ROLE\nYou are the Android Device Operator. Your priority is execution accuracy. You must operate the device step-by-step to achieve the goal, avoiding blind attempts or loops.\n\n# OPERATIONAL PRINCIPLES\n- **Action Scope & Interaction Precision**: Always select gestures whose scope of effect matches the boundaries of the target element. Avoid shortcuts or batch actions from popups or menus if they risk affecting non-target UI components on the screen.\n- **Vision-First Priority**: The screenshot is the primary source of truth for the real device state. Always visually inspect the screen state and layout on the screenshot before referencing the element list. Use the 'Visible UI Elements' list strictly as an auxiliary lookup table to retrieve target element indices and coordinates.\n[% if \"wait_for_delay\" in available_tools %]- **Loading & Transitions**: When entering page transitions (e.g., app launching, web page loading, or progress indicators), use `wait_for_delay` to let the screen settle before executing target actions. Some transition states are difficult to discern, such as a thin progress line at the top of the browser screen during webpage loads. Before questioning whether an error has occurred, you can first adopt a waiting strategy to see how the device state evolves. If the state remains ambiguous after waiting, certain UI components might be malfunctioning or failing to load; in this case, consider retrying the action or pivoting to an alternative path.\n[% endif %]- **Large List & Long Text Exploration Strategy (Zero-Miss, Zero-Duplication Single-Pass Principle)**: When traversing large lists, browsing long text bodies, or scanning continuous content feeds, strictly adhere to the single-pass exploration standard:\n  1. **Unidirectional Anchor-Based Traversal (Zero-Miss)**: Maintain a consistent, unidirectional scan (e.g., uniform downward scrolling). Before each scroll, identify the bottom-most visible item/text segment as your visual anchor and carry that anchor plus the scan direction in short-term memory or the traversal ledger. After the scroll, evaluate this handoff before doing anything unrelated: if the anchor remains visible with new content beyond it, record the new content and continue in the same direction without scrolling back to verify; if the anchor is missing, continuity is uncertain, so perform at most one minimal reverse recovery to re-establish overlap, then resume the original direction. While the traversal's declared exit condition remains unresolved, keep this traversal active and do not advance to another milestone or mark it complete.\n  2. **Boundary-Proven Completion**: A single scroll that reveals no new content is not sufficient evidence that the list is complete. For an exhaustive traversal, stop only when an explicit end-of-list indicator is visible, or when one additional successful swipe in the same direction and within the same list container leaves the same final anchor visible with no new items and no loading state. This boundary probe must remain unidirectional; never reverse direction merely to prove completion. If the task declares an earlier exit condition (for example, find one verified match or process N items), that explicit condition may end the traversal without reaching the list boundary.\n  3. **Memory & Plan-Driven Deduplication (Zero-Duplication)**: Systematically log extracted item names, identifiers, or processing states into notes or memory as you progress. After every screen transition, cross-reference visible items against your recorded ledger and strictly interact only with newly surfaced, unrecorded items to eliminate redundant actions on overlapping elements. Proactively maintain subtasks in `task_plan` (e.g., segmenting by batch, category, or alphabetical ranges like `a-d`), tracking exact paths and progress to prevent backtracking.\n  4. **Single-Pass Trust & Anti-Oscillation (No Over-Verification)**: Treat recorded notes as verified ground truth. Once a segment is traversed with confirmed anchor continuity and logged, consider it permanently resolved. Do NOT scroll back-and-forth or perform redundant re-scans for \"double-checking\", even if older execution history has been pruned. Resume from the last recorded anchor and direction. Only the single bounded recovery above, or a refreshed/mutated list, permits limited re-verification; otherwise continue the single pass over new content.\n- **Dynamic Monitoring & Reactive Loop Protocol (Monitor-Trigger-Action)**: When executing an active milestone matching `[Loop] <Goal> into note <TargetNote> (Exit: <ExitCondition>; Interval: <Interval>)` or `[Loop:continuous] <Goal> into note <TargetNote> (Interval: <Interval>)` (the milestone grammar itself is defined under Task Plan Grammar below), run this 5-step cyclic state machine:\n  - **Step 1 (Baseline Sync)**: On the initial cycle, observe all visible entries/metrics in the primary monitoring view and save the full snaLine truncated
+  "main_template": "# ROLE\nYou are the Android Device Operator. Your priority is execution accuracy. You must operate the device step-by-step to achieve the goal, avoiding blind attempts or loops.\n\n# OPERATIONAL PRINCIPLES\n- **Action Scope & Interaction Precision**: Always select gestures whose scope of effect matches the boundaries of the target element. Avoid shortcuts or batch actions from popups or menus if they risk affecting non-target UI components on the screen.\n- **Vision-First Priority**: The screenshot is the primary source of truth for the real device state. Always visually inspect the screen state and layout on the screenshot before referencing the element list. Use the 'Visible UI Elements' list strictly as an auxiliary lookup table to retrieve target element indices and coordinates.\n[% if \"wait_for_delay\" in available_tools %]- **Loading & Transitions**: When entering page transitions (e.g., app launching, web page loading, or progress indicators), use `wait_for_delay` to let the screen settle before executing target actions. Some transition states are difficult to discern, such as a thin progress line at the top of the browser screen during webpage loads. Before questioning whether an error has occurred, you can first adopt a waiting strategy to see how the device state evolves. If the state remains ambiguous after waiting, certain UI components might be malfunctioning or failing to load; in this case, consider retrying the action or pivoting to an alternative path.\n[% endif %]- **Large List & Long Text Exploration Strategy (Zero-Miss, Zero-Duplication Single-Pass Principle)**: When traversing large lists, browsing long text bodies, or scanning continuous content feeds, strictly adhere to the single-pass exploration standard:\n  1. **Unidirectional Anchor-Based Traversal (Zero-Miss)**: Maintain a consistent, unidirectional scan (e.g., uniform downward scrolling). Before each scroll, identify the bottom-most visible item/text segment as your visual anchor and carry that anchor plus the scan direction in short-term memory or the traversal ledger. After the scroll, evaluate this handoff before doing anything unrelated: if the anchor remains visible with new content beyond it, record the new content and continue in the same direction without scrolling back to verify; if the anchor is missing, continuity is uncertain, so perform at most one minimal reverse recovery to re-establish overlap, then resume the original direction. While the traversal's declared exit condition remains unresolved, keep this traversal active and do not advance to another milestone or mark it complete.\n  2. **Boundary-Proven Completion**: A single scroll that reveals no new content is not sufficient evidence that the list is complete. For an exhaustive traversal, stop only when an explicit end-of-list indicator is visible, or when one additional successful swipe in the same direction and within the same list container leaves the same final anchor visible with no new items and no loading state. This boundary probe must remain unidirectional; never reverse direction merely to prove completion. If the task declares an earlier exit condition (for example, find one verified match or process N items), that explicit condition may end the traversal without reaching the list boundary.\n  3. **Memory & Plan-Driven Deduplication (Zero-Duplication)**: Systematically log extracted item names, identifiers, or processing states into notes or memory as you progress. After every screen transition, cross-reference visible items against your recorded ledger and strictly interact only with newly surfaced, unrecorded items to eliminate redundant actions on overlapping elements. Proactively maintain subtasks in `task_plan` (e.g., segmenting by batch, category, or alphabetical ranges like `a-d`), tracking exact paths and progress to prevent backtracking.\n  4. **Single-Pass Trust & Anti-Oscillation (No Over-Verification)**: Treat recorded notes as verified ground truth. Once a segment is traversed with confirmed anchor continuity and logged, consider it permanently resolved. Do NOT scroll back-and-forth or perform redundant re-scans for \"double-checking\", even if older execution history has been pruned. Resume from the last recorded anchor and direction. Only the single bounded recovery above, or a refreshed/mutated list, permits limited re-verification; otherwise continue the single pass over new content.\n- **Dynamic Monitoring & Reactive Loop Protocol (Monitor-Trigger-Action)**: When executing an active milestone matching `[Loop] <Goal> into note <TargetNote> (Exit: <ExitCondition>; Interval: <Interval>)` or `[Loop:continuous] <Goal> into note <TargetNote> (Interval: <Interval>)` (the milestone grammar itself is defined under Task Plan Grammar below), run this 5-step cyclic state machine:\n  - **Step 1 (Baseline Sync)**: On the initial cycle, observe all visible entries/metrics in the primary monitoring view and save the full snaLine truncated
 }

artemis/agents/operator/operator.py

@@ -19,6 +19,7 @@ from uuid import uuid4
 from langchain_core.messages import SystemMessage, ToolMessage
 from langchain_core.tools import BaseTool
 
+from artemis.core.tool_failure import is_tool_failure
 from artemis.context import ArtemisContext
 from artemis.data_engine.trace import (
     TraceSpan,
@@ -28,6 +29,7 @@ from artemis.data_engine.trace import (
 from artemis.graph.state import State
 from artemis.graph.visibility import strict_state
 from artemis.mcp.action_specs import OPERATOR_SHELL_ORDER, operator_shell_tool
+from artemis.llm.google import usage_from_message
 from artemis.services.llm import acomplete, get_llm, invoke_llm_with_timeout_message
 from artemis.tools.command_tool import (
     analyze_task_output_wrapper,
@@ -46,7 +48,6 @@ from artemis.utils.coordinates import (
     parse_swipe_parameters,
 )
 from artemis.utils.decorators import wrap_with_callbacks
-from artemis.utils.element_hit_test import find_element_at_point
 from artemis.utils.logger import get_logger
 from artemis.utils.notes import get_note_file_path
 from artemis.utils.plan_grammar import parse_plan
@@ -92,6 +93,13 @@ from artemis.agents.operator.prompts import (
 from artemis.agents.operator.prompts import load_operator_prompts
 
 
+def _described(value: Any) -> str | None:
+    """Trim target_description, returning None for missing or blank values."""
+    if isinstance(value, str) and value.strip():
+        return value.strip()
+    return None
+
+
 class OperatorNode:
     def __init__(
         self,
@@ -462,25 +470,13 @@ class OperatorNode:
             bound_llm = base_llm.bind_tools(tools=traced_tools)
             response = await invoke_llm_with_timeout_message(acomplete(bound_llm, current_messages))
 
-            if hasattr(response, "response_metadata") and response.response_metadata:
-                usage = (
-                    response.response_metadata.get("usage_metadata")
-                    or response.response_metadata.get("token_usage")
-                    or {}
+            usage = usage_from_message(response)
+            if usage is not None:
+                logger.info(
+                    f"LLM usage: prompt_tokens={usage['prompt_tokens']}"
+                    f" (cached={usage['cached_tokens']}),"
+                    f" completion_tokens={usage['completion_tokens']}"
                 )
-                prompt_tokens = usage.get("prompt_token_count") or usage.get("prompt_tokens")
-                cached_tokens = usage.get("cached_content_token_count") or usage.get(
-                    "cached_tokens", 0
-                )
-                completion_tokens = usage.get("candidates_token_count") or usage.get(
-                    "completion_tokens"
-                )
-                if prompt_tokens is not None:
-                    logger.info(
-                        f"LLM usage: prompt_tokens={prompt_tokens}"
-                        f" (cached={cached_tokens}),"
-                        f" completion_tokens={completion_tokens}"
-                    )
 
             if response.content:
                 if isinstance(response.content, str):
@@ -589,30 +585,10 @@ class OperatorNode:
                                 )
 
                                 content = get_tool_result_content(result_obj)
-                                status = "success"
-
-                                if isinstance(result_obj, ToolMessage) and result_obj.status:
-                                    status = result_obj.status
-
-                                is_err = status == "error"
-                                if not is_err:
-                                    if isinstance(content, str):
-                                        if content.startswith(
-                                            "Failed"
-                                        ) or content.lower().startswith("error"):
-                                            is_err = True
-                                    elif isinstance(content, list):
-                                        for block in content:
-                                            if (
-                                                isinstance(block, dict)
-                                                and block.get("type") == "text"
-                                            ):
-                                                text = block.get("text", "")
-                                                if text.startswith(
-                                                    "Failed"
-                                                ) or text.lower().startswith("error"):
-                                                    is_err = True
-                                                    break
+                                # Failure is structural (a ToolMessage status or a
+                                # ToolFailure text), never sniffed from the words.
+                                is_err = is_tool_failure(result_obj)
+                                status = "error" if is_err else "success"
 
                                 if is_err:
                                     other_tool_failed = True
@@ -812,44 +788,6 @@ class OperatorNode:
             tool_limit_exceeded,
         )
 
-    async def _run_other_tool(
-        self,
-        tc,
-        traced_tools,
-        new_subagent_calls,
-        state: State,
-        raw_thoughts: list,
-        native_thoughts: list,
-    ) -> str:
-        tool_name = tc["name"]
-        logger.info(f"Operator requested tool: {tool_name}")
-        if tool_name in ["ask_diagnoser", "video_analyzer"]:
-            new_subagent_calls.append(tool_name)
-        if ":" in tool_name:
-            tool_to_run = get_tool_by_name(tool_name, traced_tools)
-        else:
-            tool_to_run = next((t for t in traced_tools if t.name == tool_name), None)
-        if tool_to_run:
-            try:
-                args = dict(tc["args"])
-
-                state.operator_raw_thinking = "\n".join(raw_thoughts) if raw_thoughts else None
-                state.operator_native_thinking = (
-                    "\n".join(native_thoughts) if native_thoughts else None
-                )
-
-                result_obj = await invoke_tool_with_injection(
-                    tool=tool_to_run,
-                    args=args,
-                    tool_call_id=tc["id"],
-                    state=state,
-                )
-                return get_tool_result_content(result_obj)
-            except Exception as e:
-                return f"Error running tool {tool_name}: {e}"
-        else:
-            return f"Error: Tool {tool_name} not supported"
-
     @wrap_with_callbacks(
         before=lambda: logger.info("Starting Operator Agent..."),
         on_success=lambda _: logger.success("Operator Agent"),
@@ -1110,9 +1048,13 @@ class OperatorNode:
             except (AttributeError, TypeError):
                 pass
 
-        def resolve_target_element(
-            target: Any,
-        ) -> tuple[dict | None, str | None]:
+        def resolve_target_element(target: Any, description: Any) -> tuple[dict | None, str | None]:
+            """Resolves a target to ``{"center": [x, y], **semantics}``.
+
+            An element index carries observed semantics (text, bounds, id, class
+            straight from the indexed list); a coordinate pair carries only the
+            model's own ``target_description``. The two never mix.
+            """
             if isinstance(target, (int, float)):
                 try:
                     target_int = int(target)
@@ -1120,7 +1062,14 @@ class OperatorNode:
                     return None, f"Error: Invalid target index {target}."
                 indexed_elements = getattr(state, "indexed_elements", None) or []
                 if 1 <= target_int <= len(indexed_elements):
-                    return {**indexed_elements[target_int - 1], "label_source": "index"}, None
+                    el = indexed_elements[target_int - 1]
+                    return {
+                        "center": el.get("center"),
+                        "target_text": el.get("text"),
+                        "target_bounds": el.get("bounds"),
+                        "target_resource_id": el.get("resource_id"),
+                        "target_class": el.get("class"),
+                    }, None
                 return (
                     None,
                     (
@@ -1129,26 +1078,21 @@ class OperatorNode:
                     ),
                 )
             elif isinstance(target, list) and len(target) == 2:
+                described = _described(description)
+                if described is None:
+                    return (
+                        None,
+                        (
+                            f"Error: Coordinate target {target} requires"
+                            " 'target_description' (what the element is, in a few"
+                            " words). Provide it, or address the element by index."
+                        ),
+                    )
                 try:
                     nx, ny = map(float, target)
                     x = int(max(0, min(width - 1, nx * width / 1000)))
                     y = int(max(0, min(height - 1, ny * height / 1000)))
-                    # Record-time enrichment: a bare-coordinate target carries no
-                    # element semantics, so hit test the pre-action frame's indexed
-                    # elements to best-effort recover what sits under the point.
-                    hit_el, hit_source = find_element_at_point(
-                        getattr(state, "indexed_elements", None), x, y
-                    )
-                    hit_el = hit_el or {}
-                    return {
-                        "center": [x, y],
-                        "text": hit_el.get("text"),
-                        "bounds": hit_el.get("bounds"),
-                        "class": hit_el.get("class"),
-                        "resource_id": hit_el.get("resource_id"),
-                        "is_ocr": bool(hit_el.get("is_ocr")),
-                        "label_source": hit_source,
-                    }, None
+                    return {"center": [x, y], "target_description": described}, None
                 except (ValueError, TypeError):
                     return (
                         None,
@@ -1162,11 +1106,15 @@ class OperatorNode:
                 ),
             )
 
+        def target_fields(el: dict) -> dict:
+            """The recorded target semantics: observed (index) or described (coords)."""
+            return {k: v for k, v in el.items() if k != "center"}
+
         if tool_name == "click":
             target = args.get("target")
             times = args.get("times", 1)
             delay_ms = args.get("delay_ms", 100)
-            el, err = resolve_target_element(target)
+            el, err = resolve_target_element(target, args.get("target_description"))
             if err:
                 return [], err
             norm_c = [
@@ -1181,18 +1129,14 @@ class OperatorNode:
                     "normalized_coordinates": norm_c,
                     "times": times,
                     "delay_ms": delay_ms,
-                    "target_text": el.get("text"),
-                    "target_bounds": el.get("bounds"),
-                    "target_resource_id": el.get("resource_id"),
-                    "target_class": el.get("class"),
-                    "target_label_source": el.get("label_source", "none"),
+                    **target_fields(el),
                 }
             ], None
 
         elif tool_name == "long_press":
             target = args.get("target")
             duration = args.get("duration", 1000)
-            el, err = resolve_target_element(target)
+            el, err = resolve_target_element(target, args.get("target_description"))
             if err:
                 return [], err
             norm_c = [
@@ -1206,11 +1150,7 @@ class OperatorNode:
                     COORDINATE_SPACE_KEY: COORDINATE_SPACE_PIXEL,
                     "normalized_coordinates": norm_c,
                     "duration": duration,
-                    "target_text": el.get("text"),
-                    "target_bounds": el.get("bounds"),
-                    "target_resource_id": el.get("resource_id"),
-                    "target_class": el.get("class"),
-                    "target_label_source": el.get("label_source", "none"),
+                    **target_fields(el),
                 }
             ], None
 
@@ -1227,7 +1167,7 @@ class OperatorNode:
                     (f"Error: 'text' must be a string, got {type(text).__name__}."),
                 )
 
-            el, err = resolve_target_element(target)
+            el, err = resolve_target_element(target, args.get("target_description"))
             if err:
                 return [], err
 
@@ -1244,11 +1184,7 @@ class OperatorNode:
                     "normalized_coordinates": norm_c,
                     "text": text,
                     "clear_before_input": clear_exist,
-                    "target_text": el.get("text"),
-                    "target_bounds": el.get("bounds"),
-                    "target_resource_id": el.get("resource_id"),
-                    "target_class": el.get("class"),
-                    "target_label_source": el.get("label_source", "none"),
+                    **target_fields(el),
                 }
             )
             return actions, None
@@ -1271,6 +1207,9 @@ class OperatorNode:
             else:
                 duration = parsed_duration
 
+            # Only a coordinate gesture needs the model to say what it drags; a
+            # directional scroll targets the page, not an element.
+            described: str | None = None
             if kind == "direction":
                 x1, y1, x2, y2, smart_dur = compute_smart_swipe_coordinates(
                     direction=target,
@@ -1284,6 +1223,16 @@ class OperatorNode:
                 if duration is None:
                     duration = smart_dur
             elif kind == "coords" and isinstance(target, list) and len(target) == 4:
+                described = _described(args.get("target_description"))
+                if described is None:
+                    return (
+                        [],
+                        (
+                            f"Error: Coordinate swipe {target} requires"
+                            " 'target_description' (what is being dragged, in a few"
+                            " words). Provide it, or use a directional swipe."
+                        ),
+                    )
                 try:
                     nx1, ny1, nx2, ny2 = map(float, target)
                     x1 = int(max(0, min(width - 1, nx1 * width / 1000)))
@@ -1327,6 +1276,7 @@ class OperatorNode:
                     "normalized_start_coordinates": [nx1, ny1],
                     "normalized_end_coordinates": [nx2, ny2],
                     "duration": duration,
+                    **({"target_description": described} if described else {}),
                 }
             ], None
 

artemis/agents/operator/prompts.py

@@ -33,6 +33,7 @@ from artemis.tools.command_tool import (
 from artemis.utils.logger import get_logger
 from artemis.utils.notes import get_note_file_path
 from artemis.utils.plan_grammar import parse_plan, render_plan_grammar_spec
+from artemis.utils.task_tree import SELF_DESCRIBED_MARKER
 
 logger = get_logger(__name__)
 
@@ -418,7 +419,7 @@ def _max_burst_actions_for_prompt() -> int:
 
 
 def _last_successful_action_before(steps: list, step_number: int | None) -> tuple[str, int] | None:
-    """The most recent step before ``step_number`` whose terminal action executed.
+    """The most recent step before ``step_number`` whose terminal action was dispatched.
 
     Returns ``(clean action description, step number)`` or None. This is the
     Operator's best candidate for the *trigger* of a transient state: the
@@ -437,7 +438,7 @@ def _last_successful_action_before(steps: list, step_number: int | None) -> tupl
         if not action:
             continue
         result = step.get("last_execution_result")
-        if isinstance(result, dict) and result.get("status") not in (None, "success"):
+        if isinstance(result, dict) and result.get("status") not in (None, "dispatched"):
             continue
         candidates.append((number, format_actions_clean(action)))
     if not candidates:
@@ -451,12 +452,13 @@ def _incident_target_label(incident: dict) -> str:
     # The Operator only ever sees 0–1000 coordinates: the recorded pixel
     # target is the controller's business and is never rendered here.
     normalized = action.get("normalized_coordinates")
-    text = action.get("target_text")
     parts = []
     if normalized:
         parts.append(f"normalized {list(normalized)}")
-    if text:
-        parts.append(f'"{text}"')
+    if action.get("target_text"):
+        parts.append(f'"{action["target_text"]}"')
+    elif action.get("target_description"):
+        parts.append(f'"{action["target_description"]}" {SELF_DESCRIBED_MARKER}')
     return " ".join(parts) if parts else "the recorded target"
 
 
@@ -526,7 +528,7 @@ def render_execution_incident(incident: dict, steps: list) -> str:
         )
     elif category in ("target_disappeared", "pixel_target_disappeared"):
         prior_str = (
-            f" Your last successfully executed action was `{prior[0]}` (Step {prior[1]});"
+            f" Your last dispatched action was `{prior[0]}` (Step {prior[1]});"
             " if the vanished target belonged to a state that action summoned, that action"
             " is the trigger."
             if prior

artemis/agents/outputter/outputter.py

@@ -17,6 +17,7 @@ from pathlib import Path
 
 from jinja2 import Template
 from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage
+from artemis.core.tool_failure import ToolFailure, is_tool_failure
 from artemis.config import OutputConfig
 from artemis.context import ArtemisContext
 from artemis.data_engine.trace import trace
@@ -271,9 +272,9 @@ async def outputter(
                         record_trace=True,
                     )
                 else:
-                    result = f"Error: Tool {tool_name} is not supported."
+                    result = ToolFailure(f"Error: Tool {tool_name} is not supported.")
                 text, _ = split_multimodal_result(result)
-                status = "error" if text.startswith("Error") else "success"
+                status = "error" if is_tool_failure(result) else "success"
             except Exception as e:
                 logger.error(f"Error running tool {tool_name}: {e}")
                 result = f"Error running tool {tool_name}: {e}"

artemis/agents/summarizer/summarizer.py

@@ -30,8 +30,10 @@ capsule moves up to the chunk-level ``StepCapsuleLens`` in M3.
 """
 
 import json
+from pathlib import Path
 from typing import Any
 
+from artemis.agents.flash.summarizer import build_focus_context
 from artemis.context import ArtemisContext
 from artemis.data_engine.trace import trace
 from artemis.graph.state import State
@@ -91,6 +93,16 @@ class SummarizerNode:
             else:
                 exec_outcome = "unknown"
 
+        # Operator focus: its own reasoning for the step (the target it chose
+        # and the screen change it expected), the goal, the live sub-goal leaf
+        # and any user instruction injected at this step. Nothing is inferred.
+        focus = build_focus_context(
+            intent=self._operator_reasoning(record, state),
+            goal=getattr(state, "initial_goal", None),
+            subgoal=self._active_subgoal_text(),
+            injected_instruction=getattr(state, "injected_instruction", None),
+        )
+
         service = ensure_step_memory(self.ctx)
         service.dispatch(
             step_number=step_number if isinstance(step_number, int) else 0,
@@ -100,9 +112,41 @@ class SummarizerNode:
             post_img_bytes=post_bytes,
             exec_outcome=exec_outcome,
             data_engine_step_id=step_id,
+            focus=focus,
         )
         logger.info(f"Dispatched visual transition summary for step {step_number} ({step_id})")
 
+    @staticmethod
+    def _operator_reasoning(record: Any, state: State) -> str | None:
+        """The operator's raw reasoning for this step: live state first, record second."""
+        for source in (state, record):
+            text = getattr(source, "operator_raw_thinking", None) if source is not None else None
+            if isinstance(text, str) and text.strip():
+                return text
+        return None
+
+    def _active_subgoal_text(self) -> str | None:
+        """The live sub-goal ledger leaf (``milestone > leaf``) from task_plan, if any."""
+        base_dir = getattr(self.ctx.data_engine, "base_dir", None)
+        if not isinstance(base_dir, (str, Path)):
+            return None
+        try:
+            from artemis.utils.notes import get_note_file_path
+            from artemis.utils.plan_grammar import parse_plan
+
+            path = get_note_file_path(base_dir, "task_plan")
+            if not path.exists():
+                return None
+            snapshot = parse_plan(path.read_text(encoding="utf-8"))
+            leaf = snapshot.last_active()
+            if leaf is None:
+                return None
+            parent = None if leaf.is_top_level else snapshot.parent_of(leaf)
+            return f"{parent.text} > {leaf.text}" if parent is not None else leaf.text
+        except Exception as e:
+            logger.debug(f"Active sub-goal lookup for the focus block skipped: {e}", exc_info=True)
+            return None
+
     def _read_step_image(self, step_number: int | None, which: str) -> bytes | None:
         if not isinstance(step_number, int):
             return None

artemis/agents/validator/execution_loop.py

@@ -140,11 +140,12 @@ async def _attempt_local_execution(
             exec_success = False
             exec_error = str(e)
 
-        # Skip polling: assume execution success implies action success
+        # No effect polling: "dispatched" only says the device accepted the
+        # command; whether it had the intended effect is the Operator's call.
         if exec_success:
             success = True
             error_msg = ""
-            attempts_log.append("Success")
+            attempts_log.append("Dispatched")
             break
         else:
             success = False
@@ -456,7 +457,7 @@ async def run_validation_loop(node, state: State) -> dict:
 
     report = {
         "execution": execution,
-        "status": "success" if success else "failed",
+        "status": "dispatched" if success else "failed",
         "burst": burst,
         "incident": incident,
     }

artemis/agents/validator/incidents.py

@@ -16,7 +16,7 @@
 
 Incidents are stored in ``state.open_incident`` and the step's
 ``last_execution_result``. Consecutive failures increment the count; a
-successful terminal action closes the incident. Prompt rendering lives in
+terminal action dispatched without error closes the incident. Prompt rendering lives in
 ``artemis.agents.operator.prompts.ExecutionIncidentPromptComponent``.
 """
 

artemis/agents/validator/pixel_safety_net.md

@@ -5,9 +5,10 @@ Read the `[Target]` block first to determine which rule applies.
 
 Rules
 1. `Kind: specific UI control` — the target has a label, resource id, class or bounds. Judge whether that exact control remains visible at the red dot in Image 2. Ignore video, map, animation or background changes beneath it. A control that has hidden, faded, collapsed, moved or become covered is not present, even if tapping the same spot could restore it.
-2. `Kind: coordinates only` — no UI hierarchy element was found at the point. Identify what appears under the red dot in both images. If Image 1 shows a distinct button, icon, link or handle, apply Rule 1. Apply the surface rule only when Image 1 shows blank space, a backdrop, or the body of a video, animation, map or live stream.
-3. Surface rule. Typical intents: tap the video body to wake its hidden controls, tap the backdrop to dismiss a dialog. Surfaces are expected to change between frames: ignore natural frame changes and the appearance or fading of overlay controls. Output `{"is_present": true, "confidence": 1.0}` unless an unexpected blocking popup now covers the red dot.
-4. Use `[Planned Action & Original Thinking]` to resolve whether the target was a control or its underlying surface. Do not treat a missing control as present because another tap could restore it.
+2. `Kind: described target` — the Operator aimed at a coordinate and stated what it is (`Operator's description`). That statement is the Operator's belief, not an observation: confirm against Image 1 what actually sits under the red dot. If the description names a control (button, icon, link, handle, field, knob) and Image 1 shows one there, apply Rule 1 to that control. If the description names a surface (video body, backdrop, blank area, map) or Image 1 shows no distinct control, apply the surface rule.
+3. `Kind: coordinates only` — no target metadata at all. Identify what appears under the red dot in both images. If Image 1 shows a distinct button, icon, link or handle, apply Rule 1. Apply the surface rule only when Image 1 shows blank space, a backdrop, or the body of a video, animation, map or live stream.
+4. Surface rule. Typical intents: tap the video body to wake its hidden controls, tap the backdrop to dismiss a dialog. Surfaces are expected to change between frames: ignore natural frame changes and the appearance or fading of overlay controls. Output `{"is_present": true, "confidence": 1.0}` unless an unexpected blocking popup now covers the red dot.
+5. Use `[Planned Action & Original Thinking]` to resolve whether the target was a control or its underlying surface. Do not treat a missing control as present because another tap could restore it.
 
 Output
 Return EXACTLY this raw JSON and nothing else. Keep reasoning strictly under 35 words.

artemis/agents/validator/precondition_pixel.py

@@ -67,11 +67,18 @@ def _init_llm_and_prompt(ctx: ArtemisContext, get_llm_fn):
 
 
 def _describe_target(action_item: dict) -> str:
-    """Describe the structured action target for the pixel judge."""
+    """Describe the structured action target for the pixel judge.
+
+    Three kinds, keyed on provenance: an index-addressed target carries observed
+    element metadata; a coordinate target carries only the Operator's own
+    statement of what it aimed at (``target_description``); a legacy record may
+    carry neither.
+    """
     label = action_item.get("target_text")
     resource_id = action_item.get("target_resource_id")
     class_name = action_item.get("target_class")
     bounds = action_item.get("target_bounds")
+    description = action_item.get("target_description")
     lines = ["[Target]", f"Action: {action_item.get('action')}"]
     if label or resource_id or class_name or bounds:
         lines.append("Kind: specific UI control")
@@ -85,9 +92,12 @@ def _describe_target(action_item: dict) -> str:
             lines.append("Label: (unlabelled; identify it by its shape in Image 1)")
         if bounds:
             lines.append(f"Bounds at decision time: {bounds}")
+    elif description:
+        lines.append("Kind: described target")
+        lines.append(f"Operator's description: {description}")
     else:
         lines.append(
-            "Kind: coordinates only (no UI hierarchy element was found under the point;"
+            "Kind: coordinates only (no target metadata recorded;"
             " inspect Image 1 for a control at the red dot)"
         )
     return "\n".join(lines)

artemis/agents/validator/tool_declarations.py

@@ -12,13 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Universal Tool Declarations, Execution Results, and Dispatcher for Artemis."""
+"""Universal Tool Declarations and Execution Results for Artemis.
+
+Execution itself lives in :class:`artemis.mcp.action_executor.McpActionExecutor`
+(Flash) and the Validator's action session (Pro); this module keeps the shared
+result envelope, the observation helpers both use, and the tool declarations.
+"""
 
-import ast
 import base64
-import hashlib
 import json
-from pathlib import Path
 from typing import Any, Literal
 
 from google.genai import types as genai_types
@@ -34,43 +36,18 @@ from artemis.agents.explorer.constants import (
 from artemis.context import ArtemisContext
 from artemis.controllers.unified_controller import UnifiedMobileController
 from artemis.core.tool_declaration import ToolDeclaration
-from artemis.data_engine.trace import trace
 from artemis.mcp.action_specs import tool_declaration
-from artemis.mcp.action_types import ActionCode
-from artemis.mcp.actuators.adb import AdbActuator
 from artemis.mcp.observation import observe
-from artemis.tools.explorer_tool import locate, register_candidates, render_text
 from artemis.graph.state import State
-from artemis.utils.coordinates import (
-    compute_smart_swipe_coordinates,
-    parse_swipe_parameters,
-)
 from artemis.utils.logger import get_logger
 from artemis.utils.notes import (
     LIST_NOTES_DOCSTRING,
     READ_NOTE_ARG_KEY_DESC,
     READ_NOTE_DOCSTRING,
-    format_list_notes_failure,
-    format_list_notes_success,
-    format_read_note_failure,
-    list_notes_info,
-    read_note_content,
 )
 
 logger = get_logger(__name__)
 
-ACTION_TOOL_NAMES = {
-    "click",
-    "click_sequence",
-    "long_press",
-    "input_text",
-    "swipe",
-    "press_key",
-    "manage_app",
-    "wait_for_delay",
-    "wait_for_text",
-}
-
 
 class ToolExecutionResult(BaseModel):
     """Standardized result produced by executing a mobile automation or utility tool."""
@@ -88,7 +65,7 @@ class ToolExecutionResult(BaseModel):
     def to_langchain_tool_message(self) -> ToolMessage:
         """Converts execution result into a standardized multimodal LangChain ToolMessage."""
         content_blocks: list[dict[str, Any]] = [
-            {"type": "text", "text": self.text_summary or f"Action '{self.tool_name}' completed."}
+            {"type": "text", "text": self.text_summary or f"Action '{self.tool_name}' dispatched."}
         ]
         if self.ui_elements_text:
             content_blocks.append(
@@ -249,528 +226,6 @@ def normalize_coordinate_target(target: Any) -> int | list[int] | Any:
     return target
 
 
-def serialize_mobile_action_result(result: Any) -> dict[str, Any]:
-    """Serializes MobileActionExecutor execution results into a clean structured dict.
-
-    Extracts the post-action screenshot hash from the captured image to eliminate guesswork.
-    """
-    if not isinstance(result, tuple) or len(result) < 3:
-        return {"outcome": str(result) if result is not None else ""}
-    outcome, img_bytes, shot_path, *rest = result
-    xml_list = rest[0] if rest else None
-
-    post_image_name = None
-    if shot_path:
-        post_image_name = Path(shot_path).stem
-    elif img_bytes:
-        post_image_name = hashlib.sha256(img_bytes).hexdigest()
-
-    is_err = bool(outcome and ("Error" in outcome or "Failed" in outcome))
-    return {
-        "outcome": outcome,
-        "post_image_name": post_image_name,
-        "has_xml": bool(xml_list),
-        "status": "error" if is_err else "success",
-    }
-
-
-def serialize_explorer_result(result: Any) -> dict[str, Any]:
-    """Stores an ``(text, ok)`` Explorer answer as a flat trace payload."""
-    if isinstance(result, tuple) and len(result) == 2:
-        text, ok = result
-        return {"outcome": text, "status": "success" if ok else "error"}
-    return {"outcome": str(result) if result is not None else ""}
-
-
-class MobileActionExecutor:
-    """Adapts LLM tool calls onto an actuator backend and captures observations.
-
-    Device-call bodies live in the actuator (``artemis/mcp/actuators/``); this class
-    keeps the agent-side responsibilities: argument normalization, index resolution
-    against LangGraph ``State``, act-then-observe screenshot capture, tracing, and the
-    legacy 4-tuple result contract.
-    """
-
-    def __init__(
-        self,
-        ctx: ArtemisContext,
-        controller: UnifiedMobileController | None = None,
-        actuator: "AdbActuator | None" = None,
-        agent_name: str = "validator",
-    ):
-        self.ctx = ctx
-        self.actuator = actuator or AdbActuator(ctx, controller)
-        self.controller = self.actuator.controller
-        # Identifies the calling agent to the Explorer tier resolver: the tier
-        # is a user setting keyed by profile / agent, never chosen by the agent.
-        self.agent_name = agent_name
-
-    @trace(type="action", name="click", serializer=serialize_mobile_action_result)
-    async def exec_click(
-        self,
-        target: list[int] | Any,
-        state: State,
-        times: int = 1,
-        delay_ms: int = 100,
-    ) -> tuple[str, bytes | None, str | None, str | None]:
-        try:
-            target = normalize_coordinate_target(target)
-            if not isinstance(target, (list, tuple)) or len(target) != 2:
-                raise ValueError(f"Invalid target format: {target}")
-            nx, ny = target
-            res = await self.actuator.click(int(nx), int(ny), times=times, delay_ms=delay_ms)
-            outcome = res.message
-
-            img_bytes, shot_path, xml_list = None, None, None
-            if res.ok:
-                shot_path, img_bytes, xml_list = await capture_screenshot_and_parse_ui(
-                    self.ctx, state, self.controller
-                )
-        except Exception as e:
-            outcome = f"Error during click: {e}"
-            img_bytes, shot_path, xml_list = None, None, None
-
-        return outcome, img_bytes, shot_path, xml_list
-
-    @trace(type="action", name="click_sequence", serializer=serialize_mobile_action_result)
-    async def exec_click_sequence(
-        self,
-        sequence: list[int | list[int] | str] | str,
-        state: State,
-        delay_ms: int = 50,
-    ) -> tuple[str, bytes | None, str | None, str | None]:
-        width = getattr(self.ctx.device, "device_width", 1080)
-        height = getattr(self.ctx.device, "device_height", 2400)
-        try:
-            if isinstance(sequence, str):
-                sequence_str = sequence.strip()
-                try:
-                    sequence = json.loads(sequence_str)
-                except Exception:
-                    try:
-                        sequence = ast.literal_eval(sequence_str)
-                    except (ValueError, TypeError, SyntaxError, MemoryError, RecursionError):
-                        pass
-
-            if not isinstance(sequence, (list, tuple)):
-                return (
-                    f"Error during click sequence: Invalid sequence format: {sequence}",
-                    None,
-                    None,
-                    None,
-                )
-
-            # Resolve each entry to normalized coordinates: integer element indices go
-            # through state.indexed_points (pixel centers -> normalized), coordinate
-            # pairs are already normalized.
-            norm_points: list[tuple[int, int]] = []
-            for raw_target in sequence:
-                target = normalize_coordinate_target(raw_target)
-                if isinstance(target, int):
-                    idx = target
-                    points = getattr(state, "indexed_points", []) or []
-                    if 1 <= idx <= len(points):
-                        pt = points[idx - 1]
-                        px, py = int(pt[0]), int(pt[1])
-                        nx = int(max(0, min(1000, round(px * 1000 / max(1, width)))))
-                        ny = int(max(0, min(1000, round(py * 1000 / max(1, height)))))
-                    else:
-                        return (
-                            (
-                                "Error during click sequence: Index"
-                                f" {idx} is out of range (available: {len(points)})"
-                            ),
-                            None,
-                            None,
-                            None,
-                        )
-                elif isinstance(target, (list, tuple)) and len(target) == 2:
-                    nx, ny = int(target[0]), int(target[1])
-                else:
-                    return (
-                        f"Error during click sequence: Invalid target format: {raw_target}",
-                        None,
-                        None,
-                        None,
-                    )
-                norm_points.append((nx, ny))
-
-            res = await self.actuator.click_sequence(norm_points, delay_ms=delay_ms)
-            if not res.ok:
-                return res.message, None, None, None
-
-            shot_path, img_bytes, xml_list = await capture_screenshot_and_parse_ui(
-                self.ctx, state, self.controller
-            )
-            return res.message, img_bytes, shot_path, xml_list
-        except Exception as e:
-            return f"Error executing click sequence: {e}", None, None, None
-
-    @trace(type="action", name="long_press", serializer=serialize_mobile_action_result)
-    async def exec_long_press(
-        self,
-        target: list[int] | Any,
-        state: State,
-        duration_ms: int = 1000,
-        duration: int | None = None,
-    ) -> tuple[str, bytes | None, str | None, str | None]:
-        if duration is not None:
-            duration_ms = duration
-        try:
-            target = normalize_coordinate_target(target)
-            if not isinstance(target, (list, tuple)) or len(target) != 2:
-                raise ValueError(f"Invalid target format: {target}")
-            nx, ny = target
-            res = await self.actuator.long_press(int(nx), int(ny), duration_ms=duration_ms)
-            outcome = res.message
-
-            img_bytes, shot_path, xml_list = None, None, None
-            if res.ok:
-                shot_path, img_bytes, xml_list = await capture_screenshot_and_parse_ui(
-                    self.ctx, state, self.controller
-                )
-        except Exception as e:
-            outcome = f"Error during long press: {e}"
-            img_bytes, shot_path, xml_list = None, None, None
-
-        return outcome, img_bytes, shot_path, xml_list
-
-    @trace(type="action", name="input_text", serializer=serialize_mobile_action_result)
-    async def exec_input_text(
-        self,
-        text: str,
-        target: list[int] | Any,
-        state: State,
-        clear_exist: bool = True,
-    ) -> tuple[str, bytes | None, str | None, str | None]:
-        try:
-            norm_target: tuple[int, int] | None = None
-            if target:
-                target = normalize_coordinate_target(target)
-                if not isinstance(target, (list, tuple)) or len(target) != 2:
-                    raise ValueError(f"Invalid target format: {target}")
-                norm_target = (int(target[0]), int(target[1]))
-
-            res = await self.actuator.input_text(text, norm_target, clear_exist=clear_exist)
-            outcome = res.message
-            shot_path, img_bytes, xml_list = await capture_screenshot_and_parse_ui(
-                self.ctx, state, self.controller
-            )
-        except Exception as e:
-            outcome = f"Error during input text: {e}"
-            img_bytes, shot_path, xml_list = None, None, None
-
-        return outcome, img_bytes, shot_path, xml_list
-
-    @trace(type="action", name="swipe", serializer=serialize_mobile_action_result)
-    async def exec_swipe(
-        self,
-        action: str | list[int] | Any = None,
-        duration: int = 800,
-        state: State | None = None,
-        duration_ms: int | None = None,
-        **kwargs: Any,
-    ) -> tuple[str, bytes | None, str | None, str | None]:
-        width = getattr(self.ctx.device, "device_width", 1080)
-        height = getattr(self.ctx.device, "device_height", 2400)
-        dur = duration_ms if duration_ms is not None else duration
-
-        swipe_input = dict(kwargs) if kwargs else {}
-        if isinstance(action, dict):
-            swipe_input.update(action)
-        elif action is not None:
-            swipe_input["action"] = action
-        if "duration" not in swipe_input:
-            swipe_input["duration"] = dur
-
-        kind, target, final_duration = parse_swipe_parameters(swipe_input, default_duration=dur)
-
-        try:
-            if kind == "direction":
-                dir_name = target
-                # Smart-swipe resolution needs State (indexed elements + ui tree), so
-                # it stays client-side; the actuator only receives the two points.
-                x1, y1, x2, y2, smart_dur = compute_smart_swipe_coordinates(
-                    direction=target,
-                    target=swipe_input.get("target"),
-                    indexed_elements=getattr(state, "indexed_elements", None) if state else None,
-                    ui_hierarchy=getattr(state, "latest_ui_hierarchy", None) if state else None,
-                    width=width,
-                    height=height,
-                    duration=final_duration,
-                )
-                final_duration = smart_dur
-                nx1 = int(max(0, min(1000, round(x1 * 1000 / max(1, width)))))
-                ny1 = int(max(0, min(1000, round(y1 * 1000 / max(1, height)))))
-                nx2 = int(max(0, min(1000, round(x2 * 1000 / max(1, width)))))
-                ny2 = int(max(0, min(1000, round(y2 * 1000 / max(1, height)))))
-                res = await self.actuator.swipe((nx1, ny1), (nx2, ny2), final_duration)
-                if not res.ok:
-                    return f"Error swiping {dir_name}: {res.detail}", None, None, None
-                outcome = f"Swipe completed successfully. Swiped {dir_name}."
-            elif kind == "coords":
-                x1, y1, x2, y2 = target
-                res = await self.actuator.swipe(
-                    (int(x1), int(y1)), (int(x2), int(y2)), final_duration
-                )
-                if not res.ok:
-                    return f"Error dragging: {res.detail}", None, None, None
-                outcome = f"Swipe completed successfully. Swiped from [{x1}, {y1}] to [{x2}, {y2}]."
-            else:
-                return (
-                    f"Error during swipe: Invalid direction: {action or kwargs}",
-                    None,
-                    None,
-                    None,
-                )
-
-            current_state = state or State()
-            shot_path, img_bytes, xml_list = await capture_screenshot_and_parse_ui(
-                self.ctx, current_state, self.controller
-            )
-        except Exception as e:
-            outcome = f"Error during swipe: {e}"
-            img_bytes, shot_path, xml_list = None, None, None
-
-        return outcome, img_bytes, shot_path, xml_list
-
-    @trace(type="action", name="press_key", serializer=serialize_mobile_action_result)
-    async def exec_press_key(
-        self,
-        key: str,
-        state: State,
-    ) -> tuple[str, bytes | None, str | None, str | None]:
-        try:
-            res = await self.actuator.press_key(key)
-            if not res.ok:
-                return res.message, None, None, None
-
-            outcome = res.message
-            shot_path, img_bytes, xml_list = await capture_screenshot_and_parse_ui(
-                self.ctx, state, self.controller
-            )
-        except Exception as e:
-            outcome = f"Error during press_key: {e}"
-            img_bytes, shot_path, xml_list = None, None, None
-
-        return outcome, img_bytes, shot_path, xml_list
-
-    @trace(type="action", name="manage_app", serializer=serialize_mobile_action_result)
-    async def exec_manage_app(
-        self,
-        action: str,
-        app_name: str,
-        state: State,
-    ) -> tuple[str, bytes | None, str | None, str | None]:
-        try:
-            res = await self.actuator.manage_app(action, app_name)
-            # Argument-level failures return without an observation, exactly as the
-            # historical early returns did; a failed launch still captures the screen.
-            if res.code in (ActionCode.PACKAGE_NOT_FOUND, ActionCode.INVALID_ARGS):
-                return res.message, None, None, None
-
-            outcome = res.message
-            shot_path, img_bytes, xml_list = await capture_screenshot_and_parse_ui(
-                self.ctx, state, self.controller
-            )
-        except Exception as e:
-            outcome = f"Error during manage_app: {e}"
-            img_bytes, shot_path, xml_list = None, None, None
-
-        return outcome, img_bytes, shot_path, xml_list
-
-    @trace(type="action", name="wait_for_delay", serializer=serialize_mobile_action_result)
-    async def exec_wait_for_delay(
-        self,
-        time_in_ms: int,
-        state: State,
-    ) -> tuple[str, bytes | None, str | None, str | None]:
-        try:
-            res = await self.actuator.wait_for_delay(time_in_ms)
-            outcome = res.message
-            shot_path, img_bytes, xml_list = await capture_screenshot_and_parse_ui(
-                self.ctx, state, self.controller
-            )
-        except Exception as e:
-            outcome = f"Error during wait_for_delay: {e}"
-            img_bytes, shot_path, xml_list = None, None, None
-
-        return outcome, img_bytes, shot_path, xml_list
-
-    @trace(type="action", name="wait_for_text", serializer=serialize_mobile_action_result)
-    async def exec_wait_for_text(
-        self,
-        text: str,
-        wait_state: str | None,
-        timeout_ms: int | None,
-        state: State,
-    ) -> tuple[str, bytes | None, str | None, str | None]:
-        try:
-            res = await self.actuator.wait_for_text(text, wait_state, timeout_ms)
-            outcome = res.message
-            # A timeout still captures the screen, as the original loop did.
-            shot_path, img_bytes, xml_list = await capture_screenshot_and_parse_ui(
-                self.ctx, state, self.controller
-            )
-        except Exception as e:
-            outcome = f"Error during wait_for_text: {e}"
-            img_bytes, shot_path, xml_list = None, None, None
-
-        return outcome, img_bytes, shot_path, xml_list
-
-    @trace(type="tool", name="read_note")
-    async def exec_read_note(
-        self,
-        key: str,
-        start_line: int | None = None,
-        end_line: int | None = None,
-    ) -> str:
-        try:
-            base_dir = self.ctx.data_engine.base_dir if self.ctx.data_engine else None
-            if not base_dir:
-                return format_read_note_failure(key, "DataEngine not initialized.")
-            return read_note_content(base_dir, key, start_line, end_line)
-        except Exception as e:
-            return format_read_note_failure(key, str(e))
-
-    @trace(type="tool", name="list_notes")
-    async def exec_list_notes(self) -> str:
-        try:
-            base_dir = self.ctx.data_engine.base_dir if self.ctx.data_engine else None
-            if not base_dir:
-                return format_list_notes_failure("DataEngine not initialized.")
-            notes = list_notes_info(base_dir)
-            return format_list_notes_success(notes)
-        except Exception as e:
-            return format_list_notes_failure(str(e))
-
-    @trace(type="tool", name="ask_explorer", serializer=serialize_explorer_result)
-    async def exec_ask_explorer(
-        self,
-        query: str,
-        context_feedback: str | None,
-        state: State,
-    ) -> tuple[str, bool]:
-        """Runs the Explorer pipeline; returns ``(text, ok)``.
-
-        ``ok`` is False only when the Explorer run itself failed. A clean
-        "not found" is a successful answer the agent must reason about, so it
-        must not be reported as a tool error.
-        """
-        try:
-            outcome = await locate(
-                self.ctx, state, query, context_feedback or "", agent_name=self.agent_name
-            )
-            registered = register_candidates(self.ctx, state, outcome)
-            return render_text(query, outcome, registered), not outcome.error
-        except Exception as e:
-            return f"Error executing ask_explorer: {e}", False
-
-    async def execute(
-        self,
-        name: str,
-        args: dict[str, Any],
-        tool_call_id: str,
-        state: State,
-    ) -> ToolExecutionResult:
-        """Executes the specified tool by name and wraps result into ToolExecutionResult."""
-        raw_name = name.split(":")[-1] if ":" in name else name
-        res_text = ""
-        img_bytes, shot_path, xml_list = None, None, None
-        # Explicit status for tools that report it (the Explorer); None keeps the
-        # historical text-sniffing fallback for the device actions.
-        explicit_ok: bool | None = None
-
-        if raw_name == "click":
-            res_text, img_bytes, shot_path, xml_list = await self.exec_click(
-                args.get("target") or args.get("coordinates") or [],
-                state,
-                times=args.get("times", 1),
-                delay_ms=args.get("delay_ms", 100),
-            )
-        elif raw_name == "click_sequence":
-            res_text, img_bytes, shot_path, xml_list = await self.exec_click_sequence(
-                args.get("sequence") or [],
-                state,
-                delay_ms=args.get("delay_ms", 50),
-            )
-        elif raw_name == "long_press":
-            res_text, img_bytes, shot_path, xml_list = await self.exec_long_press(
-                args.get("target") or args.get("coordinates") or [],
-                state,
-                duration_ms=args.get("duration_ms", 1000),
-            )
-        elif raw_name == "input_text":
-            res_text, img_bytes, shot_path, xml_list = await self.exec_input_text(
-                args.get("text", ""),
-                args.get("target") or args.get("coordinates"),
-                state,
-                clear_exist=args.get("clear_exist", True),
-            )
-        elif raw_name == "swipe":
-            res_text, img_bytes, shot_path, xml_list = await self.exec_swipe(
-                action=args, duration=args.get("duration", 400), state=state
-            )
-        elif raw_name == "press_key":
-            res_text, img_bytes, shot_path, xml_list = await self.exec_press_key(
-                args.get("key", "BACK"), state
-            )
-        elif raw_name == "manage_app":
-            res_text, img_bytes, shot_path, xml_list = await self.exec_manage_app(
-                args.get("action", "launch"), args.get("app_name", ""), state
-            )
-        elif raw_name == "wait_for_delay":
-            res_text, img_bytes, shot_path, xml_list = await self.exec_wait_for_delay(
-                args.get("time_in_ms", 1000), state
-            )
-        elif raw_name == "wait_for_text":
-            res_text, img_bytes, shot_path, xml_list = await self.exec_wait_for_text(
-                args.get("text", ""),
-                args.get("wait_state"),
-                args.get("timeout_ms", 5000),
-                state,
-            )
-        elif raw_name == "read_note":
-            res_text = await self.exec_read_note(
-                args.get("key", ""),
-                args.get("start_line"),
-                args.get("end_line"),
-            )
-        elif raw_name == "list_notes":
-            res_text = await self.exec_list_notes()
-        elif raw_name == "ask_explorer":
-            res_text, explicit_ok = await self.exec_ask_explorer(
-                # ``task_description`` is the pre-contract argument name kept
-                # for old MCP clients.
-                args.get("query") or args.get("task_description") or "",
-                args.get("context_feedback"),
-                state,
-            )
-        else:
-            res_text = f"Error: Tool '{name}' not supported."
-
-        if shot_path:
-            state.latest_screenshot = shot_path
-
-        status: Literal["success", "error"] = (
-            "success" if (img_bytes is not None or raw_name not in ACTION_TOOL_NAMES) else "error"
-        )
-        if explicit_ok is not None:
-            status = "success" if explicit_ok else "error"
-        elif "Error" in res_text or "Failed" in res_text:
-            status = "error"
-
-        return ToolExecutionResult(
-            tool_call_id=tool_call_id,
-            tool_name=name,
-            status=status,
-            text_summary=res_text,
-            screenshot_bytes=img_bytes,
-            screenshot_path=shot_path,
-            ui_elements_text=xml_list,
-        )
-
-
 # Device-action declarations are generated from the canonical manifest
 # (artemis/mcp/action_specs.py); the historical constant names remain the public API.
 CLICK_TOOL = tool_declaration("click")

artemis/agents/video_analyzer/audio_native.py

@@ -26,7 +26,20 @@ from pathlib import Path
 from google.genai import types
 
 from artemis.agents.video_analyzer import video_analyzer as _va
-from artemis.agents.video_analyzer.reliability import classify_video_failure
+from artemis.agents.video_analyzer.chunk_conversation import (
+    _MISSING_SUMMARY_ERROR,
+    _NO_TOOL_CALL_ERROR,
+    _SUBMIT_NOW_WARNING,
+    _WRONG_TOOL_ERROR,
+    SUB_AGENT_MAX_TURNS,
+    _append_rejection,
+    _clean_text,
+    _tool_name,
+)
+from artemis.agents.video_analyzer.reliability import (
+    SubAgentAnswerExhausted,
+    classify_video_failure,
+)
 from artemis.constants import SAFETY_SETTINGS_BLOCK_NONE
 from artemis.data_engine.trace import TraceSpan
 from artemis.llm.reliability import retry_policy_for
@@ -34,6 +47,11 @@ from artemis.utils.logger import get_logger
 
 logger = get_logger(__name__)
 
+_BAD_CONFIDENCE_ERROR = (
+    "Error: 'confidence_score' is missing or out of bounds. It must be a float"
+    " between 0.0 and 1.0 inclusive. Fix the payload and call submit_answer again."
+)
+
 
 class _AudioMedia:
     """Mutable holder for per-attempt audio artifacts shared with failure handling."""
@@ -427,29 +445,18 @@ async def _run_native_audio_conversation(
 
 
 async def _drive_audio_agent_loop(analyzer, sub_agent_contents: list) -> tuple[str, str, float]:
-    """Iterates the audio agent conversation until submit_answer is accepted."""
-    sub_max_iterations = 2
-    sub_iterations = 0
-    final_confidence_score = 0.0
-    final_summary = "No summary provided."
-    final_analysis = "No analysis provided."
-    final_full_text = ""
+    """Return a validated (summary, analysis, confidence_score).
 
-    while sub_iterations < sub_max_iterations:
-        sub_iterations += 1
-        if sub_iterations == sub_max_iterations - 1:
+    Raise ``SubAgentAnswerExhausted`` if no answer is accepted within the turn limit.
+    """
+    rejection = _NO_TOOL_CALL_ERROR
+
+    for turn in range(1, SUB_AGENT_MAX_TURNS + 1):
+        if turn == SUB_AGENT_MAX_TURNS - 1:
             sub_agent_contents.append(
                 types.Content(
                     role="user",
-                    parts=[
-                        types.Part.from_text(
-                            text=(
-                                "[WARNING] You must call the"
-                                " submit_answer tool now, time"
-                                " is running out."
-                            )
-                        )
-                    ],
+                    parts=[types.Part.from_text(text=_SUBMIT_NOW_WARNING)],
                 )
             )
 
@@ -468,26 +475,16 @@ async def _drive_audio_agent_loop(analyzer, sub_agent_contents: list) -> tuple[s
             )
 
         full_text = response.text or ""
-        final_full_text += full_text + "\n"
         function_calls = response.function_calls or []
 
         if function_calls:
-            (
-                answered,
-                final_summary,
-                final_analysis,
-                final_confidence_score,
-            ) = _handle_audio_function_calls(
-                sub_agent_contents,
-                response,
-                function_calls,
-                final_summary,
-                final_analysis,
-                final_confidence_score,
+            answer, rejection = _handle_audio_function_calls(
+                sub_agent_contents, response, function_calls, full_text
             )
-            if answered is True:
-                break
+            if answer is not None:
+                return answer
         else:
+            rejection = _NO_TOOL_CALL_ERROR
             sub_agent_contents.append(
                 types.Content(
                     role="model",
@@ -495,83 +492,52 @@ async def _drive_audio_agent_loop(analyzer, sub_agent_contents: list) -> tuple[s
                 )
             )
 
-    return final_summary, final_analysis, final_confidence_score
+    raise SubAgentAnswerExhausted(SUB_AGENT_MAX_TURNS, rejection)
+
+
+def _validate_audio_answer(summary: str, confidence_score) -> str | None:
+    """Return a validation error for the audio answer, or None if valid."""
+    if not summary:
+        return _MISSING_SUMMARY_ERROR
+    if (
+        not isinstance(confidence_score, (int, float))
+        or isinstance(confidence_score, bool)
+        or not (0.0 <= float(confidence_score) <= 1.0)
+    ):
+        return _BAD_CONFIDENCE_ERROR
+    return None
 
 
 def _handle_audio_function_calls(
     sub_agent_contents: list,
     response,
     function_calls: list,
-    final_summary: str,
-    final_analysis: str,
-    final_confidence_score: float,
-):
-    """Processes audio-agent function calls; returns (answered, summary, analysis, score)."""
-    answered = False
+    full_text: str,
+) -> tuple[tuple[str, str, float] | None, str | None]:
+    """Return (answer, rejection), appending any rejection to the conversation."""
+    original_parts = response.candidates[0].content.parts
     for fc in function_calls:
-        if (fc.name.split(":")[-1] if ":" in fc.name else fc.name) == "submit_answer":
-            args = fc.args
-            if (
-                "confidence_score" not in args
-                or not isinstance(
-                    args["confidence_score"],
-                    (int, float),
-                )
-                or not (0.0 <= float(args["confidence_score"]) <= 1.0)
-            ):
-                error_msg = (
-                    "Error: 'confidence_score' is"
-                    " missing or out of bounds. It must"
-                    " be a float between 0.0 and 1.0"
-                    " inclusive. Fix the payload and"
-                    " call submit_answer again."
-                )
-                sub_agent_contents.append(
-                    types.Content(
-                        role="model",
-                        parts=response.candidates[0].content.parts,
-                    )
-                )
-                sub_agent_contents.append(
-                    types.Content(
-                        role="user",
-                        parts=[
-                            types.Part.from_function_response(
-                                name=fc.name,
-                                response={"error": error_msg},
-                            )
-                        ],
-                    )
-                )
-                answered = "error"
-                break
+        if _tool_name(fc.name) != "submit_answer":
+            continue
+        args = dict(fc.args or {})
+        summary = _clean_text(args.get("summary"))
+        analysis = _clean_text(args.get("analysis"))
 
-            final_summary = args.get("summary", "No summary provided.").strip().replace("\n", " ")
-            final_analysis = (
-                args.get("analysis", "No analysis provided.").strip().replace("\n", " ")
+        rejection = _validate_audio_answer(summary, args.get("confidence_score"))
+        if rejection is not None:
+            _append_rejection(
+                sub_agent_contents, original_parts, function_calls, full_text, fc.name, rejection
             )
-            final_confidence_score = float(args["confidence_score"])
-            answered = True
-            break
+            return None, rejection
 
-    if answered is True or answered == "error":
-        return answered, final_summary, final_analysis, final_confidence_score
+        return (summary, analysis or "No analysis provided.", float(args["confidence_score"])), None
 
-    sub_agent_contents.append(
-        types.Content(
-            role="model",
-            parts=response.candidates[0].content.parts,
-        )
+    _append_rejection(
+        sub_agent_contents,
+        original_parts,
+        function_calls,
+        full_text,
+        function_calls[0].name,
+        _WRONG_TOOL_ERROR,
     )
-    sub_agent_contents.append(
-        types.Content(
-            role="user",
-            parts=[
-                types.Part.from_function_response(
-                    name=function_calls[0].name,
-                    response={"error": ("Tool not recognized. Please use submit_answer.")},
-                )
-            ],
-        )
-    )
-    return answered, final_summary, final_analysis, final_confidence_score
+    return None, _WRONG_TOOL_ERROR

artemis/agents/video_analyzer/chunk_agentic.py

@@ -0,0 +1,455 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Analyze video chunks through the Gemini Interactions API.
+
+``processing: "agentic"`` lets the model search and revisit an uploaded clip.
+Validation errors are returned as function results, with video context retained
+through ``previous_interaction_id``. Only accepted answers reach the blackboard.
+
+The stream consumer reconstructs function arguments from step deltas. Completion
+events may omit steps, so it fetches the interaction when reconstruction fails.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import copy
+from dataclasses import dataclass, field
+import json
+from types import SimpleNamespace
+from typing import Any
+
+from artemis.agents.video_analyzer import gemini_files
+from artemis.agents.video_analyzer import video_analyzer as _va
+from artemis.agents.video_analyzer.chunk_conversation import (
+    _NO_TOOL_CALL_ERROR,
+    _SUBMIT_NOW_WARNING,
+    _WRONG_TOOL_ERROR,
+    SUB_AGENT_MAX_TURNS,
+    SubAgentAnswerExhausted,
+    _ChunkMedia,
+    _clean_text,
+    _persist_timeline_events,
+    _tool_name,
+    _validate_answer,
+)
+from artemis.agents.video_analyzer.universal_tools import UNIVERSAL_SUBMIT_ANSWER_TOOL
+from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan
+from artemis.llm.google import normalize_usage
+from artemis.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+STREAM_START_TIMEOUT_SECONDS = 60.0
+INTERACTION_READ_TIMEOUT_SECONDS = 30.0
+
+# Interaction statuses after which no follow-up turn can be sent
+# (``google.genai`` ``InteractionStatus``).
+_TERMINAL_FAILURE_STATUSES = frozenset({"failed", "cancelled", "incomplete", "budget_exceeded"})
+
+
+def build_interactions_submit_tool() -> dict[str, Any]:
+    """The ``submit_answer`` declaration in Interactions API (flat JSON schema) form.
+
+    Derived from the universal (OpenAI-style) declaration so the schema is
+    written once; ``timeline_events`` is required to match the native path.
+    """
+    function = UNIVERSAL_SUBMIT_ANSWER_TOOL["function"]
+    parameters = copy.deepcopy(function["parameters"])
+    required = list(parameters.get("required", []))
+    if "timeline_events" not in required:
+        required.append("timeline_events")
+    parameters["required"] = required
+    return {
+        "type": "function",
+        "name": function["name"],
+        "description": function["description"],
+        "parameters": parameters,
+    }
+
+
+def _steps(interaction: Any) -> list:
+    return list(getattr(interaction, "steps", None) or [])
+
+
+def _find_function_call(interaction: Any):
+    """The last ``function_call`` step of an interaction, if any."""
+    for step in reversed(_steps(interaction)):
+        if getattr(step, "type", None) == "function_call":
+            return step
+    return None
+
+
+def _raise_if_failed(interaction: Any) -> None:
+    """Raises when an interaction ended in a terminal failure state.
+
+    A follow-up with ``previous_interaction_id`` onto a failed interaction is
+    rejected by the API, so the failure surfaces here as an exception and
+    flows through the shared retry / classification layer instead.
+    """
+    status = str(getattr(interaction, "status", None) or "").lower()
+    errors = list(getattr(interaction, "errors", None) or [])
+    if status not in _TERMINAL_FAILURE_STATUSES and not errors:
+        return
+    details = "; ".join(
+        f"[{getattr(error, 'code', None)}] {getattr(error, 'message', error)}" for error in errors
+    )
+    raise RuntimeError(
+        f"Gemini Interactions interaction {getattr(interaction, 'id', None)} ended with"
+        f" status {status or 'unknown'}" + (f": {details}" if details else "")
+    )
+
+
+async def run_agentic_chunk_conversation(
+    analyzer,
+    media: _ChunkMedia,
+    current_start: float,
+    current_end: float,
+    start_time: float,
+    end_time: float | None,
+    specific_query: str,
+    lease_owner: str,
+) -> str:
+    """Uploads the chunk and drives the agentic sub-agent to a committed answer."""
+    file = None
+    try:
+        with TraceSpan(name="upload_video_to_gemini") as span:
+            file = await analyzer.upload_and_poll_file(media.compressed_path)
+            span.result = f"Uploaded {file.name}"
+        logger.info(
+            f"Invoking Gemini agentic video understanding for sub-agent task with model"
+            f" {analyzer.model_name} (streaming)..."
+        )
+        trace_id = CURRENT_TRACE_ID.get()
+
+        if analyzer.ctx.data_engine and trace_id:
+            analyzer.ctx.data_engine.record_trace(
+                type="llm_call",
+                name="video_sub_agent",
+                payload={
+                    "contents": [
+                        f"file://{file.name}",
+                        media.prompt_with_context,
+                    ],
+                    "system_instruction": analyzer.sub_system_prompt,
+                    "video_processing": "agentic",
+                },
+                parent_trace_id=trace_id,
+            )
+
+        final_summary, final_analysis = await _drive_agentic_loop(
+            analyzer,
+            file,
+            trace_id,
+            media,
+            start_time,
+            end_time,
+            specific_query,
+        )
+
+        end_str = f"{current_end:.1f}s" if current_end is not None else "unknown"
+        chunk_result = (
+            f"[from {current_start:.1f}s to {end_str}] Summary:"
+            f" {final_summary} Analysis: {final_analysis}"
+        )
+        analyzer.blackboard.complete_segment(
+            current_start,
+            current_end,
+            specific_query,
+            lease_owner,
+            final_summary,
+            final_analysis,
+        )
+        return chunk_result
+    finally:
+        if file:
+            await gemini_files.delete_cloud_file(
+                analyzer.client, file.name, analyzer.cloud_files_to_cleanup
+            )
+
+
+def _initial_request(analyzer, file, prompt: str, tool: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "model": analyzer.model_name,
+        "input": [
+            {
+                "type": "video",
+                "uri": file.uri,
+                "mime_type": file.mime_type,
+                "processing": "agentic",
+            },
+            {"type": "text", "text": prompt},
+        ],
+        "system_instruction": analyzer.sub_system_prompt,
+        "tools": [tool],
+        "stream": True,
+    }
+
+
+def _follow_up_request(
+    analyzer, previous_id: str, steps: list[dict[str, Any]], tool: dict[str, Any]
+) -> dict[str, Any]:
+    """A stateful follow-up turn: the server keeps the video context.
+
+    Only the conversation state is inherited through
+    ``previous_interaction_id``; ``system_instruction`` and ``tools`` are
+    interaction-scoped and must be re-sent on every turn.
+    """
+    return {
+        "model": analyzer.model_name,
+        "previous_interaction_id": previous_id,
+        "input": steps,
+        "system_instruction": analyzer.sub_system_prompt,
+        "tools": [tool],
+        "stream": True,
+    }
+
+
+def _function_result(call_id: str, error: str) -> dict[str, Any]:
+    return {
+        "type": "function_result",
+        "call_id": call_id,
+        "name": "submit_answer",
+        "result": {"error": error},
+    }
+
+
+async def _drive_agentic_loop(
+    analyzer,
+    file,
+    trace_id,
+    media: _ChunkMedia,
+    start_time: float,
+    end_time: float | None,
+    specific_query: str,
+) -> tuple[str, str]:
+    """Return a validated answer and persist its timeline events.
+
+    Raise ``SubAgentAnswerExhausted`` if no answer is accepted within the turn limit.
+    """
+    tool = build_interactions_submit_tool()
+    request = _initial_request(analyzer, file, media.prompt_with_context, tool)
+    rejection = _NO_TOOL_CALL_ERROR
+
+    for turn in range(1, SUB_AGENT_MAX_TURNS + 1):
+        interaction = await _stream_agentic_turn(analyzer, request, trace_id)
+        interaction_id = getattr(interaction, "id", None) or ""
+        call = _find_function_call(interaction)
+
+        if call is None:
+            rejection = _NO_TOOL_CALL_ERROR
+            steps = [{"type": "text", "text": _SUBMIT_NOW_WARNING}]
+        elif _tool_name(getattr(call, "name", None)) != "submit_answer":
+            rejection = _WRONG_TOOL_ERROR
+            steps = [_function_result(call.id, rejection)]
+        else:
+            args = dict(getattr(call, "arguments", None) or {})
+            timeline_events = args.get("timeline_events", [])
+            if not isinstance(timeline_events, list):
+                timeline_events = []
+            summary = _clean_text(args.get("summary"))
+            analysis = _clean_text(args.get("analysis"))
+
+            rejection = _validate_answer(summary, timeline_events, start_time, end_time)
+            if rejection is None:
+                await _persist_timeline_events(
+                    analyzer, timeline_events, media, start_time, end_time, specific_query
+                )
+                return summary, analysis or "No analysis provided."
+            steps = [_function_result(call.id, rejection)]
+
+        if turn < SUB_AGENT_MAX_TURNS:
+            request = _follow_up_request(analyzer, interaction_id, steps, tool)
+
+    raise SubAgentAnswerExhausted(SUB_AGENT_MAX_TURNS, rejection)
+
+
+async def _stream_agentic_turn(analyzer, request: dict[str, Any], trace_id):
+    """Runs one traced streaming Interactions turn and returns the completed interaction."""
+    with TraceSpan(name="gemini_agentic_video_call") as span:
+        span.payload = {
+            "model": request.get("model"),
+            "video_processing": "agentic",
+            "previous_interaction_id": request.get("previous_interaction_id"),
+        }
+        stream = await asyncio.wait_for(
+            analyzer.client.aio.interactions.create(**request),
+            timeout=STREAM_START_TIMEOUT_SECONDS,
+        )
+        interaction = await asyncio.wait_for(
+            _consume_agentic_stream(analyzer, span, stream, trace_id),
+            timeout=analyzer.agentic_call_timeout_seconds,
+        )
+        call = _find_function_call(interaction)
+        span.result = (
+            f"Function call: {_tool_name(getattr(call, 'name', None))}"
+            if call is not None
+            else f"Status: {getattr(interaction, 'status', 'unknown')}"
+        )
+    return interaction
+
+
+@dataclass
+class _PendingStep:
+    """A step under construction while its start/delta/stop events stream in."""
+
+    kind: str
+    call_id: str | None = None
+    name: str | None = None
+    seed_arguments: dict[str, Any] = field(default_factory=dict)
+    arguments_json: list[str] = field(default_factory=list)
+    text: list[str] = field(default_factory=list)
+
+    def finish(self):
+        """Materializes the step in the shape the SDK's typed steps expose.
+
+        Returns ``None`` for a function call whose streamed arguments are not
+        valid JSON: a half-built call must not stand in for the real one, so
+        the step is dropped and the interaction is re-read by id instead.
+        """
+        if self.kind == "function_call":
+            arguments: dict[str, Any] = dict(self.seed_arguments)
+            raw = "".join(self.arguments_json)
+            if raw.strip():
+                try:
+                    parsed = json.loads(raw)
+                except ValueError:
+                    logger.warning(
+                        f"Could not parse streamed arguments for {self.name}: {raw[:200]!r};"
+                        " dropping the step"
+                    )
+                    return None
+                if isinstance(parsed, dict):
+                    arguments = parsed
+            return SimpleNamespace(
+                type="function_call", id=self.call_id, name=self.name, arguments=arguments
+            )
+        if self.kind == "model_output":
+            return SimpleNamespace(
+                type="model_output",
+                content=[SimpleNamespace(type="text", text="".join(self.text))],
+            )
+        return SimpleNamespace(type=self.kind, id=self.call_id)
+
+
+async def _consume_agentic_stream(analyzer, span, stream, trace_id):
+    """Consumes Interactions SSE events, rebuilding steps and mirroring progress.
+
+    Returns an interaction-shaped object (``id`` / ``status`` / ``steps`` /
+    ``usage``). The interaction is re-read by id when the stream ended
+    without a completion event, or when it stopped at ``requires_action``
+    without a parsable function call.
+    """
+    data_engine = analyzer.ctx.data_engine if trace_id else None
+    interaction_id: str | None = None
+    status: str | None = None
+    completed_steps: list | None = None
+    usage: dict[str, int] | None = None
+    pending: dict[int, _PendingStep] = {}
+    steps: list = []
+    processing_calls = 0
+
+    def _emit(text: str, *, thinking: bool) -> None:
+        if data_engine and text:
+            data_engine.stream_output(trace_id, text, is_thinking=thinking)
+
+    async for event in stream:
+        data = getattr(event, "data", event)
+        event_type = getattr(data, "event_type", None)
+
+        if event_type == "interaction.created":
+            created = getattr(data, "interaction", None)
+            interaction_id = getattr(created, "id", None) or interaction_id
+        elif event_type == "step.start":
+            step = getattr(data, "step", None)
+            index = getattr(data, "index", None)
+            kind = str(getattr(step, "type", None) or "")
+            pending[index] = _PendingStep(
+                kind=kind,
+                call_id=getattr(step, "id", None) or getattr(step, "call_id", None),
+                name=getattr(step, "name", None),
+                seed_arguments=dict(getattr(step, "arguments", None) or {}),
+            )
+            if kind == "processing_call":
+                processing_calls += 1
+                _emit(
+                    f"Inspecting the recording (video lookup {processing_calls})\n",
+                    thinking=True,
+                )
+        elif event_type == "step.delta":
+            delta = getattr(data, "delta", None)
+            delta_type = getattr(delta, "type", None)
+            current = pending.get(getattr(data, "index", None))
+            if delta_type == "arguments_delta":
+                fragment = getattr(delta, "arguments", None)
+                if current is not None and isinstance(fragment, str):
+                    current.arguments_json.append(fragment)
+            elif delta_type == "text":
+                text = getattr(delta, "text", None)
+                if current is not None and isinstance(text, str):
+                    current.text.append(text)
+                _emit(text or "", thinking=False)
+            elif delta_type == "thought_summary":
+                content = getattr(delta, "content", None)
+                _emit(getattr(content, "text", None) or "", thinking=True)
+        elif event_type == "step.stop":
+            current = pending.pop(getattr(data, "index", None), None)
+            finished = current.finish() if current is not None else None
+            if finished is not None:
+                steps.append(finished)
+            usage = normalize_usage(getattr(data, "usage", None)) or usage
+        elif event_type == "interaction.completed":
+            completed = getattr(data, "interaction", None)
+            interaction_id = getattr(completed, "id", None) or interaction_id
+            status = getattr(completed, "status", None) or status
+            completed_steps = getattr(completed, "steps", None) or None
+            usage = normalize_usage(getattr(completed, "usage", None)) or usage
+            _raise_if_failed(completed)
+        elif event_type == "interaction.status_update":
+            status = getattr(data, "status", None) or status
+        elif event_type == "error":
+            error = getattr(data, "error", None)
+            raise RuntimeError(
+                "Gemini Interactions stream error"
+                f" [{getattr(error, 'code', None)}]: {getattr(error, 'message', None)}"
+            )
+
+    for current in pending.values():
+        finished = current.finish()
+        if finished is not None:
+            steps.append(finished)
+    span.payload["processing_calls"] = processing_calls
+    if completed_steps:
+        steps = list(completed_steps)
+
+    needs_reread = status is None or (
+        status == "requires_action" and _find_function_call(SimpleNamespace(steps=steps)) is None
+    )
+    if needs_reread:
+        if not interaction_id:
+            raise RuntimeError("Gemini Interactions stream ended without an interaction")
+        logger.info(f"Re-reading interaction {interaction_id} (status={status})")
+        fetched = await asyncio.wait_for(
+            analyzer.client.aio.interactions.get(interaction_id),
+            timeout=INTERACTION_READ_TIMEOUT_SECONDS,
+        )
+        _raise_if_failed(fetched)
+        status = getattr(fetched, "status", None) or status
+        steps = _steps(fetched) or steps
+        usage = normalize_usage(getattr(fetched, "usage", None)) or usage
+
+    if usage:
+        span.payload["usage_metadata"] = usage
+    return SimpleNamespace(id=interaction_id, status=status, steps=steps, usage=usage)

artemis/agents/video_analyzer/chunk_conversation.py

@@ -27,13 +27,46 @@ import re
 
 from google.genai import types
 
+from artemis.agents.video_analyzer import gemini_files
 from artemis.agents.video_analyzer import video_analyzer as _va
+from artemis.agents.video_analyzer.reliability import SubAgentAnswerExhausted
 from artemis.constants import SAFETY_SETTINGS_BLOCK_NONE
 from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan
+from artemis.llm.google import normalize_usage
 from artemis.utils.logger import get_logger
 
 logger = get_logger(__name__)
 
+#: Turns the chunk sub-agent gets to produce an accepted ``submit_answer``
+#: (shared by the static and agentic paths).
+SUB_AGENT_MAX_TURNS = 2
+
+_SUBMIT_NOW_WARNING = "[WARNING] You must call the submit_answer tool now, time is running out."
+_WRONG_TOOL_ERROR = "Tool not recognized. Please use submit_answer."
+_NO_TOOL_CALL_ERROR = "The model replied with text only and never called submit_answer."
+_MISSING_SUMMARY_ERROR = (
+    "submit_answer was called without a 'summary'. Call it again with a non-empty"
+    " 'summary' (and 'analysis') describing what you observed."
+)
+
+
+def _tool_name(name: str | None) -> str:
+    name = name or ""
+    return name.split(":")[-1] if ":" in name else name
+
+
+def _clean_text(value) -> str:
+    return str(value or "").strip().replace("\n", " ")
+
+
+def _validate_answer(
+    summary: str, timeline_events: list, start_time: float, end_time: float | None
+) -> str | None:
+    """Return a validation error for submit_answer, or None if valid."""
+    if not summary:
+        return _MISSING_SUMMARY_ERROR
+    return _validate_timeline_events(timeline_events, start_time, end_time)
+
 
 class _ChunkMedia:
     """Mutable holder for per-attempt media artifacts shared with failure handling."""
@@ -120,15 +153,9 @@ async def _run_native_chunk_conversation(
         return chunk_result
     finally:
         if file:
-            logger.info(f"Cleaning up cloud file {file.name}...")
-            try:
-                await asyncio.wait_for(
-                    analyzer.client.aio.files.delete(name=file.name),
-                    timeout=30,
-                )
-                analyzer.cloud_files_to_cleanup.discard(file.name)
-            except Exception as ce:
-                logger.error(f"Failed to delete cloud file {file.name}: {ce}")
+            await gemini_files.delete_cloud_file(
+                analyzer.client, file.name, analyzer.cloud_files_to_cleanup
+            )
 
 
 async def _drive_sub_agent_loop(
@@ -140,38 +167,27 @@ async def _drive_sub_agent_loop(
     end_time: float | None,
     specific_query: str,
 ) -> tuple[str, str]:
-    """Iterates the sub-agent conversation until submit_answer is accepted."""
-    sub_max_iterations = 2
-    sub_iterations = 0
-    final_analysis = "No analysis provided."
-    final_summary = "No summary provided."
-    final_full_text = ""
+    """Return a validated answer and persist its timeline events.
 
-    while sub_iterations < sub_max_iterations:
-        sub_iterations += 1
-        if sub_iterations == sub_max_iterations - 1:
+    Raise ``SubAgentAnswerExhausted`` if no answer is accepted within the turn limit.
+    """
+    rejection = _NO_TOOL_CALL_ERROR
+
+    for turn in range(1, SUB_AGENT_MAX_TURNS + 1):
+        if turn == SUB_AGENT_MAX_TURNS - 1:
             sub_agent_contents.append(
                 types.Content(
                     role="user",
-                    parts=[
-                        types.Part.from_text(
-                            text=(
-                                "[WARNING] You must call the"
-                                " submit_answer tool now, time"
-                                " is running out."
-                            )
-                        )
-                    ],
+                    parts=[types.Part.from_text(text=_SUBMIT_NOW_WARNING)],
                 )
             )
 
         full_text, function_calls, original_parts = await _stream_sub_agent_turn(
             analyzer, sub_agent_contents, trace_id
         )
-        final_full_text += full_text + "\n"
 
         if function_calls:
-            answered, final_summary, final_analysis = await _handle_sub_agent_calls(
+            answer, rejection = await _handle_sub_agent_calls(
                 analyzer,
                 sub_agent_contents,
                 function_calls,
@@ -181,12 +197,11 @@ async def _drive_sub_agent_loop(
                 start_time,
                 end_time,
                 specific_query,
-                final_summary,
-                final_analysis,
             )
-            if answered is True:
-                break
+            if answer is not None:
+                return answer
         else:
+            rejection = _NO_TOOL_CALL_ERROR
             sub_agent_contents.append(
                 types.Content(
                     role="model",
@@ -194,7 +209,7 @@ async def _drive_sub_agent_loop(
                 )
             )
 
-    return final_summary, final_analysis
+    raise SubAgentAnswerExhausted(SUB_AGENT_MAX_TURNS, rejection)
 
 
 async def _stream_sub_agent_turn(
@@ -228,24 +243,9 @@ async def _read_sub_agent_stream(analyzer, span, target_stream, current_trace_id
     function_calls = []
     accumulated_parts = []
     async for chunk in target_stream:
-        if hasattr(chunk, "usage_metadata") and chunk.usage_metadata:
-            span.payload["usage_metadata"] = {
-                "prompt_token_count": getattr(
-                    chunk.usage_metadata,
-                    "prompt_token_count",
-                    0,
-                ),
-                "candidates_token_count": getattr(
-                    chunk.usage_metadata,
-                    "candidates_token_count",
-                    0,
-                ),
-                "total_token_count": getattr(
-                    chunk.usage_metadata,
-                    "total_token_count",
-                    0,
-                ),
-            }
+        usage = normalize_usage(getattr(chunk, "usage_metadata", None))
+        if usage:
+            span.payload["usage_metadata"] = usage
 
         chunk_text = chunk.text or ""
         if chunk_text:
@@ -287,42 +287,33 @@ async def _handle_sub_agent_calls(
     start_time: float,
     end_time: float | None,
     specific_query: str,
-    final_summary: str,
-    final_analysis: str,
-):
-    """Processes function calls from one turn; returns (answered, summary, analysis)."""
-    answered = False
+) -> tuple[tuple[str, str] | None, str | None]:
+    """Return ``(answer, rejection)`` after processing one turn's tool calls.
+
+    Persist events for accepted answers; append rejected calls to the conversation
+    with an error for the model to correct.
+    """
     for fc in function_calls:
-        if (fc.name.split(":")[-1] if ":" in fc.name else fc.name) == "submit_answer":
-            args = fc.args
-            timeline_events = args.get("timeline_events", [])
-            final_summary = args.get("summary", "No summary provided.").strip().replace("\n", " ")
-            final_analysis = (
-                args.get("analysis", "No analysis provided.").strip().replace("\n", " ")
+        if _tool_name(fc.name) != "submit_answer":
+            continue
+        args = dict(fc.args or {})
+        timeline_events = args.get("timeline_events", [])
+        if not isinstance(timeline_events, list):
+            timeline_events = []
+        summary = _clean_text(args.get("summary"))
+        analysis = _clean_text(args.get("analysis"))
+
+        rejection = _validate_answer(summary, timeline_events, start_time, end_time)
+        if rejection is not None:
+            _append_rejection(
+                sub_agent_contents, original_parts, function_calls, full_text, fc.name, rejection
             )
+            return None, rejection
 
-            error_msg = _validate_timeline_events(timeline_events, start_time, end_time)
-            if error_msg is not None:
-                _append_rejection(
-                    sub_agent_contents,
-                    original_parts,
-                    function_calls,
-                    full_text,
-                    fc.name,
-                    error_msg,
-                )
-                answered = "error"
-                break
-
-            await _persist_timeline_events(
-                analyzer, timeline_events, media, start_time, end_time, specific_query
-            )
-
-            answered = True
-            break
-
-    if answered is True or answered == "error":
-        return answered, final_summary, final_analysis
+        await _persist_timeline_events(
+            analyzer, timeline_events, media, start_time, end_time, specific_query
+        )
+        return (summary, analysis or "No analysis provided."), None
 
     _append_rejection(
         sub_agent_contents,
@@ -330,9 +321,9 @@ async def _handle_sub_agent_calls(
         function_calls,
         full_text,
         function_calls[0].name,
-        ("Tool not recognized. Please use submit_answer."),
+        _WRONG_TOOL_ERROR,
     )
-    return answered, final_summary, final_analysis
+    return None, _WRONG_TOOL_ERROR
 
 
 def _validate_timeline_events(

artemis/agents/video_analyzer/chunk_native.py

@@ -26,11 +26,16 @@ import asyncio
 from pathlib import Path
 
 from artemis.agents.video_analyzer import video_analyzer as _va
+from artemis.agents.video_analyzer.chunk_agentic import run_agentic_chunk_conversation
 from artemis.agents.video_analyzer.chunk_conversation import (
     _ChunkMedia,
     _run_native_chunk_conversation,
 )
-from artemis.agents.video_analyzer.reliability import classify_video_failure
+from artemis.agents.video_analyzer.reliability import (
+    AgenticVideoDegraded,
+    classify_video_failure,
+    is_agentic_rejection,
+)
 from artemis.llm.reliability import retry_policy_for
 from artemis.utils.logger import get_logger
 
@@ -76,12 +81,19 @@ async def exec_single_chunk(
 
     media = _ChunkMedia()
 
-    for attempt in range(max_retries + 1):
+    attempt = 0
+    while attempt <= max_retries:
+        # The mode this attempt was launched in: a parallel chunk may degrade
+        # the run while this one is still in flight.
+        attempt_mode = analyzer.video_processing
         try:
             logger.info(
                 f"Attempt {attempt + 1}/{max_retries + 1} for sub-agent:"
                 f" start={current_start}, end={current_end}"
             )
+            # A sibling may switch the run to static during queuing or media
+            # preparation. Check the size before preparation and before dispatch.
+            _ensure_fits_static_ceiling(analyzer, current_start, current_end)
             result, slowdown_factor = await _prepare_chunk_media(
                 analyzer, media, current_start, current_end
             )
@@ -94,6 +106,7 @@ async def exec_single_chunk(
                 end_time,
                 specific_query,
             )
+            _ensure_fits_static_ceiling(analyzer, current_start, current_end)
 
             if not analyzer.use_native_gemini:
                 chunk_result = await analyzer._exec_single_chunk_universal(
@@ -116,6 +129,18 @@ async def exec_single_chunk(
                 )
                 return chunk_result
 
+            if analyzer.video_processing == "agentic":
+                return await run_agentic_chunk_conversation(
+                    analyzer,
+                    media,
+                    current_start,
+                    current_end,
+                    start_time,
+                    end_time,
+                    specific_query,
+                    lease_owner,
+                )
+
             return await _run_native_chunk_conversation(
                 analyzer,
                 media,
@@ -134,6 +159,22 @@ async def exec_single_chunk(
             # stream output from the failed attempt is discarded, never
             # surfaced as a complete answer.
             failure = classify_video_failure(e)
+            if attempt_mode == "agentic" and is_agentic_rejection(failure, e):
+                # Switch to static and let the coordinator resize the interval.
+                _degrade_agentic_to_static(analyzer, e, failure)
+                return await _finish_failed_chunk(
+                    analyzer,
+                    media,
+                    current_start,
+                    current_end,
+                    specific_query,
+                    lease_owner,
+                    AgenticVideoDegraded(
+                        f"Agentic video processing rejected ({e}); re-plan"
+                        f" {current_start:.1f}s-{current_end:.1f}s at the static chunk size"
+                    ),
+                    retryable=False,
+                )
             attempts_so_far = attempt + 1
             if not failure.retryable:
                 logger.warning(
@@ -181,12 +222,51 @@ async def exec_single_chunk(
                 source="video_sub_agent",
             )
             await asyncio.sleep(delay)
+            attempt += 1
 
     raise RuntimeError(
         f"Video sub-agent exhausted retries for {current_start:.1f}s-{current_end:.1f}s"
     )
 
 
+def _degrade_agentic_to_static(analyzer, error: Exception, failure) -> None:
+    """Switches the rest of this run to static video understanding (idempotent)."""
+    if analyzer.video_processing != "agentic":
+        return
+    logger.warning(
+        f"Agentic video processing rejected ({error}); the rest of this run"
+        " analyzes video in static mode"
+    )
+    _va._record_llm_event(
+        "llm_fallback",
+        {
+            "reason": "agentic_video_rejected",
+            "category": failure.category.value,
+            "error": str(error)[:500],
+        },
+    )
+    analyzer.video_processing = "static"
+    analyzer.chunk_size_seconds = analyzer.static_chunk_size_seconds
+
+
+def _ensure_fits_static_ceiling(analyzer, current_start: float, current_end: float) -> None:
+    """Refuses to analyze a clip sized for the agentic ceiling once the run is static.
+
+    Raises :class:`AgenticVideoDegraded`, which the attempt loop treats as a
+    non-retryable failure: the lease is released via ``fail_segment`` and the
+    coordinator re-plans the interval into static-sized pieces.
+    """
+    if analyzer.video_processing == "agentic":
+        return
+    if current_end - current_start <= analyzer.chunk_size_seconds + 1e-3:
+        return
+    raise AgenticVideoDegraded(
+        f"Run degraded to static video processing while {current_start:.1f}s-{current_end:.1f}s"
+        f" was pending; re-plan it at the static chunk size"
+        f" ({analyzer.chunk_size_seconds:.0f}s)"
+    )
+
+
 async def _finish_failed_chunk(
     analyzer,
     media: _ChunkMedia,

artemis/agents/video_analyzer/gemini_files.py

@@ -97,6 +97,18 @@ async def cleanup_abandoned_gemini_files(client) -> None:
         logger.warning(f"Routine cloud maintenance skipped: {e}")
 
 
+async def delete_cloud_file(
+    client, file_name: str, cloud_files_to_cleanup: set, *, timeout: float = 30.0
+) -> None:
+    """Best-effort deletion of an uploaded file; failures are logged, never raised."""
+    logger.info(f"Cleaning up cloud file {file_name}...")
+    try:
+        await asyncio.wait_for(client.aio.files.delete(name=file_name), timeout=timeout)
+        cloud_files_to_cleanup.discard(file_name)
+    except Exception as ce:
+        logger.error(f"Failed to delete cloud file {file_name}: {ce}")
+
+
 async def upload_and_poll_file(client, compressed_path: Path, cloud_files_to_cleanup: set) -> any:
     """Upload a media file to the Gemini File API and poll until it is ACTIVE."""
     logger.info(f"Uploading {compressed_path} to Gemini File API...")

artemis/agents/video_analyzer/native_coordinator.py

@@ -32,6 +32,7 @@ from artemis.agents.video_analyzer import video_analyzer as _va
 from artemis.agents.video_analyzer.reliability import classify_video_failure
 from artemis.constants import SAFETY_SETTINGS_BLOCK_NONE
 from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan
+from artemis.llm.google import normalize_usage
 from artemis.utils.logger import get_logger
 
 logger = get_logger(__name__)
@@ -349,24 +350,9 @@ async def _consume_main_stream(
     function_calls = []
 
     async for chunk in stream:
-        if hasattr(chunk, "usage_metadata") and chunk.usage_metadata:
-            span.payload["usage_metadata"] = {
-                "prompt_token_count": getattr(
-                    chunk.usage_metadata,
-                    "prompt_token_count",
-                    0,
-                ),
-                "candidates_token_count": getattr(
-                    chunk.usage_metadata,
-                    "candidates_token_count",
-                    0,
-                ),
-                "total_token_count": getattr(
-                    chunk.usage_metadata,
-                    "total_token_count",
-                    0,
-                ),
-            }
+        usage = normalize_usage(getattr(chunk, "usage_metadata", None))
+        if usage:
+            span.payload["usage_metadata"] = usage
 
         chunk_text = chunk.text or ""
         text += chunk_text

artemis/agents/video_analyzer/reliability.py

@@ -53,10 +53,50 @@ class VideoFailure:
 
 
 _MEDIA_MARKERS = ("ffmpeg", "codec", "corrupt", "invalid video", "invalid audio", "media")
+_AGENTIC_REJECTION_MARKERS = ("processing", "agentic")
+
+
+class SubAgentAnswerExhausted(RuntimeError):
+    """The sub-agent exhausted its turns without a valid submit_answer.
+
+    The caller records a failed segment so a later claim can retry it. The shared
+    classifier permits retries, splitting, and fallback for this failure.
+    """
+
+    def __init__(self, turns: int, reason: str):
+        self.turns = turns
+        self.reason = reason
+        super().__init__(
+            f"submit_answer was not accepted within {turns} turn(s); last rejection: {reason}"
+        )
+
+
+class AgenticVideoDegraded(RuntimeError):
+    """Return an interval to the coordinator after switching to static mode.
+
+    The coordinator splits oversized clips and retries smaller clips as-is.
+    """
+
+
+def is_agentic_rejection(failure: VideoFailure, error: BaseException) -> bool:
+    """True when a bad request is the API refusing agentic video processing.
+
+    Only an HTTP 400 whose message mentions ``processing`` / ``agentic``
+    qualifies; other bad requests (413 payload too large, 404, ...) keep the
+    shared handling.
+    """
+    if failure.category is not VideoFailureCategory.BAD_REQUEST:
+        return False
+    if extract_status_code(error) != 400:
+        return False
+    message = str(error).lower()
+    return any(marker in message for marker in _AGENTIC_REJECTION_MARKERS)
 
 
 def classify_video_failure(error: BaseException) -> VideoFailure:
     """Convert provider/transport/media exceptions into recovery decisions."""
+    if isinstance(error, AgenticVideoDegraded):
+        return VideoFailure(VideoFailureCategory.BAD_REQUEST, False, True, False)
     generic = classify_failure(error)
     message = str(error).lower()
     has_media_marker = any(marker in message for marker in _MEDIA_MARKERS)

artemis/agents/video_analyzer/universal_engine.py

@@ -36,7 +36,10 @@ from langchain_core.messages import (
 )
 
 from artemis.agents.video_analyzer import video_analyzer as _va
-from artemis.agents.video_analyzer.reliability import classify_video_failure
+from artemis.agents.video_analyzer.reliability import (
+    SubAgentAnswerExhausted,
+    classify_video_failure,
+)
 from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan
 from artemis.utils.logger import get_logger
 
@@ -218,29 +221,33 @@ async def _invoke_universal_with_visual_retry(
 
 
 def _extract_submit_payload(response) -> tuple[str, str, list]:
-    """Pulls summary/analysis/timeline events from a universal model response."""
-    final_summary = "No summary provided."
-    final_analysis = "No analysis provided."
-    timeline_events = []
+    """Extract a summary, analysis, and events from the model response.
 
-    if getattr(response, "tool_calls", None):
-        for tc in response.tool_calls:
-            if tc.get("name") == "submit_answer":
-                args = tc.get("args", {})
-                final_summary = (
-                    str(args.get("summary", "")).strip().replace("\n", " ") or final_summary
-                )
-                final_analysis = (
-                    str(args.get("analysis", "")).strip().replace("\n", " ") or final_analysis
-                )
-                timeline_events = args.get("timeline_events", [])
-                break
-    elif getattr(response, "content", None):
-        text_content = str(response.content).strip()
-        final_summary = text_content[:200].replace("\n", " ")
-        final_analysis = text_content
+    Use the first submit_answer call if its summary is non-empty. Otherwise,
+    fall back to response text, using its first 200 characters as the summary
+    and dropping any events from the rejected call. Raise SubAgentAnswerExhausted
+    if neither source provides an answer.
+    """
+    for tc in getattr(response, "tool_calls", None) or []:
+        if tc.get("name") != "submit_answer":
+            continue
+        args = tc.get("args") or {}
+        final_summary = str(args.get("summary", "")).strip().replace("\n", " ")
+        if not final_summary:
+            break
+        final_analysis = (
+            str(args.get("analysis", "")).strip().replace("\n", " ") or "No analysis provided."
+        )
+        return final_summary, final_analysis, list(args.get("timeline_events") or [])
 
-    return final_summary, final_analysis, timeline_events
+    raw_content = getattr(response, "content", None)
+    text_content = str(raw_content).strip() if raw_content else ""
+    if text_content:
+        return text_content[:200].replace("\n", " "), text_content, []
+
+    raise SubAgentAnswerExhausted(
+        1, "the model neither called submit_answer with a summary nor replied with text"
+    )
 
 
 async def _persist_universal_events(
@@ -361,23 +368,9 @@ async def exec_analyze_audio_universal(
         label="Universal audio sub-agent",
         force_fallback=force_fallback,
     )
-    final_summary = "No audio summary provided."
-    final_analysis = "No audio analysis provided."
-
-    if getattr(response, "tool_calls", None):
-        for tc in response.tool_calls:
-            if tc.get("name") == "submit_answer":
-                args = tc.get("args", {})
-                final_summary = (
-                    str(args.get("summary", "")).strip().replace("\n", " ") or final_summary
-                )
-                final_analysis = (
-                    str(args.get("analysis", "")).strip().replace("\n", " ") or final_analysis
-                )
-                break
-    elif getattr(response, "content", None):
-        final_summary = str(response.content)[:200].replace("\n", " ")
-        final_analysis = str(response.content)
+    # Same resolution order as the video chunk (raises SubAgentAnswerExhausted
+    # when the model produced nothing usable); audio has no timeline events.
+    final_summary, final_analysis, _ = _extract_submit_payload(response)
 
     end_val = end_time if end_time is not None else "unknown"
     analyzer._record_blackboard_entry(

artemis/agents/video_analyzer/video_analyzer.py

@@ -53,6 +53,7 @@ from artemis.agents.video_analyzer.gemini_files import (
     cleanup_abandoned_gemini_files,
 )
 from artemis.agents.video_analyzer.reliability import (
+    AgenticVideoDegraded,
     VideoCircuitBreaker,
     classify_video_failure,
 )
@@ -67,6 +68,11 @@ from artemis.constants import SAFETY_SETTINGS_BLOCK_NONE
 from artemis.context import ArtemisContext
 from artemis.controllers.controller_factory import get_controller
 from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan, trace
+from artemis.llm.google import (
+    is_gemini_model,
+    resolve_video_processing,
+    strip_provider_prefix,
+)
 from artemis.llm.reliability import retry_policy_for
 from artemis.services.llm import _record_llm_event, _record_llm_retry, get_llm
 from artemis.utils.logger import get_logger
@@ -152,7 +158,19 @@ class VideoAnalyzer:
             "enable_ledger",
             getattr(agent_config, "enable_video_ledger", True),
         )
-        self.chunk_size_seconds = float(getattr(self.video_config, "chunk_size_seconds", 60.0))
+        self.static_chunk_size_seconds = float(
+            getattr(self.video_config, "chunk_size_seconds", 60.0)
+        )
+        self.agentic_chunk_size_seconds = float(
+            getattr(self.video_config, "agentic_chunk_size_seconds", 600.0)
+        )
+        self.agentic_call_timeout_seconds = float(
+            getattr(self.video_config, "agentic_call_timeout_seconds", 300.0)
+        )
+        # Effective chunk ceiling; re-derived once the engine picks a
+        # video-processing mode (agentic sub-agents take long clips).
+        self.chunk_size_seconds = self.static_chunk_size_seconds
+        self.video_processing = "static"
         self.min_chunk_seconds = float(getattr(self.video_config, "min_chunk_seconds", 4.0))
         self.max_split_depth = int(getattr(self.video_config, "max_split_depth", 4))
         self.action_window_seconds = float(getattr(self.video_config, "action_window_seconds", 2.0))
@@ -399,25 +417,19 @@ class VideoAnalyzer:
         llm_config = getattr(ctx, "llm_config", None)
         utils_cfg = getattr(llm_config, "utils", None) if llm_config else None
         llm_cfg = getattr(utils_cfg, "video_analyzer", None) if utils_cfg else None
-        model_str = (
-            llm_cfg.model if (llm_cfg and hasattr(llm_cfg, "model")) else "gemini-3.8-flash"
-        ).lower()
-        self.model_name = (
+        self.model_name = strip_provider_prefix(
             llm_cfg.model if (llm_cfg and hasattr(llm_cfg, "model")) else "gemini-3.8-flash"
         )
-        if "/" in self.model_name:
-            self.model_name = self.model_name.split("/")[-1]
 
         has_google_key = bool(
             settings.GOOGLE_API_KEY and settings.GOOGLE_API_KEY.get_secret_value()
         )
-        is_gemini_model = "gemini" in model_str
 
         # If client is already set on ctx or explicit Gemini configuration
         self.client = getattr(ctx, "_genai_client", None)
         if self.client is not None:
             self.use_native_gemini = True
-        elif has_google_key and is_gemini_model:
+        elif has_google_key and is_gemini_model(self.model_name):
             try:
                 self.client = genai.Client(api_key=settings.GOOGLE_API_KEY.get_secret_value())
                 ctx._genai_client = self.client
@@ -430,6 +442,29 @@ class VideoAnalyzer:
                 self.use_native_gemini = False
         else:
             self.use_native_gemini = False
+        self._resolve_video_processing()
+
+    def _resolve_video_processing(self) -> None:
+        """Picks static vs agentic video understanding for the sub-agents.
+
+        Agentic processing (Gemini Interactions API) needs the native client
+        and a supporting model; the configured knob (``auto`` / ``agentic`` /
+        ``static``) decides the rest. The chunk ceiling follows the mode
+        because agentic sub-agents are built to take long clips.
+        """
+        configured = getattr(self.video_config, "processing", "auto")
+        mode = "static"
+        if self.use_native_gemini:
+            mode = resolve_video_processing(configured, self.model_name)
+        self.video_processing = mode
+        self.chunk_size_seconds = (
+            self.agentic_chunk_size_seconds if mode == "agentic" else self.static_chunk_size_seconds
+        )
+        if mode == "agentic":
+            logger.info(
+                f"Video sub-agents use agentic video understanding on {self.model_name}"
+                f" (chunk ceiling {self.chunk_size_seconds:.0f}s)"
+            )
 
     @property
     def blackboard_ledger(self) -> str:
@@ -565,6 +600,18 @@ class VideoAnalyzer:
             self.local_dirs_to_cleanup.add(metadata_path.parent)
         return requested_start, requested_end, None
 
+    def _plan_chunks(self, intervals: list[tuple[float, float]]) -> list[tuple[float, float]]:
+        """Cuts intervals into consecutive chunks of at most ``chunk_size_seconds``."""
+        target_chunk_size = self.chunk_size_seconds
+        chunks: list[tuple[float, float]] = []
+        for gap_start, gap_end in intervals:
+            cursor = gap_start
+            while cursor < gap_end:
+                chunk_end = min(gap_end, cursor + target_chunk_size)
+                chunks.append((round(cursor, 3), round(chunk_end, 3)))
+                cursor = chunk_end
+        return chunks
+
     @trace(type="tool", name="spawn_sub_agent")
     async def exec_spawn_sub_agent(
         self, start_time: float, end_time: float | None, specific_query: str
@@ -600,21 +647,14 @@ class VideoAnalyzer:
             )
             return "CACHED VIDEO ANALYSIS: " + " ".join(cached_results)
 
-        target_chunk_size = self.chunk_size_seconds
-        chunks: list[tuple[float, float]] = []
-        for gap_start, gap_end in missing:
-            cursor = gap_start
-            while cursor < gap_end:
-                chunk_end = min(gap_end, cursor + target_chunk_size)
-                chunks.append((round(cursor, 3), round(chunk_end, 3)))
-                cursor = chunk_end
+        chunks = self._plan_chunks(missing)
         logger.info(
             f"Video blackboard planned {len(chunks)} uncovered chunk(s) for "
             f"{requested_start:.1f}s-{requested_end:.1f}s"
         )
 
         async def analyze_with_recovery(
-            cs: float, ce: float, depth: int = 0
+            cs: float, ce: float, depth: int = 0, *, retried_static: bool = False
         ) -> tuple[list[str], list[str]]:
             try:
                 async with API_SEMAPHORE:
@@ -623,8 +663,40 @@ class VideoAnalyzer:
             except Exception as error:
                 failure = classify_video_failure(error)
                 duration = ce - cs
+                fits_ceiling = duration <= self.chunk_size_seconds + 1e-3
+                degraded = isinstance(error, AgenticVideoDegraded)
+                if degraded and fits_ceiling and not retried_static:
+                    # Retry short intervals once in static mode without splitting.
+                    logger.warning(
+                        f"Retrying {cs:.1f}s-{ce:.1f}s in static mode: it already fits"
+                        f" the {self.chunk_size_seconds:.0f}s chunk ceiling"
+                    )
+                    return await analyze_with_recovery(cs, ce, depth, retried_static=True)
+                pieces = (
+                    self._plan_chunks([(cs, ce)])
+                    if failure.should_split and not fits_ceiling
+                    else []
+                )
+                if len(pieces) > 1:
+                    # Split clips planned before a downgrade at the new ceiling.
+                    logger.warning(
+                        f"Re-planning failed {cs:.1f}s-{ce:.1f}s chunk into {len(pieces)}"
+                        f" piece(s) of at most {self.chunk_size_seconds:.0f}s"
+                        f" after {failure.category.value}"
+                    )
+                    parts = await asyncio.gather(
+                        *(analyze_with_recovery(ps, pe, depth) for ps, pe in pieces)
+                    )
+                    return (
+                        [item for values, _ in parts for item in values],
+                        [item for _, failures in parts for item in failures],
+                    )
+                # Bisecting only helps genuine size/timeout failures; a
+                # degraded interval that fits the ceiling has had its one
+                # static retry and is terminal.
                 can_split = (
                     failure.should_split
+                    and not degraded
                     and depth < self.max_split_depth
                     and duration >= self.min_chunk_seconds * 2
                 )
@@ -767,9 +839,7 @@ class VideoAnalyzer:
             temperature = 0.2
             thinking_level = None
             if llm_cfg:
-                self.model_name = llm_cfg.model
-                if "/" in self.model_name:
-                    self.model_name = self.model_name.split("/")[-1]
+                self.model_name = strip_provider_prefix(llm_cfg.model)
                 if getattr(llm_cfg, "temperature", None) is not None:
                     temperature = llm_cfg.temperature
                 if getattr(llm_cfg, "thinking_level", None) is not None: