feat: add agentic video analysis and clarify action results

Report device actions as dispatched and preserve full failure details. Route
Flash through McpActionExecutor and use ToolFailure across helper tools and
agent loops instead of inferring failure from response text.

Require model-provided descriptions for coordinate targets. Keep observed
element metadata separate from those descriptions in safety checks, visual
summaries, history, and frontend labels.

Add Gemini Interactions API video analysis with configurable processing mode,
chunk size, and timeouts. Centralize Google provider capabilities and token
usage normalization, and update google-genai to 2.22.0.

Preserve system instructions on follow-up interactions, replan pending chunks
when falling back to static processing, and reject unanswered native video
and audio segments instead of caching placeholder results. Keep helper-tool
failure status intact through history tools and committee calls.

Validation: 1,762 Python unit tests passed (3 skipped), 40 frontend formatter
tests passed, and the production UI build completed. Ruff lint, formatting,
the quality ratchet, and git diff checks passed.

apps/showcase_ui/src/app/core/models/stream.model.ts

@@ -237,7 +237,7 @@ export interface StepReplayFrame {
   actionText?: string;
   targetText?: string;
   coords?: string;
-  status?: 'success' | 'failed' | string;
+  status?: 'dispatched' | 'failed' | string;
   isPost?: boolean;
   timestamp?: number;
   phaseId?: string;

apps/showcase_ui/src/app/utils/action-formatter.util.spec.ts

@@ -1,7 +1,11 @@
 import {
   getStepPreImageUrl,
   getStepPostImageUrl,
-  extractStepReplayFrames
+  extractStepReplayFrames,
+  getActionTargetText,
+  isActionFailed,
+  getActionErrorMessage,
+  extractActionExtraParams
 } from './action-formatter.util';
 
 describe('action-formatter.util screenshot chaining', () => {
@@ -403,3 +407,107 @@ describe('extractStepReplayFrames', () => {
     expect(frames[1].stepId).toBe('real-step-2');
   });
 });
+
+describe('getActionTargetText self-described targets', () => {
+  it('uses the model\'s target_description when no observed target_text exists', () => {
+    expect(getActionTargetText({ action: 'click', coordinates: [500, 900], target_description: 'play button' }))
+      .toBe('play button');
+    expect(getActionTargetText({ name: 'long_press', args: { target: [500, 900], target_description: 'song row' } }))
+      .toBe('song row');
+  });
+
+  it('prefers observed target_text over target_description when both are present', () => {
+    expect(getActionTargetText({
+      action: 'click',
+      target_text: 'Play',
+      target_description: 'play button'
+    })).toBe('Play');
+  });
+
+  it('chains per-point target_descriptions for click_sequence with the same arrow as coordinates', () => {
+    expect(getActionTargetText({
+      action: 'click_sequence',
+      sequence: [[500, 300], [876, 360]],
+      target_descriptions: ['play button', 'close ×']
+    })).toBe('play button → close ×');
+    expect(getActionTargetText({
+      name: 'click_sequence',
+      args: { sequence: [[500, 300], [876, 360], [100, 100]], target_descriptions: ['play button', 'close ×', 'back'] }
+    })).toBe('play button → close × → back');
+  });
+
+  it('skips blank entries and falls back when click_sequence descriptions are missing or unusable', () => {
+    expect(getActionTargetText({
+      action: 'click_sequence',
+      sequence: [[500, 300], [876, 360]],
+      target_descriptions: ['play button', '', null]
+    })).toBe('play button');
+    expect(getActionTargetText({ action: 'click_sequence', sequence: [[500, 300]], target_descriptions: [] })).toBe('');
+    expect(getActionTargetText({ action: 'click_sequence', sequence: [[500, 300]], target_descriptions: 'not-an-array' })).toBe('');
+    expect(getActionTargetText({
+      action: 'click_sequence',
+      sequence: [[500, 300]],
+      target_descriptions: [null],
+      target_description: 'single fallback'
+    })).toBe('single fallback');
+  });
+});
+
+describe('isActionFailed / getActionErrorMessage with Validator attempts', () => {
+  const stepWithAttempts = (attempts: any[]) => ({
+    step_id: 'step-attempts',
+    last_execution_result: {
+      status: 'dispatched',
+      execution: [{ attempts }]
+    }
+  });
+  const action = { action: 'click', coordinates: [500, 900] };
+
+  it('treats a retry that ended in Dispatched as not failed', () => {
+    const stepData = stepWithAttempts(['Execution error: x', 'Dispatched']);
+    expect(isActionFailed(action, stepData)).toBeFalse();
+  });
+
+  it('treats a terminal non-Dispatched attempt as failed and surfaces its text', () => {
+    const stepData = stepWithAttempts(['Execution error: y']);
+    expect(isActionFailed(action, stepData)).toBeTrue();
+    expect(getActionErrorMessage(action, stepData)).toBe('y');
+  });
+
+  it('keeps a generic error prefix verbatim and reports the last failing attempt', () => {
+    const stepData = stepWithAttempts(['Pre-execution validation failed: first', 'Error: y']);
+    expect(isActionFailed(action, stepData)).toBeTrue();
+    expect(getActionErrorMessage(action, stepData)).toBe('Error: y');
+  });
+
+  it('marks replay frames as dispatched rather than success', () => {
+    const frames = extractStepReplayFrames([
+      { step_id: 's-1', step_number: 1, pre_image_name: 'img1', action_taken: action, timestamp: 100 },
+      {
+        step_id: 's-2', step_number: 2, pre_image_name: 'img2', action_taken: action, timestamp: 200,
+        last_execution_result: { execution: [{ attempts: ['Execution error: y'] }] }
+      }
+    ]);
+    expect(frames[0].status).toBe('dispatched');
+    expect(frames[1].status).toBe('failed');
+  });
+});
+
+describe('extractActionExtraParams', () => {
+  it('hides target_description and target_descriptions from the extra parameter list', () => {
+    const single = extractActionExtraParams({
+      action: 'click',
+      coordinates: [500, 900],
+      target_description: 'play button',
+      times: 2
+    });
+    expect(single.map(p => p.key)).toEqual(['Times']);
+
+    const sequence = extractActionExtraParams({
+      name: 'click_sequence',
+      args: { sequence: [[1, 2], [3, 4]], target_descriptions: ['a', 'b'], interval_ms: 50 }
+    });
+    expect(sequence.map(p => p.key)).not.toContain('Target Descriptions');
+    expect(sequence.map(p => p.key)).toContain('Interval Ms');
+  });
+});

apps/showcase_ui/src/app/utils/action-formatter.util.ts

@@ -16,7 +16,7 @@
 
 import { ActionParam, StepReplayFrame } from '../core/models/stream.model';
 import { extractNumbersFromCoordinateValue, isPureDirectionString, parseSequenceCoordinates, unwrapTraceAction } from './image-overlay.util';
-import { cleanErrorMessage } from './tool-formatter.util';
+import { cleanErrorMessage, joinTargetDescriptions } from './tool-formatter.util';
 
 /**
  * Safely parse JSON string if it looks like an object/array, otherwise return original value
@@ -247,7 +247,16 @@ export function getActionTargetText(action: any): string {
     const ms = act.time_in_ms || act.delay_ms || act.delay_seconds || args.time_in_ms || args.delay_ms || args.delay_seconds || args.duration;
     if (ms) return `${ms}ms`;
   }
-  return act.target_text || act.target_class || act.element_id || args.target_text || args.element_id || '';
+  // Flash click_sequence carries one self-described target per point; render
+  // them in the same ' → ' chain that getActionCoords uses for the points.
+  if (name === 'click_sequence' || name === 'tap_sequence') {
+    const joined = joinTargetDescriptions(act.target_descriptions ?? args.target_descriptions);
+    if (joined) return joined;
+  }
+  // target_text is observed element text (index targets); target_description is
+  // what the model said it aimed at (coordinate targets). Same label either way.
+  return act.target_text || act.target_description || act.target_class || act.element_id
+    || args.target_text || args.target_description || args.element_id || '';
 }
 
 /**
@@ -360,6 +369,22 @@ export function getActionCoords(action: any): string {
   return '';
 }
 
+/**
+ * The Validator records "Dispatched" as the terminal attempt of an action the device
+ * accepted (kept only when a retry preceded it); any other terminal attempt is the
+ * failure text. Skipped burst members count as not dispatched.
+ */
+function attemptsFailed(attempts: any): boolean {
+  if (!Array.isArray(attempts) || attempts.length === 0) return false;
+  return String(attempts[attempts.length - 1]) !== 'Dispatched';
+}
+
+function lastFailedAttempt(attempts: any): string | null {
+  if (!Array.isArray(attempts)) return null;
+  const failing = attempts.filter((a: any) => String(a) !== 'Dispatched');
+  return failing.length > 0 ? String(failing[failing.length - 1]) : null;
+}
+
 /**
  * Check if the action execution failed or encountered an execution failure/interception
  */
@@ -384,7 +409,7 @@ export function isActionFailed(action: any, stepData?: any): boolean {
         }
         if (Array.isArray(res.execution) && res.execution.length > 0) {
           const firstExec = res.execution[0];
-          if (firstExec && (firstExec.status === 'failed' || firstExec.status === 'error' || firstExec.error || (Array.isArray(firstExec.attempts) && firstExec.attempts.length > 0))) {
+          if (firstExec && (firstExec.status === 'failed' || firstExec.status === 'error' || firstExec.error || attemptsFailed(firstExec.attempts))) {
             return true;
           }
         }
@@ -411,8 +436,9 @@ export function getActionErrorMessage(action: any, stepData?: any): string {
       if (Array.isArray(res.execution) && res.execution.length > 0) {
         const firstExec = res.execution[0];
         if (firstExec) {
-          if (Array.isArray(firstExec.attempts) && firstExec.attempts.length > 0) {
-            return cleanErrorMessage(firstExec.attempts[0]);
+          const failedAttempt = lastFailedAttempt(firstExec.attempts);
+          if (failedAttempt) {
+            return cleanErrorMessage(failedAttempt);
           }
           if (firstExec.error || firstExec.failure_reason || firstExec.message) {
             return cleanErrorMessage(firstExec.error || firstExec.failure_reason || firstExec.message);
@@ -469,7 +495,8 @@ export function extractActionExtraParams(action: any, cache?: WeakMap<any, Actio
   }
 
   const standardKeys = new Set([
-    'action', 'name', 'type', 'target_text', 'text', 'input_text', 'target',
+    'action', 'name', 'type', 'target_text', 'target_description', 'target_descriptions',
+    'text', 'input_text', 'target',
     'coordinates', 'coords', 'target_bounds', 'bounds', 'target_resource_id',
     'resource_id', 'target_class', 'class_name', 'normalized_coordinates',
     'normalized_start_coordinates', 'normalized_end_coordinates',
@@ -980,7 +1007,7 @@ export function extractStepReplayFrames(logsOrSteps: any[]): StepReplayFrame[] {
       isPost: false,
       timestamp: stepData.timestamp,
       summary: stepData.summary || '',
-      status: failed ? 'failed' : 'success'
+      status: failed ? 'failed' : 'dispatched'
     });
   }
 

apps/showcase_ui/src/app/utils/tool-formatter.util.spec.ts

@@ -1,10 +1,13 @@
 import {
   cleanErrorMessage,
+  extractToolExtraParams,
   getCompressionLabel,
   getToolDisplayLabel,
   getToolIcon,
+  getToolTargetText,
   getUniqueGenericTools,
   getVideoAnalysisView,
+  joinTargetDescriptions,
   shouldShowTool
 } from './tool-formatter.util';
 
@@ -255,3 +258,59 @@ describe('shouldShowTool (Option A Single Source of Truth)', () => {
     expect(shouldShowTool(plumbingTool)).toBeFalse();
   });
 });
+
+describe('getToolTargetText self-described targets', () => {
+  it('uses target_description for coordinate tools when no target_text is observed', () => {
+    expect(getToolTargetText({
+      name: 'click',
+      payload: { args: { coordinates: [500, 900], target_description: 'play button' } }
+    })).toBe('play button');
+    expect(getToolTargetText({
+      name: 'exec_long_press',
+      args: { coordinates: [500, 900], target_description: 'song row' }
+    })).toBe('song row');
+  });
+
+  it('prefers observed target_text over target_description', () => {
+    expect(getToolTargetText({
+      name: 'click',
+      payload: { args: { target_text: 'Play', target_description: 'play button' } }
+    })).toBe('Play');
+  });
+
+  it('chains click_sequence target_descriptions with the coordinate arrow', () => {
+    expect(getToolTargetText({
+      name: 'click_sequence',
+      payload: { args: { sequence: [[500, 300], [876, 360]], target_descriptions: ['play button', 'close ×'] } }
+    })).toBe('play button → close ×');
+  });
+
+  it('falls back gracefully when click_sequence descriptions are missing or short', () => {
+    expect(getToolTargetText({
+      name: 'click_sequence',
+      payload: { args: { sequence: [[500, 300], [876, 360]] } }
+    })).toBe('');
+    expect(getToolTargetText({
+      name: 'click_sequence',
+      payload: { args: { sequence: [[500, 300], [876, 360]], target_descriptions: ['play button'] } }
+    })).toBe('play button');
+    expect(getToolTargetText({
+      name: 'click_sequence',
+      payload: { args: { sequence: [[500, 300]], target_descriptions: [null, ''], target_description: 'fallback' } }
+    })).toBe('fallback');
+    expect(joinTargetDescriptions(undefined)).toBe('');
+    expect(joinTargetDescriptions('play button')).toBe('');
+  });
+});
+
+describe('extractToolExtraParams', () => {
+  it('hides target_description and target_descriptions from the extra parameter list', () => {
+    const keys = extractToolExtraParams({
+      name: 'click_sequence',
+      args: { sequence: [[1, 2]], target_descriptions: ['a'], target_description: 'b', interval_ms: 50 }
+    }).map(p => p.key);
+    expect(keys).not.toContain('Target Descriptions');
+    expect(keys).not.toContain('Target Description');
+    expect(keys).toContain('Interval Ms');
+  });
+});

apps/showcase_ui/src/app/utils/tool-formatter.util.ts

@@ -911,6 +911,19 @@ export function getToolTitle(tool: any): string {
   }
 }
 
+/**
+ * Join per-point self-described targets (click_sequence `target_descriptions`)
+ * into one label, chained with ' → ' like the coordinate rendering.
+ * Returns '' when the value is missing or holds no usable text.
+ */
+export function joinTargetDescriptions(descriptions: any): string {
+  if (!Array.isArray(descriptions)) return '';
+  const parts = descriptions
+    .map(d => (d === null || d === undefined) ? '' : String(d).trim())
+    .filter(d => d.length > 0);
+  return parts.join(' → ');
+}
+
 /**
  * Get target text description for tools
  */
@@ -950,6 +963,11 @@ export function getToolTargetText(tool: any): string {
   if (name === 'report_failure_analysis') {
     return args.status || args.reason || args.analysis || '';
   }
+  if (name === 'click_sequence' || name === 'tap_sequence') {
+    // One self-described target per point, chained like getToolCoords does.
+    const joined = joinTargetDescriptions(args.target_descriptions);
+    if (joined) return joined;
+  }
   if (args.target && typeof args.target === 'string') {
     return args.target;
   }
@@ -959,7 +977,7 @@ export function getToolTargetText(tool: any): string {
   if (args.index !== undefined) {
     return `Element #${args.index}`;
   }
-  return args.target_text || args.target_class || args.element || args.element_text || (name !== 'input_text' ? args.text : '') || '';
+  return args.target_text || args.target_description || args.target_class || args.element || args.element_text || (name !== 'input_text' ? args.text : '') || '';
 }
 
 /**
@@ -1162,7 +1180,8 @@ export function extractToolExtraParams(toolData: any, cache?: WeakMap<any, Actio
   }
 
   const standardKeys = new Set([
-    'action', 'name', 'type', 'target_text', 'text', 'input_text', 'target',
+    'action', 'name', 'type', 'target_text', 'target_description', 'target_descriptions',
+    'text', 'input_text', 'target',
     'coordinates', 'coords', 'target_bounds', 'bounds', 'target_resource_id',
     'resource_id', 'target_class', 'class_name', 'normalized_coordinates',
     'pre_image_name', 'post_image_name', 'pre_screenshot', 'post_screenshot',

artemis/agents/checker/checker.py

@@ -44,6 +44,7 @@ from langchain_core.tools import BaseTool, tool
 
 from pydantic import BaseModel, Field, ValidationError
 
+from artemis.core.tool_failure import ToolFailure, is_tool_failure
 from artemis.constants import CHECKER_MAX_ITERATIONS
 from artemis.context import ArtemisContext
 from artemis.data_engine.context_vars import CURRENT_TRACE_ID
@@ -57,6 +58,7 @@ from artemis.tools.tool_wrapper import (
     tool_result_messages,
 )
 from artemis.utils.logger import get_logger
+from artemis.utils.task_tree import format_actions_clean
 from artemis.utils.ocr_api import is_ocr_configured, perform_ocr
 from artemis.utils.ocr_xml_fusion import (
     _crop_image_remove_status_bar,
@@ -204,7 +206,11 @@ def build_probe_argv(kind: str, params: dict[str, Any] | None = None) -> list[st
 
 
 async def _execute_probe(ctx: ArtemisContext, argv: list[str]) -> str:
-    """Executes a validated probe argv over the ADB device channel."""
+    """Executes a validated probe argv over the ADB device channel.
+
+    A failed probe answers with a :class:`ToolFailure` so the tool loop marks
+    the result as an error structurally rather than by its wording.
+    """
     try:
         adb_client = ctx.get_adb_client()
         device = adb_client.device(serial=ctx.device.device_id)
@@ -214,7 +220,7 @@ async def _execute_probe(ctx: ArtemisContext, argv: list[str]) -> str:
             text = text[:_PROBE_OUTPUT_LIMIT] + "\n... [output truncated]"
         return text
     except Exception as e:
-        return f"Error executing probe: {e}"
+        return ToolFailure(f"Error executing probe: {e}")
 
 
 def get_probe_tool(ctx: ArtemisContext) -> BaseTool:
@@ -230,7 +236,7 @@ def get_probe_tool(ctx: ArtemisContext) -> BaseTool:
         try:
             argv = build_probe_argv(kind, params)
         except ValueError as e:
-            return f"Error: {e}"
+            return ToolFailure(f"Error: {e}")
         return await _execute_probe(ctx, argv)
 
     return probe_device
@@ -330,8 +336,11 @@ def _format_history(ctx: ArtemisContext, limit: int = 60) -> str:
         action = s.get("action_taken")
         action_str = ""
         if action and not summary:
+            # The shared renderer keeps the target's provenance visible: a
+            # self-described coordinate target reads "'play button' (self-described)",
+            # an observed index target reads "'Login'" with no marker.
             try:
-                action_str = json.dumps(action, ensure_ascii=False)[:160]
+                action_str = format_actions_clean(action)[:160]
             except Exception:
                 action_str = str(action)[:160]
         lines.append(f"- Step {num} ({rel}): {summary or action_str}".rstrip())
@@ -456,7 +465,7 @@ async def _run_check_loop(
                     tool=tool_to_run, args=args, tool_call_id=tc["id"]
                 )
                 text, _ = split_multimodal_result(result_obj)
-                status = "error" if text.startswith("Error") else "success"
+                status = "error" if is_tool_failure(result_obj) else "success"
             except Exception as e:
                 result_obj = f"Error running tool {tool_name}: {e}"
                 status = "error"

artemis/agents/diagnoser/diagnoser.py

@@ -22,6 +22,7 @@ import uuid
 
 from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage, ToolMessage
 
+from artemis.core.tool_failure import is_tool_failure
 from artemis.context import ArtemisContext
 from artemis.data_engine.trace import (
     CURRENT_TRACE_ID,
@@ -30,6 +31,7 @@ from artemis.data_engine.trace import (
     trace_langchain_tool,
 )
 from artemis.graph.state import State
+from artemis.llm.google import usage_from_message
 from artemis.services.llm import acomplete, get_llm, invoke_llm_with_timeout_message
 from artemis.tools.command_tool import get_run_short_adb_command_tool
 from artemis.tools.diagnoser_submit_answer_tool import get_submit_answer_tool
@@ -304,24 +306,8 @@ class Diagnoser:
         return base_llm.bind_tools(tools=traced_tools)
 
     def _token_usage_payload(self, response) -> dict | None:
-        """Extracts token-usage counters from the response metadata, if present."""
-        if not hasattr(response, "usage_metadata") or response.usage_metadata is None:
-            return None
-        usage = response.usage_metadata
-        if isinstance(usage, dict):
-            prompt_token_count = usage.get("prompt_token_count")
-            candidates_token_count = usage.get("candidates_token_count")
-            cached_content_token_count = usage.get("cached_content_token_count")
-        else:
-            prompt_token_count = getattr(usage, "prompt_token_count", None)
-            candidates_token_count = getattr(usage, "candidates_token_count", None)
-            cached_content_token_count = getattr(usage, "cached_content_token_count", None)
-
-        return {
-            "prompt_token_count": prompt_token_count,
-            "candidates_token_count": candidates_token_count,
-            "cached_content_token_count": (cached_content_token_count),
-        }
+        """Extracts normalized token-usage counters from the response, if present."""
+        return usage_from_message(response)
 
     async def _invoke_model(
         self,
@@ -495,7 +481,7 @@ class Diagnoser:
                         result = get_tool_result_content(result_obj)
                         result_str, _ = split_multimodal_result(result)
                         span.result = result
-                        if result_str.startswith("Error"):
+                        if is_tool_failure(result_obj):
                             status = "error"
                             span.status = "failed"
                             span.error = result

artemis/agents/explorer/explorer.py

@@ -49,6 +49,7 @@ from artemis.config import settings
 from artemis.context import ArtemisContext
 from artemis.data_engine.trace import trace
 from artemis.graph.state import State
+from artemis.llm.google import is_gemini_model, strip_provider_prefix
 from artemis.utils.logger import get_logger
 from artemis.utils.ocr_api import is_ocr_configured
 
@@ -118,9 +119,7 @@ class Explorer(PerceptionToolsMixin, UniversalRunnerMixin, RunSetupMixin, Native
         llm_cfg = getattr(llm_config, "explorer", None) if llm_config else None
         model = getattr(llm_cfg, "model", None) if llm_cfg else None
         model_name = model if isinstance(model, str) and model else DEFAULT_EXPLORER_MODEL
-        if "/" in model_name:
-            model_name = model_name.split("/")[-1]
-        return model_name
+        return strip_provider_prefix(model_name)
 
     def _detect_native_engine(self) -> bool:
         """True when the native google-genai SDK should drive the reasoning loop.
@@ -129,7 +128,7 @@ class Explorer(PerceptionToolsMixin, UniversalRunnerMixin, RunSetupMixin, Native
         client already shared on the context) the SDK cannot be constructed,
         so such configurations run through the universal LangChain engine.
         """
-        if "gemini" not in self.model_name.lower():
+        if not is_gemini_model(self.model_name):
             return False
         if getattr(self.ctx, "_genai_client", None) is not None:
             return True

artemis/agents/flash/context_compressor.py

@@ -313,7 +313,7 @@ class ScrubEdgeCompressor:
 
         # Defensive fallback: never leave ToolMessage or HumanMessage content empty
         if not new_blocks:
-            new_blocks = [{"type": "text", "text": "Action completed." if rec["is_tool"] else ""}]
+            new_blocks = [{"type": "text", "text": "Action dispatched." if rec["is_tool"] else ""}]
 
         msg.content = new_blocks
         self._frozen.add(rec["idx"])

artemis/agents/flash/flash_runner.md

@@ -23,7 +23,7 @@ Keep in mind that the mobile device is an asynchronous, constantly evolving envi
 ---
 
 # 3. READING YOUR HISTORY & THE SESSION CLOCK
-- **Conversation = execution history**: The earlier turns of this conversation are your raw execution history. Each observation opens with `# CURRENT OBSERVATION [T+mm:ss]`, and each turn that executed a device action ends with an `--- Action Execution Result (T+mm:ss) ---` message (`Status: success` means the action was dispatched and observed; `Status: failed` carries the `Error:` line). Every timestamp is a session-relative `T+mm:ss` offset from the start of this session; it is the only clock you have, for pacing waits as well as for reasoning about elapsed time.
+- **Conversation = execution history**: The earlier turns of this conversation are your raw execution history. Each observation opens with `# CURRENT OBSERVATION [T+mm:ss]`, and each turn that executed a device action ends with an `--- Action Execution Result (T+mm:ss) ---` message (`Status: dispatched` means the device accepted the action; whether it had the intended effect is yours to judge from the observation that follows. `Status: failed` carries the `Error:` line). Every timestamp is a session-relative `T+mm:ss` offset from the start of this session; it is the only clock you have, for pacing waits as well as for reasoning about elapsed time.
 - **Compression**: Screenshots older than a few turns are replaced by short `--- Historical Visual Transition ---` summaries and old `--- Visible UI Elements ---` lists are dropped. Long spans render as `[Era N | Steps a–b | T+.. → T+..]` blocks (a merged synopsis, segment titles, and a per-step action ledger). Because of this, if you spot critical text that you will need later (e.g., verification codes, specific titles, search lists), **you must explicitly verbalize and write those details into your text reasoning** so they persist across turns.
 {% if "search_history" in available_tools %}- **History Lookup (`search_history`)**: Compressed history keeps the step numbers and `T+` offsets but not every detail. When you need exactly what you did or saw at an earlier step (a value you read, the result of an action, which screen you were on), call `search_history` with keywords and/or the `[start, end]` step range shown in a chunk block; it searches the full stored history (screen descriptions, actions, results, reasoning, tool calls, notes, on-screen text) and every hit carries its step number. It is a pre-decision helper: read its answer, then act.{% endif %}
 {% if "replay_steps" in available_tools %}- **Step Replay (`replay_steps`)**: Replays one earlier step (or a short range) exactly as your context showed it — screen description, reasoning, tool calls with their results, the action and its execution result. Use the step numbers shown in the compressed history{% if "search_history" in available_tools %} or returned by `search_history`{% endif %}.{% endif %}
@@ -39,7 +39,7 @@ Keep in mind that the mobile device is an asynchronous, constantly evolving envi
 ---
 
 # 5. ACTION RULES
-1. **Normalized Coordinates (`0-1000` Scale)**: Always use normalized `[x, y]` coordinates in `0-1000` scale for any location-based target (e.g., `target=[500, 600]` or `click_sequence=[[300, 400], [500, 600]]`).
+1. **Normalized Coordinates (`0-1000` Scale)**: Always use normalized `[x, y]` coordinates in `0-1000` scale for any location-based target (e.g., `target=[500, 600]` or `click_sequence=[[300, 400], [500, 600]]`). Every coordinate target must carry `target_description` (one entry per point for `click_sequence`): a short, meaningful name for what you are aiming at, as a user would call it (e.g. 'play button', 'video body'), never a generic 'element' or 'button'. Nothing is inferred on your behalf: it is recorded as your own statement, and your history shows it marked as self-described.
 2. **Do not determine element coordinates based on guesswork.** If the element is visible in the screenshot but its coordinates are missing from the provided `--- Visible UI Elements ---` list, please use the `ask_explorer` tool to locate the element first.
 {% if "manage_app" in available_tools %}3. **App Launching**: To start or reset an application, always use `manage_app(action="launch", app_name="...")` or `action="stop"`. Do not manually swipe across home screens.{% endif %}
 

artemis/agents/flash/flash_summarizer.md

@@ -5,7 +5,7 @@ You are the Step Summarizer for an Android UI automation agent. Your task is to
 
 # PERSPECTIVE & FORMAT CONSTRAINTS
 1. **First-Person Perspective**: Write strictly from the agent's first-person perspective using **"I"** (e.g., "In Step {{ step_number }}, I tapped...", "I swiped up on... and observed..."). NEVER use third-person terms like "The agent", "The operator", or "The system".
-2. **Single Continuous Paragraph**: Your output MUST be exactly **one compact, continuous paragraph** (1–3 sentences, 35–65 words).
+2. **Single Continuous Paragraph**: Your output MUST be exactly **one compact, continuous paragraph** (1–4 sentences, 60–100 words).
 3. **No Lists or Formatting**: Do NOT use bullet points, numbered lists, markdown headers, bold labels, or line breaks in your output. Your output must NEVER contain `---` separators or section-marker lines — those belong to the input, not the summary.
 
 ---
@@ -21,6 +21,15 @@ You are the Step Summarizer for an Android UI automation agent. Your task is to
 
 ---
 
+# OPERATOR FOCUS (ATTENTION GUIDANCE)
+The input may open with an `--- [0] OPERATOR FOCUS ... ---` block: the operator's own words about this step — the task goal, the active sub-goal, a user instruction, and the operator's reasoning naming the target it chose and the screen change it expected. The action line shows the operator's target as `'...' (self-described)` (its own statement) or as `'...'` (an element observed in the UI tree). Use all of this to direct your attention, never to supply facts:
+1. **Focused details are mandatory**: Every visible detail that relates to the named target, the stated expectation, the sub-goal or the goal — exact labels, values, amounts, counts, toggle/selection/enabled/loading states, positions — must be transcribed verbatim. Never paraphrase or skip them. If the target control is visible, name its exact label.
+2. **Absence is a fact**: If something the operator expected or looked for is NOT visible, say so plainly (e.g., "no confirmation dialog is visible", "the 'Pay' button is not on this screen"). Never describe an expected element or change merely because the operator expected it.
+3. **The rest stays complete**: Focus adds emphasis; it never narrows the description. Keep describing the whole screen and the whole transition at full detail.
+4. **Not evidence**: The focus block is intention, not observation. A self-described target is what the operator believed it aimed at; describe what the red marker actually landed on.
+
+---
+
 # EXAMPLES OF HIGH-QUALITY SUMMARIES
 
 - **Click / Navigation**:
@@ -35,6 +44,9 @@ You are the Step Summarizer for an Android UI automation agent. Your task is to
 - **Modal / Alert State**:
   "In Step 4, I tapped the 'Confirm Booking' button; an alert dialog titled 'Payment Method Required' appeared over the view with an 'Add Card' option."
 
+- **With operator focus** (action line `Tapped 'Wi-Fi toggle' (self-described) at [880, 410]`, reasoning expected the toggle to switch on):
+  "In Step 6, I tapped the switch marked in red on the 'Wi-Fi' row, which read 'Off' before the tap; afterwards the switch showed 'On' with 'Searching for networks…' beneath it, the network list below stayed empty, and the remaining rows ('Mobile network', 'Hotspot & tethering') were unchanged."
+
 ---
 
 # TASK

artemis/agents/flash/flash_summarizer_single.md

@@ -12,7 +12,7 @@ You are the Step Summarizer for an Android UI automation agent. Exactly ONE scre
 
 # PERSPECTIVE & FORMAT CONSTRAINTS
 1. **First-Person Perspective**: Write strictly from the agent's first-person perspective using **"I"** (e.g., "In Step {{ step_number }}, I tapped... on a screen showing..."). NEVER use third-person terms like "The agent", "The operator", or "The system".
-2. **Single Continuous Paragraph**: Your output MUST be exactly **one compact, continuous paragraph** (1–3 sentences, 35–65 words).
+2. **Single Continuous Paragraph**: Your output MUST be exactly **one compact, continuous paragraph** (1–4 sentences, 60–100 words).
 3. **No Lists or Formatting**: Do NOT use bullet points, numbered lists, markdown headers, bold labels, or line breaks. Your output must NEVER contain `---` separators or section-marker lines — those belong to the input, not the summary.
 
 ---
@@ -25,6 +25,15 @@ You are the Step Summarizer for an Android UI automation agent. Exactly ONE scre
 
 ---
 
+# OPERATOR FOCUS (ATTENTION GUIDANCE)
+The input may open with an `--- [0] OPERATOR FOCUS ... ---` block: the operator's own words about this step — the task goal, the active sub-goal, a user instruction, and the operator's reasoning naming the target it chose and the screen change it expected. The action line shows the operator's target as `'...' (self-described)` (its own statement) or as `'...'` (an element observed in the UI tree). Use all of this to direct your attention, never to supply facts:
+1. **Focused details are mandatory**: Every visible detail that relates to the named target, the stated expectation (as it stands on THIS frame only), the sub-goal or the goal — exact labels, values, amounts, counts, toggle/selection/enabled/loading states, positions — must be transcribed verbatim. Never paraphrase or skip them. If the target control is visible, name its exact label.
+2. **Absence is a fact**: If something the operator expected or looked for is NOT visible, say so plainly (e.g., "no confirmation dialog is visible", "the 'Pay' button is not on this screen"). Never describe an expected element or change merely because the operator expected it.
+3. **The rest stays complete**: Focus adds emphasis; it never narrows the description. Keep describing the whole screen and the whole transition at full detail.
+4. **Not evidence**: The focus block is intention, not observation. A self-described target is what the operator believed it aimed at; describe what the red marker actually landed on.
+
+---
+
 # EXAMPLES OF HIGH-QUALITY SINGLE-FRAME SUMMARIES
 
 - **Click on a list row**:
@@ -36,6 +45,9 @@ You are the Step Summarizer for an Android UI automation agent. Exactly ONE scre
 - **Swipe / Scroll**:
   "In Step 3, I swiped up starting from the red-marked point on the flight results list, which at that moment displayed 4 morning flights priced from '$310' under the header 'Best departing flights'."
 
+- **With operator focus** (action line `Tapped 'Wi-Fi toggle' (self-described) at [880, 410]`, sub-goal "Enable Wi-Fi"):
+  "In Step 6, I tapped the switch marked in red at the right end of the 'Wi-Fi' row, which read 'Off' at that moment; the Network & internet screen also listed 'Mobile network' with 'T-Mobile' beneath it and 'Hotspot & tethering' with 'Off', and no network list or dialog was visible."
+
 ---
 
 # TASK

artemis/agents/flash/runner.py

@@ -44,7 +44,7 @@ from langchain_core.messages import (
     ToolMessage,
 )
 
-from artemis.agents.flash.summarizer import VisualStepSummarizer
+from artemis.agents.flash.summarizer import VisualStepSummarizer, build_focus_context
 from artemis.agents.validator.tool_declarations import (
     ASK_EXPLORER_TOOL,
     CLICK_SEQUENCE_TOOL,
@@ -123,8 +123,10 @@ class _TurnRecord:
             return None
         for name, status, text in self.actions:
             if status != "success":
-                return {"status": "failed", "error": f"{name}: {text}"}
-        return {"status": "success"}
+                # Executor messages name the action themselves ("Error
+                # executing click: ..."); prefix only the ones that don't.
+                return {"status": "failed", "error": text if name in text else f"{name}: {text}"}
+        return {"status": "dispatched"}
 
 
 class FlashRunner:
@@ -284,10 +286,17 @@ class FlashRunner:
     # Per-turn helpers (observe / think)
     # ------------------------------------------------------------------
 
-    async def _read_injected_instruction(self) -> str | None:
-        """Returns the real-time injected instruction text for this turn, if any."""
+    async def _read_injected_instruction(self) -> tuple[str | None, str | None]:
+        """Consumes this turn's injected instruction: ``(instruction, notice)``.
+
+        ``instruction`` is the user's text verbatim — what the step record,
+        the chunk ledger and the visual-transition lens receive (the same
+        shape Pro stamps). ``notice`` wraps it with the operator-facing
+        directive and rides only on the observation tail. Both are None
+        when nothing was injected.
+        """
         if not (self.ctx.data_engine and self.ctx.data_engine.base_dir):
-            return None
+            return None, None
         try:
             injected_payload = await asyncio.to_thread(
                 _check_injected_instruction_file,
@@ -295,21 +304,22 @@ class FlashRunner:
             )
         except (OSError, ValueError, AttributeError) as e:
             logger.warning(f"Failed to check injected instruction in FlashRunner: {e}")
-            return None
+            return None, None
         if not (injected_payload and injected_payload.get("instruction")):
-            return None
-        injected_text = (
+            return None, None
+        instruction = str(injected_payload["instruction"])
+        notice = (
             "[REAL-TIME INJECTED INSTRUCTION from user]:"
-            f" {injected_payload['instruction']}\nYou MUST immediately"
+            f" {instruction}\nYou MUST immediately"
             " follow this instruction and adjust your plan/actions."
         )
         if injected_payload.get("release_loop"):
-            injected_text += (
+            notice += (
                 "\nThe user has explicitly authorized stopping any"
                 " ongoing monitoring loop; you may now wrap up and"
                 " complete the task."
             )
-        return injected_text
+        return instruction, notice
 
     def _build_tail(
         self,
@@ -597,16 +607,13 @@ class FlashRunner:
                 action_dict["normalized_start_coordinates"] = norm_start
                 action_dict["normalized_end_coordinates"] = norm_end
 
-            # Record-time enrichment computed by the executor from
-            # the pre-action frame (target_text / target_class /
-            # target_resource_id / target_label_source).
-            target_semantics = (exec_result.metadata or {}).get("target_semantics")
-            if isinstance(target_semantics, dict):
-                action_dict.update(target_semantics)
+            # The model's own statement of what it aimed at (target_description),
+            # validated and shaped by the executor into the fields Pro records.
+            action_dict.update((exec_result.metadata or {}).get("target_semantics") or {})
 
             succeeded = exec_result.status == "success"
             last_execution_result = {
-                "status": "success" if succeeded else "failed",
+                "status": "dispatched" if succeeded else "failed",
                 "result": exec_result.text_summary,
             }
             if not succeeded:
@@ -704,6 +711,13 @@ class FlashRunner:
                     exec_outcome=exec_result.text_summary,
                     action_key=str(tc_id),
                     data_engine_step_id=recorded_step_id,
+                    # The model's own reasoning for this turn steers the lens's
+                    # attention; the target it named rides on ``args``.
+                    focus=build_focus_context(
+                        intent=raw_text,
+                        goal=self.goal,
+                        injected_instruction=injected,
+                    ),
                 )
 
             if post_img_bytes:
@@ -719,7 +733,7 @@ class FlashRunner:
             content = (
                 raw_blocks
                 if isinstance(raw_blocks, list) and raw_blocks
-                else exec_result.text_summary or f"Action '{name}' completed."
+                else exec_result.text_summary or f"Action '{name}' dispatched."
             )
             messages.extend(
                 tool_result_messages(
@@ -868,8 +882,9 @@ class FlashRunner:
             self._commit_turn(ledger, previous_turn)
             previous_turn = None
 
-            # Check for real-time injected instructions
-            injected = await self._read_injected_instruction()
+            # Check for real-time injected instructions: the verbatim text
+            # is stamped on the step, the wrapped notice goes to the tail.
+            injected, injected_notice = await self._read_injected_instruction()
 
             # Tool restriction on the final turn (bounded loops only)
             is_final = limit is not None and turns == limit
@@ -878,7 +893,7 @@ class FlashRunner:
                 turns,
                 current_pre_screenshot_bytes,
                 current_xml_list,
-                injected=injected,
+                injected=injected_notice,
                 notices=pending_notices,
                 is_final=is_final,
             )

artemis/agents/flash/summarizer.py

@@ -39,6 +39,7 @@ from artemis.memory.step_memory import JobKey, StepMemoryService
 from artemis.services.llm import RobustChatModelWrapper, get_google_llm, get_llm
 from artemis.services.token_meter import record_llm_usage
 from artemis.utils.logger import get_logger
+from artemis.utils.task_tree import format_actions_clean
 from artemis.utils.visualization import draw_action_overlay_on_image
 
 logger = get_logger(__name__)
@@ -62,6 +63,79 @@ def degenerate_summary_reason(text: str) -> str | None:
     return None
 
 
+#: Header of the operator-focus block that leads the lens input. It shares
+#: the ``--- [`` shape of the frame headers so an echoed header trips the
+#: same degenerate-output guard.
+FOCUS_BLOCK_HEADER = "--- [0] OPERATOR FOCUS (context for attention, not evidence) ---"
+
+#: Cap on the operator reasoning excerpt carried in the focus block. The
+#: expectation statement closes the reasoning, so the tail is kept intact
+#: and the cut lands in the middle.
+FOCUS_INTENT_MAX_CHARS = 900
+FOCUS_INTENT_TAIL_CHARS = 600
+
+
+def _cap_intent(text: str) -> str:
+    if len(text) <= FOCUS_INTENT_MAX_CHARS:
+        return text
+    head_len = FOCUS_INTENT_MAX_CHARS - FOCUS_INTENT_TAIL_CHARS
+    head, tail = text[:head_len], text[-FOCUS_INTENT_TAIL_CHARS:]
+    cut = len(text) - len(head) - len(tail)
+    return f"{head}\n[… {cut} characters cut here …]\n{tail}"
+
+
+def build_focus_context(
+    *,
+    intent: str | None = None,
+    goal: str | None = None,
+    subgoal: str | None = None,
+    injected_instruction: str | None = None,
+) -> dict[str, str] | None:
+    """Collect operator context to guide the visual summary's focus.
+
+    Target descriptions remain on the action. Return None if all fields are empty.
+    """
+    focus: dict[str, str] = {}
+    if isinstance(goal, str) and goal.strip():
+        focus["goal"] = goal.strip()
+    if isinstance(subgoal, str) and subgoal.strip():
+        focus["subgoal"] = subgoal.strip()
+    if isinstance(injected_instruction, str) and injected_instruction.strip():
+        focus["injected_instruction"] = injected_instruction.strip()
+    if isinstance(intent, str) and intent.strip():
+        focus["intent"] = _cap_intent(intent.strip())
+    return focus or None
+
+
+def render_focus_block(focus: dict[str, str], *, dual: bool = True) -> str:
+    """Plain-text focus block (goal, sub-goal, instruction, then reasoning).
+
+    ``dual=False`` (a single decision frame, every Pro step) labels the
+    reasoning so its expectation is read as the operator's aim only: the
+    outcome is not in the frame and must not be described.
+    """
+    lines = [FOCUS_BLOCK_HEADER]
+    if focus.get("goal"):
+        lines.append(f"Task goal: {focus['goal']}")
+    if focus.get("subgoal"):
+        lines.append(f"Active sub-goal: {focus['subgoal']}")
+    if focus.get("injected_instruction"):
+        lines.append(f"User instruction at this step: {focus['injected_instruction']}")
+    if focus.get("intent"):
+        if dual:
+            lines.append(
+                "Operator reasoning for this step (why this target, what screen"
+                " change it expected):"
+            )
+        else:
+            lines.append(
+                "Operator reasoning for this step (why this target; the expected"
+                " outcome is NOT in this frame and must not be described):"
+            )
+        lines.append(focus["intent"])
+    return "\n".join(lines)
+
+
 class VisualStepSummarizer(StepMemoryService):
     """Visual-transition lens on top of the shared step-memory runtime.
 
@@ -148,6 +222,7 @@ class VisualStepSummarizer(StepMemoryService):
         *,
         action_key: str | None = None,
         data_engine_step_id: UUID | str | None = None,
+        focus: dict[str, str] | None = None,
     ) -> None:
         """Dispatches an asynchronous summarization task without blocking the caller.
 
@@ -155,6 +230,10 @@ class VisualStepSummarizer(StepMemoryService):
         tool_call_id (``action_key``) is retained as an alias so the message
         compressor can keep querying by it. Callers with neither provide the
         step ordinal, matching the legacy keying.
+
+        ``focus`` (:func:`build_focus_context`) is the operator's own context
+        for the step; the lens renders it ahead of the frames so the details
+        the operator was after are transcribed rather than summarized away.
         """
         key: JobKey
         aliases: tuple[JobKey, ...] = ()
@@ -173,9 +252,32 @@ class VisualStepSummarizer(StepMemoryService):
             "post_img_bytes": post_img_bytes,
             "exec_outcome": exec_outcome,
             "data_engine_step_id": data_engine_step_id,
+            "focus": focus,
         }
         self.submit(key, payload, aliases=aliases)
 
+    @staticmethod
+    def _action_phrase(action_name: str, action_args: dict[str, Any] | None) -> str:
+        """The action as every other history reader sees it (``format_actions_clean``).
+
+        A described coordinate target renders as ``'play button' (self-described)``,
+        an observed element as ``'Play'``; a Pro burst lists every member.
+        """
+        args = dict(action_args or {})
+        extra = args.pop("additional_actions", None)
+        # Flash's coordinate swipe names its points ``start``/``end``; the
+        # history renderer reads ``start_coordinates``/``end_coordinates``.
+        if action_name == "swipe" and "start" in args and "end" in args:
+            args.setdefault("start_coordinates", args["start"])
+            args.setdefault("end_coordinates", args["end"])
+        try:
+            first = {"action": action_name, **args}
+            actions = [first, *extra] if isinstance(extra, list) and extra else first
+            return format_actions_clean(actions)
+        except Exception as exc:
+            logger.debug(f"Action phrase rendering fell back to raw args: {exc}", exc_info=True)
+            return f"{action_name}({action_args})"
+
     def _meter_lens_call(self, response: Any) -> None:
         """Meter one raw-model lens call as an ``llm_usage`` trace, best-effort.
 
@@ -236,15 +338,15 @@ class VisualStepSummarizer(StepMemoryService):
             dual = bool(pre_bytes) and bool(post_bytes)
             template = self._prompt_template if dual else self._single_prompt_template
             rendered_prompt = Template(template).render(step_number=step_number)
-            content_blocks: list[dict[str, Any]] = [
-                {
-                    "type": "text",
-                    "text": (
-                        f"Step {step_number} Physical Action: {action_name}({action_args})\n"
-                        f"Controller Outcome: {exec_outcome}"
-                    ),
-                }
+            lead_lines = [
+                f"Step {step_number} Physical Action: {self._action_phrase(action_name, action_args)}",
+                f"Action arguments (verbatim): {action_name}({action_args})",
+                f"Controller Outcome: {exec_outcome}",
             ]
+            focus = input_data.get("focus")
+            if isinstance(focus, dict) and focus:
+                lead_lines.extend(["", render_focus_block(focus, dual=dual)])
+            content_blocks: list[dict[str, Any]] = [{"type": "text", "text": "\n".join(lead_lines)}]
 
             if pre_bytes:
                 # 🎨 Visually mark the exact action (tap ripple, sequence numbers, swipe arrow) on the BEFORE screenshot
@@ -318,10 +420,12 @@ class VisualStepSummarizer(StepMemoryService):
                     f"VisualStepSummarizer: Generated summary for Step {step_number}: {summary_text[:80]}..."
                 )
 
-                # Free binary image buffers from memory once summary is secured
+                # Free binary image buffers (and the focus text) once the
+                # summary is secured; a landed job is never re-rendered.
                 if key in self._step_inputs:
                     self._step_inputs[key]["pre_img_bytes"] = None
                     self._step_inputs[key]["post_img_bytes"] = None
+                    self._step_inputs[key]["focus"] = None
 
                 # Update DataEngine telemetry if active
                 if self.ctx.data_engine:

artemis/agents/history_analyzer/history_analyzer.py

@@ -25,6 +25,7 @@ from pathlib import Path
 from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage
 from langchain_core.tools import BaseTool
 
+from artemis.core.tool_failure import is_tool_failure
 from artemis.context import ArtemisContext
 from artemis.data_engine.trace import TraceSpan, trace
 from artemis.memory.context_policy import build_history_for
@@ -161,7 +162,7 @@ class HistoryAnalyzer:
                                 tool=tool, args=args, tool_call_id=tc["id"]
                             )
                             text, _ = split_multimodal_result(result)
-                            status = "error" if text.startswith("Error") else "success"
+                            status = "error" if is_tool_failure(result) else "success"
                             span.result = text
                             if status == "error":
                                 span.status = "failed"

artemis/agents/log_analyzer/log_analyzer.py

@@ -51,6 +51,7 @@ class SpawnLogReaderArgs(BaseModel):
 
 from langgraph.prebuilt import InjectedState
 
+from artemis.core.tool_failure import ToolFailure, is_tool_failure
 from artemis.context import ArtemisContext
 from artemis.data_engine.trace import (
     CURRENT_TRACE_ID,
@@ -102,7 +103,7 @@ class LogAnalyzerNode:
                     return self._raw_log_cache[cache_key]
 
                 result = await original_coroutine(**kwargs)
-                if isinstance(result, str) and not result.startswith("Failed to read logs"):
+                if isinstance(result, str) and not is_tool_failure(result):
                     self._raw_log_cache[cache_key] = result
                 return result
 
@@ -130,7 +131,7 @@ class LogAnalyzerNode:
                         )
                     except Exception as e:
                         logger.error(f"Failed to search logs locally: {e}")
-                        return f"Error searching logs: {e}"
+                        return ToolFailure(f"Error searching logs: {e}")
 
                 # If not cached, fetch them once via read_logs (or the original tool, which will fetch them)
                 # Note: search_logs itself fetches and filters. We can let the original_coroutine run,
@@ -161,7 +162,7 @@ class LogAnalyzerNode:
                     )
                 except Exception as e:
                     logger.error(f"Failed to fetch and search logs: {e}")
-                    return f"Error searching logs: {e}"
+                    return ToolFailure(f"Error searching logs: {e}")
 
             # Fallback to standard tool caching for other tools
             serialized_args = json.dumps(kwargs, sort_keys=True)
@@ -276,7 +277,7 @@ class LogAnalyzerNode:
                                 elif not isinstance(result, str):
                                     result = str(result)
                                 span.result = result
-                                if result.startswith("Error"):
+                                if is_tool_failure(result_obj):
                                     status = "error"
                                     span.status = "failed"
                                     span.error = result

artemis/agents/log_analyzer/output_analyzer.py

@@ -16,6 +16,7 @@ from pathlib import Path
 from typing import Annotated
 from langchain_core.messages import HumanMessage, SystemMessage, ToolMessage
 from langchain_core.tools import tool
+from artemis.core.tool_failure import ToolFailure, is_tool_failure
 from artemis.context import ArtemisContext
 from artemis.services.llm import get_llm, invoke_llm_with_timeout_message
 from artemis.tools.mobile.search_logs import search_and_merge_logs
@@ -147,15 +148,15 @@ class TaskOutputAnalyzerNode:
                     try:
                         result = tool_to_run.invoke(args)
                     except Exception as e:
-                        result = f"Failed to run tool {tool_name}: {e}"
+                        result = ToolFailure(f"Failed to run tool {tool_name}: {e}")
                 else:
-                    result = f"Error: Tool {tool_name} not found."
+                    result = ToolFailure(f"Error: Tool {tool_name} not found.")
 
                 current_messages.append(
                     ToolMessage(
                         tool_call_id=tc["id"],
                         content=str(result),
-                        status="success" if not str(result).startswith("Error") else "error",
+                        status="error" if is_tool_failure(result) else "success",
                     )
                 )
 

artemis/agents/operator/operator.json

@@ -1,3 +1,3 @@
 {
-  "main_template": "# ROLE\nYou are the Android Device Operator. Your priority is execution accuracy. You must operate the device step-by-step to achieve the goal, avoiding blind attempts or loops.\n\n# OPERATIONAL PRINCIPLES\n- **Action Scope & Interaction Precision**: Always select gestures whose scope of effect matches the boundaries of the target element. Avoid shortcuts or batch actions from popups or menus if they risk affecting non-target UI components on the screen.\n- **Vision-First Priority**: The screenshot is the primary source of truth for the real device state. Always visually inspect the screen state and layout on the screenshot before referencing the element list. Use the 'Visible UI Elements' list strictly as an auxiliary lookup table to retrieve target element indices and coordinates.\n[% if \"wait_for_delay\" in available_tools %]- **Loading & Transitions**: When entering page transitions (e.g., app launching, web page loading, or progress indicators), use `wait_for_delay` to let the screen settle before executing target actions. Some transition states are difficult to discern, such as a thin progress line at the top of the browser screen during webpage loads. Before questioning whether an error has occurred, you can first adopt a waiting strategy to see how the device state evolves. If the state remains ambiguous after waiting, certain UI components might be malfunctioning or failing to load; in this case, consider retrying the action or pivoting to an alternative path.\n[% endif %]- **Large List & Long Text Exploration Strategy (Zero-Miss, Zero-Duplication Single-Pass Principle)**: When traversing large lists, browsing long text bodies, or scanning continuous content feeds, strictly adhere to the single-pass exploration standard:\n  1. **Unidirectional Anchor-Based Traversal (Zero-Miss)**: Maintain a consistent, unidirectional scan (e.g., uniform downward scrolling). Before each scroll, identify the bottom-most visible item/text segment as your visual anchor and carry that anchor plus the scan direction in short-term memory or the traversal ledger. After the scroll, evaluate this handoff before doing anything unrelated: if the anchor remains visible with new content beyond it, record the new content and continue in the same direction without scrolling back to verify; if the anchor is missing, continuity is uncertain, so perform at most one minimal reverse recovery to re-establish overlap, then resume the original direction. While the traversal's declared exit condition remains unresolved, keep this traversal active and do not advance to another milestone or mark it complete.\n  2. **Boundary-Proven Completion**: A single scroll that reveals no new content is not sufficient evidence that the list is complete. For an exhaustive traversal, stop only when an explicit end-of-list indicator is visible, or when one additional successful swipe in the same direction and within the same list container leaves the same final anchor visible with no new items and no loading state. This boundary probe must remain unidirectional; never reverse direction merely to prove completion. If the task declares an earlier exit condition (for example, find one verified match or process N items), that explicit condition may end the traversal without reaching the list boundary.\n  3. **Memory & Plan-Driven Deduplication (Zero-Duplication)**: Systematically log extracted item names, identifiers, or processing states into notes or memory as you progress. After every screen transition, cross-reference visible items against your recorded ledger and strictly interact only with newly surfaced, unrecorded items to eliminate redundant actions on overlapping elements. Proactively maintain subtasks in `task_plan` (e.g., segmenting by batch, category, or alphabetical ranges like `a-d`), tracking exact paths and progress to prevent backtracking.\n  4. **Single-Pass Trust & Anti-Oscillation (No Over-Verification)**: Treat recorded notes as verified ground truth. Once a segment is traversed with confirmed anchor continuity and logged, consider it permanently resolved. Do NOT scroll back-and-forth or perform redundant re-scans for \"double-checking\", even if older execution history has been pruned. Resume from the last recorded anchor and direction. Only the single bounded recovery above, or a refreshed/mutated list, permits limited re-verification; otherwise continue the single pass over new content.\n- **Dynamic Monitoring & Reactive Loop Protocol (Monitor-Trigger-Action)**: When executing an active milestone matching `[Loop] <Goal> into note <TargetNote> (Exit: <ExitCondition>; Interval: <Interval>)` or `[Loop:continuous] <Goal> into note <TargetNote> (Interval: <Interval>)` (the milestone grammar itself is defined under Task Plan Grammar below), run this 5-step cyclic state machine:\n  - **Step 1 (Baseline Sync)**: On the initial cycle, observe all visible entries/metrics in the primary monitoring view and save the full snaLine truncated
+  "main_template": "# ROLE\nYou are the Android Device Operator. Your priority is execution accuracy. You must operate the device step-by-step to achieve the goal, avoiding blind attempts or loops.\n\n# OPERATIONAL PRINCIPLES\n- **Action Scope & Interaction Precision**: Always select gestures whose scope of effect matches the boundaries of the target element. Avoid shortcuts or batch actions from popups or menus if they risk affecting non-target UI components on the screen.\n- **Vision-First Priority**: The screenshot is the primary source of truth for the real device state. Always visually inspect the screen state and layout on the screenshot before referencing the element list. Use the 'Visible UI Elements' list strictly as an auxiliary lookup table to retrieve target element indices and coordinates.\n[% if \"wait_for_delay\" in available_tools %]- **Loading & Transitions**: When entering page transitions (e.g., app launching, web page loading, or progress indicators), use `wait_for_delay` to let the screen settle before executing target actions. Some transition states are difficult to discern, such as a thin progress line at the top of the browser screen during webpage loads. Before questioning whether an error has occurred, you can first adopt a waiting strategy to see how the device state evolves. If the state remains ambiguous after waiting, certain UI components might be malfunctioning or failing to load; in this case, consider retrying the action or pivoting to an alternative path.\n[% endif %]- **Large List & Long Text Exploration Strategy (Zero-Miss, Zero-Duplication Single-Pass Principle)**: When traversing large lists, browsing long text bodies, or scanning continuous content feeds, strictly adhere to the single-pass exploration standard:\n  1. **Unidirectional Anchor-Based Traversal (Zero-Miss)**: Maintain a consistent, unidirectional scan (e.g., uniform downward scrolling). Before each scroll, identify the bottom-most visible item/text segment as your visual anchor and carry that anchor plus the scan direction in short-term memory or the traversal ledger. After the scroll, evaluate this handoff before doing anything unrelated: if the anchor remains visible with new content beyond it, record the new content and continue in the same direction without scrolling back to verify; if the anchor is missing, continuity is uncertain, so perform at most one minimal reverse recovery to re-establish overlap, then resume the original direction. While the traversal's declared exit condition remains unresolved, keep this traversal active and do not advance to another milestone or mark it complete.\n  2. **Boundary-Proven Completion**: A single scroll that reveals no new content is not sufficient evidence that the list is complete. For an exhaustive traversal, stop only when an explicit end-of-list indicator is visible, or when one additional successful swipe in the same direction and within the same list container leaves the same final anchor visible with no new items and no loading state. This boundary probe must remain unidirectional; never reverse direction merely to prove completion. If the task declares an earlier exit condition (for example, find one verified match or process N items), that explicit condition may end the traversal without reaching the list boundary.\n  3. **Memory & Plan-Driven Deduplication (Zero-Duplication)**: Systematically log extracted item names, identifiers, or processing states into notes or memory as you progress. After every screen transition, cross-reference visible items against your recorded ledger and strictly interact only with newly surfaced, unrecorded items to eliminate redundant actions on overlapping elements. Proactively maintain subtasks in `task_plan` (e.g., segmenting by batch, category, or alphabetical ranges like `a-d`), tracking exact paths and progress to prevent backtracking.\n  4. **Single-Pass Trust & Anti-Oscillation (No Over-Verification)**: Treat recorded notes as verified ground truth. Once a segment is traversed with confirmed anchor continuity and logged, consider it permanently resolved. Do NOT scroll back-and-forth or perform redundant re-scans for \"double-checking\", even if older execution history has been pruned. Resume from the last recorded anchor and direction. Only the single bounded recovery above, or a refreshed/mutated list, permits limited re-verification; otherwise continue the single pass over new content.\n- **Dynamic Monitoring & Reactive Loop Protocol (Monitor-Trigger-Action)**: When executing an active milestone matching `[Loop] <Goal> into note <TargetNote> (Exit: <ExitCondition>; Interval: <Interval>)` or `[Loop:continuous] <Goal> into note <TargetNote> (Interval: <Interval>)` (the milestone grammar itself is defined under Task Plan Grammar below), run this 5-step cyclic state machine:\n  - **Step 1 (Baseline Sync)**: On the initial cycle, observe all visible entries/metrics in the primary monitoring view and save the full snaLine truncated
 }

artemis/agents/operator/operator.py

@@ -19,6 +19,7 @@ from uuid import uuid4
 from langchain_core.messages import SystemMessage, ToolMessage
 from langchain_core.tools import BaseTool
 
+from artemis.core.tool_failure import is_tool_failure
 from artemis.context import ArtemisContext
 from artemis.data_engine.trace import (
     TraceSpan,
@@ -28,6 +29,7 @@ from artemis.data_engine.trace import (
 from artemis.graph.state import State
 from artemis.graph.visibility import strict_state
 from artemis.mcp.action_specs import OPERATOR_SHELL_ORDER, operator_shell_tool
+from artemis.llm.google import usage_from_message
 from artemis.services.llm import acomplete, get_llm, invoke_llm_with_timeout_message
 from artemis.tools.command_tool import (
     analyze_task_output_wrapper,
@@ -46,7 +48,6 @@ from artemis.utils.coordinates import (
     parse_swipe_parameters,
 )
 from artemis.utils.decorators import wrap_with_callbacks
-from artemis.utils.element_hit_test import find_element_at_point
 from artemis.utils.logger import get_logger
 from artemis.utils.notes import get_note_file_path
 from artemis.utils.plan_grammar import parse_plan
@@ -92,6 +93,13 @@ from artemis.agents.operator.prompts import (
 from artemis.agents.operator.prompts import load_operator_prompts
 
 
+def _described(value: Any) -> str | None:
+    """Trim target_description, returning None for missing or blank values."""
+    if isinstance(value, str) and value.strip():
+        return value.strip()
+    return None
+
+
 class OperatorNode:
     def __init__(
         self,
@@ -462,24 +470,12 @@ class OperatorNode:
             bound_llm = base_llm.bind_tools(tools=traced_tools)
             response = await invoke_llm_with_timeout_message(acomplete(bound_llm, current_messages))
 
-            if hasattr(response, "response_metadata") and response.response_metadata:
-                usage = (
-                    response.response_metadata.get("usage_metadata")
-                    or response.response_metadata.get("token_usage")
-                    or {}
-                )
-                prompt_tokens = usage.get("prompt_token_count") or usage.get("prompt_tokens")
-                cached_tokens = usage.get("cached_content_token_count") or usage.get(
-                    "cached_tokens", 0
-                )
-                completion_tokens = usage.get("candidates_token_count") or usage.get(
-                    "completion_tokens"
-                )
-                if prompt_tokens is not None:
+            usage = usage_from_message(response)
+            if usage is not None:
                 logger.info(
-                        f"LLM usage: prompt_tokens={prompt_tokens}"
-                        f" (cached={cached_tokens}),"
-                        f" completion_tokens={completion_tokens}"
+                    f"LLM usage: prompt_tokens={usage['prompt_tokens']}"
+                    f" (cached={usage['cached_tokens']}),"
+                    f" completion_tokens={usage['completion_tokens']}"
                 )
 
             if response.content:
@@ -589,30 +585,10 @@ class OperatorNode:
                                 )
 
                                 content = get_tool_result_content(result_obj)
-                                status = "success"
-
-                                if isinstance(result_obj, ToolMessage) and result_obj.status:
-                                    status = result_obj.status
-
-                                is_err = status == "error"
-                                if not is_err:
-                                    if isinstance(content, str):
-                                        if content.startswith(
-                                            "Failed"
-                                        ) or content.lower().startswith("error"):
-                                            is_err = True
-                                    elif isinstance(content, list):
-                                        for block in content:
-                                            if (
-                                                isinstance(block, dict)
-                                                and block.get("type") == "text"
-                                            ):
-                                                text = block.get("text", "")
-                                                if text.startswith(
-                                                    "Failed"
-                                                ) or text.lower().startswith("error"):
-                                                    is_err = True
-                                                    break
+                                # Failure is structural (a ToolMessage status or a
+                                # ToolFailure text), never sniffed from the words.
+                                is_err = is_tool_failure(result_obj)
+                                status = "error" if is_err else "success"
 
                                 if is_err:
                                     other_tool_failed = True
@@ -812,44 +788,6 @@ class OperatorNode:
             tool_limit_exceeded,
         )
 
-    async def _run_other_tool(
-        self,
-        tc,
-        traced_tools,
-        new_subagent_calls,
-        state: State,
-        raw_thoughts: list,
-        native_thoughts: list,
-    ) -> str:
-        tool_name = tc["name"]
-        logger.info(f"Operator requested tool: {tool_name}")
-        if tool_name in ["ask_diagnoser", "video_analyzer"]:
-            new_subagent_calls.append(tool_name)
-        if ":" in tool_name:
-            tool_to_run = get_tool_by_name(tool_name, traced_tools)
-        else:
-            tool_to_run = next((t for t in traced_tools if t.name == tool_name), None)
-        if tool_to_run:
-            try:
-                args = dict(tc["args"])
-
-                state.operator_raw_thinking = "\n".join(raw_thoughts) if raw_thoughts else None
-                state.operator_native_thinking = (
-                    "\n".join(native_thoughts) if native_thoughts else None
-                )
-
-                result_obj = await invoke_tool_with_injection(
-                    tool=tool_to_run,
-                    args=args,
-                    tool_call_id=tc["id"],
-                    state=state,
-                )
-                return get_tool_result_content(result_obj)
-            except Exception as e:
-                return f"Error running tool {tool_name}: {e}"
-        else:
-            return f"Error: Tool {tool_name} not supported"
-
     @wrap_with_callbacks(
         before=lambda: logger.info("Starting Operator Agent..."),
         on_success=lambda _: logger.success("Operator Agent"),
@@ -1110,9 +1048,13 @@ class OperatorNode:
             except (AttributeError, TypeError):
                 pass
 
-        def resolve_target_element(
-            target: Any,
-        ) -> tuple[dict | None, str | None]:
+        def resolve_target_element(target: Any, description: Any) -> tuple[dict | None, str | None]:
+            """Resolves a target to ``{"center": [x, y], **semantics}``.
+
+            An element index carries observed semantics (text, bounds, id, class
+            straight from the indexed list); a coordinate pair carries only the
+            model's own ``target_description``. The two never mix.
+            """
             if isinstance(target, (int, float)):
                 try:
                     target_int = int(target)
@@ -1120,7 +1062,14 @@ class OperatorNode:
                     return None, f"Error: Invalid target index {target}."
                 indexed_elements = getattr(state, "indexed_elements", None) or []
                 if 1 <= target_int <= len(indexed_elements):
-                    return {**indexed_elements[target_int - 1], "label_source": "index"}, None
+                    el = indexed_elements[target_int - 1]
+                    return {
+                        "center": el.get("center"),
+                        "target_text": el.get("text"),
+                        "target_bounds": el.get("bounds"),
+                        "target_resource_id": el.get("resource_id"),
+                        "target_class": el.get("class"),
+                    }, None
                 return (
                     None,
                     (
@@ -1129,26 +1078,21 @@ class OperatorNode:
                     ),
                 )
             elif isinstance(target, list) and len(target) == 2:
+                described = _described(description)
+                if described is None:
+                    return (
+                        None,
+                        (
+                            f"Error: Coordinate target {target} requires"
+                            " 'target_description' (what the element is, in a few"
+                            " words). Provide it, or address the element by index."
+                        ),
+                    )
                 try:
                     nx, ny = map(float, target)
                     x = int(max(0, min(width - 1, nx * width / 1000)))
                     y = int(max(0, min(height - 1, ny * height / 1000)))
-                    # Record-time enrichment: a bare-coordinate target carries no
-                    # element semantics, so hit test the pre-action frame's indexed
-                    # elements to best-effort recover what sits under the point.
-                    hit_el, hit_source = find_element_at_point(
-                        getattr(state, "indexed_elements", None), x, y
-                    )
-                    hit_el = hit_el or {}
-                    return {
-                        "center": [x, y],
-                        "text": hit_el.get("text"),
-                        "bounds": hit_el.get("bounds"),
-                        "class": hit_el.get("class"),
-                        "resource_id": hit_el.get("resource_id"),
-                        "is_ocr": bool(hit_el.get("is_ocr")),
-                        "label_source": hit_source,
-                    }, None
+                    return {"center": [x, y], "target_description": described}, None
                 except (ValueError, TypeError):
                     return (
                         None,
@@ -1162,11 +1106,15 @@ class OperatorNode:
                 ),
             )
 
+        def target_fields(el: dict) -> dict:
+            """The recorded target semantics: observed (index) or described (coords)."""
+            return {k: v for k, v in el.items() if k != "center"}
+
         if tool_name == "click":
             target = args.get("target")
             times = args.get("times", 1)
             delay_ms = args.get("delay_ms", 100)
-            el, err = resolve_target_element(target)
+            el, err = resolve_target_element(target, args.get("target_description"))
             if err:
                 return [], err
             norm_c = [
@@ -1181,18 +1129,14 @@ class OperatorNode:
                     "normalized_coordinates": norm_c,
                     "times": times,
                     "delay_ms": delay_ms,
-                    "target_text": el.get("text"),
-                    "target_bounds": el.get("bounds"),
-                    "target_resource_id": el.get("resource_id"),
-                    "target_class": el.get("class"),
-                    "target_label_source": el.get("label_source", "none"),
+                    **target_fields(el),
                 }
             ], None
 
         elif tool_name == "long_press":
             target = args.get("target")
             duration = args.get("duration", 1000)
-            el, err = resolve_target_element(target)
+            el, err = resolve_target_element(target, args.get("target_description"))
             if err:
                 return [], err
             norm_c = [
@@ -1206,11 +1150,7 @@ class OperatorNode:
                     COORDINATE_SPACE_KEY: COORDINATE_SPACE_PIXEL,
                     "normalized_coordinates": norm_c,
                     "duration": duration,
-                    "target_text": el.get("text"),
-                    "target_bounds": el.get("bounds"),
-                    "target_resource_id": el.get("resource_id"),
-                    "target_class": el.get("class"),
-                    "target_label_source": el.get("label_source", "none"),
+                    **target_fields(el),
                 }
             ], None
 
@@ -1227,7 +1167,7 @@ class OperatorNode:
                     (f"Error: 'text' must be a string, got {type(text).__name__}."),
                 )
 
-            el, err = resolve_target_element(target)
+            el, err = resolve_target_element(target, args.get("target_description"))
             if err:
                 return [], err
 
@@ -1244,11 +1184,7 @@ class OperatorNode:
                     "normalized_coordinates": norm_c,
                     "text": text,
                     "clear_before_input": clear_exist,
-                    "target_text": el.get("text"),
-                    "target_bounds": el.get("bounds"),
-                    "target_resource_id": el.get("resource_id"),
-                    "target_class": el.get("class"),
-                    "target_label_source": el.get("label_source", "none"),
+                    **target_fields(el),
                 }
             )
             return actions, None
@@ -1271,6 +1207,9 @@ class OperatorNode:
             else:
                 duration = parsed_duration
 
+            # Only a coordinate gesture needs the model to say what it drags; a
+            # directional scroll targets the page, not an element.
+            described: str | None = None
             if kind == "direction":
                 x1, y1, x2, y2, smart_dur = compute_smart_swipe_coordinates(
                     direction=target,
@@ -1284,6 +1223,16 @@ class OperatorNode:
                 if duration is None:
                     duration = smart_dur
             elif kind == "coords" and isinstance(target, list) and len(target) == 4:
+                described = _described(args.get("target_description"))
+                if described is None:
+                    return (
+                        [],
+                        (
+                            f"Error: Coordinate swipe {target} requires"
+                            " 'target_description' (what is being dragged, in a few"
+                            " words). Provide it, or use a directional swipe."
+                        ),
+                    )
                 try:
                     nx1, ny1, nx2, ny2 = map(float, target)
                     x1 = int(max(0, min(width - 1, nx1 * width / 1000)))
@@ -1327,6 +1276,7 @@ class OperatorNode:
                     "normalized_start_coordinates": [nx1, ny1],
                     "normalized_end_coordinates": [nx2, ny2],
                     "duration": duration,
+                    **({"target_description": described} if described else {}),
                 }
             ], None
 

artemis/agents/operator/prompts.py

@@ -33,6 +33,7 @@ from artemis.tools.command_tool import (
 from artemis.utils.logger import get_logger
 from artemis.utils.notes import get_note_file_path
 from artemis.utils.plan_grammar import parse_plan, render_plan_grammar_spec
+from artemis.utils.task_tree import SELF_DESCRIBED_MARKER
 
 logger = get_logger(__name__)
 
@@ -418,7 +419,7 @@ def _max_burst_actions_for_prompt() -> int:
 
 
 def _last_successful_action_before(steps: list, step_number: int | None) -> tuple[str, int] | None:
-    """The most recent step before ``step_number`` whose terminal action executed.
+    """The most recent step before ``step_number`` whose terminal action was dispatched.
 
     Returns ``(clean action description, step number)`` or None. This is the
     Operator's best candidate for the *trigger* of a transient state: the
@@ -437,7 +438,7 @@ def _last_successful_action_before(steps: list, step_number: int | None) -> tupl
         if not action:
             continue
         result = step.get("last_execution_result")
-        if isinstance(result, dict) and result.get("status") not in (None, "success"):
+        if isinstance(result, dict) and result.get("status") not in (None, "dispatched"):
             continue
         candidates.append((number, format_actions_clean(action)))
     if not candidates:
@@ -451,12 +452,13 @@ def _incident_target_label(incident: dict) -> str:
     # The Operator only ever sees 0–1000 coordinates: the recorded pixel
     # target is the controller's business and is never rendered here.
     normalized = action.get("normalized_coordinates")
-    text = action.get("target_text")
     parts = []
     if normalized:
         parts.append(f"normalized {list(normalized)}")
-    if text:
-        parts.append(f'"{text}"')
+    if action.get("target_text"):
+        parts.append(f'"{action["target_text"]}"')
+    elif action.get("target_description"):
+        parts.append(f'"{action["target_description"]}" {SELF_DESCRIBED_MARKER}')
     return " ".join(parts) if parts else "the recorded target"
 
 
@@ -526,7 +528,7 @@ def render_execution_incident(incident: dict, steps: list) -> str:
         )
     elif category in ("target_disappeared", "pixel_target_disappeared"):
         prior_str = (
-            f" Your last successfully executed action was `{prior[0]}` (Step {prior[1]});"
+            f" Your last dispatched action was `{prior[0]}` (Step {prior[1]});"
             " if the vanished target belonged to a state that action summoned, that action"
             " is the trigger."
             if prior

artemis/agents/outputter/outputter.py

@@ -17,6 +17,7 @@ from pathlib import Path
 
 from jinja2 import Template
 from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage
+from artemis.core.tool_failure import ToolFailure, is_tool_failure
 from artemis.config import OutputConfig
 from artemis.context import ArtemisContext
 from artemis.data_engine.trace import trace
@@ -271,9 +272,9 @@ async def outputter(
                         record_trace=True,
                     )
                 else:
-                    result = f"Error: Tool {tool_name} is not supported."
+                    result = ToolFailure(f"Error: Tool {tool_name} is not supported.")
                 text, _ = split_multimodal_result(result)
-                status = "error" if text.startswith("Error") else "success"
+                status = "error" if is_tool_failure(result) else "success"
             except Exception as e:
                 logger.error(f"Error running tool {tool_name}: {e}")
                 result = f"Error running tool {tool_name}: {e}"

artemis/agents/summarizer/summarizer.py

@@ -30,8 +30,10 @@ capsule moves up to the chunk-level ``StepCapsuleLens`` in M3.
 """
 
 import json
+from pathlib import Path
 from typing import Any
 
+from artemis.agents.flash.summarizer import build_focus_context
 from artemis.context import ArtemisContext
 from artemis.data_engine.trace import trace
 from artemis.graph.state import State
@@ -91,6 +93,16 @@ class SummarizerNode:
             else:
                 exec_outcome = "unknown"
 
+        # Operator focus: its own reasoning for the step (the target it chose
+        # and the screen change it expected), the goal, the live sub-goal leaf
+        # and any user instruction injected at this step. Nothing is inferred.
+        focus = build_focus_context(
+            intent=self._operator_reasoning(record, state),
+            goal=getattr(state, "initial_goal", None),
+            subgoal=self._active_subgoal_text(),
+            injected_instruction=getattr(state, "injected_instruction", None),
+        )
+
         service = ensure_step_memory(self.ctx)
         service.dispatch(
             step_number=step_number if isinstance(step_number, int) else 0,
@@ -100,9 +112,41 @@ class SummarizerNode:
             post_img_bytes=post_bytes,
             exec_outcome=exec_outcome,
             data_engine_step_id=step_id,
+            focus=focus,
         )
         logger.info(f"Dispatched visual transition summary for step {step_number} ({step_id})")
 
+    @staticmethod
+    def _operator_reasoning(record: Any, state: State) -> str | None:
+        """The operator's raw reasoning for this step: live state first, record second."""
+        for source in (state, record):
+            text = getattr(source, "operator_raw_thinking", None) if source is not None else None
+            if isinstance(text, str) and text.strip():
+                return text
+        return None
+
+    def _active_subgoal_text(self) -> str | None:
+        """The live sub-goal ledger leaf (``milestone > leaf``) from task_plan, if any."""
+        base_dir = getattr(self.ctx.data_engine, "base_dir", None)
+        if not isinstance(base_dir, (str, Path)):
+            return None
+        try:
+            from artemis.utils.notes import get_note_file_path
+            from artemis.utils.plan_grammar import parse_plan
+
+            path = get_note_file_path(base_dir, "task_plan")
+            if not path.exists():
+                return None
+            snapshot = parse_plan(path.read_text(encoding="utf-8"))
+            leaf = snapshot.last_active()
+            if leaf is None:
+                return None
+            parent = None if leaf.is_top_level else snapshot.parent_of(leaf)
+            return f"{parent.text} > {leaf.text}" if parent is not None else leaf.text
+        except Exception as e:
+            logger.debug(f"Active sub-goal lookup for the focus block skipped: {e}", exc_info=True)
+            return None
+
     def _read_step_image(self, step_number: int | None, which: str) -> bytes | None:
         if not isinstance(step_number, int):
             return None

artemis/agents/validator/execution_loop.py

@@ -140,11 +140,12 @@ async def _attempt_local_execution(
             exec_success = False
             exec_error = str(e)
 
-        # Skip polling: assume execution success implies action success
+        # No effect polling: "dispatched" only says the device accepted the
+        # command; whether it had the intended effect is the Operator's call.
         if exec_success:
             success = True
             error_msg = ""
-            attempts_log.append("Success")
+            attempts_log.append("Dispatched")
             break
         else:
             success = False
@@ -456,7 +457,7 @@ async def run_validation_loop(node, state: State) -> dict:
 
     report = {
         "execution": execution,
-        "status": "success" if success else "failed",
+        "status": "dispatched" if success else "failed",
         "burst": burst,
         "incident": incident,
     }

artemis/agents/validator/incidents.py

@@ -16,7 +16,7 @@
 
 Incidents are stored in ``state.open_incident`` and the step's
 ``last_execution_result``. Consecutive failures increment the count; a
-successful terminal action closes the incident. Prompt rendering lives in
+terminal action dispatched without error closes the incident. Prompt rendering lives in
 ``artemis.agents.operator.prompts.ExecutionIncidentPromptComponent``.
 """
 

artemis/agents/validator/pixel_safety_net.md

@@ -5,9 +5,10 @@ Read the `[Target]` block first to determine which rule applies.
 
 Rules
 1. `Kind: specific UI control` — the target has a label, resource id, class or bounds. Judge whether that exact control remains visible at the red dot in Image 2. Ignore video, map, animation or background changes beneath it. A control that has hidden, faded, collapsed, moved or become covered is not present, even if tapping the same spot could restore it.
-2. `Kind: coordinates only` — no UI hierarchy element was found at the point. Identify what appears under the red dot in both images. If Image 1 shows a distinct button, icon, link or handle, apply Rule 1. Apply the surface rule only when Image 1 shows blank space, a backdrop, or the body of a video, animation, map or live stream.
-3. Surface rule. Typical intents: tap the video body to wake its hidden controls, tap the backdrop to dismiss a dialog. Surfaces are expected to change between frames: ignore natural frame changes and the appearance or fading of overlay controls. Output `{"is_present": true, "confidence": 1.0}` unless an unexpected blocking popup now covers the red dot.
-4. Use `[Planned Action & Original Thinking]` to resolve whether the target was a control or its underlying surface. Do not treat a missing control as present because another tap could restore it.
+2. `Kind: described target` — the Operator aimed at a coordinate and stated what it is (`Operator's description`). That statement is the Operator's belief, not an observation: confirm against Image 1 what actually sits under the red dot. If the description names a control (button, icon, link, handle, field, knob) and Image 1 shows one there, apply Rule 1 to that control. If the description names a surface (video body, backdrop, blank area, map) or Image 1 shows no distinct control, apply the surface rule.
+3. `Kind: coordinates only` — no target metadata at all. Identify what appears under the red dot in both images. If Image 1 shows a distinct button, icon, link or handle, apply Rule 1. Apply the surface rule only when Image 1 shows blank space, a backdrop, or the body of a video, animation, map or live stream.
+4. Surface rule. Typical intents: tap the video body to wake its hidden controls, tap the backdrop to dismiss a dialog. Surfaces are expected to change between frames: ignore natural frame changes and the appearance or fading of overlay controls. Output `{"is_present": true, "confidence": 1.0}` unless an unexpected blocking popup now covers the red dot.
+5. Use `[Planned Action & Original Thinking]` to resolve whether the target was a control or its underlying surface. Do not treat a missing control as present because another tap could restore it.
 
 Output
 Return EXACTLY this raw JSON and nothing else. Keep reasoning strictly under 35 words.

artemis/agents/validator/precondition_pixel.py

@@ -67,11 +67,18 @@ def _init_llm_and_prompt(ctx: ArtemisContext, get_llm_fn):
 
 
 def _describe_target(action_item: dict) -> str:
-    """Describe the structured action target for the pixel judge."""
+    """Describe the structured action target for the pixel judge.
+
+    Three kinds, keyed on provenance: an index-addressed target carries observed
+    element metadata; a coordinate target carries only the Operator's own
+    statement of what it aimed at (``target_description``); a legacy record may
+    carry neither.
+    """
     label = action_item.get("target_text")
     resource_id = action_item.get("target_resource_id")
     class_name = action_item.get("target_class")
     bounds = action_item.get("target_bounds")
+    description = action_item.get("target_description")
     lines = ["[Target]", f"Action: {action_item.get('action')}"]
     if label or resource_id or class_name or bounds:
         lines.append("Kind: specific UI control")
@@ -85,9 +92,12 @@ def _describe_target(action_item: dict) -> str:
             lines.append("Label: (unlabelled; identify it by its shape in Image 1)")
         if bounds:
             lines.append(f"Bounds at decision time: {bounds}")
+    elif description:
+        lines.append("Kind: described target")
+        lines.append(f"Operator's description: {description}")
     else:
         lines.append(
-            "Kind: coordinates only (no UI hierarchy element was found under the point;"
+            "Kind: coordinates only (no target metadata recorded;"
             " inspect Image 1 for a control at the red dot)"
         )
     return "\n".join(lines)

artemis/agents/validator/tool_declarations.py

@@ -12,13 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Universal Tool Declarations, Execution Results, and Dispatcher for Artemis."""
+"""Universal Tool Declarations and Execution Results for Artemis.
+
+Execution itself lives in :class:`artemis.mcp.action_executor.McpActionExecutor`
+(Flash) and the Validator's action session (Pro); this module keeps the shared
+result envelope, the observation helpers both use, and the tool declarations.
+"""
 
-import ast
 import base64
-import hashlib
 import json
-from pathlib import Path
 from typing import Any, Literal
 
 from google.genai import types as genai_types
@@ -34,43 +36,18 @@ from artemis.agents.explorer.constants import (
 from artemis.context import ArtemisContext
 from artemis.controllers.unified_controller import UnifiedMobileController
 from artemis.core.tool_declaration import ToolDeclaration
-from artemis.data_engine.trace import trace
 from artemis.mcp.action_specs import tool_declaration
-from artemis.mcp.action_types import ActionCode
-from artemis.mcp.actuators.adb import AdbActuator
 from artemis.mcp.observation import observe
-from artemis.tools.explorer_tool import locate, register_candidates, render_text
 from artemis.graph.state import State
-from artemis.utils.coordinates import (
-    compute_smart_swipe_coordinates,
-    parse_swipe_parameters,
-)
 from artemis.utils.logger import get_logger
 from artemis.utils.notes import (
     LIST_NOTES_DOCSTRING,
     READ_NOTE_ARG_KEY_DESC,
     READ_NOTE_DOCSTRING,
-    format_list_notes_failure,
-    format_list_notes_success,
-    format_read_note_failure,
-    list_notes_info,
-    read_note_content,
 )
 
 logger = get_logger(__name__)
 
-ACTION_TOOL_NAMES = {
-    "click",
-    "click_sequence",
-    "long_press",
-    "input_text",
-    "swipe",
-    "press_key",
-    "manage_app",
-    "wait_for_delay",
-    "wait_for_text",
-}
-
 
 class ToolExecutionResult(BaseModel):
     """Standardized result produced by executing a mobile automation or utility tool."""
@@ -88,7 +65,7 @@ class ToolExecutionResult(BaseModel):
     def to_langchain_tool_message(self) -> ToolMessage:
         """Converts execution result into a standardized multimodal LangChain ToolMessage."""
         content_blocks: list[dict[str, Any]] = [
-            {"type": "text", "text": self.text_summary or f"Action '{self.tool_name}' completed."}
+            {"type": "text", "text": self.text_summary or f"Action '{self.tool_name}' dispatched."}
         ]
         if self.ui_elements_text:
             content_blocks.append(
@@ -249,528 +226,6 @@ def normalize_coordinate_target(target: Any) -> int | list[int] | Any:
     return target
 
 
-def serialize_mobile_action_result(result: Any) -> dict[str, Any]:
-    """Serializes MobileActionExecutor execution results into a clean structured dict.
-
-    Extracts the post-action screenshot hash from the captured image to eliminate guesswork.
-    """
-    if not isinstance(result, tuple) or len(result) < 3:
-        return {"outcome": str(result) if result is not None else ""}
-    outcome, img_bytes, shot_path, *rest = result
-    xml_list = rest[0] if rest else None
-
-    post_image_name = None
-    if shot_path:
-        post_image_name = Path(shot_path).stem
-    elif img_bytes:
-        post_image_name = hashlib.sha256(img_bytes).hexdigest()
-
-    is_err = bool(outcome and ("Error" in outcome or "Failed" in outcome))
-    return {
-        "outcome": outcome,
-        "post_image_name": post_image_name,
-        "has_xml": bool(xml_list),
-        "status": "error" if is_err else "success",
-    }
-
-
-def serialize_explorer_result(result: Any) -> dict[str, Any]:
-    """Stores an ``(text, ok)`` Explorer answer as a flat trace payload."""
-    if isinstance(result, tuple) and len(result) == 2:
-        text, ok = result
-        return {"outcome": text, "status": "success" if ok else "error"}
-    return {"outcome": str(result) if result is not None else ""}
-
-
-class MobileActionExecutor:
-    """Adapts LLM tool calls onto an actuator backend and captures observations.
-
-    Device-call bodies live in the actuator (``artemis/mcp/actuators/``); this class
-    keeps the agent-side responsibilities: argument normalization, index resolution
-    against LangGraph ``State``, act-then-observe screenshot capture, tracing, and the
-    legacy 4-tuple result contract.
-    """
-
-    def __init__(
-        self,
-        ctx: ArtemisContext,
-        controller: UnifiedMobileController | None = None,
-        actuator: "AdbActuator | None" = None,
-        agent_name: str = "validator",
-    ):
-        self.ctx = ctx
-        self.actuator = actuator or AdbActuator(ctx, controller)
-        self.controller = self.actuator.controller
-        # Identifies the calling agent to the Explorer tier resolver: the tier
-        # is a user setting keyed by profile / agent, never chosen by the agent.
-        self.agent_name = agent_name
-
-    @trace(type="action", name="click", serializer=serialize_mobile_action_result)
-    async def exec_click(
-        self,
-        target: list[int] | Any,
-        state: State,
-        times: int = 1,
-        delay_ms: int = 100,
-    ) -> tuple[str, bytes | None, str | None, str | None]:
-        try:
-            target = normalize_coordinate_target(target)
-            if not isinstance(target, (list, tuple)) or len(target) != 2:
-                raise ValueError(f"Invalid target format: {target}")
-            nx, ny = target
-            res = await self.actuator.click(int(nx), int(ny), times=times, delay_ms=delay_ms)
-            outcome = res.message
-
-            img_bytes, shot_path, xml_list = None, None, None
-            if res.ok:
-                shot_path, img_bytes, xml_list = await capture_screenshot_and_parse_ui(
-                    self.ctx, state, self.controller
-                )
-        except Exception as e:
-            outcome = f"Error during click: {e}"
-            img_bytes, shot_path, xml_list = None, None, None
-
-        return outcome, img_bytes, shot_path, xml_list
-
-    @trace(type="action", name="click_sequence", serializer=serialize_mobile_action_result)
-    async def exec_click_sequence(
-        self,
-        sequence: list[int | list[int] | str] | str,
-        state: State,
-        delay_ms: int = 50,
-    ) -> tuple[str, bytes | None, str | None, str | None]:
-        width = getattr(self.ctx.device, "device_width", 1080)
-        height = getattr(self.ctx.device, "device_height", 2400)
-        try:
-            if isinstance(sequence, str):
-                sequence_str = sequence.strip()
-                try:
-                    sequence = json.loads(sequence_str)
-                except Exception:
-                    try:
-                        sequence = ast.literal_eval(sequence_str)
-                    except (ValueError, TypeError, SyntaxError, MemoryError, RecursionError):
-                        pass
-
-            if not isinstance(sequence, (list, tuple)):
-                return (
-                    f"Error during click sequence: Invalid sequence format: {sequence}",
-                    None,
-                    None,
-                    None,
-                )
-
-            # Resolve each entry to normalized coordinates: integer element indices go
-            # through state.indexed_points (pixel centers -> normalized), coordinate
-            # pairs are already normalized.
-            norm_points: list[tuple[int, int]] = []
-            for raw_target in sequence:
-                target = normalize_coordinate_target(raw_target)
-                if isinstance(target, int):
-                    idx = target
-                    points = getattr(state, "indexed_points", []) or []
-                    if 1 <= idx <= len(points):
-                        pt = points[idx - 1]
-                        px, py = int(pt[0]), int(pt[1])
-                        nx = int(max(0, min(1000, round(px * 1000 / max(1, width)))))
-                        ny = int(max(0, min(1000, round(py * 1000 / max(1, height)))))
-                    else:
-                        return (
-                            (
-                                "Error during click sequence: Index"
-                                f" {idx} is out of range (available: {len(points)})"
-                            ),
-                            None,
-                            None,
-                            None,
-                        )
-                elif isinstance(target, (list, tuple)) and len(target) == 2:
-                    nx, ny = int(target[0]), int(target[1])
-                else:
-                    return (
-                        f"Error during click sequence: Invalid target format: {raw_target}",
-                        None,
-                        None,
-                        None,
-                    )
-                norm_points.append((nx, ny))
-
-            res = await self.actuator.click_sequence(norm_points, delay_ms=delay_ms)
-            if not res.ok:
-                return res.message, None, None, None
-
-            shot_path, img_bytes, xml_list = await capture_screenshot_and_parse_ui(
-                self.ctx, state, self.controller
-            )
-            return res.message, img_bytes, shot_path, xml_list
-        except Exception as e:
-            return f"Error executing click sequence: {e}", None, None, None
-
-    @trace(type="action", name="long_press", serializer=serialize_mobile_action_result)
-    async def exec_long_press(
-        self,
-        target: list[int] | Any,
-        state: State,
-        duration_ms: int = 1000,
-        duration: int | None = None,
-    ) -> tuple[str, bytes | None, str | None, str | None]:
-        if duration is not None:
-            duration_ms = duration
-        try:
-            target = normalize_coordinate_target(target)
-            if not isinstance(target, (list, tuple)) or len(target) != 2:
-                raise ValueError(f"Invalid target format: {target}")
-            nx, ny = target
-            res = await self.actuator.long_press(int(nx), int(ny), duration_ms=duration_ms)
-            outcome = res.message
-
-            img_bytes, shot_path, xml_list = None, None, None
-            if res.ok:
-                shot_path, img_bytes, xml_list = await capture_screenshot_and_parse_ui(
-                    self.ctx, state, self.controller
-                )
-        except Exception as e:
-            outcome = f"Error during long press: {e}"
-            img_bytes, shot_path, xml_list = None, None, None
-
-        return outcome, img_bytes, shot_path, xml_list
-
-    @trace(type="action", name="input_text", serializer=serialize_mobile_action_result)
-    async def exec_input_text(
-        self,
-        text: str,
-        target: list[int] | Any,
-        state: State,
-        clear_exist: bool = True,
-    ) -> tuple[str, bytes | None, str | None, str | None]:
-        try:
-            norm_target: tuple[int, int] | None = None
-            if target:
-                target = normalize_coordinate_target(target)
-                if not isinstance(target, (list, tuple)) or len(target) != 2:
-                    raise ValueError(f"Invalid target format: {target}")
-                norm_target = (int(target[0]), int(target[1]))
-
-            res = await self.actuator.input_text(text, norm_target, clear_exist=clear_exist)
-            outcome = res.message
-            shot_path, img_bytes, xml_list = await capture_screenshot_and_parse_ui(
-                self.ctx, state, self.controller
-            )
-        except Exception as e:
-            outcome = f"Error during input text: {e}"
-            img_bytes, shot_path, xml_list = None, None, None
-
-        return outcome, img_bytes, shot_path, xml_list
-
-    @trace(type="action", name="swipe", serializer=serialize_mobile_action_result)
-    async def exec_swipe(
-        self,
-        action: str | list[int] | Any = None,
-        duration: int = 800,
-        state: State | None = None,
-        duration_ms: int | None = None,
-        **kwargs: Any,
-    ) -> tuple[str, bytes | None, str | None, str | None]:
-        width = getattr(self.ctx.device, "device_width", 1080)
-        height = getattr(self.ctx.device, "device_height", 2400)
-        dur = duration_ms if duration_ms is not None else duration
-
-        swipe_input = dict(kwargs) if kwargs else {}
-        if isinstance(action, dict):
-            swipe_input.update(action)
-        elif action is not None:
-            swipe_input["action"] = action
-        if "duration" not in swipe_input:
-            swipe_input["duration"] = dur
-
-        kind, target, final_duration = parse_swipe_parameters(swipe_input, default_duration=dur)
-
-        try:
-            if kind == "direction":
-                dir_name = target
-                # Smart-swipe resolution needs State (indexed elements + ui tree), so
-                # it stays client-side; the actuator only receives the two points.
-                x1, y1, x2, y2, smart_dur = compute_smart_swipe_coordinates(
-                    direction=target,
-                    target=swipe_input.get("target"),
-                    indexed_elements=getattr(state, "indexed_elements", None) if state else None,
-                    ui_hierarchy=getattr(state, "latest_ui_hierarchy", None) if state else None,
-                    width=width,
-                    height=height,
-                    duration=final_duration,
-                )
-                final_duration = smart_dur
-                nx1 = int(max(0, min(1000, round(x1 * 1000 / max(1, width)))))
-                ny1 = int(max(0, min(1000, round(y1 * 1000 / max(1, height)))))
-                nx2 = int(max(0, min(1000, round(x2 * 1000 / max(1, width)))))
-                ny2 = int(max(0, min(1000, round(y2 * 1000 / max(1, height)))))
-                res = await self.actuator.swipe((nx1, ny1), (nx2, ny2), final_duration)
-                if not res.ok:
-                    return f"Error swiping {dir_name}: {res.detail}", None, None, None
-                outcome = f"Swipe completed successfully. Swiped {dir_name}."
-            elif kind == "coords":
-                x1, y1, x2, y2 = target
-                res = await self.actuator.swipe(
-                    (int(x1), int(y1)), (int(x2), int(y2)), final_duration
-                )
-                if not res.ok:
-                    return f"Error dragging: {res.detail}", None, None, None
-                outcome = f"Swipe completed successfully. Swiped from [{x1}, {y1}] to [{x2}, {y2}]."
-            else:
-                return (
-                    f"Error during swipe: Invalid direction: {action or kwargs}",
-                    None,
-                    None,
-                    None,
-                )
-
-            current_state = state or State()
-            shot_path, img_bytes, xml_list = await capture_screenshot_and_parse_ui(
-                self.ctx, current_state, self.controller
-            )
-        except Exception as e:
-            outcome = f"Error during swipe: {e}"
-            img_bytes, shot_path, xml_list = None, None, None
-
-        return outcome, img_bytes, shot_path, xml_list
-
-    @trace(type="action", name="press_key", serializer=serialize_mobile_action_result)
-    async def exec_press_key(
-        self,
-        key: str,
-        state: State,
-    ) -> tuple[str, bytes | None, str | None, str | None]:
-        try:
-            res = await self.actuator.press_key(key)
-            if not res.ok:
-                return res.message, None, None, None
-
-            outcome = res.message
-            shot_path, img_bytes, xml_list = await capture_screenshot_and_parse_ui(
-                self.ctx, state, self.controller
-            )
-        except Exception as e:
-            outcome = f"Error during press_key: {e}"
-            img_bytes, shot_path, xml_list = None, None, None
-
-        return outcome, img_bytes, shot_path, xml_list
-
-    @trace(type="action", name="manage_app", serializer=serialize_mobile_action_result)
-    async def exec_manage_app(
-        self,
-        action: str,
-        app_name: str,
-        state: State,
-    ) -> tuple[str, bytes | None, str | None, str | None]:
-        try:
-            res = await self.actuator.manage_app(action, app_name)
-            # Argument-level failures return without an observation, exactly as the
-            # historical early returns did; a failed launch still captures the screen.
-            if res.code in (ActionCode.PACKAGE_NOT_FOUND, ActionCode.INVALID_ARGS):
-                return res.message, None, None, None
-
-            outcome = res.message
-            shot_path, img_bytes, xml_list = await capture_screenshot_and_parse_ui(
-                self.ctx, state, self.controller
-            )
-        except Exception as e:
-            outcome = f"Error during manage_app: {e}"
-            img_bytes, shot_path, xml_list = None, None, None
-
-        return outcome, img_bytes, shot_path, xml_list
-
-    @trace(type="action", name="wait_for_delay", serializer=serialize_mobile_action_result)
-    async def exec_wait_for_delay(
-        self,
-        time_in_ms: int,
-        state: State,
-    ) -> tuple[str, bytes | None, str | None, str | None]:
-        try:
-            res = await self.actuator.wait_for_delay(time_in_ms)
-            outcome = res.message
-            shot_path, img_bytes, xml_list = await capture_screenshot_and_parse_ui(
-                self.ctx, state, self.controller
-            )
-        except Exception as e:
-            outcome = f"Error during wait_for_delay: {e}"
-            img_bytes, shot_path, xml_list = None, None, None
-
-        return outcome, img_bytes, shot_path, xml_list
-
-    @trace(type="action", name="wait_for_text", serializer=serialize_mobile_action_result)
-    async def exec_wait_for_text(
-        self,
-        text: str,
-        wait_state: str | None,
-        timeout_ms: int | None,
-        state: State,
-    ) -> tuple[str, bytes | None, str | None, str | None]:
-        try:
-            res = await self.actuator.wait_for_text(text, wait_state, timeout_ms)
-            outcome = res.message
-            # A timeout still captures the screen, as the original loop did.
-            shot_path, img_bytes, xml_list = await capture_screenshot_and_parse_ui(
-                self.ctx, state, self.controller
-            )
-        except Exception as e:
-            outcome = f"Error during wait_for_text: {e}"
-            img_bytes, shot_path, xml_list = None, None, None
-
-        return outcome, img_bytes, shot_path, xml_list
-
-    @trace(type="tool", name="read_note")
-    async def exec_read_note(
-        self,
-        key: str,
-        start_line: int | None = None,
-        end_line: int | None = None,
-    ) -> str:
-        try:
-            base_dir = self.ctx.data_engine.base_dir if self.ctx.data_engine else None
-            if not base_dir:
-                return format_read_note_failure(key, "DataEngine not initialized.")
-            return read_note_content(base_dir, key, start_line, end_line)
-        except Exception as e:
-            return format_read_note_failure(key, str(e))
-
-    @trace(type="tool", name="list_notes")
-    async def exec_list_notes(self) -> str:
-        try:
-            base_dir = self.ctx.data_engine.base_dir if self.ctx.data_engine else None
-            if not base_dir:
-                return format_list_notes_failure("DataEngine not initialized.")
-            notes = list_notes_info(base_dir)
-            return format_list_notes_success(notes)
-        except Exception as e:
-            return format_list_notes_failure(str(e))
-
-    @trace(type="tool", name="ask_explorer", serializer=serialize_explorer_result)
-    async def exec_ask_explorer(
-        self,
-        query: str,
-        context_feedback: str | None,
-        state: State,
-    ) -> tuple[str, bool]:
-        """Runs the Explorer pipeline; returns ``(text, ok)``.
-
-        ``ok`` is False only when the Explorer run itself failed. A clean
-        "not found" is a successful answer the agent must reason about, so it
-        must not be reported as a tool error.
-        """
-        try:
-            outcome = await locate(
-                self.ctx, state, query, context_feedback or "", agent_name=self.agent_name
-            )
-            registered = register_candidates(self.ctx, state, outcome)
-            return render_text(query, outcome, registered), not outcome.error
-        except Exception as e:
-            return f"Error executing ask_explorer: {e}", False
-
-    async def execute(
-        self,
-        name: str,
-        args: dict[str, Any],
-        tool_call_id: str,
-        state: State,
-    ) -> ToolExecutionResult:
-        """Executes the specified tool by name and wraps result into ToolExecutionResult."""
-        raw_name = name.split(":")[-1] if ":" in name else name
-        res_text = ""
-        img_bytes, shot_path, xml_list = None, None, None
-        # Explicit status for tools that report it (the Explorer); None keeps the
-        # historical text-sniffing fallback for the device actions.
-        explicit_ok: bool | None = None
-
-        if raw_name == "click":
-            res_text, img_bytes, shot_path, xml_list = await self.exec_click(
-                args.get("target") or args.get("coordinates") or [],
-                state,
-                times=args.get("times", 1),
-                delay_ms=args.get("delay_ms", 100),
-            )
-        elif raw_name == "click_sequence":
-            res_text, img_bytes, shot_path, xml_list = await self.exec_click_sequence(
-                args.get("sequence") or [],
-                state,
-                delay_ms=args.get("delay_ms", 50),
-            )
-        elif raw_name == "long_press":
-            res_text, img_bytes, shot_path, xml_list = await self.exec_long_press(
-                args.get("target") or args.get("coordinates") or [],
-                state,
-                duration_ms=args.get("duration_ms", 1000),
-            )
-        elif raw_name == "input_text":
-            res_text, img_bytes, shot_path, xml_list = await self.exec_input_text(
-                args.get("text", ""),
-                args.get("target") or args.get("coordinates"),
-                state,
-                clear_exist=args.get("clear_exist", True),
-            )
-        elif raw_name == "swipe":
-            res_text, img_bytes, shot_path, xml_list = await self.exec_swipe(
-                action=args, duration=args.get("duration", 400), state=state
-            )
-        elif raw_name == "press_key":
-            res_text, img_bytes, shot_path, xml_list = await self.exec_press_key(
-                args.get("key", "BACK"), state
-            )
-        elif raw_name == "manage_app":
-            res_text, img_bytes, shot_path, xml_list = await self.exec_manage_app(
-                args.get("action", "launch"), args.get("app_name", ""), state
-            )
-        elif raw_name == "wait_for_delay":
-            res_text, img_bytes, shot_path, xml_list = await self.exec_wait_for_delay(
-                args.get("time_in_ms", 1000), state
-            )
-        elif raw_name == "wait_for_text":
-            res_text, img_bytes, shot_path, xml_list = await self.exec_wait_for_text(
-                args.get("text", ""),
-                args.get("wait_state"),
-                args.get("timeout_ms", 5000),
-                state,
-            )
-        elif raw_name == "read_note":
-            res_text = await self.exec_read_note(
-                args.get("key", ""),
-                args.get("start_line"),
-                args.get("end_line"),
-            )
-        elif raw_name == "list_notes":
-            res_text = await self.exec_list_notes()
-        elif raw_name == "ask_explorer":
-            res_text, explicit_ok = await self.exec_ask_explorer(
-                # ``task_description`` is the pre-contract argument name kept
-                # for old MCP clients.
-                args.get("query") or args.get("task_description") or "",
-                args.get("context_feedback"),
-                state,
-            )
-        else:
-            res_text = f"Error: Tool '{name}' not supported."
-
-        if shot_path:
-            state.latest_screenshot = shot_path
-
-        status: Literal["success", "error"] = (
-            "success" if (img_bytes is not None or raw_name not in ACTION_TOOL_NAMES) else "error"
-        )
-        if explicit_ok is not None:
-            status = "success" if explicit_ok else "error"
-        elif "Error" in res_text or "Failed" in res_text:
-            status = "error"
-
-        return ToolExecutionResult(
-            tool_call_id=tool_call_id,
-            tool_name=name,
-            status=status,
-            text_summary=res_text,
-            screenshot_bytes=img_bytes,
-            screenshot_path=shot_path,
-            ui_elements_text=xml_list,
-        )
-
-
 # Device-action declarations are generated from the canonical manifest
 # (artemis/mcp/action_specs.py); the historical constant names remain the public API.
 CLICK_TOOL = tool_declaration("click")

artemis/agents/video_analyzer/audio_native.py

@@ -26,7 +26,20 @@ from pathlib import Path
 from google.genai import types
 
 from artemis.agents.video_analyzer import video_analyzer as _va
-from artemis.agents.video_analyzer.reliability import classify_video_failure
+from artemis.agents.video_analyzer.chunk_conversation import (
+    _MISSING_SUMMARY_ERROR,
+    _NO_TOOL_CALL_ERROR,
+    _SUBMIT_NOW_WARNING,
+    _WRONG_TOOL_ERROR,
+    SUB_AGENT_MAX_TURNS,
+    _append_rejection,
+    _clean_text,
+    _tool_name,
+)
+from artemis.agents.video_analyzer.reliability import (
+    SubAgentAnswerExhausted,
+    classify_video_failure,
+)
 from artemis.constants import SAFETY_SETTINGS_BLOCK_NONE
 from artemis.data_engine.trace import TraceSpan
 from artemis.llm.reliability import retry_policy_for
@@ -34,6 +47,11 @@ from artemis.utils.logger import get_logger
 
 logger = get_logger(__name__)
 
+_BAD_CONFIDENCE_ERROR = (
+    "Error: 'confidence_score' is missing or out of bounds. It must be a float"
+    " between 0.0 and 1.0 inclusive. Fix the payload and call submit_answer again."
+)
+
 
 class _AudioMedia:
     """Mutable holder for per-attempt audio artifacts shared with failure handling."""
@@ -427,29 +445,18 @@ async def _run_native_audio_conversation(
 
 
 async def _drive_audio_agent_loop(analyzer, sub_agent_contents: list) -> tuple[str, str, float]:
-    """Iterates the audio agent conversation until submit_answer is accepted."""
-    sub_max_iterations = 2
-    sub_iterations = 0
-    final_confidence_score = 0.0
-    final_summary = "No summary provided."
-    final_analysis = "No analysis provided."
-    final_full_text = ""
+    """Return a validated (summary, analysis, confidence_score).
 
-    while sub_iterations < sub_max_iterations:
-        sub_iterations += 1
-        if sub_iterations == sub_max_iterations - 1:
+    Raise ``SubAgentAnswerExhausted`` if no answer is accepted within the turn limit.
+    """
+    rejection = _NO_TOOL_CALL_ERROR
+
+    for turn in range(1, SUB_AGENT_MAX_TURNS + 1):
+        if turn == SUB_AGENT_MAX_TURNS - 1:
             sub_agent_contents.append(
                 types.Content(
                     role="user",
-                    parts=[
-                        types.Part.from_text(
-                            text=(
-                                "[WARNING] You must call the"
-                                " submit_answer tool now, time"
-                                " is running out."
-                            )
-                        )
-                    ],
+                    parts=[types.Part.from_text(text=_SUBMIT_NOW_WARNING)],
                 )
             )
 
@@ -468,26 +475,16 @@ async def _drive_audio_agent_loop(analyzer, sub_agent_contents: list) -> tuple[s
             )
 
         full_text = response.text or ""
-        final_full_text += full_text + "\n"
         function_calls = response.function_calls or []
 
         if function_calls:
-            (
-                answered,
-                final_summary,
-                final_analysis,
-                final_confidence_score,
-            ) = _handle_audio_function_calls(
-                sub_agent_contents,
-                response,
-                function_calls,
-                final_summary,
-                final_analysis,
-                final_confidence_score,
+            answer, rejection = _handle_audio_function_calls(
+                sub_agent_contents, response, function_calls, full_text
             )
-            if answered is True:
-                break
+            if answer is not None:
+                return answer
         else:
+            rejection = _NO_TOOL_CALL_ERROR
             sub_agent_contents.append(
                 types.Content(
                     role="model",
@@ -495,83 +492,52 @@ async def _drive_audio_agent_loop(analyzer, sub_agent_contents: list) -> tuple[s
                 )
             )
 
-    return final_summary, final_analysis, final_confidence_score
+    raise SubAgentAnswerExhausted(SUB_AGENT_MAX_TURNS, rejection)
+
+
+def _validate_audio_answer(summary: str, confidence_score) -> str | None:
+    """Return a validation error for the audio answer, or None if valid."""
+    if not summary:
+        return _MISSING_SUMMARY_ERROR
+    if (
+        not isinstance(confidence_score, (int, float))
+        or isinstance(confidence_score, bool)
+        or not (0.0 <= float(confidence_score) <= 1.0)
+    ):
+        return _BAD_CONFIDENCE_ERROR
+    return None
 
 
 def _handle_audio_function_calls(
     sub_agent_contents: list,
     response,
     function_calls: list,
-    final_summary: str,
-    final_analysis: str,
-    final_confidence_score: float,
-):
-    """Processes audio-agent function calls; returns (answered, summary, analysis, score)."""
-    answered = False
+    full_text: str,
+) -> tuple[tuple[str, str, float] | None, str | None]:
+    """Return (answer, rejection), appending any rejection to the conversation."""
+    original_parts = response.candidates[0].content.parts
     for fc in function_calls:
-        if (fc.name.split(":")[-1] if ":" in fc.name else fc.name) == "submit_answer":
-            args = fc.args
-            if (
-                "confidence_score" not in args
-                or not isinstance(
-                    args["confidence_score"],
-                    (int, float),
-                )
-                or not (0.0 <= float(args["confidence_score"]) <= 1.0)
-            ):
-                error_msg = (
-                    "Error: 'confidence_score' is"
-                    " missing or out of bounds. It must"
-                    " be a float between 0.0 and 1.0"
-                    " inclusive. Fix the payload and"
-                    " call submit_answer again."
-                )
-                sub_agent_contents.append(
-                    types.Content(
-                        role="model",
-                        parts=response.candidates[0].content.parts,
-                    )
-                )
-                sub_agent_contents.append(
-                    types.Content(
-                        role="user",
-                        parts=[
-                            types.Part.from_function_response(
-                                name=fc.name,
-                                response={"error": error_msg},
-                            )
-                        ],
-                    )
-                )
-                answered = "error"
-                break
+        if _tool_name(fc.name) != "submit_answer":
+            continue
+        args = dict(fc.args or {})
+        summary = _clean_text(args.get("summary"))
+        analysis = _clean_text(args.get("analysis"))
 
-            final_summary = args.get("summary", "No summary provided.").strip().replace("\n", " ")
-            final_analysis = (
-                args.get("analysis", "No analysis provided.").strip().replace("\n", " ")
+        rejection = _validate_audio_answer(summary, args.get("confidence_score"))
+        if rejection is not None:
+            _append_rejection(
+                sub_agent_contents, original_parts, function_calls, full_text, fc.name, rejection
             )
-            final_confidence_score = float(args["confidence_score"])
-            answered = True
-            break
+            return None, rejection
 
-    if answered is True or answered == "error":
-        return answered, final_summary, final_analysis, final_confidence_score
+        return (summary, analysis or "No analysis provided.", float(args["confidence_score"])), None
 
-    sub_agent_contents.append(
-        types.Content(
-            role="model",
-            parts=response.candidates[0].content.parts,
+    _append_rejection(
+        sub_agent_contents,
+        original_parts,
+        function_calls,
+        full_text,
+        function_calls[0].name,
+        _WRONG_TOOL_ERROR,
     )
-    )
-    sub_agent_contents.append(
-        types.Content(
-            role="user",
-            parts=[
-                types.Part.from_function_response(
-                    name=function_calls[0].name,
-                    response={"error": ("Tool not recognized. Please use submit_answer.")},
-                )
-            ],
-        )
-    )
-    return answered, final_summary, final_analysis, final_confidence_score
+    return None, _WRONG_TOOL_ERROR

artemis/agents/video_analyzer/chunk_agentic.py

@@ -0,0 +1,455 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Analyze video chunks through the Gemini Interactions API.
+
+``processing: "agentic"`` lets the model search and revisit an uploaded clip.
+Validation errors are returned as function results, with video context retained
+through ``previous_interaction_id``. Only accepted answers reach the blackboard.
+
+The stream consumer reconstructs function arguments from step deltas. Completion
+events may omit steps, so it fetches the interaction when reconstruction fails.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import copy
+from dataclasses import dataclass, field
+import json
+from types import SimpleNamespace
+from typing import Any
+
+from artemis.agents.video_analyzer import gemini_files
+from artemis.agents.video_analyzer import video_analyzer as _va
+from artemis.agents.video_analyzer.chunk_conversation import (
+    _NO_TOOL_CALL_ERROR,
+    _SUBMIT_NOW_WARNING,
+    _WRONG_TOOL_ERROR,
+    SUB_AGENT_MAX_TURNS,
+    SubAgentAnswerExhausted,
+    _ChunkMedia,
+    _clean_text,
+    _persist_timeline_events,
+    _tool_name,
+    _validate_answer,
+)
+from artemis.agents.video_analyzer.universal_tools import UNIVERSAL_SUBMIT_ANSWER_TOOL
+from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan
+from artemis.llm.google import normalize_usage
+from artemis.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+STREAM_START_TIMEOUT_SECONDS = 60.0
+INTERACTION_READ_TIMEOUT_SECONDS = 30.0
+
+# Interaction statuses after which no follow-up turn can be sent
+# (``google.genai`` ``InteractionStatus``).
+_TERMINAL_FAILURE_STATUSES = frozenset({"failed", "cancelled", "incomplete", "budget_exceeded"})
+
+
+def build_interactions_submit_tool() -> dict[str, Any]:
+    """The ``submit_answer`` declaration in Interactions API (flat JSON schema) form.
+
+    Derived from the universal (OpenAI-style) declaration so the schema is
+    written once; ``timeline_events`` is required to match the native path.
+    """
+    function = UNIVERSAL_SUBMIT_ANSWER_TOOL["function"]
+    parameters = copy.deepcopy(function["parameters"])
+    required = list(parameters.get("required", []))
+    if "timeline_events" not in required:
+        required.append("timeline_events")
+    parameters["required"] = required
+    return {
+        "type": "function",
+        "name": function["name"],
+        "description": function["description"],
+        "parameters": parameters,
+    }
+
+
+def _steps(interaction: Any) -> list:
+    return list(getattr(interaction, "steps", None) or [])
+
+
+def _find_function_call(interaction: Any):
+    """The last ``function_call`` step of an interaction, if any."""
+    for step in reversed(_steps(interaction)):
+        if getattr(step, "type", None) == "function_call":
+            return step
+    return None
+
+
+def _raise_if_failed(interaction: Any) -> None:
+    """Raises when an interaction ended in a terminal failure state.
+
+    A follow-up with ``previous_interaction_id`` onto a failed interaction is
+    rejected by the API, so the failure surfaces here as an exception and
+    flows through the shared retry / classification layer instead.
+    """
+    status = str(getattr(interaction, "status", None) or "").lower()
+    errors = list(getattr(interaction, "errors", None) or [])
+    if status not in _TERMINAL_FAILURE_STATUSES and not errors:
+        return
+    details = "; ".join(
+        f"[{getattr(error, 'code', None)}] {getattr(error, 'message', error)}" for error in errors
+    )
+    raise RuntimeError(
+        f"Gemini Interactions interaction {getattr(interaction, 'id', None)} ended with"
+        f" status {status or 'unknown'}" + (f": {details}" if details else "")
+    )
+
+
+async def run_agentic_chunk_conversation(
+    analyzer,
+    media: _ChunkMedia,
+    current_start: float,
+    current_end: float,
+    start_time: float,
+    end_time: float | None,
+    specific_query: str,
+    lease_owner: str,
+) -> str:
+    """Uploads the chunk and drives the agentic sub-agent to a committed answer."""
+    file = None
+    try:
+        with TraceSpan(name="upload_video_to_gemini") as span:
+            file = await analyzer.upload_and_poll_file(media.compressed_path)
+            span.result = f"Uploaded {file.name}"
+        logger.info(
+            f"Invoking Gemini agentic video understanding for sub-agent task with model"
+            f" {analyzer.model_name} (streaming)..."
+        )
+        trace_id = CURRENT_TRACE_ID.get()
+
+        if analyzer.ctx.data_engine and trace_id:
+            analyzer.ctx.data_engine.record_trace(
+                type="llm_call",
+                name="video_sub_agent",
+                payload={
+                    "contents": [
+                        f"file://{file.name}",
+                        media.prompt_with_context,
+                    ],
+                    "system_instruction": analyzer.sub_system_prompt,
+                    "video_processing": "agentic",
+                },
+                parent_trace_id=trace_id,
+            )
+
+        final_summary, final_analysis = await _drive_agentic_loop(
+            analyzer,
+            file,
+            trace_id,
+            media,
+            start_time,
+            end_time,
+            specific_query,
+        )
+
+        end_str = f"{current_end:.1f}s" if current_end is not None else "unknown"
+        chunk_result = (
+            f"[from {current_start:.1f}s to {end_str}] Summary:"
+            f" {final_summary} Analysis: {final_analysis}"
+        )
+        analyzer.blackboard.complete_segment(
+            current_start,
+            current_end,
+            specific_query,
+            lease_owner,
+            final_summary,
+            final_analysis,
+        )
+        return chunk_result
+    finally:
+        if file:
+            await gemini_files.delete_cloud_file(
+                analyzer.client, file.name, analyzer.cloud_files_to_cleanup
+            )
+
+
+def _initial_request(analyzer, file, prompt: str, tool: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "model": analyzer.model_name,
+        "input": [
+            {
+                "type": "video",
+                "uri": file.uri,
+                "mime_type": file.mime_type,
+                "processing": "agentic",
+            },
+            {"type": "text", "text": prompt},
+        ],
+        "system_instruction": analyzer.sub_system_prompt,
+        "tools": [tool],
+        "stream": True,
+    }
+
+
+def _follow_up_request(
+    analyzer, previous_id: str, steps: list[dict[str, Any]], tool: dict[str, Any]
+) -> dict[str, Any]:
+    """A stateful follow-up turn: the server keeps the video context.
+
+    Only the conversation state is inherited through
+    ``previous_interaction_id``; ``system_instruction`` and ``tools`` are
+    interaction-scoped and must be re-sent on every turn.
+    """
+    return {
+        "model": analyzer.model_name,
+        "previous_interaction_id": previous_id,
+        "input": steps,
+        "system_instruction": analyzer.sub_system_prompt,
+        "tools": [tool],
+        "stream": True,
+    }
+
+
+def _function_result(call_id: str, error: str) -> dict[str, Any]:
+    return {
+        "type": "function_result",
+        "call_id": call_id,
+        "name": "submit_answer",
+        "result": {"error": error},
+    }
+
+
+async def _drive_agentic_loop(
+    analyzer,
+    file,
+    trace_id,
+    media: _ChunkMedia,
+    start_time: float,
+    end_time: float | None,
+    specific_query: str,
+) -> tuple[str, str]:
+    """Return a validated answer and persist its timeline events.
+
+    Raise ``SubAgentAnswerExhausted`` if no answer is accepted within the turn limit.
+    """
+    tool = build_interactions_submit_tool()
+    request = _initial_request(analyzer, file, media.prompt_with_context, tool)
+    rejection = _NO_TOOL_CALL_ERROR
+
+    for turn in range(1, SUB_AGENT_MAX_TURNS + 1):
+        interaction = await _stream_agentic_turn(analyzer, request, trace_id)
+        interaction_id = getattr(interaction, "id", None) or ""
+        call = _find_function_call(interaction)
+
+        if call is None:
+            rejection = _NO_TOOL_CALL_ERROR
+            steps = [{"type": "text", "text": _SUBMIT_NOW_WARNING}]
+        elif _tool_name(getattr(call, "name", None)) != "submit_answer":
+            rejection = _WRONG_TOOL_ERROR
+            steps = [_function_result(call.id, rejection)]
+        else:
+            args = dict(getattr(call, "arguments", None) or {})
+            timeline_events = args.get("timeline_events", [])
+            if not isinstance(timeline_events, list):
+                timeline_events = []
+            summary = _clean_text(args.get("summary"))
+            analysis = _clean_text(args.get("analysis"))
+
+            rejection = _validate_answer(summary, timeline_events, start_time, end_time)
+            if rejection is None:
+                await _persist_timeline_events(
+                    analyzer, timeline_events, media, start_time, end_time, specific_query
+                )
+                return summary, analysis or "No analysis provided."
+            steps = [_function_result(call.id, rejection)]
+
+        if turn < SUB_AGENT_MAX_TURNS:
+            request = _follow_up_request(analyzer, interaction_id, steps, tool)
+
+    raise SubAgentAnswerExhausted(SUB_AGENT_MAX_TURNS, rejection)
+
+
+async def _stream_agentic_turn(analyzer, request: dict[str, Any], trace_id):
+    """Runs one traced streaming Interactions turn and returns the completed interaction."""
+    with TraceSpan(name="gemini_agentic_video_call") as span:
+        span.payload = {
+            "model": request.get("model"),
+            "video_processing": "agentic",
+            "previous_interaction_id": request.get("previous_interaction_id"),
+        }
+        stream = await asyncio.wait_for(
+            analyzer.client.aio.interactions.create(**request),
+            timeout=STREAM_START_TIMEOUT_SECONDS,
+        )
+        interaction = await asyncio.wait_for(
+            _consume_agentic_stream(analyzer, span, stream, trace_id),
+            timeout=analyzer.agentic_call_timeout_seconds,
+        )
+        call = _find_function_call(interaction)
+        span.result = (
+            f"Function call: {_tool_name(getattr(call, 'name', None))}"
+            if call is not None
+            else f"Status: {getattr(interaction, 'status', 'unknown')}"
+        )
+    return interaction
+
+
+@dataclass
+class _PendingStep:
+    """A step under construction while its start/delta/stop events stream in."""
+
+    kind: str
+    call_id: str | None = None
+    name: str | None = None
+    seed_arguments: dict[str, Any] = field(default_factory=dict)
+    arguments_json: list[str] = field(default_factory=list)
+    text: list[str] = field(default_factory=list)
+
+    def finish(self):
+        """Materializes the step in the shape the SDK's typed steps expose.
+
+        Returns ``None`` for a function call whose streamed arguments are not
+        valid JSON: a half-built call must not stand in for the real one, so
+        the step is dropped and the interaction is re-read by id instead.
+        """
+        if self.kind == "function_call":
+            arguments: dict[str, Any] = dict(self.seed_arguments)
+            raw = "".join(self.arguments_json)
+            if raw.strip():
+                try:
+                    parsed = json.loads(raw)
+                except ValueError:
+                    logger.warning(
+                        f"Could not parse streamed arguments for {self.name}: {raw[:200]!r};"
+                        " dropping the step"
+                    )
+                    return None
+                if isinstance(parsed, dict):
+                    arguments = parsed
+            return SimpleNamespace(
+                type="function_call", id=self.call_id, name=self.name, arguments=arguments
+            )
+        if self.kind == "model_output":
+            return SimpleNamespace(
+                type="model_output",
+                content=[SimpleNamespace(type="text", text="".join(self.text))],
+            )
+        return SimpleNamespace(type=self.kind, id=self.call_id)
+
+
+async def _consume_agentic_stream(analyzer, span, stream, trace_id):
+    """Consumes Interactions SSE events, rebuilding steps and mirroring progress.
+
+    Returns an interaction-shaped object (``id`` / ``status`` / ``steps`` /
+    ``usage``). The interaction is re-read by id when the stream ended
+    without a completion event, or when it stopped at ``requires_action``
+    without a parsable function call.
+    """
+    data_engine = analyzer.ctx.data_engine if trace_id else None
+    interaction_id: str | None = None
+    status: str | None = None
+    completed_steps: list | None = None
+    usage: dict[str, int] | None = None
+    pending: dict[int, _PendingStep] = {}
+    steps: list = []
+    processing_calls = 0
+
+    def _emit(text: str, *, thinking: bool) -> None:
+        if data_engine and text:
+            data_engine.stream_output(trace_id, text, is_thinking=thinking)
+
+    async for event in stream:
+        data = getattr(event, "data", event)
+        event_type = getattr(data, "event_type", None)
+
+        if event_type == "interaction.created":
+            created = getattr(data, "interaction", None)
+            interaction_id = getattr(created, "id", None) or interaction_id
+        elif event_type == "step.start":
+            step = getattr(data, "step", None)
+            index = getattr(data, "index", None)
+            kind = str(getattr(step, "type", None) or "")
+            pending[index] = _PendingStep(
+                kind=kind,
+                call_id=getattr(step, "id", None) or getattr(step, "call_id", None),
+                name=getattr(step, "name", None),
+                seed_arguments=dict(getattr(step, "arguments", None) or {}),
+            )
+            if kind == "processing_call":
+                processing_calls += 1
+                _emit(
+                    f"Inspecting the recording (video lookup {processing_calls})\n",
+                    thinking=True,
+                )
+        elif event_type == "step.delta":
+            delta = getattr(data, "delta", None)
+            delta_type = getattr(delta, "type", None)
+            current = pending.get(getattr(data, "index", None))
+            if delta_type == "arguments_delta":
+                fragment = getattr(delta, "arguments", None)
+                if current is not None and isinstance(fragment, str):
+                    current.arguments_json.append(fragment)
+            elif delta_type == "text":
+                text = getattr(delta, "text", None)
+                if current is not None and isinstance(text, str):
+                    current.text.append(text)
+                _emit(text or "", thinking=False)
+            elif delta_type == "thought_summary":
+                content = getattr(delta, "content", None)
+                _emit(getattr(content, "text", None) or "", thinking=True)
+        elif event_type == "step.stop":
+            current = pending.pop(getattr(data, "index", None), None)
+            finished = current.finish() if current is not None else None
+            if finished is not None:
+                steps.append(finished)
+            usage = normalize_usage(getattr(data, "usage", None)) or usage
+        elif event_type == "interaction.completed":
+            completed = getattr(data, "interaction", None)
+            interaction_id = getattr(completed, "id", None) or interaction_id
+            status = getattr(completed, "status", None) or status
+            completed_steps = getattr(completed, "steps", None) or None
+            usage = normalize_usage(getattr(completed, "usage", None)) or usage
+            _raise_if_failed(completed)
+        elif event_type == "interaction.status_update":
+            status = getattr(data, "status", None) or status
+        elif event_type == "error":
+            error = getattr(data, "error", None)
+            raise RuntimeError(
+                "Gemini Interactions stream error"
+                f" [{getattr(error, 'code', None)}]: {getattr(error, 'message', None)}"
+            )
+
+    for current in pending.values():
+        finished = current.finish()
+        if finished is not None:
+            steps.append(finished)
+    span.payload["processing_calls"] = processing_calls
+    if completed_steps:
+        steps = list(completed_steps)
+
+    needs_reread = status is None or (
+        status == "requires_action" and _find_function_call(SimpleNamespace(steps=steps)) is None
+    )
+    if needs_reread:
+        if not interaction_id:
+            raise RuntimeError("Gemini Interactions stream ended without an interaction")
+        logger.info(f"Re-reading interaction {interaction_id} (status={status})")
+        fetched = await asyncio.wait_for(
+            analyzer.client.aio.interactions.get(interaction_id),
+            timeout=INTERACTION_READ_TIMEOUT_SECONDS,
+        )
+        _raise_if_failed(fetched)
+        status = getattr(fetched, "status", None) or status
+        steps = _steps(fetched) or steps
+        usage = normalize_usage(getattr(fetched, "usage", None)) or usage
+
+    if usage:
+        span.payload["usage_metadata"] = usage
+    return SimpleNamespace(id=interaction_id, status=status, steps=steps, usage=usage)

artemis/agents/video_analyzer/chunk_conversation.py

@@ -27,13 +27,46 @@ import re
 
 from google.genai import types
 
+from artemis.agents.video_analyzer import gemini_files
 from artemis.agents.video_analyzer import video_analyzer as _va
+from artemis.agents.video_analyzer.reliability import SubAgentAnswerExhausted
 from artemis.constants import SAFETY_SETTINGS_BLOCK_NONE
 from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan
+from artemis.llm.google import normalize_usage
 from artemis.utils.logger import get_logger
 
 logger = get_logger(__name__)
 
+#: Turns the chunk sub-agent gets to produce an accepted ``submit_answer``
+#: (shared by the static and agentic paths).
+SUB_AGENT_MAX_TURNS = 2
+
+_SUBMIT_NOW_WARNING = "[WARNING] You must call the submit_answer tool now, time is running out."
+_WRONG_TOOL_ERROR = "Tool not recognized. Please use submit_answer."
+_NO_TOOL_CALL_ERROR = "The model replied with text only and never called submit_answer."
+_MISSING_SUMMARY_ERROR = (
+    "submit_answer was called without a 'summary'. Call it again with a non-empty"
+    " 'summary' (and 'analysis') describing what you observed."
+)
+
+
+def _tool_name(name: str | None) -> str:
+    name = name or ""
+    return name.split(":")[-1] if ":" in name else name
+
+
+def _clean_text(value) -> str:
+    return str(value or "").strip().replace("\n", " ")
+
+
+def _validate_answer(
+    summary: str, timeline_events: list, start_time: float, end_time: float | None
+) -> str | None:
+    """Return a validation error for submit_answer, or None if valid."""
+    if not summary:
+        return _MISSING_SUMMARY_ERROR
+    return _validate_timeline_events(timeline_events, start_time, end_time)
+
 
 class _ChunkMedia:
     """Mutable holder for per-attempt media artifacts shared with failure handling."""
@@ -120,15 +153,9 @@ async def _run_native_chunk_conversation(
         return chunk_result
     finally:
         if file:
-            logger.info(f"Cleaning up cloud file {file.name}...")
-            try:
-                await asyncio.wait_for(
-                    analyzer.client.aio.files.delete(name=file.name),
-                    timeout=30,
+            await gemini_files.delete_cloud_file(
+                analyzer.client, file.name, analyzer.cloud_files_to_cleanup
             )
-                analyzer.cloud_files_to_cleanup.discard(file.name)
-            except Exception as ce:
-                logger.error(f"Failed to delete cloud file {file.name}: {ce}")
 
 
 async def _drive_sub_agent_loop(
@@ -140,38 +167,27 @@ async def _drive_sub_agent_loop(
     end_time: float | None,
     specific_query: str,
 ) -> tuple[str, str]:
-    """Iterates the sub-agent conversation until submit_answer is accepted."""
-    sub_max_iterations = 2
-    sub_iterations = 0
-    final_analysis = "No analysis provided."
-    final_summary = "No summary provided."
-    final_full_text = ""
+    """Return a validated answer and persist its timeline events.
 
-    while sub_iterations < sub_max_iterations:
-        sub_iterations += 1
-        if sub_iterations == sub_max_iterations - 1:
+    Raise ``SubAgentAnswerExhausted`` if no answer is accepted within the turn limit.
+    """
+    rejection = _NO_TOOL_CALL_ERROR
+
+    for turn in range(1, SUB_AGENT_MAX_TURNS + 1):
+        if turn == SUB_AGENT_MAX_TURNS - 1:
             sub_agent_contents.append(
                 types.Content(
                     role="user",
-                    parts=[
-                        types.Part.from_text(
-                            text=(
-                                "[WARNING] You must call the"
-                                " submit_answer tool now, time"
-                                " is running out."
-                            )
-                        )
-                    ],
+                    parts=[types.Part.from_text(text=_SUBMIT_NOW_WARNING)],
                 )
             )
 
         full_text, function_calls, original_parts = await _stream_sub_agent_turn(
             analyzer, sub_agent_contents, trace_id
         )
-        final_full_text += full_text + "\n"
 
         if function_calls:
-            answered, final_summary, final_analysis = await _handle_sub_agent_calls(
+            answer, rejection = await _handle_sub_agent_calls(
                 analyzer,
                 sub_agent_contents,
                 function_calls,
@@ -181,12 +197,11 @@ async def _drive_sub_agent_loop(
                 start_time,
                 end_time,
                 specific_query,
-                final_summary,
-                final_analysis,
             )
-            if answered is True:
-                break
+            if answer is not None:
+                return answer
         else:
+            rejection = _NO_TOOL_CALL_ERROR
             sub_agent_contents.append(
                 types.Content(
                     role="model",
@@ -194,7 +209,7 @@ async def _drive_sub_agent_loop(
                 )
             )
 
-    return final_summary, final_analysis
+    raise SubAgentAnswerExhausted(SUB_AGENT_MAX_TURNS, rejection)
 
 
 async def _stream_sub_agent_turn(
@@ -228,24 +243,9 @@ async def _read_sub_agent_stream(analyzer, span, target_stream, current_trace_id
     function_calls = []
     accumulated_parts = []
     async for chunk in target_stream:
-        if hasattr(chunk, "usage_metadata") and chunk.usage_metadata:
-            span.payload["usage_metadata"] = {
-                "prompt_token_count": getattr(
-                    chunk.usage_metadata,
-                    "prompt_token_count",
-                    0,
-                ),
-                "candidates_token_count": getattr(
-                    chunk.usage_metadata,
-                    "candidates_token_count",
-                    0,
-                ),
-                "total_token_count": getattr(
-                    chunk.usage_metadata,
-                    "total_token_count",
-                    0,
-                ),
-            }
+        usage = normalize_usage(getattr(chunk, "usage_metadata", None))
+        if usage:
+            span.payload["usage_metadata"] = usage
 
         chunk_text = chunk.text or ""
         if chunk_text:
@@ -287,42 +287,33 @@ async def _handle_sub_agent_calls(
     start_time: float,
     end_time: float | None,
     specific_query: str,
-    final_summary: str,
-    final_analysis: str,
-):
-    """Processes function calls from one turn; returns (answered, summary, analysis)."""
-    answered = False
-    for fc in function_calls:
-        if (fc.name.split(":")[-1] if ":" in fc.name else fc.name) == "submit_answer":
-            args = fc.args
-            timeline_events = args.get("timeline_events", [])
-            final_summary = args.get("summary", "No summary provided.").strip().replace("\n", " ")
-            final_analysis = (
-                args.get("analysis", "No analysis provided.").strip().replace("\n", " ")
-            )
+) -> tuple[tuple[str, str] | None, str | None]:
+    """Return ``(answer, rejection)`` after processing one turn's tool calls.
 
-            error_msg = _validate_timeline_events(timeline_events, start_time, end_time)
-            if error_msg is not None:
+    Persist events for accepted answers; append rejected calls to the conversation
+    with an error for the model to correct.
+    """
+    for fc in function_calls:
+        if _tool_name(fc.name) != "submit_answer":
+            continue
+        args = dict(fc.args or {})
+        timeline_events = args.get("timeline_events", [])
+        if not isinstance(timeline_events, list):
+            timeline_events = []
+        summary = _clean_text(args.get("summary"))
+        analysis = _clean_text(args.get("analysis"))
+
+        rejection = _validate_answer(summary, timeline_events, start_time, end_time)
+        if rejection is not None:
             _append_rejection(
-                    sub_agent_contents,
-                    original_parts,
-                    function_calls,
-                    full_text,
-                    fc.name,
-                    error_msg,
+                sub_agent_contents, original_parts, function_calls, full_text, fc.name, rejection
             )
-                answered = "error"
-                break
+            return None, rejection
 
         await _persist_timeline_events(
             analyzer, timeline_events, media, start_time, end_time, specific_query
         )
-
-            answered = True
-            break
-
-    if answered is True or answered == "error":
-        return answered, final_summary, final_analysis
+        return (summary, analysis or "No analysis provided."), None
 
     _append_rejection(
         sub_agent_contents,
@@ -330,9 +321,9 @@ async def _handle_sub_agent_calls(
         function_calls,
         full_text,
         function_calls[0].name,
-        ("Tool not recognized. Please use submit_answer."),
+        _WRONG_TOOL_ERROR,
     )
-    return answered, final_summary, final_analysis
+    return None, _WRONG_TOOL_ERROR
 
 
 def _validate_timeline_events(

artemis/agents/video_analyzer/chunk_native.py

@@ -26,11 +26,16 @@ import asyncio
 from pathlib import Path
 
 from artemis.agents.video_analyzer import video_analyzer as _va
+from artemis.agents.video_analyzer.chunk_agentic import run_agentic_chunk_conversation
 from artemis.agents.video_analyzer.chunk_conversation import (
     _ChunkMedia,
     _run_native_chunk_conversation,
 )
-from artemis.agents.video_analyzer.reliability import classify_video_failure
+from artemis.agents.video_analyzer.reliability import (
+    AgenticVideoDegraded,
+    classify_video_failure,
+    is_agentic_rejection,
+)
 from artemis.llm.reliability import retry_policy_for
 from artemis.utils.logger import get_logger
 
@@ -76,12 +81,19 @@ async def exec_single_chunk(
 
     media = _ChunkMedia()
 
-    for attempt in range(max_retries + 1):
+    attempt = 0
+    while attempt <= max_retries:
+        # The mode this attempt was launched in: a parallel chunk may degrade
+        # the run while this one is still in flight.
+        attempt_mode = analyzer.video_processing
         try:
             logger.info(
                 f"Attempt {attempt + 1}/{max_retries + 1} for sub-agent:"
                 f" start={current_start}, end={current_end}"
             )
+            # A sibling may switch the run to static during queuing or media
+            # preparation. Check the size before preparation and before dispatch.
+            _ensure_fits_static_ceiling(analyzer, current_start, current_end)
             result, slowdown_factor = await _prepare_chunk_media(
                 analyzer, media, current_start, current_end
             )
@@ -94,6 +106,7 @@ async def exec_single_chunk(
                 end_time,
                 specific_query,
             )
+            _ensure_fits_static_ceiling(analyzer, current_start, current_end)
 
             if not analyzer.use_native_gemini:
                 chunk_result = await analyzer._exec_single_chunk_universal(
@@ -116,6 +129,18 @@ async def exec_single_chunk(
                 )
                 return chunk_result
 
+            if analyzer.video_processing == "agentic":
+                return await run_agentic_chunk_conversation(
+                    analyzer,
+                    media,
+                    current_start,
+                    current_end,
+                    start_time,
+                    end_time,
+                    specific_query,
+                    lease_owner,
+                )
+
             return await _run_native_chunk_conversation(
                 analyzer,
                 media,
@@ -134,6 +159,22 @@ async def exec_single_chunk(
             # stream output from the failed attempt is discarded, never
             # surfaced as a complete answer.
             failure = classify_video_failure(e)
+            if attempt_mode == "agentic" and is_agentic_rejection(failure, e):
+                # Switch to static and let the coordinator resize the interval.
+                _degrade_agentic_to_static(analyzer, e, failure)
+                return await _finish_failed_chunk(
+                    analyzer,
+                    media,
+                    current_start,
+                    current_end,
+                    specific_query,
+                    lease_owner,
+                    AgenticVideoDegraded(
+                        f"Agentic video processing rejected ({e}); re-plan"
+                        f" {current_start:.1f}s-{current_end:.1f}s at the static chunk size"
+                    ),
+                    retryable=False,
+                )
             attempts_so_far = attempt + 1
             if not failure.retryable:
                 logger.warning(
@@ -181,12 +222,51 @@ async def exec_single_chunk(
                 source="video_sub_agent",
             )
             await asyncio.sleep(delay)
+            attempt += 1
 
     raise RuntimeError(
         f"Video sub-agent exhausted retries for {current_start:.1f}s-{current_end:.1f}s"
     )
 
 
+def _degrade_agentic_to_static(analyzer, error: Exception, failure) -> None:
+    """Switches the rest of this run to static video understanding (idempotent)."""
+    if analyzer.video_processing != "agentic":
+        return
+    logger.warning(
+        f"Agentic video processing rejected ({error}); the rest of this run"
+        " analyzes video in static mode"
+    )
+    _va._record_llm_event(
+        "llm_fallback",
+        {
+            "reason": "agentic_video_rejected",
+            "category": failure.category.value,
+            "error": str(error)[:500],
+        },
+    )
+    analyzer.video_processing = "static"
+    analyzer.chunk_size_seconds = analyzer.static_chunk_size_seconds
+
+
+def _ensure_fits_static_ceiling(analyzer, current_start: float, current_end: float) -> None:
+    """Refuses to analyze a clip sized for the agentic ceiling once the run is static.
+
+    Raises :class:`AgenticVideoDegraded`, which the attempt loop treats as a
+    non-retryable failure: the lease is released via ``fail_segment`` and the
+    coordinator re-plans the interval into static-sized pieces.
+    """
+    if analyzer.video_processing == "agentic":
+        return
+    if current_end - current_start <= analyzer.chunk_size_seconds + 1e-3:
+        return
+    raise AgenticVideoDegraded(
+        f"Run degraded to static video processing while {current_start:.1f}s-{current_end:.1f}s"
+        f" was pending; re-plan it at the static chunk size"
+        f" ({analyzer.chunk_size_seconds:.0f}s)"
+    )
+
+
 async def _finish_failed_chunk(
     analyzer,
     media: _ChunkMedia,

artemis/agents/video_analyzer/gemini_files.py

@@ -97,6 +97,18 @@ async def cleanup_abandoned_gemini_files(client) -> None:
         logger.warning(f"Routine cloud maintenance skipped: {e}")
 
 
+async def delete_cloud_file(
+    client, file_name: str, cloud_files_to_cleanup: set, *, timeout: float = 30.0
+) -> None:
+    """Best-effort deletion of an uploaded file; failures are logged, never raised."""
+    logger.info(f"Cleaning up cloud file {file_name}...")
+    try:
+        await asyncio.wait_for(client.aio.files.delete(name=file_name), timeout=timeout)
+        cloud_files_to_cleanup.discard(file_name)
+    except Exception as ce:
+        logger.error(f"Failed to delete cloud file {file_name}: {ce}")
+
+
 async def upload_and_poll_file(client, compressed_path: Path, cloud_files_to_cleanup: set) -> any:
     """Upload a media file to the Gemini File API and poll until it is ACTIVE."""
     logger.info(f"Uploading {compressed_path} to Gemini File API...")

artemis/agents/video_analyzer/native_coordinator.py

@@ -32,6 +32,7 @@ from artemis.agents.video_analyzer import video_analyzer as _va
 from artemis.agents.video_analyzer.reliability import classify_video_failure
 from artemis.constants import SAFETY_SETTINGS_BLOCK_NONE
 from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan
+from artemis.llm.google import normalize_usage
 from artemis.utils.logger import get_logger
 
 logger = get_logger(__name__)
@@ -349,24 +350,9 @@ async def _consume_main_stream(
     function_calls = []
 
     async for chunk in stream:
-        if hasattr(chunk, "usage_metadata") and chunk.usage_metadata:
-            span.payload["usage_metadata"] = {
-                "prompt_token_count": getattr(
-                    chunk.usage_metadata,
-                    "prompt_token_count",
-                    0,
-                ),
-                "candidates_token_count": getattr(
-                    chunk.usage_metadata,
-                    "candidates_token_count",
-                    0,
-                ),
-                "total_token_count": getattr(
-                    chunk.usage_metadata,
-                    "total_token_count",
-                    0,
-                ),
-            }
+        usage = normalize_usage(getattr(chunk, "usage_metadata", None))
+        if usage:
+            span.payload["usage_metadata"] = usage
 
         chunk_text = chunk.text or ""
         text += chunk_text

artemis/agents/video_analyzer/reliability.py

@@ -53,10 +53,50 @@ class VideoFailure:
 
 
 _MEDIA_MARKERS = ("ffmpeg", "codec", "corrupt", "invalid video", "invalid audio", "media")
+_AGENTIC_REJECTION_MARKERS = ("processing", "agentic")
+
+
+class SubAgentAnswerExhausted(RuntimeError):
+    """The sub-agent exhausted its turns without a valid submit_answer.
+
+    The caller records a failed segment so a later claim can retry it. The shared
+    classifier permits retries, splitting, and fallback for this failure.
+    """
+
+    def __init__(self, turns: int, reason: str):
+        self.turns = turns
+        self.reason = reason
+        super().__init__(
+            f"submit_answer was not accepted within {turns} turn(s); last rejection: {reason}"
+        )
+
+
+class AgenticVideoDegraded(RuntimeError):
+    """Return an interval to the coordinator after switching to static mode.
+
+    The coordinator splits oversized clips and retries smaller clips as-is.
+    """
+
+
+def is_agentic_rejection(failure: VideoFailure, error: BaseException) -> bool:
+    """True when a bad request is the API refusing agentic video processing.
+
+    Only an HTTP 400 whose message mentions ``processing`` / ``agentic``
+    qualifies; other bad requests (413 payload too large, 404, ...) keep the
+    shared handling.
+    """
+    if failure.category is not VideoFailureCategory.BAD_REQUEST:
+        return False
+    if extract_status_code(error) != 400:
+        return False
+    message = str(error).lower()
+    return any(marker in message for marker in _AGENTIC_REJECTION_MARKERS)
 
 
 def classify_video_failure(error: BaseException) -> VideoFailure:
     """Convert provider/transport/media exceptions into recovery decisions."""
+    if isinstance(error, AgenticVideoDegraded):
+        return VideoFailure(VideoFailureCategory.BAD_REQUEST, False, True, False)
     generic = classify_failure(error)
     message = str(error).lower()
     has_media_marker = any(marker in message for marker in _MEDIA_MARKERS)

artemis/agents/video_analyzer/universal_engine.py

@@ -36,7 +36,10 @@ from langchain_core.messages import (
 )
 
 from artemis.agents.video_analyzer import video_analyzer as _va
-from artemis.agents.video_analyzer.reliability import classify_video_failure
+from artemis.agents.video_analyzer.reliability import (
+    SubAgentAnswerExhausted,
+    classify_video_failure,
+)
 from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan
 from artemis.utils.logger import get_logger
 
@@ -218,29 +221,33 @@ async def _invoke_universal_with_visual_retry(
 
 
 def _extract_submit_payload(response) -> tuple[str, str, list]:
-    """Pulls summary/analysis/timeline events from a universal model response."""
-    final_summary = "No summary provided."
-    final_analysis = "No analysis provided."
-    timeline_events = []
+    """Extract a summary, analysis, and events from the model response.
 
-    if getattr(response, "tool_calls", None):
-        for tc in response.tool_calls:
-            if tc.get("name") == "submit_answer":
-                args = tc.get("args", {})
-                final_summary = (
-                    str(args.get("summary", "")).strip().replace("\n", " ") or final_summary
-                )
-                final_analysis = (
-                    str(args.get("analysis", "")).strip().replace("\n", " ") or final_analysis
-                )
-                timeline_events = args.get("timeline_events", [])
+    Use the first submit_answer call if its summary is non-empty. Otherwise,
+    fall back to response text, using its first 200 characters as the summary
+    and dropping any events from the rejected call. Raise SubAgentAnswerExhausted
+    if neither source provides an answer.
+    """
+    for tc in getattr(response, "tool_calls", None) or []:
+        if tc.get("name") != "submit_answer":
+            continue
+        args = tc.get("args") or {}
+        final_summary = str(args.get("summary", "")).strip().replace("\n", " ")
+        if not final_summary:
             break
-    elif getattr(response, "content", None):
-        text_content = str(response.content).strip()
-        final_summary = text_content[:200].replace("\n", " ")
-        final_analysis = text_content
+        final_analysis = (
+            str(args.get("analysis", "")).strip().replace("\n", " ") or "No analysis provided."
+        )
+        return final_summary, final_analysis, list(args.get("timeline_events") or [])
 
-    return final_summary, final_analysis, timeline_events
+    raw_content = getattr(response, "content", None)
+    text_content = str(raw_content).strip() if raw_content else ""
+    if text_content:
+        return text_content[:200].replace("\n", " "), text_content, []
+
+    raise SubAgentAnswerExhausted(
+        1, "the model neither called submit_answer with a summary nor replied with text"
+    )
 
 
 async def _persist_universal_events(
@@ -361,23 +368,9 @@ async def exec_analyze_audio_universal(
         label="Universal audio sub-agent",
         force_fallback=force_fallback,
     )
-    final_summary = "No audio summary provided."
-    final_analysis = "No audio analysis provided."
-
-    if getattr(response, "tool_calls", None):
-        for tc in response.tool_calls:
-            if tc.get("name") == "submit_answer":
-                args = tc.get("args", {})
-                final_summary = (
-                    str(args.get("summary", "")).strip().replace("\n", " ") or final_summary
-                )
-                final_analysis = (
-                    str(args.get("analysis", "")).strip().replace("\n", " ") or final_analysis
-                )
-                break
-    elif getattr(response, "content", None):
-        final_summary = str(response.content)[:200].replace("\n", " ")
-        final_analysis = str(response.content)
+    # Same resolution order as the video chunk (raises SubAgentAnswerExhausted
+    # when the model produced nothing usable); audio has no timeline events.
+    final_summary, final_analysis, _ = _extract_submit_payload(response)
 
     end_val = end_time if end_time is not None else "unknown"
     analyzer._record_blackboard_entry(

artemis/agents/video_analyzer/video_analyzer.py

@@ -53,6 +53,7 @@ from artemis.agents.video_analyzer.gemini_files import (
     cleanup_abandoned_gemini_files,
 )
 from artemis.agents.video_analyzer.reliability import (
+    AgenticVideoDegraded,
     VideoCircuitBreaker,
     classify_video_failure,
 )
@@ -67,6 +68,11 @@ from artemis.constants import SAFETY_SETTINGS_BLOCK_NONE
 from artemis.context import ArtemisContext
 from artemis.controllers.controller_factory import get_controller
 from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan, trace
+from artemis.llm.google import (
+    is_gemini_model,
+    resolve_video_processing,
+    strip_provider_prefix,
+)
 from artemis.llm.reliability import retry_policy_for
 from artemis.services.llm import _record_llm_event, _record_llm_retry, get_llm
 from artemis.utils.logger import get_logger
@@ -152,7 +158,19 @@ class VideoAnalyzer:
             "enable_ledger",
             getattr(agent_config, "enable_video_ledger", True),
         )
-        self.chunk_size_seconds = float(getattr(self.video_config, "chunk_size_seconds", 60.0))
+        self.static_chunk_size_seconds = float(
+            getattr(self.video_config, "chunk_size_seconds", 60.0)
+        )
+        self.agentic_chunk_size_seconds = float(
+            getattr(self.video_config, "agentic_chunk_size_seconds", 600.0)
+        )
+        self.agentic_call_timeout_seconds = float(
+            getattr(self.video_config, "agentic_call_timeout_seconds", 300.0)
+        )
+        # Effective chunk ceiling; re-derived once the engine picks a
+        # video-processing mode (agentic sub-agents take long clips).
+        self.chunk_size_seconds = self.static_chunk_size_seconds
+        self.video_processing = "static"
         self.min_chunk_seconds = float(getattr(self.video_config, "min_chunk_seconds", 4.0))
         self.max_split_depth = int(getattr(self.video_config, "max_split_depth", 4))
         self.action_window_seconds = float(getattr(self.video_config, "action_window_seconds", 2.0))
@@ -399,25 +417,19 @@ class VideoAnalyzer:
         llm_config = getattr(ctx, "llm_config", None)
         utils_cfg = getattr(llm_config, "utils", None) if llm_config else None
         llm_cfg = getattr(utils_cfg, "video_analyzer", None) if utils_cfg else None
-        model_str = (
-            llm_cfg.model if (llm_cfg and hasattr(llm_cfg, "model")) else "gemini-3.8-flash"
-        ).lower()
-        self.model_name = (
+        self.model_name = strip_provider_prefix(
             llm_cfg.model if (llm_cfg and hasattr(llm_cfg, "model")) else "gemini-3.8-flash"
         )
-        if "/" in self.model_name:
-            self.model_name = self.model_name.split("/")[-1]
 
         has_google_key = bool(
             settings.GOOGLE_API_KEY and settings.GOOGLE_API_KEY.get_secret_value()
         )
-        is_gemini_model = "gemini" in model_str
 
         # If client is already set on ctx or explicit Gemini configuration
         self.client = getattr(ctx, "_genai_client", None)
         if self.client is not None:
             self.use_native_gemini = True
-        elif has_google_key and is_gemini_model:
+        elif has_google_key and is_gemini_model(self.model_name):
             try:
                 self.client = genai.Client(api_key=settings.GOOGLE_API_KEY.get_secret_value())
                 ctx._genai_client = self.client
@@ -430,6 +442,29 @@ class VideoAnalyzer:
                 self.use_native_gemini = False
         else:
             self.use_native_gemini = False
+        self._resolve_video_processing()
+
+    def _resolve_video_processing(self) -> None:
+        """Picks static vs agentic video understanding for the sub-agents.
+
+        Agentic processing (Gemini Interactions API) needs the native client
+        and a supporting model; the configured knob (``auto`` / ``agentic`` /
+        ``static``) decides the rest. The chunk ceiling follows the mode
+        because agentic sub-agents are built to take long clips.
+        """
+        configured = getattr(self.video_config, "processing", "auto")
+        mode = "static"
+        if self.use_native_gemini:
+            mode = resolve_video_processing(configured, self.model_name)
+        self.video_processing = mode
+        self.chunk_size_seconds = (
+            self.agentic_chunk_size_seconds if mode == "agentic" else self.static_chunk_size_seconds
+        )
+        if mode == "agentic":
+            logger.info(
+                f"Video sub-agents use agentic video understanding on {self.model_name}"
+                f" (chunk ceiling {self.chunk_size_seconds:.0f}s)"
+            )
 
     @property
     def blackboard_ledger(self) -> str:
@@ -565,6 +600,18 @@ class VideoAnalyzer:
             self.local_dirs_to_cleanup.add(metadata_path.parent)
         return requested_start, requested_end, None
 
+    def _plan_chunks(self, intervals: list[tuple[float, float]]) -> list[tuple[float, float]]:
+        """Cuts intervals into consecutive chunks of at most ``chunk_size_seconds``."""
+        target_chunk_size = self.chunk_size_seconds
+        chunks: list[tuple[float, float]] = []
+        for gap_start, gap_end in intervals:
+            cursor = gap_start
+            while cursor < gap_end:
+                chunk_end = min(gap_end, cursor + target_chunk_size)
+                chunks.append((round(cursor, 3), round(chunk_end, 3)))
+                cursor = chunk_end
+        return chunks
+
     @trace(type="tool", name="spawn_sub_agent")
     async def exec_spawn_sub_agent(
         self, start_time: float, end_time: float | None, specific_query: str
@@ -600,21 +647,14 @@ class VideoAnalyzer:
             )
             return "CACHED VIDEO ANALYSIS: " + " ".join(cached_results)
 
-        target_chunk_size = self.chunk_size_seconds
-        chunks: list[tuple[float, float]] = []
-        for gap_start, gap_end in missing:
-            cursor = gap_start
-            while cursor < gap_end:
-                chunk_end = min(gap_end, cursor + target_chunk_size)
-                chunks.append((round(cursor, 3), round(chunk_end, 3)))
-                cursor = chunk_end
+        chunks = self._plan_chunks(missing)
         logger.info(
             f"Video blackboard planned {len(chunks)} uncovered chunk(s) for "
             f"{requested_start:.1f}s-{requested_end:.1f}s"
         )
 
         async def analyze_with_recovery(
-            cs: float, ce: float, depth: int = 0
+            cs: float, ce: float, depth: int = 0, *, retried_static: bool = False
         ) -> tuple[list[str], list[str]]:
             try:
                 async with API_SEMAPHORE:
@@ -623,8 +663,40 @@ class VideoAnalyzer:
             except Exception as error:
                 failure = classify_video_failure(error)
                 duration = ce - cs
+                fits_ceiling = duration <= self.chunk_size_seconds + 1e-3
+                degraded = isinstance(error, AgenticVideoDegraded)
+                if degraded and fits_ceiling and not retried_static:
+                    # Retry short intervals once in static mode without splitting.
+                    logger.warning(
+                        f"Retrying {cs:.1f}s-{ce:.1f}s in static mode: it already fits"
+                        f" the {self.chunk_size_seconds:.0f}s chunk ceiling"
+                    )
+                    return await analyze_with_recovery(cs, ce, depth, retried_static=True)
+                pieces = (
+                    self._plan_chunks([(cs, ce)])
+                    if failure.should_split and not fits_ceiling
+                    else []
+                )
+                if len(pieces) > 1:
+                    # Split clips planned before a downgrade at the new ceiling.
+                    logger.warning(
+                        f"Re-planning failed {cs:.1f}s-{ce:.1f}s chunk into {len(pieces)}"
+                        f" piece(s) of at most {self.chunk_size_seconds:.0f}s"
+                        f" after {failure.category.value}"
+                    )
+                    parts = await asyncio.gather(
+                        *(analyze_with_recovery(ps, pe, depth) for ps, pe in pieces)
+                    )
+                    return (
+                        [item for values, _ in parts for item in values],
+                        [item for _, failures in parts for item in failures],
+                    )
+                # Bisecting only helps genuine size/timeout failures; a
+                # degraded interval that fits the ceiling has had its one
+                # static retry and is terminal.
                 can_split = (
                     failure.should_split
+                    and not degraded
                     and depth < self.max_split_depth
                     and duration >= self.min_chunk_seconds * 2
                 )
@@ -767,9 +839,7 @@ class VideoAnalyzer:
             temperature = 0.2
             thinking_level = None
             if llm_cfg:
-                self.model_name = llm_cfg.model
-                if "/" in self.model_name:
-                    self.model_name = self.model_name.split("/")[-1]
+                self.model_name = strip_provider_prefix(llm_cfg.model)
                 if getattr(llm_cfg, "temperature", None) is not None:
                     temperature = llm_cfg.temperature
                 if getattr(llm_cfg, "thinking_level", None) is not None:

artemis/config/agent.py

@@ -26,6 +26,7 @@ from artemis.config.constants import (
     ExplorerVersion,
 )
 from artemis.config.paths import ROOT_DIR, get_config_path
+from artemis.llm.google import VideoProcessing
 from artemis.utils.file import load_jsonc
 from artemis.utils.logger import get_logger
 
@@ -109,6 +110,30 @@ class VideoAnalyzerConfig(BaseModel):
         le=600.0,
         description="Hard timeout for one native or universal video-model response.",
     )
+    processing: VideoProcessing = Field(
+        default="auto",
+        description=(
+            "How Gemini sub-agents watch a clip. 'agentic' lets the model search,"
+            " scan and re-watch the clip on demand (Gemini Interactions API);"
+            " 'static' samples it at a fixed frame rate; 'auto' picks agentic on"
+            " Gemini 3.6+ Flash models and static everywhere else."
+        ),
+    )
+    agentic_chunk_size_seconds: float = Field(
+        default=600.0,
+        ge=30.0,
+        le=1800.0,
+        description=(
+            "Maximum clip length handed to one agentic sub-agent. Static mode keeps"
+            " using chunk_size_seconds."
+        ),
+    )
+    agentic_call_timeout_seconds: float = Field(
+        default=300.0,
+        ge=30.0,
+        le=1200.0,
+        description="Hard timeout for one agentic sub-agent response.",
+    )
     model_config = {"extra": "allow"}
 
 

artemis/config/settings.py

@@ -186,9 +186,11 @@ class Settings(BaseSettings):
         Returns:
             SecretStr containing the API key or None if not configured.
         """
+        from artemis.llm.google.provider import is_google_family_provider
+
         provider_lower = provider.lower()
         key: SecretStr | None = None
-        if provider_lower in ("google", "gemini", "vertexai"):
+        if is_google_family_provider(provider_lower):
             key = self.GOOGLE_API_KEY or self.GEMINI_API_KEY or self.GCP_API_KEY
         elif provider_lower in ("ocr", "vision", "google_vision"):
             key = self.OCR_API_KEY or self.VISION_API_KEY
@@ -214,10 +216,12 @@ class Settings(BaseSettings):
             persist_to_env: Whether to save the key to the app directory's .env file.
         """
         secret = SecretStr(key)
+        from artemis.llm.google.provider import is_google_family_provider
+
         provider_lower = provider.lower()
         env_key_name = None
 
-        if provider_lower in ("google", "gemini", "vertexai"):
+        if is_google_family_provider(provider_lower):
             self.GOOGLE_API_KEY = secret
             self.GEMINI_API_KEY = secret
             self.GCP_API_KEY = secret
@@ -265,7 +269,7 @@ class Settings(BaseSettings):
 
                     # Determine keys to update
                     keys_to_update = [env_key_name]
-                    if provider_lower in ("google", "gemini", "vertexai"):
+                    if is_google_family_provider(provider_lower):
                         keys_to_update = [ENV_GEMINI_API_KEY, ENV_GOOGLE_API_KEY, ENV_GCP_API_KEY]
                     elif provider_lower in ("ocr", "vision", "google_vision"):
                         keys_to_update = [ENV_OCR_API_KEY, ENV_VISION_API_KEY]

artemis/core/tool_failure.py

@@ -0,0 +1,56 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Typed failures for text-returning helper tools.
+
+Tools return ``ToolFailure`` on failure; callers use ``is_tool_failure`` to
+check the result. The str subclass keeps results compatible with text consumers
+without treating words such as "Error" in tool output as a failure signal.
+
+Pydantic converts str subclasses to plain strings in ``ToolMessage.content``
+and ``ToolExecutionResult.text_summary``. Check the raw result before wrapping
+it and pass ``status="error"`` to preserve the failure state.
+"""
+
+from typing import Any
+
+from langchain_core.messages import ToolMessage
+
+
+class ToolFailure(str):
+    """A helper tool's failure report, carried as the text the agent reads."""
+
+    __slots__ = ()
+
+
+def is_tool_failure(result: Any) -> bool:
+    """Whether a tool result reports failure.
+
+    True for a ``ToolMessage`` whose ``status`` is ``"error"``, for a
+    :class:`ToolFailure` text, and for a content-block list whose text is a
+    :class:`ToolFailure`. Plain strings are never failures, whatever they say.
+    """
+    if isinstance(result, ToolMessage):
+        if result.status == "error":
+            return True
+        result = result.content
+    if isinstance(result, ToolFailure):
+        return True
+    if isinstance(result, list):
+        for block in result:
+            if isinstance(block, ToolFailure):
+                return True
+            if isinstance(block, dict) and isinstance(block.get("text"), ToolFailure):
+                return True
+    return False

artemis/llm/google/__init__.py

@@ -0,0 +1,57 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Google / Gemini specific knowledge shared by every call site.
+
+Two concerns live here so they are defined exactly once:
+
+* :mod:`artemis.llm.google.provider` - "is this endpoint Google?" and the
+  Gemini model capability table (thinking levels, agentic video).
+* :mod:`artemis.llm.google.usage` - token-usage normalization across the
+  LangChain, ``generate_content`` and Interactions API response shapes.
+
+Both modules are dependency-free (no settings, no SDK client) so they can be
+imported from configuration code without creating import cycles.
+"""
+
+from artemis.llm.google.provider import (
+    VideoProcessing,
+    gemini_version,
+    is_agentic_video_auto_eligible,
+    is_gemini_model,
+    is_google_chat_model,
+    is_google_family_provider,
+    is_google_provider,
+    resolve_video_processing,
+    strip_provider_prefix,
+    supports_agentic_video,
+    supports_thinking_level,
+)
+from artemis.llm.google.usage import normalize_usage, usage_from_message
+
+__all__ = [
+    "VideoProcessing",
+    "gemini_version",
+    "is_agentic_video_auto_eligible",
+    "is_gemini_model",
+    "is_google_chat_model",
+    "is_google_family_provider",
+    "is_google_provider",
+    "normalize_usage",
+    "resolve_video_processing",
+    "strip_provider_prefix",
+    "supports_agentic_video",
+    "supports_thinking_level",
+    "usage_from_message",
+]

artemis/llm/google/provider.py

@@ -0,0 +1,185 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Google provider identity and the Gemini model capability table.
+
+Provider identity has two tiers:
+
+* :func:`is_google_provider` - the Gemini Developer API (``google`` /
+  ``gemini``). Native ``google-genai`` features (Files API, explicit caching,
+  the Interactions API) are only reachable here.
+* :func:`is_google_family_provider` - additionally Vertex AI, which shares
+  Google credentials but not the developer-API-only features.
+
+Model IDs use ``gemini-<major>.<minor>-<tier>[<suffix>]``. Release suffixes
+(``preview``, ``latest``, ``exp``, and numeric tokens) retain base-family
+capabilities. Variants such as ``image`` and ``tts`` are excluded from agentic
+video support, but their version still determines thinking-level support.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Literal, NamedTuple
+
+VideoProcessing = Literal["auto", "agentic", "static"]
+ResolvedVideoProcessing = Literal["agentic", "static"]
+
+_GOOGLE_API_PROVIDERS = frozenset({"google", "gemini"})
+_GOOGLE_FAMILY_PROVIDERS = _GOOGLE_API_PROVIDERS | frozenset({"vertexai", "vertex"})
+
+_GEMINI_MODEL_RE = re.compile(r"gemini-(\d+)\.(\d+)-(flash-lite|flash|pro)", re.IGNORECASE)
+_RELEASE_SUFFIX_TOKENS = frozenset({"preview", "latest", "exp"})
+
+
+class _GeminiModel(NamedTuple):
+    version: tuple[int, int]
+    tier: str
+    #: False for variant ids (``-image``, ``-tts``, ...) that carry their own
+    #: capabilities and must not inherit the family's answers.
+    is_base: bool
+
+
+# Agentic video understanding (Interactions API, ``processing: "agentic"``)
+# launched on Gemini 3.6/3.7/3.8 Flash and 3.5 Flash-Lite. Later releases in
+# the same families are assumed to keep it.
+_AGENTIC_VIDEO_MIN_FLASH = (3, 6)
+_AGENTIC_VIDEO_MIN_FLASH_LITE = (3, 5)
+
+
+def _normalize_provider(value: Any) -> str:
+    if value is None:
+        return ""
+    value = getattr(value, "value", value)
+    return str(value).lower().replace("_", "").replace("-", "").strip()
+
+
+def is_google_provider(value: Any) -> bool:
+    """True for the Gemini Developer API (``google`` / ``gemini``).
+
+    Accepts a provider name, a provider enum, or anything exposing ``.value``.
+    """
+    return _normalize_provider(value) in _GOOGLE_API_PROVIDERS
+
+
+def is_google_family_provider(value: Any) -> bool:
+    """True for every provider that authenticates with Google credentials."""
+    return _normalize_provider(value) in _GOOGLE_FAMILY_PROVIDERS
+
+
+def is_google_chat_model(model: Any) -> bool:
+    """True when a LangChain chat model is backed by Google (by class name).
+
+    Only used when no endpoint metadata is available; the class-name sniff is
+    the historical fallback and is kept in one place here.
+    """
+    cls = getattr(model, "__class__", None)
+    name = cls.__name__ if cls else ""
+    return "Google" in name or "Gemini" in name or "VertexAI" in name
+
+
+def strip_provider_prefix(model_name: Any) -> str:
+    """Drops a ``provider/`` prefix (``google/gemini-3.8-flash`` -> ``gemini-3.8-flash``)."""
+    name = str(model_name or "")
+    return name.rsplit("/", 1)[-1] if "/" in name else name
+
+
+def is_gemini_model(model_name: Any) -> bool:
+    """True when the model name refers to any Gemini model."""
+    return "gemini" in strip_provider_prefix(model_name).lower()
+
+
+def _is_release_suffix(rest: str) -> bool:
+    """Whether what follows the tier is a release suffix (see the module docstring)."""
+    if not rest:
+        return True
+    if not rest.startswith("-"):
+        return False
+    return all(token in _RELEASE_SUFFIX_TOKENS or token.isdigit() for token in rest[1:].split("-"))
+
+
+def _parse_gemini_model(model_name: Any) -> _GeminiModel | None:
+    name = strip_provider_prefix(model_name).lower()
+    match = _GEMINI_MODEL_RE.search(name)
+    if not match:
+        return None
+    return _GeminiModel(
+        version=(int(match.group(1)), int(match.group(2))),
+        tier=match.group(3),
+        is_base=_is_release_suffix(name[match.end() :]),
+    )
+
+
+def gemini_version(model_name: Any) -> tuple[int, int] | None:
+    """``(major, minor)`` parsed from a ``gemini-X.Y-*`` name, else ``None``."""
+    parsed = _parse_gemini_model(model_name)
+    return parsed.version if parsed else None
+
+
+def _gemini_tier(model_name: Any) -> str | None:
+    """The tier of a *base* model id; ``None`` for variants and non-Gemini names."""
+    parsed = _parse_gemini_model(model_name)
+    return parsed.tier if parsed and parsed.is_base else None
+
+
+def supports_thinking_level(model_name: Any) -> bool:
+    """Whether the model accepts ``thinking_level`` (Gemini 3 and later).
+
+    Gemini 1.x/2.x only understand ``thinking_budget``. Names that do not
+    carry a parsable version (e.g. the robotics ER models) are assumed to be
+    current and therefore accept it.
+    """
+    version = gemini_version(model_name)
+    return version is None or version[0] >= 3
+
+
+def supports_agentic_video(model_name: Any) -> bool:
+    """Whether the Gemini API accepts ``processing: "agentic"`` for this model.
+
+    Only base Flash / Flash-Lite ids qualify; variants (``-image``, ``-tts``,
+    ``-live``, ...) do not inherit the family's support.
+    """
+    parsed = _parse_gemini_model(model_name)
+    if parsed is None or not parsed.is_base:
+        return False
+    if parsed.tier == "flash":
+        return parsed.version >= _AGENTIC_VIDEO_MIN_FLASH
+    if parsed.tier == "flash-lite":
+        return parsed.version >= _AGENTIC_VIDEO_MIN_FLASH_LITE
+    return False
+
+
+def is_agentic_video_auto_eligible(model_name: Any) -> bool:
+    """Models that get agentic video processing under ``processing: "auto"``.
+
+    Flash-Lite is deliberately excluded: it accepts the mode but mis-reads
+    timestamps in practice, so it must be opted into explicitly.
+    """
+    return supports_agentic_video(model_name) and _gemini_tier(model_name) == "flash"
+
+
+def resolve_video_processing(configured: Any, model_name: Any) -> ResolvedVideoProcessing:
+    """Turns the configured video-processing knob into a concrete mode.
+
+    ``static`` is always honoured. ``agentic`` is honoured only for models
+    that support it and silently degrades otherwise. ``auto`` (and any
+    unrecognised value) enables agentic processing for the auto-eligible
+    models and keeps static processing for everything else.
+    """
+    mode = str(configured or "auto").strip().lower() if isinstance(configured, str) else "auto"
+    if mode == "static":
+        return "static"
+    if mode == "agentic":
+        return "agentic" if supports_agentic_video(model_name) else "static"
+    return "agentic" if is_agentic_video_auto_eligible(model_name) else "static"

artemis/llm/google/usage.py

@@ -0,0 +1,148 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Token-usage normalization across every response shape ARTEMIS sees.
+
+Google alone reports usage under three different vocabularies:
+
+* LangChain ``usage_metadata`` - ``input_tokens`` / ``output_tokens`` /
+  ``input_token_details.cache_read`` (OpenAI-style names appear via
+  ``response_metadata.token_usage``).
+* ``generate_content`` ``usage_metadata`` - ``prompt_token_count`` /
+  ``candidates_token_count`` / ``cached_content_token_count`` /
+  ``thoughts_token_count`` / ``tool_use_prompt_token_count``.
+* Interactions API ``usage`` - ``total_input_tokens`` /
+  ``total_output_tokens`` / ``total_cached_tokens`` / ``total_thought_tokens``
+  / ``total_tool_use_tokens``.
+
+:func:`normalize_usage` folds all of them into one dict so meters, traces and
+the UI read a single set of keys.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+_PROMPT_KEYS = ("prompt_tokens", "input_tokens", "prompt_token_count", "total_input_tokens")
+_COMPLETION_KEYS = (
+    "completion_tokens",
+    "output_tokens",
+    "candidates_token_count",
+    "total_output_tokens",
+)
+_TOTAL_KEYS = ("total_tokens", "total_token_count")
+_CACHED_KEYS = (
+    "cached_tokens",
+    "cached_content_token_count",
+    "cachedContentTokenCount",
+    "total_cached_tokens",
+)
+_THOUGHT_KEYS = (
+    "thought_tokens",
+    "thoughts_token_count",
+    "reasoning_tokens",
+    "total_thought_tokens",
+)
+_TOOL_USE_KEYS = ("tool_use_tokens", "tool_use_prompt_token_count", "total_tool_use_tokens")
+
+NORMALIZED_USAGE_KEYS = (
+    "prompt_tokens",
+    "completion_tokens",
+    "total_tokens",
+    "cached_tokens",
+    "thought_tokens",
+    "tool_use_tokens",
+)
+
+
+def _int(value: Any) -> int:
+    if isinstance(value, bool):
+        return 0
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return 0
+
+
+def _read(raw: Any, key: str) -> Any:
+    if isinstance(raw, dict):
+        return raw.get(key)
+    return getattr(raw, key, None)
+
+
+def _first(raw: Any, keys: tuple[str, ...]) -> int:
+    for key in keys:
+        value = _int(_read(raw, key))
+        if value:
+            return value
+    return 0
+
+
+def normalize_usage(raw: Any) -> dict[str, int] | None:
+    """Normalizes a usage payload (dict or object) into the shared key set.
+
+    Returns ``None`` when the payload carries no usable token counts, so
+    callers can distinguish "no usage reported" from an all-zero report.
+    """
+    if raw is None:
+        return None
+
+    prompt = _first(raw, _PROMPT_KEYS)
+    completion = _first(raw, _COMPLETION_KEYS)
+    total = _first(raw, _TOTAL_KEYS) or (prompt + completion)
+
+    cached = 0
+    details = _read(raw, "input_token_details")
+    if details is not None:
+        cached = _int(_read(details, "cache_read"))
+    if not cached:
+        cached = _first(raw, _CACHED_KEYS)
+
+    thought = 0
+    output_details = _read(raw, "output_token_details")
+    if output_details is not None:
+        thought = _int(_read(output_details, "reasoning"))
+    if not thought:
+        thought = _first(raw, _THOUGHT_KEYS)
+
+    tool_use = _first(raw, _TOOL_USE_KEYS)
+
+    if total <= 0:
+        return None
+    return {
+        "prompt_tokens": prompt,
+        "completion_tokens": completion,
+        "total_tokens": total,
+        "cached_tokens": cached,
+        "thought_tokens": thought,
+        "tool_use_tokens": tool_use,
+    }
+
+
+def usage_from_message(message: Any) -> dict[str, int] | None:
+    """Extracts normalized usage from a LangChain message, best-effort.
+
+    Prefers LangChain's ``usage_metadata``; falls back to the raw provider
+    payload stored under ``response_metadata`` (``usage_metadata`` for Google,
+    ``token_usage`` for OpenAI-compatible providers).
+    """
+    raw = getattr(message, "usage_metadata", None)
+    if isinstance(raw, dict) and raw:
+        return normalize_usage(raw)
+    meta = getattr(message, "response_metadata", None)
+    if isinstance(meta, dict):
+        candidate = meta.get("usage_metadata") or meta.get("token_usage")
+        if isinstance(candidate, dict) and candidate:
+            return normalize_usage(candidate)
+    return None

artemis/llm/router.py

@@ -28,6 +28,7 @@ from langchain_core.language_models.chat_models import BaseChatModel
 from pydantic import BaseModel, Field
 
 from artemis.config.settings import settings
+from artemis.llm.google.provider import supports_thinking_level
 from artemis.utils.logger import get_logger
 
 logger = get_logger(__name__)
@@ -234,9 +235,10 @@ class ModelFactory:
                 or os.environ.get("GOOGLE_API_KEY")
                 or os.environ.get("GEMINI_API_KEY")
             )
-            thinking_level = endpoint.thinking_level
-            if endpoint.model_name and any(v in endpoint.model_name for v in ("2.5", "2.0", "1.5")):
-                thinking_level = None
+            # Gemini 1.x/2.x only understand thinking_budget.
+            thinking_level = (
+                endpoint.thinking_level if supports_thinking_level(endpoint.model_name) else None
+            )
 
             kwargs: dict[str, Any] = {
                 "model": endpoint.model_name,

artemis/mcp/action_executor.py

@@ -14,22 +14,12 @@
 
 """MCP-backed action executor for the FlashRunner.
 
-Same public contract as the legacy ``MobileActionExecutor`` --
-``execute(name, args, tool_call_id, state) -> ToolExecutionResult`` -- but execution
-routes through the in-process action MCP session, and dispatch is table-driven so a
-backend's extension tools are callable without touching this file.
+Device actions use the in-process MCP session. Argument normalization,
+coordinate descriptions, smart swipes, post-action observations, state updates,
+tracing, and helper tools are handled here on the agent side.
 
-Kept agent-side (by design, not omission): argument normalization, element-index
-resolution against LangGraph ``State``, smart-swipe resolution, act-then-observe
-capture and ``State`` write-back, tracing (a ``ContextVar`` cannot cross the server's
-task boundary -- see the plan's R2), and the non-device tools (``read_note``,
-``list_notes``, ``ask_explorer``, ``video_analyzer`` and the shared history tools
-``search_history`` / ``replay_steps`` / ``get_step_screenshot``).
-
-Status is carried by ``ActionResult.ok`` -- the historical substring sniffing on
-``"Error"``/``"Failed"`` is gone, so typing the literal text "Failed" into a field no
-longer marks the step as failed, and a successful tap with a failed screenshot is no
-longer misreported as an error.
+Device action status comes from ``ActionResult.ok``; helper tools report failures
+through ``ToolFailure`` or an explicit status.
 """
 
 import ast
@@ -38,6 +28,7 @@ import json
 from pathlib import Path
 from typing import Any
 
+from artemis.core.tool_failure import ToolFailure, is_tool_failure
 from artemis.context import ArtemisContext
 from artemis.controllers.unified_controller import UnifiedMobileController
 from artemis.data_engine.trace import TraceSpan
@@ -56,7 +47,6 @@ from artemis.utils.coordinates import (
     compute_smart_swipe_coordinates,
     parse_swipe_parameters,
 )
-from artemis.utils.element_hit_test import hit_test_semantics
 from artemis.utils.logger import get_logger
 from artemis.utils.notes import (
     format_list_notes_failure,
@@ -153,19 +143,13 @@ class McpActionExecutor:
         img_bytes: bytes | None = None
         shot_path: str | None = None
         xml_list: str | None = None
-        target_semantics: dict[str, Any] | None = None
-
+        target_semantics: dict[str, Any] = {}
         with span:
             try:
                 session = await self._session_or_start()
-                wire_name, wire_args, finalize = self._translate(raw_name, args, state)
-                # Record-time enrichment: bare-coordinate targeted actions get
-                # best-effort element semantics from the pre-action frame (the
-                # post-action observe below overwrites state.indexed_elements,
-                # so this must happen before the device call).
-                target_semantics = self._target_semantics(raw_name, wire_args, state)
-                if target_semantics:
-                    span.payload = {"args": {**args, **target_semantics}}
+                wire_name, wire_args, finalize, target_semantics = self._translate(
+                    raw_name, args, state
+                )
                 extension = raw_name not in (REQUIRED_ACTIONS | OPTIONAL_ACTIONS)
 
                 if extension:
@@ -233,71 +217,11 @@ class McpActionExecutor:
             },
         )
 
-    def _target_semantics(
-        self, raw_name: str, wire_args: dict[str, Any], state: Any
-    ) -> dict[str, Any] | None:
-        """Best-effort element semantics for a coordinate-targeted action.
-
-        Hit tests the normalized (0-1000) target point(s) against the
-        pre-action frame's indexed elements. Single-target actions get the
-        flat ``target_*`` fields; multi-point actions (``click_sequence``,
-        ``swipe``) hit test every point best-effort (M5), hoisting the FIRST
-        point's semantics to the main ``target_*`` label and keeping the
-        per-point results alongside. Returns None for actions without a
-        resolvable coordinate target; degrades to
-        ``target_label_source: "none"`` when no element data covers the point.
-        """
-        try:
-            elements = getattr(state, "indexed_elements", None) if state else None
-
-            if raw_name in ("click", "long_press", "input_text"):
-                target = wire_args.get("target")
-                if not (isinstance(target, (list, tuple)) and len(target) == 2):
-                    return None
-                return self._hit_test_point(elements, target)
-
-            if raw_name == "click_sequence":
-                sequence = wire_args.get("sequence")
-                if not (isinstance(sequence, (list, tuple)) and sequence):
-                    return None
-                points = [self._hit_test_point(elements, p) for p in sequence]
-                semantics: dict[str, Any] = dict(points[0])
-                semantics["points_semantics"] = points
-                return semantics
-
-            if raw_name == "swipe":
-                start = wire_args.get("start")
-                end = wire_args.get("end")
-                if not (isinstance(start, (list, tuple)) and len(start) == 2):
-                    return None
-                semantics = dict(self._hit_test_point(elements, start))
-                semantics["start_semantics"] = self._hit_test_point(elements, start)
-                if isinstance(end, (list, tuple)) and len(end) == 2:
-                    semantics["end_semantics"] = self._hit_test_point(elements, end)
-                return semantics
-
-            return None
-        except Exception as enrich_err:
-            logger.debug(f"Target semantics enrichment skipped for {raw_name}: {enrich_err}")
-            return {"target_label_source": "none"}
-
-    def _hit_test_point(self, elements: Any, target: Any) -> dict[str, Any]:
-        """Hit tests one normalized (0-1000) point; never raises."""
-        try:
-            width = getattr(self.ctx.device, "device_width", 1080) if self.ctx.device else 1080
-            height = getattr(self.ctx.device, "device_height", 2400) if self.ctx.device else 2400
-            x_px = int(max(0, min(width - 1, float(target[0]) * width / 1000)))
-            y_px = int(max(0, min(height - 1, float(target[1]) * height / 1000)))
-            return hit_test_semantics(elements, x_px, y_px)
-        except Exception:
-            return {"target_label_source": "none"}
-
     @staticmethod
     def _observe_despite_failure(raw_name: str, res: ActionResult) -> bool:
-        """Failures that still observe the screen, mirroring the legacy executor.
+        """Capture the screen after launch failures and wait_for_text timeouts.
 
-        A failed app launch and a wait_for_text timeout captured the screen; argument
-        and package errors returned without one.
+        Argument and package lookup errors do not require an observation.
         """
         if raw_name == "manage_app":
             return res.code not in (ActionCode.PACKAGE_NOT_FOUND, ActionCode.INVALID_ARGS)
@@ -309,16 +233,19 @@ class McpActionExecutor:
 
     def _translate(
         self, raw_name: str, args: dict[str, Any], state: Any
-    ) -> tuple[str, dict[str, Any], Any]:
-        """Maps agent-facing args to wire args; returns (name, args, finalize).
+    ) -> tuple[str, dict[str, Any], Any, dict[str, Any]]:
+        """Maps agent-facing args to wire args; returns (name, args, finalize, recorded).
 
-        ``finalize(res)`` post-processes the ActionResult message where the historical
-        wording depended on client-side context (direction swipes).
+        ``finalize(res)`` adds client-side context to swipe result messages.
+        ``recorded`` contains validated target descriptions in the Pro Operator's
+        record format. Focused input and directional swipes have no coordinate
+        target, so they do not record a description.
         """
         if raw_name == "click":
             target = self._require_pair(
                 args.get("target") or args.get("coordinates") or [], "click"
             )
+            recorded = self._require_description(args, "click")
             return (
                 "click",
                 {
@@ -327,12 +254,14 @@ class McpActionExecutor:
                     "delay_ms": args.get("delay_ms", 100),
                 },
                 None,
+                recorded,
             )
 
         if raw_name == "long_press":
             target = self._require_pair(
                 args.get("target") or args.get("coordinates") or [], "long press"
             )
+            recorded = self._require_description(args, "long press")
             return (
                 "long_press",
                 {
@@ -340,13 +269,16 @@ class McpActionExecutor:
                     "duration_ms": args.get("duration_ms", args.get("duration", 1000)),
                 },
                 None,
+                recorded,
             )
 
         if raw_name == "input_text":
             raw_target = args.get("target") or args.get("coordinates")
             target = None
+            recorded = {}
             if raw_target:
                 target = list(self._require_pair(raw_target, "input text"))
+                recorded = self._require_description(args, "input text")
             return (
                 "input_text",
                 {
@@ -355,23 +287,27 @@ class McpActionExecutor:
                     "clear_exist": args.get("clear_exist", True),
                 },
                 None,
+                recorded,
             )
 
         if raw_name == "click_sequence":
+            sequence = self._resolve_sequence(args.get("sequence") or [])
+            recorded = self._require_descriptions(args, len(sequence))
             return (
                 "click_sequence",
                 {
-                    "sequence": self._resolve_sequence(args.get("sequence") or [], state),
+                    "sequence": sequence,
                     "delay_ms": args.get("delay_ms", 50),
                 },
                 None,
+                recorded,
             )
 
         if raw_name == "swipe":
             return self._translate_swipe(args, state)
 
         if raw_name == "press_key":
-            return "press_key", {"key": args.get("key", "BACK")}, None
+            return "press_key", {"key": args.get("key", "BACK")}, None, {}
 
         if raw_name == "manage_app":
             return (
@@ -381,10 +317,11 @@ class McpActionExecutor:
                     "app_name": args.get("app_name", ""),
                 },
                 None,
+                {},
             )
 
         if raw_name == "wait_for_delay":
-            return "wait_for_delay", {"time_in_ms": args.get("time_in_ms", 1000)}, None
+            return "wait_for_delay", {"time_in_ms": args.get("time_in_ms", 1000)}, None, {}
 
         if raw_name == "wait_for_text":
             return (
@@ -395,10 +332,11 @@ class McpActionExecutor:
                     "timeout_ms": args.get("timeout_ms", 5000),
                 },
                 None,
+                {},
             )
 
         # Backend extension: pass the arguments straight through.
-        return raw_name, dict(args), None
+        return raw_name, dict(args), None, {}
 
     @staticmethod
     def _require_pair(raw: Any, label: str) -> tuple[int, int]:
@@ -412,8 +350,40 @@ class McpActionExecutor:
             raise _ArgError(f"{prefix}: Invalid target format: {raw}")
         return int(target[0]), int(target[1])
 
-    def _resolve_sequence(self, sequence: Any, state: Any) -> list[list[int]]:
-        """Resolves click_sequence entries (indices or pairs) to normalized pairs."""
+    @staticmethod
+    def _require_description(args: dict[str, Any], label: str) -> dict[str, Any]:
+        """Validate and trim the coordinate target description for the action record."""
+        description = args.get("target_description")
+        if not (isinstance(description, str) and description.strip()):
+            raise _ArgError(
+                f"Error during {label}: 'target_description' is required (what the"
+                " target is, in a few words)."
+            )
+        return {"target_description": description.strip()}
+
+    @staticmethod
+    def _require_descriptions(args: dict[str, Any], count: int) -> dict[str, Any]:
+        """``click_sequence``: one description per sequence entry, in order.
+
+        Returns the cleaned statements as the recorded ``target_descriptions`` field.
+        """
+        descriptions = args.get("target_descriptions")
+        if isinstance(descriptions, str):
+            descriptions = [descriptions]
+        if (
+            not isinstance(descriptions, (list, tuple))
+            or len(descriptions) != count
+            or not all(isinstance(d, str) and d.strip() for d in descriptions)
+        ):
+            raise _ArgError(
+                "Error during click sequence: 'target_descriptions' is required, one"
+                f" non-empty entry per sequence entry ({count} expected)."
+            )
+        return {"target_descriptions": [d.strip() for d in descriptions]}
+
+    @staticmethod
+    def _resolve_sequence(sequence: Any) -> list[list[int]]:
+        """Normalize click_sequence entries to coordinate pairs; reject element indices."""
         if isinstance(sequence, str):
             sequence_str = sequence.strip()
             try:
@@ -427,31 +397,26 @@ class McpActionExecutor:
         if not isinstance(sequence, (list, tuple)):
             raise _ArgError(f"Error during click sequence: Invalid sequence format: {sequence}")
 
-        width = getattr(self.ctx.device, "device_width", 1080) if self.ctx.device else 1080
-        height = getattr(self.ctx.device, "device_height", 2400) if self.ctx.device else 2400
-
         resolved: list[list[int]] = []
-        for raw_target in sequence:
+        for position, raw_target in enumerate(sequence, start=1):
             target = normalize_coordinate_target(raw_target)
-            if isinstance(target, int):
-                idx = target
-                points = getattr(state, "indexed_points", []) or []
-                if not 1 <= idx <= len(points):
+            if isinstance(target, (int, float)) and not isinstance(target, bool):
                 raise _ArgError(
-                        "Error during click sequence: Index"
-                        f" {idx} is out of range (available: {len(points)})"
+                    f"Error during click sequence: entry {position} ({raw_target!r}) is an"
+                    " element index; click_sequence takes normalized [x, y] coordinate"
+                    " pairs only. Use the element's coordinates from the element list,"
+                    " or ask_explorer to locate it."
                 )
-                px, py = int(points[idx - 1][0]), int(points[idx - 1][1])
-                nx = int(max(0, min(1000, round(px * 1000 / max(1, width)))))
-                ny = int(max(0, min(1000, round(py * 1000 / max(1, height)))))
-            elif isinstance(target, (list, tuple)) and len(target) == 2:
+            if isinstance(target, (list, tuple)) and len(target) == 2:
                 nx, ny = int(target[0]), int(target[1])
             else:
                 raise _ArgError(f"Error during click sequence: Invalid target format: {raw_target}")
             resolved.append([nx, ny])
         return resolved
 
-    def _translate_swipe(self, args: dict[str, Any], state: Any) -> tuple[str, dict[str, Any], Any]:
+    def _translate_swipe(
+        self, args: dict[str, Any], state: Any
+    ) -> tuple[str, dict[str, Any], Any, dict[str, Any]]:
         width = getattr(self.ctx.device, "device_width", 1080) if self.ctx.device else 1080
         height = getattr(self.ctx.device, "device_height", 2400) if self.ctx.device else 2400
         default_duration = args.get("duration", 400)
@@ -477,7 +442,7 @@ class McpActionExecutor:
             def finalize(res: ActionResult) -> str:
                 if not res.ok:
                     return f"Error swiping {dir_name}: {res.detail}"
-                return f"Swipe completed successfully. Swiped {dir_name}."
+                return f"Swiped {dir_name}."
 
             return (
                 "swipe",
@@ -487,15 +452,17 @@ class McpActionExecutor:
                     "duration_ms": smart_dur,
                 },
                 finalize,
+                {},
             )
 
         if kind == "coords":
             x1, y1, x2, y2 = target
+            recorded = self._require_description(args, "swipe")
 
             def finalize(res: ActionResult) -> str:
                 if not res.ok:
                     return f"Error dragging: {res.detail}"
-                return f"Swipe completed successfully. Swiped from [{x1}, {y1}] to [{x2}, {y2}]."
+                return f"Swiped from [{x1}, {y1}] to [{x2}, {y2}] (normalized)."
 
             return (
                 "swipe",
@@ -505,6 +472,7 @@ class McpActionExecutor:
                     "duration_ms": final_duration,
                 },
                 finalize,
+                recorded,
             )
 
         raise _ArgError(f"Error during swipe: Invalid direction: {args}")
@@ -537,7 +505,6 @@ class McpActionExecutor:
                 )
             elif raw_name in HISTORY_TOOL_NAMES:
                 text, blocks = await self._history_tool(raw_name, args)
-                ok = True
             else:
                 text, ok = await self._ask_explorer(
                     # ``task_description`` is the pre-contract argument name
@@ -551,13 +518,11 @@ class McpActionExecutor:
                 span.status = "failed"
                 span.error = text
 
-        # Note texts come from our own failure formatters, so the historical
-        # "Error" marker check is reliable there (unlike free-form device
-        # output); the Explorer and the video analyzer report their status
-        # explicitly and a history lookup's "no match" / "not found" answer
-        # is not an error.
+        # The Explorer and the video analyzer report their status explicitly;
+        # note and history texts carry it structurally (a ``ToolFailure``), so a
+        # history lookup's "no match" answer is an answer, not an error.
         if ok is None:
-            ok = "Error" not in (text or "")
+            ok = not is_tool_failure(text)
         return ToolExecutionResult(
             tool_call_id=tool_call_id,
             tool_name=name,
@@ -597,13 +562,13 @@ class McpActionExecutor:
         """
         tool = history_tool_by_name(raw_name)
         if tool is None:
-            return f"Error: Tool '{raw_name}' not supported.", None
+            return ToolFailure(f"Error: Tool '{raw_name}' not supported."), None
         accepted = {k: v for k, v in args.items() if k in tool.args_schema.model_fields}
         try:
             result = await tool.execute(ctx=self.ctx, **accepted)
         except Exception as e:
             logger.error(f"Error running {raw_name}: {e}")
-            return f"{raw_name} failed: {e}", None
+            return ToolFailure(f"{raw_name} failed: {e}"), None
         text, images = split_multimodal_result(result)
         if images:
             return text or "Screenshot attached.", result

artemis/mcp/action_manifest.py

@@ -83,7 +83,7 @@ OPTIONAL_ACTIONS: frozenset[str] = frozenset(
         "manage_app",
         "wait_for_delay",
         # The four below appear in no prompt at all. `wait_for_text` corroborates the
-        # classification: it is listed in the legacy ACTION_TOOL_NAMES but has never had
+        # classification: the former legacy executor dispatched it, but it has never had
         # a ToolDeclaration, i.e. it has always been unreachable by any LLM.
         "wait_for_text",
         "open_link",

artemis/mcp/action_specs.py

@@ -36,9 +36,19 @@ and declared**, not accidental:
   be returned automatically").
 * ``wire`` -- what the action MCP server serves, mirroring the actuator protocol
   one-to-one: normalized coordinates, millisecond durations, no addressing sugar.
-  Client-side executors (``McpActionExecutor``, ``MobileActionExecutor``) lower the
+  The client-side executor (``McpActionExecutor``) lowers the
   agent dialects onto it.
 
+A coordinate target names nothing by itself, so both agent dialects pair it with
+``target_description`` (``target_descriptions`` for a ``click_sequence``): the
+model's own statement of what it is aiming at. It is required whenever the target
+is a coordinate (always, in the declaration dialect), recorded verbatim on the
+action, and never sent to the wire. An element index needs none -- its text,
+bounds and id are read from the indexed list -- and the recorded fields stay
+separate (``target_text``/``target_bounds``/... for observed elements,
+``target_description`` for described coordinates) so no reader can mistake the
+model's belief for an observation.
+
 Each ``ActionSpec.param_bridge`` maps operator parameter names onto declaration
 parameter names, so every rename (``duration`` -> ``duration_ms``, ``gesture`` ->
 ``action``) is written down exactly once. ``tests/unit/mcp/test_action_specs.py``
@@ -164,6 +174,22 @@ _COORD_PAIR_JSON = {
     "items": {"type": "integer"},
 }
 
+#: Operator-dialect wording for the description that a coordinate target must carry.
+#: An element index already names its element (its text, id and bounds are recorded
+#: from the indexed list); a bare coordinate names nothing, so the model states what
+#: it is aiming at and that statement is recorded as ``target_description``.
+_OPERATOR_TARGET_DESCRIPTION = (
+    "What the target is, in a few words (e.g. 'play button', 'search input',"
+    " 'video body'). REQUIRED when target is a coordinate pair; ignored for an"
+    " element index."
+)
+
+#: Declaration-dialect wording (coordinates-only dialect: always required).
+_DECLARATION_TARGET_DESCRIPTION = (
+    "What the target is, in a few words (e.g. 'play button', 'search input',"
+    " 'video body'). Required: a coordinate names nothing by itself."
+)
+
 
 # --- Wire bindings -------------------------------------------------------------------
 # Each binding reproduces the exact conversion of the historical hand-written FastMCP
@@ -245,6 +271,13 @@ _SPECS: tuple[ActionSpec, ...] = (
                     "Click target. Can be an element index number (int, e.g. 3) OR"
                     " normalized coordinates (list of 2 integers, e.g. [500, 600]).",
                 ),
+                ParamSpec(
+                    "target_description",
+                    str | None,
+                    _OPERATOR_TARGET_DESCRIPTION,
+                    required=False,
+                    default=None,
+                ),
                 ParamSpec(
                     "times",
                     int,
@@ -278,6 +311,10 @@ _SPECS: tuple[ActionSpec, ...] = (
                         "items": {"type": "integer"},
                         "description": "Normalized coordinates [x, y] in 0-1000 scale.",
                     },
+                    "target_description": {
+                        "type": "string",
+                        "description": _DECLARATION_TARGET_DESCRIPTION,
+                    },
                     "times": {
                         "type": "integer",
                         "description": (
@@ -289,7 +326,7 @@ _SPECS: tuple[ActionSpec, ...] = (
                         "description": "Delay between taps in milliseconds (default 100).",
                     },
                 },
-                "required": ["target"],
+                "required": ["target", "target_description"],
             },
         ),
         wire=WireDialect(
@@ -301,10 +338,17 @@ _SPECS: tuple[ActionSpec, ...] = (
             ),
             bind=_wire_click,
         ),
-        param_bridge={"target": "target", "times": "times", "delay_ms": "delay_ms"},
+        param_bridge={
+            "target": "target",
+            "target_description": "target_description",
+            "times": "times",
+            "delay_ms": "delay_ms",
+        },
         differences=(
             "target: operator accepts an element index or a coordinate pair; the"
             " declaration and wire dialects take coordinates only."
+            " target_description: recorded, never sent to the wire; the operator"
+            " requires it only for coordinate targets, the declaration always."
         ),
     ),
     ActionSpec(
@@ -324,14 +368,22 @@ _SPECS: tuple[ActionSpec, ...] = (
                             "type": "array",
                             "items": {"type": "integer"},
                             "description": (
-                                "Normalized coordinates [x, y] in 0-1000 scale (e.g., [500, 280]),"
-                                " or a single integer element index."
+                                "Normalized coordinates [x, y] in 0-1000 scale (e.g., [500, 280])."
                             ),
                         },
                         "description": (
                             "List of targets to tap in sequence, e.g. [[500, 280], [885, 362]]."
                         ),
                     },
+                    "target_descriptions": {
+                        "type": "array",
+                        "items": {"type": "string"},
+                        "description": (
+                            "What each target is, in a few words, one entry per sequence"
+                            " entry in the same order (e.g. ['video body', 'skip button'])."
+                            " Required: a coordinate names nothing by itself."
+                        ),
+                    },
                     "delay_ms": {
                         "type": "integer",
                         "description": (
@@ -339,7 +391,7 @@ _SPECS: tuple[ActionSpec, ...] = (
                         ),
                     },
                 },
-                "required": ["sequence"],
+                "required": ["sequence", "target_descriptions"],
             },
         ),
         wire=WireDialect(
@@ -352,8 +404,9 @@ _SPECS: tuple[ActionSpec, ...] = (
         ),
         differences=(
             "Declared to Flash only (the Pro Operator chains actions as a fast-action"
-            " burst instead); sequence entries may be element indices at the declaration layer"
-            " (resolved client-side), while the wire takes coordinate pairs only."
+            " burst instead); sequence entries are normalized coordinate pairs in both"
+            " dialects (an element index is refused, never resolved client-side)."
+            " target_descriptions is recorded per entry and never sent to the wire."
         ),
     ),
     ActionSpec(
@@ -371,6 +424,13 @@ _SPECS: tuple[ActionSpec, ...] = (
                     " 3) OR normalized coordinates (list of 2 integers, e.g. [500,"
                     " 600]).",
                 ),
+                ParamSpec(
+                    "target_description",
+                    str | None,
+                    _OPERATOR_TARGET_DESCRIPTION,
+                    required=False,
+                    default=None,
+                ),
                 ParamSpec(
                     "duration",
                     int,
@@ -393,12 +453,16 @@ _SPECS: tuple[ActionSpec, ...] = (
                         "items": {"type": "integer"},
                         "description": "Normalized coordinates [x, y] in 0-1000 scale.",
                     },
+                    "target_description": {
+                        "type": "string",
+                        "description": _DECLARATION_TARGET_DESCRIPTION,
+                    },
                     "duration_ms": {
                         "type": "integer",
                         "description": "Duration of press in milliseconds (default 1000ms).",
                     },
                 },
-                "required": ["target"],
+                "required": ["target", "target_description"],
             },
         ),
         wire=WireDialect(
@@ -409,9 +473,15 @@ _SPECS: tuple[ActionSpec, ...] = (
             ),
             bind=_wire_long_press,
         ),
-        param_bridge={"target": "target", "duration": "duration_ms"},
+        param_bridge={
+            "target": "target",
+            "target_description": "target_description",
+            "duration": "duration_ms",
+        },
         differences=(
             "target: operator accepts an element index or a coordinate pair."
+            " target_description: recorded, never sent to the wire; the operator"
+            " requires it only for coordinate targets, the declaration always."
             " duration: the operator spells the duration parameter `duration` (its"
             " structured decisions and recorded traces carry that key); the"
             " declaration and wire dialects spell it `duration_ms`. Executors accept"
@@ -439,6 +509,13 @@ _SPECS: tuple[ActionSpec, ...] = (
                     " (int, e.g. 3) OR normalized coordinates (list of 2 integers,"
                     " e.g. [500, 600]).",
                 ),
+                ParamSpec(
+                    "target_description",
+                    str | None,
+                    _OPERATOR_TARGET_DESCRIPTION,
+                    required=False,
+                    default=None,
+                ),
                 ParamSpec(
                     "clear_exist",
                     bool,
@@ -468,6 +545,10 @@ _SPECS: tuple[ActionSpec, ...] = (
                         "items": {"type": "integer"},
                         "description": "Coordinates [x, y] of the input box in 0-1000 scale.",
                     },
+                    "target_description": {
+                        "type": "string",
+                        "description": _DECLARATION_TARGET_DESCRIPTION,
+                    },
                     "clear_exist": {
                         "type": "boolean",
                         "description": (
@@ -476,7 +557,7 @@ _SPECS: tuple[ActionSpec, ...] = (
                         ),
                     },
                 },
-                "required": ["text", "target"],
+                "required": ["text", "target", "target_description"],
             },
         ),
         wire=WireDialect(
@@ -488,11 +569,18 @@ _SPECS: tuple[ActionSpec, ...] = (
             ),
             bind=_wire_input_text,
         ),
-        param_bridge={"target": "target", "text": "text", "clear_exist": "clear_exist"},
+        param_bridge={
+            "target": "target",
+            "target_description": "target_description",
+            "text": "text",
+            "clear_exist": "clear_exist",
+        },
         differences=(
             "target: operator accepts an element index or a coordinate pair, and"
             " requires a target; the wire dialect allows omitting the target to type"
-            " into the already-focused field."
+            " into the already-focused field. target_description: recorded, never"
+            " sent to the wire; the operator requires it only for coordinate targets,"
+            " the declaration always."
         ),
     ),
     ActionSpec(
@@ -510,6 +598,7 @@ _SPECS: tuple[ActionSpec, ...] = (
                 "    end: End normalized coordinates [end_x, end_y] in 0-1000 scale.\n"
                 "    target: Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within.\n"
                 "    gesture: Backward-compatible parameter: direction string OR custom coordinates list [start_x, start_y, end_x, end_y] in 0-1000 scale.\n"
+                "    target_description: What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling.\n"
                 "    duration: Optional gesture duration in milliseconds (default 800)."
             ),
             params=(
@@ -554,6 +643,15 @@ _SPECS: tuple[ActionSpec, ...] = (
                     required=False,
                     default=None,
                 ),
+                ParamSpec(
+                    "target_description",
+                    str | None,
+                    "What is being dragged, in a few words (e.g. 'brightness slider"
+                    " knob'). REQUIRED for coordinate gestures ('start'/'end' or a"
+                    " coordinates list); ignored for directional scrolling.",
+                    required=False,
+                    default=None,
+                ),
                 ParamSpec(
                     "duration",
                     int | None,
@@ -609,6 +707,14 @@ _SPECS: tuple[ActionSpec, ...] = (
                             " OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale."
                         ),
                     },
+                    "target_description": {
+                        "type": "string",
+                        "description": (
+                            "What is being dragged, in a few words (e.g. 'brightness slider"
+                            " knob'). Required for coordinate gestures ('start'/'end' or a"
+                            " coordinates list); ignored for directional scrolling."
+                        ),
+                    },
                     "duration": {
                         "type": "integer",
                         "description": (
@@ -635,9 +741,12 @@ _SPECS: tuple[ActionSpec, ...] = (
             "end": "end",
             "target": "target",
             "gesture": "action",
+            "target_description": "target_description",
             "duration": "duration",
         },
         differences=(
+            "target_description: recorded, never sent to the wire; required by both"
+            " agent dialects for coordinate gestures only."
             " gesture/action: the legacy combined direction-or-coordinates parameter is"
             " spelled `gesture` in the operator dialect and `action` in the"
             " declaration dialect (`action` collides with the operator's structured"

artemis/mcp/actuators/adb.py

@@ -14,10 +14,9 @@
 
 """ADB-backed actuator: the reference implementation of the actuator contract.
 
-The device-call bodies here were extracted from
-``artemis.agents.validator.tool_declarations.MobileActionExecutor``; the outcome
-message wording is preserved byte-for-byte because agent transcripts, trace payloads,
-and existing tests assert on it.
+Outcome messages describe the dispatched command, such as
+"Tapped at [x, y] (normalized)." The agent verifies the effect from subsequent
+observations.
 
 Error convention: device-level refusals the controller *reports* (an ``error`` field,
 a falsy success flag) come back as ``ActionResult(ok=False)`` with the historical
@@ -147,13 +146,13 @@ class AdbActuator:
                     f"Error executing click at step {i + 1}: {err}",
                     detail=str(err),
                 )
-            outcomes.append(f"Tapped at [{x}, {y}]")
+            outcomes.append(f"[{nx}, {ny}]")
             if i < len(points) - 1:
                 await asyncio.sleep(max(0, delay_ms) / 1000.0)
 
         return ActionResult.success(
             "click_sequence",
-            f"Sequence clicked successfully: {'; '.join(outcomes)}",
+            f"Tapped in sequence at {'; '.join(outcomes)} (normalized).",
         )
 
     # --- Optional actions ------------------------------------------------------------
@@ -166,7 +165,7 @@ class AdbActuator:
             return ActionResult.failure("click", f"Error executing click: {err}", detail=str(err))
         return ActionResult.success(
             "click",
-            f"Clicked at [{nx}, {ny}] (normalized) successfully.",
+            f"Tapped at [{nx}, {ny}] (normalized).",
             normalized_coordinates=[int(nx), int(ny)],
         )
 
@@ -182,7 +181,7 @@ class AdbActuator:
             )
         return ActionResult.success(
             "long_press",
-            f"Long pressed at [{nx}, {ny}] (normalized) for {duration_ms}ms successfully.",
+            f"Long-pressed at [{nx}, {ny}] (normalized) for {duration_ms}ms.",
             normalized_coordinates=[int(nx), int(ny)],
             duration_ms=duration_ms,
         )
@@ -227,7 +226,8 @@ class AdbActuator:
 
         return ActionResult.success(
             "input_text",
-            f"Executed typing '{text}'.",
+            f"Typed '{text}'"
+            + (f" at [{coords[0]}, {coords[1]}] (normalized)." if coords else "."),
             normalized_coordinates=coords,
         )
 
@@ -246,7 +246,7 @@ class AdbActuator:
             return ActionResult.failure("swipe", f"Error dragging: {err}", detail=str(err))
         return ActionResult.success(
             "swipe",
-            f"Swipe completed successfully. Swiped from [{nx1}, {ny1}] to [{nx2}, {ny2}].",
+            f"Swiped from [{nx1}, {ny1}] to [{nx2}, {ny2}] (normalized).",
             normalized_coordinates=[int(nx1), int(ny1), int(nx2), int(ny2)],
             duration_ms=duration_ms,
         )
@@ -281,7 +281,7 @@ class AdbActuator:
             if not res:
                 return ActionResult.failure("press_key", f"Error executing key press '{key}'.")
 
-        return ActionResult.success("press_key", f"Executed key press '{key}'.")
+        return ActionResult.success("press_key", f"Pressed key '{key}'.")
 
     async def manage_app(self, action: str, app_name: str) -> ActionResult:
         # Imported lazily: launch_app pulls in tool wrappers that are costly at import.
@@ -310,7 +310,7 @@ class AdbActuator:
             if success:
                 return ActionResult.success(
                     "manage_app",
-                    f"Launched app '{app_name}' ({target_pkg}) successfully.",
+                    f"Launched app '{app_name}' ({target_pkg}); foreground confirmed.",
                 )
             return ActionResult.failure(
                 "manage_app",
@@ -322,7 +322,9 @@ class AdbActuator:
                 res_term = self.controller.terminate_app(target_pkg)
                 if inspect.iscoroutine(res_term):
                     await res_term
-            return ActionResult.success("manage_app", f"Terminated app '{app_name}' successfully.")
+            return ActionResult.success(
+                "manage_app", f"Force-stopped app '{app_name}' ({target_pkg})."
+            )
         return ActionResult.failure(
             "manage_app",
             f"Invalid manage_app action: {action}",
@@ -334,7 +336,7 @@ class AdbActuator:
         await asyncio.sleep(delay_s)
         return ActionResult.success(
             "wait_for_delay",
-            f"Waited for {time_in_ms}ms successfully.",
+            f"Waited {time_in_ms}ms.",
             duration_ms=int(time_in_ms),
         )
 
@@ -360,7 +362,8 @@ class AdbActuator:
         if found:
             return ActionResult.success(
                 "wait_for_text",
-                f"Successfully waited for text '{text}' to {target_state}.",
+                f"Text '{text}' {'appeared' if target_state == 'appear' else 'disappeared'}"
+                f" in the UI tree after {int((time.time() - start) * 1000)}ms.",
             )
         return ActionResult.failure(
             "wait_for_text",
@@ -371,7 +374,7 @@ class AdbActuator:
     async def open_link(self, url: str) -> ActionResult:
         success = await self.controller.open_url(url)
         if success:
-            return ActionResult.success("open_link", f"Opened link '{url}' successfully.")
+            return ActionResult.success("open_link", f"Opened link '{url}'.")
         return ActionResult.failure("open_link", f"Failed to open link '{url}'.")
 
     async def erase_one_char(self) -> ActionResult:
@@ -394,7 +397,7 @@ class AdbActuator:
         if success:
             return ActionResult.success(
                 "focus_and_clear_text",
-                f"Cleared text at [{nx}, {ny}] (normalized) successfully.",
+                f"Cleared text at [{nx}, {ny}] (normalized).",
                 normalized_coordinates=[int(nx), int(ny)],
             )
         return ActionResult.failure("focus_and_clear_text", "Failed to erase text.")

artemis/memory/chunking.py

@@ -39,6 +39,7 @@ from uuid import uuid4
 
 from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage
 
+from artemis.llm.google import is_google_provider
 from artemis.memory.step_memory import JobKey, StepLens, StepMemoryService
 from artemis.memory.transcript import format_session_offset
 from artemis.utils.logger import get_logger
@@ -80,9 +81,7 @@ def _result_phrase(result: Any) -> str:
     if detail:
         return detail
     status = result.get("status")
-    if status == "success":
-        return "executed"
-    return str(status) if status else "executed"
+    return str(status) if status else "dispatched"
 
 
 def _action_phrase(step: dict) -> str:
@@ -963,7 +962,7 @@ class HistoryChunkManager:
             fallback = getattr(getattr(llm_cfg, "summarizer", None), "fallback", None)
             provider = str(getattr(fallback, "provider", "") or "")
             model = getattr(fallback, "model", None)
-            if model and provider in ("google", "gemini") and model != self._model_name:
+            if model and is_google_provider(provider) and model != self._model_name:
                 return str(model)
         except Exception as exc:
             logger.debug(f"Capsule fallback model resolution skipped: {exc}", exc_info=True)

artemis/memory/transcript.py

@@ -367,8 +367,8 @@ class TranscriptLedger:
             try:
                 from artemis.utils.task_tree import format_result_clean
 
-                # format_result_clean reports only errors/repairs; success
-                # renders as the bare status line.
+                # format_result_clean reports only errors/repairs; a dispatched
+                # action renders as the bare status line.
                 detail = format_result_clean(result)
             except Exception:
                 detail = None

artemis/resources/config/artemis.jsonc

@@ -65,7 +65,8 @@
 
     // 🎬 Video Analyzer: Multimodal screen recording verification and temporal analysis
     // NOTE: Dual-path engine automatically routes:
-    //  - Native Gemini Files API for Gemini models with Google API keys (long video file stream)
+    //  - Native Gemini Files API for Gemini models with Google API keys (long video file stream);
+    //    Gemini 3.6+ Flash sub-agents use agentic video understanding by default (see video_analyzer.processing below)
     //  - Universal Multi-Model Engine for Claude 3.5/3.7, GPT-4o, OpenRouter, Vertex AI (dynamic keyframe extraction + multimodal tool calling)
     "video_analyzer": {},
 
@@ -244,6 +245,9 @@
     "video_analyzer": {
       "enabled": true, // Whether continuous screen recording via scrcpy is active (default: auto-detected)
       "enable_ledger": true
+      // "processing": "auto", // How Gemini sub-agents watch a clip: auto (agentic on Gemini 3.6+ Flash, static elsewhere) | agentic | static
+      // "agentic_chunk_size_seconds": 600, // Longest clip one agentic sub-agent takes; static mode keeps chunk_size_seconds (60)
+      // "agentic_call_timeout_seconds": 300 // Hard timeout for one agentic sub-agent response
     },
     // 📝 Outputter: Post-execution report synthesis and structured schema extraction
     // Reviews action history, notes, and verifies goal achievement to synthesize final output.

artemis/services/llm.py

@@ -45,6 +45,7 @@ from artemis.config import (
 )
 from artemis.context import ArtemisContext
 from artemis.data_engine.trace import CURRENT_TRACE_ID, DataEngineCallbackHandler
+from artemis.llm.google import is_google_chat_model, is_google_provider
 from artemis.llm.reliability import (
     CircuitBreaker,
     FailureCategory,
@@ -550,20 +551,10 @@ class RobustChatModelWrapper:
         processed_tools = list(tools_list) if tools_list is not None else []
 
         # Determine if the underlying model provider is Google / Gemini
-        is_google_provider = False
         if self.endpoint is not None:
-            is_google_provider = self.endpoint.provider in (
-                ModelProvider.GOOGLE,
-                ModelProvider.GEMINI,
-            )
+            google_backed = is_google_provider(self.endpoint.provider)
         else:
-            base_model_cls = getattr(self.base_model, "__class__", None)
-            base_model_name = base_model_cls.__name__ if base_model_cls else ""
-            is_google_provider = (
-                "Google" in base_model_name
-                or "Gemini" in base_model_name
-                or "VertexAI" in base_model_name
-            )
+            google_backed = is_google_chat_model(self.base_model)
 
         has_explicit_google_search = any(
             isinstance(t, dict) and "google_search" in t for t in processed_tools
@@ -572,7 +563,7 @@ class RobustChatModelWrapper:
             self.endpoint is not None and self.endpoint.enable_grounding
         ) or has_explicit_google_search
 
-        if is_google_provider:
+        if google_backed:
             if should_ground:
                 # Add native Google Search Grounding tool if not already present
                 if not has_explicit_google_search:

artemis/services/token_meter.py

@@ -34,6 +34,7 @@ import threading
 from typing import Any
 
 from artemis.data_engine.context_vars import CURRENT_NODE_NAME
+from artemis.llm.google import usage_from_message
 from artemis.utils.logger import get_logger
 
 logger = get_logger(__name__)
@@ -55,59 +56,18 @@ OPERATOR_NODE_NAMES = frozenset({"operator", "flashrunner"})
 def extract_usage(response: Any) -> dict[str, int] | None:
     """Extracts unified token usage from an LLM response message, best-effort.
 
-    Prefers LangChain's normalized ``usage_metadata``; falls back to raw
-    provider shapes in ``response_metadata``. Returns None when the response
-    carries no usable usage numbers.
+    Thin wrapper over :func:`artemis.llm.google.usage.usage_from_message`
+    that keeps the meter's historical four-key shape. Returns None when the
+    response carries no usable usage numbers.
     """
-    usage: dict[str, Any] | None = None
-    raw = getattr(response, "usage_metadata", None)
-    if isinstance(raw, dict) and raw:
-        usage = raw
-    else:
-        meta = getattr(response, "response_metadata", None)
-        if isinstance(meta, dict):
-            candidate = meta.get("usage_metadata") or meta.get("token_usage")
-            if isinstance(candidate, dict) and candidate:
-                usage = candidate
+    usage = usage_from_message(response)
     if usage is None:
         return None
-
-    def _int(value: Any) -> int:
-        try:
-            return int(value)
-        except (TypeError, ValueError):
-            return 0
-
-    prompt = _int(
-        usage.get("input_tokens") or usage.get("prompt_tokens") or usage.get("prompt_token_count")
-    )
-    completion = _int(
-        usage.get("output_tokens")
-        or usage.get("completion_tokens")
-        or usage.get("candidates_token_count")
-    )
-    total = _int(usage.get("total_tokens") or usage.get("total_token_count")) or (
-        prompt + completion
-    )
-
-    cached = 0
-    details = usage.get("input_token_details")
-    if isinstance(details, dict):
-        cached = _int(details.get("cache_read"))
-    if not cached:
-        cached = _int(
-            usage.get("cached_content_token_count")
-            or usage.get("cachedContentTokenCount")
-            or usage.get("cached_tokens")
-        )
-
-    if total <= 0:
-        return None
     return {
-        "prompt_tokens": prompt,
-        "completion_tokens": completion,
-        "total_tokens": total,
-        "cached_tokens": cached,
+        "prompt_tokens": usage["prompt_tokens"],
+        "completion_tokens": usage["completion_tokens"],
+        "total_tokens": usage["total_tokens"],
+        "cached_tokens": usage["cached_tokens"],
     }
 
 

artemis/tools/command_tool.py

@@ -47,6 +47,7 @@ import uuid
 from langchain_core.tools import BaseTool
 from pydantic import BaseModel, Field
 
+from artemis.core.tool_failure import ToolFailure
 from artemis.agents.log_analyzer.output_analyzer import TaskOutputAnalyzerNode
 from artemis.context import ArtemisContext
 from artemis.controllers.platform_specific_commands_controller import (
@@ -681,7 +682,7 @@ class RunAdbCommandTool(ArtemisTool):
                     timeout=wait_seconds,
                 )
             except TimeoutError:
-                return (
+                return ToolFailure(
                     f"Error: Command did not finish within {wait_seconds:.1f}s."
                     " Background tasks are not available on this device"
                     " connection; bound the command (logcat -d, top -n 1,"
@@ -689,7 +690,7 @@ class RunAdbCommandTool(ArtemisTool):
                 )
             except Exception as e:  # pylint: disable=broad-exception-caught
                 logger.error(f"Failed to execute virtual adb shell for '{cmd_line}': {e}")
-                return f"Failed to execute adb command: {e}"
+                return ToolFailure(f"Failed to execute adb command: {e}")
             clean_output, new_env, script_exit = BackgroundTask.parse_output(str(output))
             if run_persistent and new_env is not None:
                 registry.persistent_envs[terminal_id] = _filter_persistent_env(new_env)
@@ -720,7 +721,7 @@ class RunAdbCommandTool(ArtemisTool):
             )
         except Exception as e:  # pylint: disable=broad-exception-caught
             logger.error(f"Failed to spawn adb shell process for command '{cmd_line}': {e}")
-            return f"Failed to spawn adb process: {e}"
+            return ToolFailure(f"Failed to spawn adb process: {e}")
 
         stdout_data: list[str] = []
         try:
@@ -916,7 +917,7 @@ class ManageTaskTool(ArtemisTool):
             )
 
         if not task_id:
-            return "Error: TaskId is required for status, kill, or send_input actions."
+            return ToolFailure("Error: TaskId is required for status, kill, or send_input actions.")
 
         task = registry.background.get(task_id)
 
@@ -952,15 +953,15 @@ class ManageTaskTool(ArtemisTool):
                 await task.stop(ctx, status="killed")
                 return f"Task {task_id} successfully terminated."
             except Exception as e:  # pylint: disable=broad-exception-caught
-                return f"Failed to terminate task {task_id}: {e}"
+                return ToolFailure(f"Failed to terminate task {task_id}: {e}")
 
         if action == "send_input":
             if not task:
                 return f"Task {task_id} is not active."
             if not input_str:
-                return "Error: Input is required for send_input action."
+                return ToolFailure("Error: Input is required for send_input action.")
             if not task.interactive or task.process.stdin is None:
-                return (
+                return ToolFailure(
                     f"Error: Task {task_id} was launched without Interactive=true, so its"
                     " stdin is closed. Relaunch the command with Interactive=true."
                 )
@@ -969,7 +970,7 @@ class ManageTaskTool(ArtemisTool):
                 await task.process.stdin.drain()
                 return f"Input successfully sent to task {task_id}."
             except Exception as e:  # pylint: disable=broad-exception-caught
-                return f"Failed to send input: {e}"
+                return ToolFailure(f"Failed to send input: {e}")
 
         return "Unsupported action."
 
@@ -1065,9 +1066,9 @@ class RunShortAdbCommandTool(ArtemisTool):
                 )
                 return f"ADB command completed.\nOutput:\n{output}"
             except TimeoutError:
-                return f"Error: Command timed out after {timeout:.0f} seconds."
+                return ToolFailure(f"Error: Command timed out after {timeout:.0f} seconds.")
             except Exception as e:  # pylint: disable=broad-exception-caught
-                return f"Error running command: {e}"
+                return ToolFailure(f"Error running command: {e}")
 
         try:
             process = await asyncio.create_subprocess_exec(
@@ -1084,7 +1085,7 @@ class RunShortAdbCommandTool(ArtemisTool):
             )
         except Exception as e:  # pylint: disable=broad-exception-caught
             logger.error(f"Failed to spawn adb shell process for command '{cmd_line}': {e}")
-            return f"Failed to spawn adb process: {e}"
+            return ToolFailure(f"Failed to spawn adb process: {e}")
 
         try:
             stdout_data, _ = await asyncio.wait_for(process.communicate(), timeout=timeout)
@@ -1101,9 +1102,9 @@ class RunShortAdbCommandTool(ArtemisTool):
                     process.kill()
                 except (ProcessLookupError, OSError):
                     pass
-            return f"Error: Command timed out after {timeout:.0f} seconds."
+            return ToolFailure(f"Error: Command timed out after {timeout:.0f} seconds.")
         except Exception as e:  # pylint: disable=broad-exception-caught
-            return f"Error running command: {e}"
+            return ToolFailure(f"Error running command: {e}")
 
 
 # Universal tool instance & aliases
@@ -1181,7 +1182,7 @@ class AnalyzeTaskOutputTool(ArtemisTool):
 
         task_info = _get_task_info(task_id, ctx)
         if not task_info:
-            return f"Error: Task {task_id} not found."
+            return ToolFailure(f"Error: Task {task_id} not found.")
 
         analyzer = TaskOutputAnalyzerNode(ctx)
         result = await analyzer.run(

artemis/tools/committee_tool.py

@@ -26,6 +26,7 @@ from langchain_core.messages import HumanMessage, SystemMessage
 from langchain_core.tools import BaseTool
 from pydantic import BaseModel, Field
 
+from artemis.core.tool_failure import ToolFailure, is_tool_failure
 from artemis.context import ArtemisContext
 from artemis.data_engine.trace import (
     CURRENT_TRACE_ID,
@@ -43,7 +44,6 @@ from artemis.tools.tool_wrapper import (
     ToolWrapper,
     get_tool_result_content,
     invoke_tool_with_injection,
-    split_multimodal_result,
     tool_result_messages,
 )
 from artemis.tools.types import CyFunctionDetector
@@ -302,43 +302,42 @@ async def _execute_committee(
                 messages.append(response)
 
                 async def run_tool(tc):
+                    """Keep the raw result so the caller can read ToolMessage.status."""
                     tool_name = tc["name"]
                     logger.info(f"{agent_name} requested tool: {tool_name}")
                     tool_to_run = next((t for t in tools if t.name == tool_name), None)
                     if tool_to_run:
                         try:
                             args = dict(tc["args"])
-                            result = await invoke_tool_with_injection(
+                            return await invoke_tool_with_injection(
                                 tool=tool_to_run,
                                 args=args,
                                 tool_call_id=tc["id"],
                                 state=state,
                             )
-                            return get_tool_result_content(result)
                         except Exception as e:  # pylint: disable=broad-exception-caught
-                            return f"Error: {e}"
+                            return ToolFailure(f"Error: {e}")
                     else:
-                        return f"Error: Tool {tool_name} not found"
+                        return ToolFailure(f"Error: Tool {tool_name} not found")
 
                 active_tool_calls = [
                     tc for tc in response.tool_calls if tc["name"] != "google_search"
                 ]
                 tool_outputs = await asyncio.gather(*(run_tool(tc) for tc in active_tool_calls))
 
-                for tc, result in zip(active_tool_calls, tool_outputs):
-                    # A step screenshot (content blocks) travels in the carrier
-                    # the member's provider accepts; text stays a ToolMessage.
-                    text, _ = split_multimodal_result(result)
+                for tc, raw_result in zip(active_tool_calls, tool_outputs):
+                    # Read status before unwrapping ToolMessage.content.
+                    status = "error" if is_tool_failure(raw_result) else "success"
                     messages.extend(
                         tool_result_messages(
                             tc["id"],
-                            result,
+                            get_tool_result_content(raw_result),
                             name=tc["name"],
-                            status="error" if text.startswith("Error") else "success",
+                            status=status,
                             llm=llm,
                         )
                     )
-            return "Error: Reached max iterations in turn."
+            return ToolFailure("Error: Reached max iterations in turn.")
 
         for r in range(1, rounds + 1):
             logger.info(f"Committee Round {r}")
@@ -407,13 +406,13 @@ async def _run_committee_logic(
 ) -> str:
     """Executes multi-agent committee debate logic."""
     if ctx is None:
-        return "Error: ArtemisContext is required for ask_committee."
+        return ToolFailure("Error: ArtemisContext is required for ask_committee.")
 
     try:
         return await asyncio.wait_for(_execute_committee(ctx, state, avatar_directive), timeout=300)
     except TimeoutError:
         logger.error("Committee timed out after 300 seconds.")
-        return "Error: Committee timed out after 300 seconds."
+        return ToolFailure("Error: Committee timed out after 300 seconds.")
 
 
 ask_committee_wrapper = ToolWrapper(

artemis/tools/diagnostic_tool.py

@@ -19,6 +19,7 @@ from typing import Any
 from langchain_core.tools import BaseTool
 from pydantic import BaseModel, Field
 
+from artemis.core.tool_failure import ToolFailure
 from artemis.agents.diagnoser.diagnoser import Diagnoser
 from artemis.context import ArtemisContext
 from artemis.data_engine.trace import trace_langchain_tool
@@ -53,7 +54,7 @@ async def _run_diagnoser_logic(
 ) -> str:
     logger.info(f"ask_diagnoser tool called with query: {query}")
     if ctx is None:
-        return "Error: ArtemisContext is required for Diagnoser."
+        return ToolFailure("Error: ArtemisContext is required for Diagnoser.")
     agent = Diagnoser(ctx)
     try:
         result = await agent.run(query, state)

artemis/tools/history/__init__.py

@@ -26,6 +26,7 @@ from typing import Any, Literal
 from langchain_core.tools import BaseTool
 from pydantic import BaseModel, Field
 
+from artemis.core.tool_failure import ToolFailure
 from artemis.core.tool_declaration import ToolDeclaration
 from artemis.data_engine.trace import trace_langchain_tool
 from artemis.tools.base import ArtemisTool
@@ -176,9 +177,9 @@ class SearchHistoryTool(ArtemisTool):
     ) -> str:
         reader = getattr(ctx, "data_engine", None) if ctx else None
         if reader is None:
-            return "search_history unavailable: no active execution history."
+            return ToolFailure("search_history unavailable: no active execution history.")
         if not query and not step_range:
-            return (
+            return ToolFailure(
                 "search_history needs a query and/or a step_range — e.g."
                 ' search_history(query="login timeout") or'
                 ' search_history(query="", step_range=[1, 40]).'
@@ -193,7 +194,7 @@ class SearchHistoryTool(ArtemisTool):
             )
         except Exception as e:
             logger.error(f"search_history failed: {e}")
-            return f"search_history failed: {e}"
+            return ToolFailure(f"search_history failed: {e}")
 
 
 # --- replay_steps ------------------------------------------------------------------------
@@ -246,7 +247,7 @@ class ReplayStepsTool(ArtemisTool):
     ) -> str:
         reader = getattr(ctx, "data_engine", None) if ctx else None
         if reader is None:
-            return "Error: no execution history available."
+            return ToolFailure("Error: no execution history available.")
         cfg = _replay_config()
         try:
             return replay_steps_text(
@@ -258,7 +259,7 @@ class ReplayStepsTool(ArtemisTool):
             )
         except Exception as e:
             logger.error(f"replay_steps failed: {e}")
-            return f"replay_steps failed: {e}"
+            return ToolFailure(f"replay_steps failed: {e}")
 
 
 # --- get_step_screenshot -----------------------------------------------------------------
@@ -310,12 +311,12 @@ class GetStepScreenshotTool(ArtemisTool):
     ) -> list[dict[str, Any]] | str:
         reader = getattr(ctx, "data_engine", None) if ctx else None
         if reader is None:
-            return "Error: no execution history available."
+            return ToolFailure("Error: no execution history available.")
         try:
             return load_step_screenshot(reader, step_number, which).to_content_blocks()
         except Exception as e:
             logger.error(f"get_step_screenshot failed: {e}")
-            return f"get_step_screenshot failed: {e}"
+            return ToolFailure(f"get_step_screenshot failed: {e}")
 
 
 # --- Instances, declarations, exports ------------------------------------------------------

artemis/tools/history/replay.py

@@ -28,6 +28,7 @@ from __future__ import annotations
 
 from typing import Any
 
+from artemis.core.tool_failure import ToolFailure
 from artemis.utils.task_tree import render_step_replay
 
 DEFAULT_MAX_STEPS = 5
@@ -48,12 +49,12 @@ def replay_steps_text(
 ) -> str:
     """Replays steps ``start_step``..``end_step`` (inclusive, 1-based) from ``reader``."""
     if reader is None:
-        return "Error: no execution history available."
+        return ToolFailure("Error: no execution history available.")
     try:
         start = int(start_step)
         end = start if end_step is None else int(end_step)
     except (TypeError, ValueError):
-        return (
+        return ToolFailure(
             f"Error: start_step and end_step must be integers, got {start_step!r} and {end_step!r}."
         )
     if start > end:
@@ -71,11 +72,11 @@ def replay_steps_text(
     try:
         steps: list[dict] = reader.get_agent_friendly_steps_in_range(start, end) or []
     except Exception as e:
-        return f"Error loading steps {start}–{end}: {e}"
+        return ToolFailure(f"Error loading steps {start}–{end}: {e}")
     if not steps:
         if start == end:
-            return f"Error: step {start} not found."
-        return f"Error: no recorded steps in range {start}–{end}."
+            return ToolFailure(f"Error: step {start} not found.")
+        return ToolFailure(f"Error: no recorded steps in range {start}–{end}.")
 
     budget_tokens = max(1, int(max_tokens or DEFAULT_MAX_TOKENS))
     rendered: list[str] = []
@@ -89,7 +90,7 @@ def replay_steps_text(
         try:
             text = render_step_replay(step)
         except Exception as e:
-            return f"Error rendering steps {start}–{end}: {e}"
+            return ToolFailure(f"Error rendering steps {start}–{end}: {e}")
         cost = _estimate_tokens(text) + 1
         # The first step is always replayed; afterwards the budget drops whole
         # trailing steps rather than truncating a step in the middle.

artemis/tools/history/search.py

@@ -40,7 +40,9 @@ import json
 import sqlite3
 from typing import Any
 
+from artemis.core.tool_failure import ToolFailure
 from artemis.utils.logger import get_logger
+from artemis.utils.task_tree import format_actions_clean
 
 logger = get_logger(__name__)
 
@@ -133,13 +135,27 @@ def _events_text(step: dict) -> str:
     return "\n".join(parts)
 
 
+def _action_text(action_taken: Any) -> str:
+    """Combine the rendered action with raw fields for search.
+
+    Rendered excerpts identify self-described targets. Raw JSON also makes
+    resource IDs, bounds, and other fields searchable.
+    """
+    if not action_taken:
+        return ""
+    try:
+        rendered = format_actions_clean(action_taken)
+    except Exception:
+        rendered = ""
+    raw = json.dumps(action_taken, ensure_ascii=False, default=str)
+    return f"{rendered}\n{raw}" if rendered else raw
+
+
 def _step_haystack(step: dict) -> str:
     extra = step.get("extra_metadata") or {}
     fields = [
         str(step.get("summary") or ""),
-        json.dumps(step.get("action_taken"), ensure_ascii=False, default=str)
-        if step.get("action_taken")
-        else "",
+        _action_text(step.get("action_taken")),
         json.dumps(step.get("last_execution_result"), ensure_ascii=False, default=str)
         if step.get("last_execution_result")
         else "",
@@ -262,7 +278,7 @@ def search_history_text(
     try:
         steps: list[dict] = reader.get_agent_friendly_steps() or []
     except Exception as e:
-        return f"search_history failed to load history: {e}"
+        return ToolFailure(f"search_history failed to load history: {e}")
 
     range_start, range_end = _parse_range(step_range)
 

artemis/tools/log_tool.py

@@ -19,6 +19,7 @@ from typing import Any
 from langchain_core.tools import BaseTool
 from pydantic import BaseModel, Field
 
+from artemis.core.tool_failure import ToolFailure
 from artemis.agents.log_analyzer.log_analyzer import LogAnalyzerNode
 from artemis.context import ArtemisContext
 from artemis.data_engine.trace import trace_langchain_tool
@@ -83,7 +84,7 @@ class AnalyzeLogsTool(ArtemisTool):
         st = state if state is not None else kwargs.get("state")
         logger.info(f"analyze_logs tool called with query: '{query}'")
         if ctx is None:
-            return "Error: ArtemisContext is required for LogAnalyzer."
+            return ToolFailure("Error: ArtemisContext is required for LogAnalyzer.")
         analyst = LogAnalyzerNode(ctx)
         result = await analyst.run(query, st)
         return result

artemis/tools/mobile/launch_app.py

@@ -153,17 +153,13 @@ class LaunchAppTool(ArtemisTool):
                         ctx=ctx, app_package=package_name
                     )
                     outcome = (
-                        f"App '{app}' launched successfully."
+                        f"Launched app '{app}' ({package_name}); foreground confirmed."
                         if success
                         else f"Failed to launch app '{app}': {error_msg}"
                     )
             elif driver is not None and hasattr(driver, "launch_app"):
                 success = await driver.launch_app(app)
-                outcome = (
-                    f"App '{app}' launched successfully."
-                    if success
-                    else f"Failed to launch app '{app}'."
-                )
+                outcome = f"Launched app '{app}'." if success else f"Failed to launch app '{app}'."
                 error_msg = None if success else "Launch failed."
             else:
                 success = False
@@ -198,6 +194,6 @@ def get_launch_app_tool(ctx: ArtemisContext) -> BaseTool:
 
 launch_app_wrapper = ToolWrapper(
     tool_fn_getter=get_launch_app_tool,
-    on_success_fn=lambda *args, **kwargs: "Success",
+    on_success_fn=lambda *args, **kwargs: "Launch dispatched",
     on_failure_fn=lambda *args, **kwargs: "Failure",
 )

artemis/tools/mobile/read_hierarchy.py

@@ -18,6 +18,7 @@ from typing import Any
 from langchain_core.tools import BaseTool
 from pydantic import BaseModel
 
+from artemis.core.tool_failure import ToolFailure
 from artemis.context import ArtemisContext
 from artemis.data_engine.trace import trace_langchain_tool
 from artemis.drivers.base import BaseDeviceDriver
@@ -75,16 +76,18 @@ class GetUiHierarchyTool(ArtemisTool):
                 xml_hierarchy = getattr(screen_data, "ui_hierarchy_xml", None)
                 if xml_hierarchy:
                     return str(xml_hierarchy)
-                return "Error retrieving UI hierarchy: No UI hierarchy in screen data."
+                return ToolFailure("Error retrieving UI hierarchy: No UI hierarchy in screen data.")
             if driver is not None and hasattr(driver, "get_ui_hierarchy"):
                 xml_hierarchy = await driver.get_ui_hierarchy()
                 if inspect.iscoroutine(xml_hierarchy):
                     xml_hierarchy = await xml_hierarchy
                 return str(xml_hierarchy)
-            return "Error retrieving UI hierarchy: No UI Automator client or driver provided."
+            return ToolFailure(
+                "Error retrieving UI hierarchy: No UI Automator client or driver provided."
+            )
         except Exception as e:  # pylint: disable=broad-exception-caught
             logger.error(f"Failed to get UI hierarchy: {e}")
-            return f"Error retrieving UI hierarchy: {e}"
+            return ToolFailure(f"Error retrieving UI hierarchy: {e}")
 
 
 # Universal tool instance & aliases

artemis/tools/mobile/read_logs.py

@@ -17,6 +17,7 @@ from typing import Any
 from langchain_core.tools.base import BaseTool
 from pydantic import BaseModel, Field
 
+from artemis.core.tool_failure import ToolFailure
 from artemis.context import ArtemisContext
 from artemis.data_engine.trace import trace_langchain_tool
 from artemis.drivers.base import BaseDeviceDriver
@@ -101,7 +102,7 @@ class ReadLogsTool(ArtemisTool):
             return logs
         except Exception as e:  # pylint: disable=broad-exception-caught
             logger.error(f"Failed to read logs: {e}")
-            return f"Failed to read logs: {e}"
+            return ToolFailure(f"Failed to read logs: {e}")
 
 
 # Universal tool instance & aliases

artemis/tools/mobile/search_logs.py

@@ -18,6 +18,7 @@ from typing import Any
 from langchain_core.tools.base import BaseTool
 from pydantic import BaseModel, Field
 
+from artemis.core.tool_failure import ToolFailure
 from artemis.context import ArtemisContext
 from artemis.data_engine.trace import trace_langchain_tool
 from artemis.drivers.base import BaseDeviceDriver
@@ -43,7 +44,7 @@ def search_and_merge_logs(
         try:
             pattern = re.compile(keyword, re.IGNORECASE)
         except re.error as e:
-            return f"Error: Invalid regex: {e}"
+            return ToolFailure(f"Error: Invalid regex: {e}")
     else:
         keyword_lower = keyword.lower()
 
@@ -222,7 +223,7 @@ class SearchLogsTool(ArtemisTool):
             )
         except Exception as e:  # pylint: disable=broad-exception-caught
             logger.error(f"Failed to search logs: {e}")
-            return f"Error searching logs: {e}"
+            return ToolFailure(f"Error searching logs: {e}")
 
 
 # Universal tool instance & aliases

artemis/tools/scratchpad.py

@@ -18,6 +18,7 @@ from langchain_core.messages import ToolMessage
 from langchain_core.tools.base import BaseTool
 from pydantic import BaseModel, Field
 
+from artemis.core.tool_failure import ToolFailure
 from artemis.context import ArtemisContext
 from artemis.data_engine.trace import trace_langchain_tool
 from artemis.drivers.base import BaseDeviceDriver
@@ -643,7 +644,7 @@ class SaveNotePureTool(ArtemisTool):
             save_note_content(base_dir, k, c)
             return f"Successfully saved note to {k}.md."
         except Exception as e:  # pylint: disable=broad-exception-caught
-            return f"Failed to save note {k}.md: {e}"
+            return ToolFailure(f"Failed to save note {k}.md: {e}")
 
 
 # Universal pure tool instance & aliases
@@ -711,7 +712,7 @@ class UpdateNotePureTool(ArtemisTool):
                 return f"Successfully updated note '{k}'.\nWARNING: {warning}"
             return f"Successfully updated note '{k}'."
         except Exception as e:  # pylint: disable=broad-exception-caught
-            return f"Failed to update note '{k}': {e}"
+            return ToolFailure(f"Failed to update note '{k}': {e}")
 
 
 # Universal pure tool instance & aliases
@@ -775,7 +776,7 @@ class AppendNotePureTool(ArtemisTool):
             append_note_content(base_dir, k, c)
             return f"Successfully appended note to {k}.md."
         except Exception as e:  # pylint: disable=broad-exception-caught
-            return f"Failed to append note {k}.md: {e}"
+            return ToolFailure(f"Failed to append note {k}.md: {e}")
 
 
 # Universal pure tool instance & aliases

artemis/tools/tool_wrapper.py

@@ -26,6 +26,7 @@ from langchain_core.tools.base import InjectedToolCallId
 from langgraph.prebuilt import InjectedState
 from pydantic import BaseModel
 
+from artemis.core.tool_failure import is_tool_failure
 from artemis.context import ArtemisContext
 from artemis.data_engine import engine as engine_mod
 from artemis.data_engine.trace import CURRENT_TRACE_ID, smart_serialize
@@ -123,8 +124,13 @@ async def invoke_tool_with_injection(
                 result = await func(**final_args)
             else:
                 result = func(**final_args)
-            execution_status = "success"
             execution_result = result
+            if is_tool_failure(result):
+                # A structural failure is traced as one; the text is the error.
+                execution_status = "failed"
+                execution_error = result
+            else:
+                execution_status = "success"
             return result
         except Exception as e:
             execution_status = "failed"

artemis/tools/video_tool.py

@@ -25,6 +25,7 @@ from langchain_core.messages import ToolMessage
 from langchain_core.tools import BaseTool
 from pydantic import BaseModel, Field
 
+from artemis.core.tool_failure import ToolFailure
 from artemis.agents.video_analyzer.video_analyzer import VideoAnalyzer
 from artemis.context import ArtemisContext
 from artemis.core.tool_declaration import ToolDeclaration
@@ -220,11 +221,11 @@ class VideoAnalyzerPureTool(ArtemisTool):
             agent = VideoAnalyzer(ctx)
             agent_outcome, status = await agent.run(time_desc, purp)
             if status == "failed":
-                return f"Video analysis failed: {agent_outcome}"
+                return ToolFailure(f"Video analysis failed: {agent_outcome}")
             return agent_outcome
         except Exception as e:  # pylint: disable=broad-exception-caught
             logger.error(f"Error in pure video_analyzer tool: {e}")
-            return f"Error running video analyzer: {e}"
+            return ToolFailure(f"Error running video analyzer: {e}")
 
 
 # Universal pure tool instance & aliases

artemis/tools/wait_tool.py

@@ -76,7 +76,7 @@ class WaitTool(ArtemisTool):
 
         logger.info(f"Diagnoser waiting for {sec} seconds...")
         await asyncio.sleep(sec)
-        return f"Successfully waited for {sec} seconds."
+        return f"Waited {sec} seconds."
 
 
 # Universal tool instance & aliases

artemis/utils/credentials_validator.py

@@ -16,6 +16,8 @@
 
 import httpx
 
+from artemis.llm.google.provider import is_google_family_provider
+
 from artemis.utils.logger import get_logger
 
 logger = get_logger(__name__)
@@ -89,7 +91,7 @@ async def validate_api_key(
 
     try:
         async with httpx.AsyncClient(timeout=timeout) as client:
-            if clean_provider in ("google", "gemini", "vertexai", "vertex"):
+            if is_google_family_provider(clean_provider):
                 # Test Google Gemini via generativelanguage API
                 url = f"https://generativelanguage.googleapis.com/v1beta/models?key={clean_key}&pageSize=1"
                 resp = await client.get(url)

artemis/utils/element_hit_test.py

@@ -12,35 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Record-time action semantics enrichment via element hit testing.
+"""Element hit testing: which indexed element covers a pixel point.
 
-An action addressed by bare coordinates carries no element semantics
-("tap(632,1180)" says nothing about what was tapped). Before such an action is
-persisted to the DataEngine, these helpers resolve the smallest indexed element
-covering the tap point on the pre-action frame and attach best-effort
-``target_text`` / ``target_class`` / ``target_resource_id`` fields, plus a
-``target_label_source`` marker distinguishing model-named targets from
-after-the-fact inference:
-
-- ``"index"``: the model addressed the element by its perception index — the
-  semantics come straight from the indexed element (highest confidence).
-- ``"hit_test"``: inferred by hit testing the coordinates against XML-derived
-  indexed elements on the pre-action frame.
-- ``"ocr"``: inferred by hit testing against OCR-derived indexed elements
-  (used only when no XML-derived element covers the point).
-- ``"none"``: no element covers the point (or no perception data was
-  available) — the action stays a bare-coordinate action.
-
-Enrichment is strictly best-effort: missing/malformed element data degrades to
-``"none"`` and never raises.
+Used by the Explorer to deduplicate candidates against the screen index.
+Recorded coordinate actions use the model's ``target_description``.
 """
 
 from typing import Any
 
 from artemis.utils.visualization import parse_bounds
 
-_LABEL_SOURCE_KEY = "target_label_source"
-
 
 def _element_bounds(element: dict[str, Any]) -> tuple[int, int, int, int] | None:
     """Returns (left, top, right, bottom) pixel bounds of an indexed element.
@@ -69,7 +50,8 @@ def find_element_at_point(
     XML-derived elements are preferred over OCR-derived ones (OCR boxes carry
     text but weaker structure); within each group the smallest covering area
     wins. Returns ``(element, source)`` where source is ``"hit_test"``,
-    ``"ocr"``, or ``"none"`` (with element None).
+    ``"ocr"``, or ``"none"`` (with element None). Malformed element data is
+    skipped; the function never raises on it.
     """
     best_xml: tuple[int, dict[str, Any]] | None = None
     best_ocr: tuple[int, dict[str, Any]] | None = None
@@ -96,26 +78,3 @@ def find_element_at_point(
     if best_ocr is not None:
         return best_ocr[1], "ocr"
     return None, "none"
-
-
-def hit_test_semantics(elements: list[dict[str, Any]] | None, x: int, y: int) -> dict[str, Any]:
-    """Best-effort semantic fields for a bare-coordinate action at pixel (x, y).
-
-    Always returns a dict containing ``target_label_source``; the
-    ``target_text`` / ``target_class`` / ``target_resource_id`` fields are only
-    present when the covering element carries them.
-    """
-    try:
-        element, source = find_element_at_point(elements, int(x), int(y))
-    except Exception:
-        return {_LABEL_SOURCE_KEY: "none"}
-    semantics: dict[str, Any] = {_LABEL_SOURCE_KEY: source}
-    if element is None:
-        return semantics
-    if element.get("text"):
-        semantics["target_text"] = element["text"]
-    if element.get("class"):
-        semantics["target_class"] = element["class"]
-    if element.get("resource_id"):
-        semantics["target_resource_id"] = element["resource_id"]
-    return semantics

artemis/utils/notes.py

@@ -18,6 +18,7 @@ import os
 from pathlib import Path
 import re
 
+from artemis.core.tool_failure import ToolFailure
 from artemis.data_engine.engine import _CURRENT_DATA_ENGINE
 from artemis.utils.logger import get_logger
 from artemis.utils.plan_grammar import parse_plan
@@ -385,8 +386,8 @@ def format_read_note_success(
 def format_read_note_failure(key: str, error: str) -> str:
     """Formats the failure message for reading a note."""
     if "not found" in error.lower():
-        return f"Note '{key}' not found in scratchpad."
-    return f"Failed to read note '{key}': {error}"
+        return ToolFailure(f"Note '{key}' not found in scratchpad.")
+    return ToolFailure(f"Failed to read note '{key}': {error}")
 
 
 def format_list_notes_success(notes_info: dict[str, int]) -> str:
@@ -399,7 +400,7 @@ def format_list_notes_success(notes_info: dict[str, int]) -> str:
 
 def format_list_notes_failure(error: str) -> str:
     """Formats the failure message for listing notes."""
-    return f"Failed to list notes: {error}"
+    return ToolFailure(f"Failed to list notes: {error}")
 
 
 def record_subgoal_hash_chain(

artemis/utils/task_tree.py

@@ -23,6 +23,23 @@ from artemis.utils.plan_grammar import parse_plan
 
 logger = logging.getLogger(__name__)
 
+#: Suffix on a rendered target label whose text is the acting model's own
+#: ``target_description`` (a coordinate target) rather than the element text
+#: observed in the UI tree (an index target). Every later reader of the history
+#: (Checker, Planner, Summarizer, incident block) sees which kind it is.
+SELF_DESCRIBED_MARKER = "(self-described)"
+
+
+def _target_label(action_obj: dict, *, fallback: str) -> str:
+    """The quoted target label, provenance-marked, or ``fallback`` when unnamed."""
+    observed = action_obj.get("target_text")
+    if observed:
+        return f"'{observed}'"
+    described = action_obj.get("target_description")
+    if described:
+        return f"'{described}' {SELF_DESCRIBED_MARKER}"
+    return fallback
+
 
 def format_action_clean(action_obj) -> str:
     if not action_obj:
@@ -49,7 +66,6 @@ def format_action_clean(action_obj) -> str:
         merged.setdefault("intent", raw_args.get("action"))
         action_obj = merged
 
-    target_text = action_obj.get("target_text") or action_obj.get("text")
     coords = action_obj.get("coordinates") or action_obj.get("target")
     app_name = (
         action_obj.get("app_name")
@@ -60,9 +76,13 @@ def format_action_clean(action_obj) -> str:
     )
     keycode = action_obj.get("keycode") or action_obj.get("key")
 
-    if act_type in ("tap", "click", "long_press"):
-        label = f"'{target_text}'" if target_text else "element"
-        if act_type == "long_press":
+    if act_type in ("tap", "click", "long_press", "long_press_on"):
+        # Legacy records carried the label under ``text``; the typed text of an
+        # input action never reaches this branch.
+        label = _target_label(action_obj, fallback="element")
+        if label == "element" and action_obj.get("text"):
+            label = f"'{action_obj['text']}'"
+        if act_type in ("long_press", "long_press_on"):
             duration = action_obj.get("duration") or action_obj.get("duration_ms")
             duration_str = f" for {duration}ms" if duration else ""
             return f"Long pressed {label} at {coords}{duration_str}"
@@ -82,10 +102,9 @@ def format_action_clean(action_obj) -> str:
             else:
                 return f"Tapped {label} at {coords}"
     elif act_type in ("input_text", "focus_and_input_text"):
-        # The field label is the resolved target's text only — the typed text
-        # is not the field's name.
-        field_text = action_obj.get("target_text")
-        label = f"'{field_text}'" if field_text else "field"
+        # The field label is the resolved target's text (or the model's own
+        # description of the field) only — the typed text is not the field's name.
+        label = _target_label(action_obj, fallback="field")
         text_val = action_obj.get("text")
         clear_exist = action_obj.get("clear_exist") or action_obj.get("clear_before_input")
         clear_str = " (without clearing)" if clear_exist is False else ""
@@ -110,7 +129,9 @@ def format_action_clean(action_obj) -> str:
             if isinstance(coords_list, list) and len(coords_list) == 4:
                 start_coords = coords_list[:2]
                 end_coords = coords_list[2:]
-        return f"Swiped from {start_coords} to {end_coords}{duration_str}"
+        dragged = _target_label(action_obj, fallback="")
+        dragged_str = f" {dragged}" if dragged else ""
+        return f"Swiped{dragged_str} from {start_coords} to {end_coords}{duration_str}"
     elif act_type == "press_key":
         return f"Pressed key '{keycode}'"
     elif act_type == "launch_app":
@@ -142,7 +163,18 @@ def format_action_clean(action_obj) -> str:
         )
         return f"Waited for {delay_val}"
     elif act_type in ("click_sequence", "tap_sequence"):
-        seq = action_obj.get("target") or action_obj.get("sequence") or []
+        seq = (
+            action_obj.get("target")
+            or action_obj.get("sequence")
+            or action_obj.get("coordinates")
+            or []
+        )
+        descriptions = action_obj.get("target_descriptions")
+        if isinstance(descriptions, (list, tuple)) and len(descriptions) == len(seq) and seq:
+            labelled = ", ".join(
+                f"'{d}' {SELF_DESCRIBED_MARKER} at {p}" for d, p in zip(descriptions, seq)
+            )
+            return f"Tapped sequence of targets: {labelled}"
         return f"Tapped sequence of targets: {seq}"
     else:
         return f"Action: {act_type} with args: {json.dumps(action_obj, ensure_ascii=False)}"
@@ -164,7 +196,7 @@ def _is_terminal_attempt_failure(attempts) -> bool:
     if not attempts:
         return False
     last = str(attempts[-1])
-    return last != "Success" and not last.startswith("Skipped")
+    return last != "Dispatched" and not last.startswith("Skipped")
 
 
 def failed_execution_error(result_obj) -> str | None:
@@ -217,13 +249,19 @@ def format_result_clean(result_obj) -> str | None:
 
     status = result_obj.get("status")
     error = failed_execution_error(result_obj)
-    if status in ("failed", "error"):
-        return f"Error: {error}" if error else "Error"
     if error:
-        return f"Error: {error}"
+        return _error_line(error)
+    if status in ("failed", "error"):
+        return "Error"
     return None
 
 
+def _error_line(error: str) -> str:
+    """One ``Error: ...`` line; executor messages already start with "Error"."""
+    text = str(error).strip()
+    return text if text.lower().startswith("error") else f"Error: {text}"
+
+
 def safe_parse_validation_result(result: Any) -> list:
     if isinstance(result, (list, tuple)):
         return list(result)
@@ -315,6 +353,8 @@ def format_tool_call_clean(
             "target_text": (
                 args.get("text") or args.get("target_text") or args.get("target_text_or_desc")
             ),
+            "target_description": args.get("target_description"),
+            "target_descriptions": args.get("target_descriptions"),
             "text": args.get("text"),
             "clear_exist": (args.get("clear_exist") or args.get("clear_before_input")),
             "clear_before_input": args.get("clear_before_input"),
@@ -492,13 +532,13 @@ def _burst_member_status(result: Any, index: int) -> str:
         return ""
     exec_list = result.get("execution") or []
     if index >= len(exec_list) or not isinstance(exec_list[index], dict):
-        return " (not executed)"
+        return " (not dispatched)"
     attempts = exec_list[index].get("attempts") or []
     if not attempts:
-        return " (executed)"
+        return " (dispatched)"
     last = str(attempts[-1])
-    if last == "Success":
-        return " (executed)"
+    if last == "Dispatched":
+        return " (dispatched)"
     if last.startswith("Skipped"):
         return " (skipped)"
     return f" (FAILED: {last.strip()})"

config/artemis.jsonc

@@ -65,7 +65,8 @@
 
     // 🎬 Video Analyzer: Multimodal screen recording verification and temporal analysis
     // NOTE: Dual-path engine automatically routes:
-    //  - Native Gemini Files API for Gemini models with Google API keys (long video file stream)
+    //  - Native Gemini Files API for Gemini models with Google API keys (long video file stream);
+    //    Gemini 3.6+ Flash sub-agents use agentic video understanding by default (see video_analyzer.processing below)
     //  - Universal Multi-Model Engine for Claude 3.5/3.7, GPT-4o, OpenRouter, Vertex AI (dynamic keyframe extraction + multimodal tool calling)
     "video_analyzer": {},
 
@@ -244,6 +245,9 @@
     "video_analyzer": {
       "enabled": true, // Whether continuous screen recording via scrcpy is active (default: auto-detected)
       "enable_ledger": true
+      // "processing": "auto", // How Gemini sub-agents watch a clip: auto (agentic on Gemini 3.6+ Flash, static elsewhere) | agentic | static
+      // "agentic_chunk_size_seconds": 600, // Longest clip one agentic sub-agent takes; static mode keeps chunk_size_seconds (60)
+      // "agentic_call_timeout_seconds": 300 // Hard timeout for one agentic sub-agent response
     },
     // 📝 Outputter: Post-execution report synthesis and structured schema extraction
     // Reviews action history, notes, and verifies goal achievement to synthesize final output.

pyproject.toml

@@ -34,7 +34,7 @@ dependencies = [
     "langchain-mcp-adapters>=0.2.0",
     "mcp>=1.26.0",
     # LLM Providers & GenAI SDKs
-    "google-genai>=0.1.1",
+    "google-genai>=2.22.0",
     "langchain-google-genai>=4.0.0",
     "langchain-google-vertexai>=3.0.0",
     "langchain-openai>=1.0.0",

tests/fixtures/action_surfaces/operator_shells.json

@@ -20,6 +20,18 @@
             ],
             "description": "Click target. Can be an element index number (int, e.g. 3) OR normalized coordinates (list of 2 integers, e.g. [500, 600])."
           },
+          "target_description": {
+            "anyOf": [
+              {
+                "type": "string"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "default": null,
+            "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index."
+          },
           "times": {
             "default": 1,
             "description": "Number of consecutive clicks on this target. Use this for double-clicks or multi-clicks (e.g. 7 to enter developer mode). Default is 1.",
@@ -63,6 +75,18 @@
             ],
             "description": "Input target field. Can be an input box element index number (int, e.g. 3) OR normalized coordinates (list of 2 integers, e.g. [500, 600])."
           },
+          "target_description": {
+            "anyOf": [
+              {
+                "type": "string"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "default": null,
+            "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index."
+          },
           "clear_exist": {
             "default": true,
             "description": "Whether to clear existing text before typing. True (default): clear/replace entire text. False: append at the end of existing content.",
@@ -81,7 +105,7 @@
     "type": "function",
     "function": {
       "name": "swipe",
-      "description": "[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen.\n\n• Directional Scrolling ('direction'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n• Precise Coordinate Gestures ('start', 'end'): Best for local, fine-grained interactions such as adjusting sliders/SeekBars (e.g., volume, brightness, progress bars), drag-and-drop / list reordering, or as a reliable fallback when directional scrolling fails on specific containers. Always drag slightly PAST the target position to overcome touch slop and reliably trigger the update. When setting a slider to Maximum (100%) or Minimum (0%), swipe fully to the extreme boundary.\n\nArgs:\n    direction: Smart directional scrolling ('up', 'down', 'left', 'right'). Automatically computes safe swipe vectors, retaining 40% visual overlap: 'up' (reveal content below), 'down' (reveal content above), 'left', 'right'.\n    start: Start normalized coordinates [start_x, start_y] in 0-1000 scale.\n    end: End normalized coordinates [end_x, end_y] in 0-1000 scale.\n    target: Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within.\n    gesture: Backward-compatible parameter: direction string OR custom coordinates list [start_x, start_y, end_x, end_y] in 0-1000 scale.\n    duration: Optional gesture duration in milliseconds (default 800).",
+      "description": "[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen.\n\n• Directional Scrolling ('direction'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n• Precise Coordinate Gestures ('start', 'end'): Best for local, fine-grained interactions such as adjusting sliders/SeekBars (e.g., volume, brightness, progress bars), drag-and-drop / list reordering, or as a reliable fallback when directional scrolling fails on specific containers. Always drag slightly PAST the target position to overcome touch slop and reliably trigger the update. When setting a slider to Maximum (100%) or Minimum (0%), swipe fully to the extreme boundary.\n\nArgs:\n    direction: Smart directional scrolling ('up', 'down', 'left', 'right'). Automatically computes safe swipe vectors, retaining 40% visual overlap: 'up' (reveal content below), 'down' (reveal content above), 'left', 'right'.\n    start: Start normalized coordinates [start_x, start_y] in 0-1000 scale.\n    end: End normalized coordinates [end_x, end_y] in 0-1000 scale.\n    target: Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within.\n    gesture: Backward-compatible parameter: direction string OR custom coordinates list [start_x, start_y, end_x, end_y] in 0-1000 scale.\n    target_description: What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling.\n    duration: Optional gesture duration in milliseconds (default 800).",
       "parameters": {
         "properties": {
           "direction": {
@@ -177,6 +201,18 @@
             "default": null,
             "description": "Backward-compatible swipe gesture: smart direction string ('up', 'down', 'left', 'right') OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale."
           },
+          "target_description": {
+            "anyOf": [
+              {
+                "type": "string"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "default": null,
+            "description": "What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling."
+          },
           "duration": {
             "anyOf": [
               {
@@ -287,6 +323,18 @@
             ],
             "description": "Long press target. Can be an element index number (int, e.g. 3) OR normalized coordinates (list of 2 integers, e.g. [500, 600])."
           },
+          "target_description": {
+            "anyOf": [
+              {
+                "type": "string"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "default": null,
+            "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index."
+          },
           "duration": {
             "default": 1000,
             "description": "Long press duration in milliseconds (default 1000).",

tests/fixtures/action_surfaces/tool_declarations.json

@@ -14,6 +14,10 @@
             },
             "description": "Normalized coordinates [x, y] in 0-1000 scale."
           },
+          "target_description": {
+            "type": "string",
+            "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). Required: a coordinate names nothing by itself."
+          },
           "times": {
             "type": "integer",
             "description": "Number of taps to perform (default 1). Set to 2 for double click."
@@ -24,7 +28,8 @@
           }
         },
         "required": [
-          "target"
+          "target",
+          "target_description"
         ]
       }
     }
@@ -44,17 +49,25 @@
               "items": {
                 "type": "integer"
               },
-              "description": "Normalized coordinates [x, y] in 0-1000 scale (e.g., [500, 280]), or a single integer element index."
+              "description": "Normalized coordinates [x, y] in 0-1000 scale (e.g., [500, 280])."
             },
             "description": "List of targets to tap in sequence, e.g. [[500, 280], [885, 362]]."
           },
+          "target_descriptions": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "description": "What each target is, in a few words, one entry per sequence entry in the same order (e.g. ['video body', 'skip button']). Required: a coordinate names nothing by itself."
+          },
           "delay_ms": {
             "type": "integer",
             "description": "Delay between consecutive taps in milliseconds (default 50ms)."
           }
         },
         "required": [
-          "sequence"
+          "sequence",
+          "target_descriptions"
         ]
       }
     }
@@ -74,13 +87,18 @@
             },
             "description": "Normalized coordinates [x, y] in 0-1000 scale."
           },
+          "target_description": {
+            "type": "string",
+            "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). Required: a coordinate names nothing by itself."
+          },
           "duration_ms": {
             "type": "integer",
             "description": "Duration of press in milliseconds (default 1000ms)."
           }
         },
         "required": [
-          "target"
+          "target",
+          "target_description"
         ]
       }
     }
@@ -104,6 +122,10 @@
             },
             "description": "Coordinates [x, y] of the input box in 0-1000 scale."
           },
+          "target_description": {
+            "type": "string",
+            "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). Required: a coordinate names nothing by itself."
+          },
           "clear_exist": {
             "type": "boolean",
             "description": "Whether to clear existing text in the input box before typing (default True)."
@@ -111,7 +133,8 @@
         },
         "required": [
           "text",
-          "target"
+          "target",
+          "target_description"
         ]
       }
     }
@@ -154,6 +177,10 @@
           "action": {
             "description": "Backward-compatible swipe gesture: smart direction string ('up', 'down', 'left', 'right') OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale."
           },
+          "target_description": {
+            "type": "string",
+            "description": "What is being dragged, in a few words (e.g. 'brightness slider knob'). Required for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling."
+          },
           "duration": {
             "type": "integer",
             "description": "Optional swipe/drag duration in milliseconds (default 800). For drag-and-drop, list reordering, or sliding/adjusting sliders (e.g., volume, brightness, SeekBars), set duration >= 1000 (e.g. 1500). If omitted for directional swipe, duration is computed automatically."

tests/unit/agents/test_checker.py

@@ -14,12 +14,15 @@
 
 from unittest.mock import AsyncMock, MagicMock, patch
 
+from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
+from langchain_core.tools import StructuredTool
 import pytest
 
 from artemis.agents.checker.checker import (
     CheckReport,
     CheckVerdict,
     _normalize_report,
+    _run_check_loop,
     assemble_checker_prompt_segments,
     build_checker_tools,
     build_probe_argv,
@@ -28,6 +31,7 @@ from artemis.agents.checker.checker import (
     verdicts_allow_release,
 )
 from artemis.context import ArtemisContext, ExecutionSetup
+from artemis.core.tool_failure import ToolFailure
 from artemis.graph.checkpoints import EvidenceAnchor
 from artemis.utils.plan_grammar import CheckItem
 
@@ -569,3 +573,126 @@ async def test_no_transcript_without_attempt_id_or_streamed_text(tmp_path):
 
     assert _read_streams(tmp_path) == []
     assert not (tmp_path / "check_streams.jsonl").exists()
+
+
+# --- Probe failures are structural; history rows keep target provenance ---------------
+
+
+@pytest.mark.asyncio
+async def test_probe_device_invalid_kind_is_a_tool_failure():
+    """A refused probe answers with a ToolFailure, so the tool loop marks the
+    ToolMessage as an error structurally rather than by sniffing its wording."""
+    from artemis.agents.checker.checker import get_probe_tool
+    from artemis.core.tool_failure import is_tool_failure
+
+    tool = get_probe_tool(_mock_ctx(disable_device_probes=False))
+    result = await tool.ainvoke({"kind": "shell", "params": None})
+    assert is_tool_failure(result)
+    assert "Error" in str(result)
+
+
+@pytest.mark.asyncio
+async def test_probe_device_execution_error_is_a_tool_failure():
+    from artemis.agents.checker.checker import get_probe_tool
+    from artemis.core.tool_failure import is_tool_failure
+
+    ctx = _mock_ctx(disable_device_probes=False)
+    ctx.get_adb_client.side_effect = RuntimeError("no adb")
+    tool = get_probe_tool(ctx)
+    result = await tool.ainvoke({"kind": "battery", "params": None})
+    assert is_tool_failure(result)
+    assert "no adb" in str(result)
+
+
+def test_format_history_marks_self_described_targets_only():
+    """A step without a summary renders its action through the shared renderer,
+    so the Checker sees which target labels are the model's own statement."""
+    from artemis.agents.checker.checker import _format_history
+    from artemis.utils.task_tree import SELF_DESCRIBED_MARKER
+
+    ctx = _mock_ctx()
+    ctx.data_engine.get_agent_friendly_steps.return_value = [
+        {
+            "step_number": 1,
+            "relative_time": "T+00:01",
+            "summary": "",
+            "action_taken": {
+                "action": "click",
+                "coordinates": [500, 600],
+                "target_description": "play button",
+            },
+        },
+        {
+            "step_number": 2,
+            "relative_time": "T+00:05",
+            "summary": None,
+            "action_taken": {
+                "action": "click",
+                "coordinates": [100, 200],
+                "target_text": "Login",
+                "target_resource_id": "com.app:id/login",
+            },
+        },
+    ]
+
+    text = _format_history(ctx)
+    lines = text.splitlines()
+    assert lines[0].startswith("- Step 1 (T+00:01): ")
+    assert f"'play button' {SELF_DESCRIBED_MARKER}" in lines[0]
+    assert lines[1].startswith("- Step 2 (T+00:05): ")
+    assert "'Login'" in lines[1]
+    assert SELF_DESCRIBED_MARKER not in lines[1]
+    # Neither row is the raw JSON record.
+    assert "target_description" not in text
+    assert "target_resource_id" not in text
+
+
+# --- tool results: status is structural, never sniffed from the words -----------------
+
+
+# A helper tool reports failure structurally (``ToolFailure``); free-form text
+# that merely starts with "Error" is an ordinary answer.
+_STATUS_CASES = [
+    pytest.param(ToolFailure("Error: note 'progress' not found"), "error", id="tool_failure"),
+    pytest.param("Error 404 was typed into the search box", "success", id="plain_error_text"),
+]
+
+
+def _text_tool(name: str, result):
+    async def _run(key: str) -> str:
+        return result
+
+    return StructuredTool.from_function(coroutine=_run, name=name, description=name)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("result, expected_status", _STATUS_CASES)
+async def test_check_loop_tool_message_status_is_structural(result, expected_status):
+    ctx = _mock_ctx()
+    tool = _text_tool("read_note", result)
+    tool_turn = AIMessage(
+        content="", tool_calls=[{"name": "read_note", "args": {"key": "progress"}, "id": "c1"}]
+    )
+    final_turn = AIMessage(content="done", tool_calls=[])
+
+    llm = MagicMock()
+    llm.bind_tools.return_value = llm
+    llm.with_structured_output.return_value.ainvoke = AsyncMock(
+        return_value=CheckReport(verdicts=[])
+    )
+    messages = [SystemMessage(content="s"), HumanMessage(content="h")]
+
+    with (
+        patch("artemis.agents.checker.checker.get_llm", return_value=llm),
+        patch(
+            "artemis.agents.checker.checker.acomplete",
+            new=AsyncMock(side_effect=[tool_turn, final_turn]),
+        ),
+    ):
+        await _run_check_loop(ctx, messages, [tool], check_items=[])
+
+    tool_msgs = [m for m in messages if isinstance(m, ToolMessage)]
+    assert len(tool_msgs) == 1
+    assert tool_msgs[0].tool_call_id == "c1"
+    assert tool_msgs[0].status == expected_status
+    assert tool_msgs[0].content == str(result)

tests/unit/agents/test_diagnoser.py

@@ -14,8 +14,10 @@
 
 from unittest.mock import AsyncMock, Mock, patch
 from langchain_core.messages import AIMessage, ToolMessage
+from langchain_core.tools import StructuredTool
 from artemis.agents.diagnoser.diagnoser import Diagnoser
 from artemis.context import ArtemisContext
+from artemis.core.tool_failure import ToolFailure
 from artemis.graph.state import State
 import pytest
 
@@ -293,3 +295,40 @@ async def test_diagnoser_background_job_lifecycle(
     assert job["status"] == "completed"
     assert job["result"] == "Video shows step successfully completed."
     assert job["consumed"] is True
+
+
+# --- tool results: status is structural, never sniffed from the words -----------------
+
+
+# A helper tool reports failure structurally (``ToolFailure``); free-form text
+# that merely starts with "Error" is an ordinary answer.
+_STATUS_CASES = [
+    pytest.param(ToolFailure("Error: note 'progress' not found"), "error", id="tool_failure"),
+    pytest.param("Error 404 was typed into the search box", "success", id="plain_error_text"),
+]
+
+
+def _text_tool(name: str, result):
+    async def _run(key: str) -> str:
+        return result
+
+    return StructuredTool.from_function(coroutine=_run, name=name, description=name)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("result, expected_status", _STATUS_CASES)
+async def test_run_tool_message_status_is_structural(
+    result, expected_status, mock_context, mock_state
+):
+    agent = Diagnoser(mock_context)
+    tool = _text_tool("read_note", result)
+
+    messages = await agent._run_tool(
+        {"name": "read_note", "args": {"key": "progress"}, "id": "c1"}, [tool], mock_state
+    )
+
+    assert len(messages) == 1
+    assert isinstance(messages[0], ToolMessage)
+    assert messages[0].tool_call_id == "c1"
+    assert messages[0].status == expected_status
+    assert messages[0].content == str(result)

tests/unit/agents/test_diagnoser_failures.py

@@ -17,6 +17,7 @@ from adbutils import AdbClient
 from langchain_core.messages import AIMessage, ToolMessage
 from artemis.agents.diagnoser.diagnoser import Diagnoser
 from artemis.context import ArtemisContext
+from artemis.core.tool_failure import ToolFailure
 from artemis.graph.state import State
 import pytest
 
@@ -107,7 +108,7 @@ async def test_maestro_uninstall_timeout(
     mock_ui_tool = AsyncMock()
     mock_ui_tool.name = "get_ui_hierarchy"
     mock_ui_tool.args = {}
-    mock_ui_tool.ainvoke.return_value = "Error: pm list packages timed out"
+    mock_ui_tool.ainvoke.return_value = ToolFailure("Error: pm list packages timed out")
     mock_get_ui_tool.return_value = mock_ui_tool
 
     # Run agent
@@ -255,7 +256,7 @@ async def test_ui_hierarchy_dump_stall(
     mock_ui_tool = AsyncMock()
     mock_ui_tool.name = "get_ui_hierarchy"
     mock_ui_tool.args = {}
-    mock_ui_tool.ainvoke.return_value = (
+    mock_ui_tool.ainvoke.return_value = ToolFailure(
         "Error retrieving UI hierarchy: dump_hierarchy timed out after 3 seconds"
     )
     mock_get_ui_tool.return_value = mock_ui_tool

tests/unit/agents/test_execution_incident_prompt.py

@@ -37,7 +37,7 @@ STEPS = [
     {
         "step_number": 11,
         "action_taken": [{"action": "click", "coordinates": [540, 1800], "target_text": "player"}],
-        "last_execution_result": {"status": "success"},
+        "last_execution_result": {"status": "dispatched"},
     },
     {
         "step_number": 12,
@@ -98,6 +98,20 @@ def test_open_incident_continues_the_previous_count():
 # --- rendering ----------------------------------------------------------------------
 
 
+def test_render_labels_a_described_coordinate_target_by_its_description():
+    incident = _disappeared()
+    incident["action"] = {
+        "action": "tap",
+        "coordinates": [950, 288],
+        "normalized_coordinates": [880, 120],
+        "target_description": "skip ad button",
+    }
+    text = render_execution_incident(incident, STEPS)
+    assert 'normalized [880, 120] "skip ad button" (self-described)' in text
+    # An observed (index) target carries no marker.
+    assert '"Skip" (self-described)' not in render_execution_incident(_disappeared(), STEPS)
+
+
 def test_render_disappeared_names_target_trigger_and_burst_exception():
     text = render_execution_incident(_disappeared(), STEPS)
     assert text.startswith(EXECUTION_INCIDENT_MARKER)
@@ -106,7 +120,7 @@ def test_render_disappeared_names_target_trigger_and_burst_exception():
     # Recorded target with normalized coordinates and text.
     assert 'normalized [880, 120] "Skip"' in text
     # The last successful action before the incident is the trigger candidate.
-    assert "Your last successfully executed action was `Tapped 'player' at [540, 1800]`" in text
+    assert "Your last dispatched action was `Tapped 'player' at [540, 1800]`" in text
     assert "(Step 11)" in text
     # Facts and evidence only: the response protocol is stated once, in the
     # static system prompt, so the per-turn block never restates it.
@@ -145,7 +159,7 @@ def test_render_burst_abort_and_exec_error():
 
 def test_render_without_history_omits_the_trigger_hint():
     text = render_execution_incident(_disappeared(), [])
-    assert "last successfully executed action" not in text
+    assert "last dispatched action" not in text
     assert 'Its recorded target was normalized [880, 120] "Skip".' in text
 
 

tests/unit/agents/test_flash_runner.py

@@ -143,14 +143,17 @@ def test_flash_runner_screenshot_pruning(mock_context):
 # --- Recorded action shape (coordinate space + ledger rendering) ---------------------
 
 
-def _record(runner, name, args):
+def _record(runner, name, args, metadata=None):
     from types import SimpleNamespace
 
     runner.ctx.data_engine = Mock()
     runner.ctx.data_engine.current_step_id = None
     runner.ctx.data_engine.record_step.return_value = "step-1"
     exec_result = SimpleNamespace(
-        status="success", text_summary=f"{name} executed", metadata={}, ui_elements_text="ui"
+        status="success",
+        text_summary=f"{name} executed",
+        metadata=metadata or {},
+        ui_elements_text="ui",
     )
     step_id = runner._record_action_step(
         name, args, exec_result, "thought", {}, b"pre", ["xml"], b"post"
@@ -205,3 +208,41 @@ def test_flash_records_render_in_the_ledger_with_their_arguments(mock_context):
     assert format_actions_clean(swipe_fa) == "Swiped down"
     swipe_coords = _record(runner, "swipe", {"coordinates": [556, 289, 556, 124]})
     assert format_actions_clean(swipe_coords) == "Swiped from [556, 289] to [556, 124]"
+
+
+def test_flash_records_the_executors_target_semantics(mock_context):
+    """Flash targets are coordinates, so the model's own description (validated
+    and shaped by the executor into ``metadata.target_semantics``) is the only
+    target semantics on record: top level, the shape Pro records, rendered as the
+    action's label. Nothing is inferred when the executor hands back nothing."""
+    from artemis.utils.task_tree import format_actions_clean
+
+    with patch("artemis.controllers.unified_controller.get_driver"):
+        runner = FlashRunner(mock_context, goal="g")
+
+    click_args = {"target": [320, 399], "target_description": "play button"}
+    click = _record(
+        runner,
+        "click",
+        click_args,
+        metadata={"target_semantics": {"target_description": "play button"}},
+    )
+    assert click["target_description"] == "play button"
+    assert "target_text" not in click
+    assert format_actions_clean(click) == "Tapped 'play button' (self-described) at [320, 399]"
+
+    burst = _record(
+        runner,
+        "click_sequence",
+        {"sequence": [[500, 280], [885, 362]], "target_descriptions": ["video body", "skip"]},
+        metadata={"target_semantics": {"target_descriptions": ["video body", "skip"]}},
+    )
+    assert burst["target_descriptions"] == ["video body", "skip"]
+    assert format_actions_clean(burst) == (
+        "Tapped sequence of targets: 'video body' (self-described) at [500, 280],"
+        " 'skip' (self-described) at [885, 362]"
+    )
+
+    bare = _record(runner, "click", {"target": [320, 399]}, metadata={"target_semantics": {}})
+    assert "target_description" not in bare
+    assert "target_text" not in bare

tests/unit/agents/test_flash_runner_ledger.py

@@ -31,7 +31,7 @@ from unittest.mock import AsyncMock, Mock, patch
 import pytest
 from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
 
-from artemis.agents.flash.runner import FlashRunner
+from artemis.agents.flash.runner import FlashRunner, _TurnRecord
 from artemis.agents.validator.tool_declarations import ToolExecutionResult
 from artemis.context import ArtemisContext
 from artemis.graph.state import State
@@ -43,7 +43,7 @@ from artemis.memory.transcript import (
 
 OBSERVATION_HEADER_RE = re.compile(r"^# CURRENT OBSERVATION \[T\+\d{2,}:\d{2}\]$")
 RESULT_RE = re.compile(
-    rf"^{re.escape(EXECUTION_RESULT_MARKER)} \(T\+\d{{2,}}:\d{{2}}\) ---\nStatus: success$"
+    rf"^{re.escape(EXECUTION_RESULT_MARKER)} \(T\+\d{{2,}}:\d{{2}}\) ---\nStatus: dispatched$"
 )
 
 
@@ -198,6 +198,44 @@ def test_observation_tail_has_pro_shape(mock_context):
         ]
 
 
+# ---------------------------------------------------------------------------
+# Injected instruction: verbatim text for the record and the lens, wrapped
+# notice for the observation tail only
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_injected_instruction_splits_verbatim_text_from_operator_notice(
+    mock_context, tmp_path
+):
+    """The user's words are what the step record, the chunk ledger and the
+    visual lens receive (Pro parity); the 'You MUST immediately ...' directive
+    is operator-facing and rides on the observation tail alone."""
+    import json
+
+    (tmp_path / "injected_instruction.json").write_text(
+        json.dumps({"instruction": "Skip the popup and log in", "release_loop": True}),
+        encoding="utf-8",
+    )
+    mock_context.data_engine = Mock()
+    mock_context.data_engine.base_dir = str(tmp_path)
+
+    with patch("artemis.controllers.unified_controller.get_driver"):
+        runner = FlashRunner(mock_context, goal="Log in")
+        instruction, notice = await runner._read_injected_instruction()
+
+    assert instruction == "Skip the popup and log in"
+    assert notice.startswith(
+        "[REAL-TIME INJECTED INSTRUCTION from user]: Skip the popup and log in"
+    )
+    assert "You MUST immediately follow this instruction" in notice
+    assert "authorized stopping any ongoing monitoring loop" in notice
+    assert not (tmp_path / "injected_instruction.json").exists()
+
+    with patch("artemis.controllers.unified_controller.get_driver"):
+        assert await runner._read_injected_instruction() == (None, None)
+
+
 # ---------------------------------------------------------------------------
 # End-to-end loop over the ledger (mocked model / executor)
 # ---------------------------------------------------------------------------
@@ -258,7 +296,11 @@ async def test_run_builds_prompt_from_ledger_with_session_offsets(mock_context):
         AIMessage(
             content="I will tap then wait.",
             tool_calls=[
-                {"name": "click", "args": {"target": [500, 600]}, "id": "tc1"},
+                {
+                    "name": "click",
+                    "args": {"target": [500, 600], "target_description": "Settings"},
+                    "id": "tc1",
+                },
                 {"name": "wait_for_delay", "args": {"seconds": 1}, "id": "tc2"},
             ],
         ),
@@ -358,3 +400,48 @@ async def test_run_no_tool_call_turn_is_nudged_not_terminated(mock_context):
         and any(EXECUTION_RESULT_MARKER in b.get("text", "") for b in m.content)
         for m in turn2_messages
     )
+
+
+# ---------------------------------------------------------------------------
+# _TurnRecord.result: the turn's execution result in the validator-report shape
+# ---------------------------------------------------------------------------
+
+
+def test_turn_record_result_is_none_for_helper_only_turns():
+    assert _TurnRecord().result() is None
+
+
+def test_turn_record_result_is_dispatched_when_every_action_was_accepted():
+    record = _TurnRecord(
+        actions=[("click", "success", "Tapped at [500, 500]"), ("swipe", "success", "Swiped up.")]
+    )
+    assert record.result() == {"status": "dispatched"}
+
+
+def test_turn_record_failed_result_keeps_an_executor_message_that_names_the_action():
+    """Executor messages already name the action; they are not prefixed again."""
+    record = _TurnRecord(
+        actions=[
+            ("click", "success", "Tapped at [500, 500]"),
+            ("swipe", "error", "Error executing swipe: direction 'sideways' is not valid"),
+        ]
+    )
+    assert record.result() == {
+        "status": "failed",
+        "error": "Error executing swipe: direction 'sideways' is not valid",
+    }
+
+
+def test_turn_record_failed_result_prefixes_an_error_that_does_not_name_the_action():
+    record = _TurnRecord(actions=[("type_text", "error", "Device is offline")])
+    assert record.result() == {"status": "failed", "error": "type_text: Device is offline"}
+
+
+def test_turn_record_reports_the_first_failure_of_a_multi_action_turn():
+    record = _TurnRecord(
+        actions=[
+            ("click", "error", "click rejected: coordinates out of bounds"),
+            ("swipe", "error", "never reached"),
+        ]
+    )
+    assert record.result()["error"] == "click rejected: coordinates out of bounds"

tests/unit/agents/test_flash_scrub_edge.py

@@ -316,4 +316,4 @@ def test_untracked_messages_are_never_touched(mock_context):
     assert messages[0].content == "[REAL-TIME INJECTED INSTRUCTION from user]: stay put"
     assert messages[2].content == "thinking"
     # Image-only tool message froze with the defensive fallback text.
-    assert messages[1].content == [{"type": "text", "text": "Action completed."}]
+    assert messages[1].content == [{"type": "text", "text": "Action dispatched."}]

tests/unit/agents/test_flash_step_summarizer.py

@@ -20,7 +20,13 @@ import pytest
 from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
 
 from artemis.agents.flash.context_compressor import ScrubEdgeCompressor
-from artemis.agents.flash.summarizer import VisualStepSummarizer
+from artemis.agents.flash.summarizer import (
+    FOCUS_BLOCK_HEADER,
+    FOCUS_INTENT_MAX_CHARS,
+    VisualStepSummarizer,
+    build_focus_context,
+    render_focus_block,
+)
 from artemis.context import ArtemisContext
 from artemis.sdk.builders import Builders
 
@@ -560,6 +566,175 @@ async def test_raw_lens_call_meters_llm_usage_without_touching_context_base(mock
     assert meter.last_prompt_tokens == 777
 
 
+@pytest.mark.asyncio
+async def test_focus_block_and_provenance_marked_action_phrase_reach_the_lens(mock_context):
+    """The lens input opens with the action as every history reader sees it
+    (self-described target marked), the verbatim arguments, the outcome, and
+    the operator focus block; the system prompt teaches how to use it."""
+    summarizer = VisualStepSummarizer(mock_context)
+    captured: list = []
+
+    async def mock_ainvoke(messages):
+        captured.append(messages)
+        return AIMessage(
+            content=(
+                "In Step 3, I tapped the switch marked in red on the 'Wi-Fi' row, which"
+                " read 'Off' before; afterwards it showed 'On' and the list stayed empty."
+            )
+        )
+
+    summarizer._llm = Mock()
+    summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
+
+    summarizer.dispatch(
+        step_number=3,
+        action_name="click",
+        action_args={"target": [880, 410], "target_description": "Wi-Fi toggle"},
+        pre_img_bytes=b"pre",
+        post_img_bytes=b"post",
+        exec_outcome="Tapped at [880, 410] (normalized).",
+        focus=build_focus_context(
+            intent="The Wi-Fi row shows Off. I will tap its switch and expect it to read On.",
+            goal="Turn on Wi-Fi",
+            subgoal="Enable Wi-Fi > Tap the Wi-Fi switch",
+            injected_instruction="Do not join any network yet.",
+        ),
+    )
+    await summarizer.flush()
+
+    assert summarizer.has_summary(3)
+    system_text = str(captured[0][0].content)
+    lead_text = captured[0][1].content[0]["text"]
+    assert "# OPERATOR FOCUS" in system_text
+    assert "Absence is a fact" in system_text
+    assert lead_text.startswith(
+        "Step 3 Physical Action: Tapped 'Wi-Fi toggle' (self-described) at [880, 410]\n"
+    )
+    assert "Action arguments (verbatim): click({'target': [880, 410]" in lead_text
+    assert "Controller Outcome: Tapped at [880, 410] (normalized)." in lead_text
+    assert FOCUS_BLOCK_HEADER in lead_text
+    assert "Task goal: Turn on Wi-Fi" in lead_text
+    assert "Active sub-goal: Enable Wi-Fi > Tap the Wi-Fi switch" in lead_text
+    assert "User instruction at this step: Do not join any network yet." in lead_text
+    assert "expect it to read On." in lead_text
+    # The focus precedes the frames: context first, evidence second.
+    assert captured[0][1].content[1]["text"].startswith("--- [1] BEFORE ACTION SCREEN")
+
+
+@pytest.mark.asyncio
+async def test_dispatch_without_focus_carries_no_focus_block(mock_context):
+    """Callers without operator context (older call sites, tests) get the
+    plain lead block: no focus header, observed target rendered unmarked."""
+    summarizer = VisualStepSummarizer(mock_context)
+    captured: list = []
+
+    async def mock_ainvoke(messages):
+        captured.append(messages)
+        return AIMessage(
+            content="In Step 1, I tapped the 'Search' field marked in red; the keyboard opened."
+        )
+
+    summarizer._llm = Mock()
+    summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
+
+    summarizer.dispatch(
+        step_number=1,
+        action_name="tap",
+        action_args={"coordinates": [500, 600], "target_text": "Search"},
+        pre_img_bytes=b"pre",
+        post_img_bytes=None,
+        exec_outcome="dispatched",
+    )
+    await summarizer.flush()
+
+    lead_text = captured[0][1].content[0]["text"]
+    assert lead_text.startswith("Step 1 Physical Action: Tapped 'Search' at [500, 600]\n")
+    assert "(self-described)" not in lead_text
+    assert FOCUS_BLOCK_HEADER not in lead_text
+    assert summarizer.get_job_payload(1)["focus"] is None
+
+
+@pytest.mark.asyncio
+async def test_single_frame_focus_label_forbids_describing_the_outcome(mock_context):
+    """With one decision frame (every Pro step) the reasoning is labelled as the
+    operator's aim only, never as an expected screen change to narrate."""
+    summarizer = VisualStepSummarizer(mock_context)
+    captured: list = []
+
+    async def mock_ainvoke(messages):
+        captured.append(messages)
+        return AIMessage(
+            content="In Step 2, I tapped the 'Pay' button marked in red on a checkout screen totalling '$42.00'."
+        )
+
+    summarizer._llm = Mock()
+    summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
+
+    summarizer.dispatch(
+        step_number=2,
+        action_name="tap",
+        action_args={"coordinates": [500, 900], "target_text": "Pay"},
+        pre_img_bytes=b"decision_frame",
+        post_img_bytes=None,
+        exec_outcome="dispatched",
+        focus=build_focus_context(intent="Tap Pay; I expect a confirmation dialog."),
+    )
+    await summarizer.flush()
+
+    lead_text = captured[0][1].content[0]["text"]
+    assert "the expected outcome is NOT in this frame and must not be described" in lead_text
+    assert "what screen change it expected" not in lead_text
+    # The focus text is released once the summary landed, like the frames.
+    assert summarizer.get_job_payload(2)["focus"] is None
+
+
+def test_action_phrase_renders_flash_swipe_points_and_bursts():
+    """Flash's ``start``/``end`` swipe points and a Pro burst's additional
+    actions render through the shared history phrase, never as None."""
+    phrase = VisualStepSummarizer._action_phrase(
+        "swipe", {"start": [500, 800], "end": [500, 200], "target_description": "video list"}
+    )
+    assert phrase == "Swiped 'video list' (self-described) from [500, 800] to [500, 200]"
+    assert VisualStepSummarizer._action_phrase("swipe", {"direction": "up"}) == "Swiped up"
+
+    burst = VisualStepSummarizer._action_phrase(
+        "tap",
+        {
+            "coordinates": [100, 200],
+            "target_description": "Skip ad",
+            "additional_actions": [
+                {"action": "tap", "coordinates": [300, 400], "target_text": "Play"}
+            ],
+        },
+    )
+    assert burst.startswith("Fast-action burst (2 actions, unvetted): ")
+    assert "Tapped 'Skip ad' (self-described) at [100, 200] -> Tapped 'Play' at [300, 400]" in burst
+
+
+def test_build_focus_context_drops_blank_fields_and_caps_reasoning_tail_intact():
+    """Blank/None fields never appear; an over-long reasoning keeps its tail
+    (where the operator states the expected screen change) and announces
+    the cut in place."""
+    assert (
+        build_focus_context(intent="  ", goal=None, subgoal="", injected_instruction=None) is None
+    )
+
+    head = "H" * 500
+    tail = "I expect the toggle to read On."
+    filler = "x" * (FOCUS_INTENT_MAX_CHARS * 2)
+    focus = build_focus_context(intent=head + filler + tail, goal="Turn on Wi-Fi")
+    assert set(focus) == {"goal", "intent"}
+    assert focus["intent"].startswith("HHHH")
+    assert focus["intent"].endswith(tail)
+    assert "characters cut here" in focus["intent"]
+    assert len(focus["intent"]) < FOCUS_INTENT_MAX_CHARS + 80
+
+    block = render_focus_block(focus)
+    assert block.splitlines()[0] == FOCUS_BLOCK_HEADER
+    assert "Task goal: Turn on Wi-Fi" in block
+    assert "Active sub-goal" not in block
+
+
 def test_flash_config_and_builder():
     """Verify Flash profile configuration model and SDK builder fluent API."""
     cfg = Builders.AgentConfig.with_flash_config(

tests/unit/agents/test_history_analyzer.py

@@ -16,8 +16,10 @@ from pathlib import Path
 from unittest.mock import MagicMock, patch
 
 from langchain_core.messages import ToolMessage
+from langchain_core.tools import StructuredTool
 from artemis.agents.history_analyzer.history_analyzer import HistoryAnalyzer
 from artemis.context import ArtemisContext
+from artemis.core.tool_failure import ToolFailure
 import pytest
 
 
@@ -374,3 +376,65 @@ async def test_history_analyzer_integration_with_task_tree():
         assert args[3] == expected_hash
 
         assert kwargs.get("engine") is mock_ctx.data_engine
+
+
+# --- tool results: status is structural, never sniffed from the words -----------------
+
+
+# A helper tool reports failure structurally (``ToolFailure``); free-form text
+# that merely starts with "Error" is an ordinary answer.
+_STATUS_CASES = [
+    pytest.param(ToolFailure("Error: note 'progress' not found"), "error", id="tool_failure"),
+    pytest.param("Error 404 was typed into the search box", "success", id="plain_error_text"),
+]
+
+
+def _text_tool(name: str, result):
+    async def _run(key: str) -> str:
+        return result
+
+    return StructuredTool.from_function(coroutine=_run, name=name, description=name)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("result, expected_status", _STATUS_CASES)
+async def test_history_analyzer_tool_message_status_is_structural(result, expected_status):
+    mock_ctx = MagicMock(spec=ArtemisContext)
+    mock_ctx.data_engine = MagicMock()
+    mock_ctx.data_engine.base_dir = "/tmp/fake_traces"
+    mock_ctx.data_engine.get_agent_friendly_steps.return_value = [
+        {"step_number": 1, "relative_time": "1.2s", "summary": "Opened the settings app"}
+    ]
+
+    tool_turn = MagicMock(content="")
+    tool_turn.tool_calls = [{"name": "read_note", "args": {"key": "progress"}, "id": "c1"}]
+    final_turn = MagicMock(content="done")
+    final_turn.tool_calls = []
+    responses = [tool_turn, final_turn]
+
+    async def mock_astream(*args, **kwargs):
+        yield responses.pop(0)
+
+    mock_llm = MagicMock()
+    mock_llm.bind_tools.return_value = mock_llm
+    mock_llm.astream.side_effect = mock_astream
+
+    with (
+        patch(
+            "artemis.agents.history_analyzer.history_analyzer.get_llm",
+            return_value=mock_llm,
+        ),
+        patch.object(
+            HistoryAnalyzer, "_build_tools", return_value=[_text_tool("read_note", result)]
+        ),
+        patch("pathlib.Path.exists", return_value=False),
+    ):
+        answer = await HistoryAnalyzer(mock_ctx).run("What did the note say?")
+
+    assert answer == "done"
+    second_turn_messages = mock_llm.astream.call_args_list[1][0][0]
+    tool_msgs = [m for m in second_turn_messages if isinstance(m, ToolMessage)]
+    assert len(tool_msgs) == 1
+    assert tool_msgs[0].tool_call_id == "c1"
+    assert tool_msgs[0].status == expected_status
+    assert tool_msgs[0].content == str(result)

tests/unit/agents/test_log_analyzer.py

@@ -0,0 +1,78 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""LogAnalyzerNode's ReAct loop: a helper tool's failure is structural
+(``ToolFailure``), never sniffed from the words of its text."""
+
+from unittest.mock import AsyncMock, MagicMock, patch
+
+from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
+from langchain_core.tools import StructuredTool
+import pytest
+
+from artemis.agents.log_analyzer.log_analyzer import LogAnalyzerNode
+from artemis.context import ArtemisContext
+from artemis.core.tool_failure import ToolFailure
+
+_STATUS_CASES = [
+    pytest.param(ToolFailure("Error searching logs: device gone"), "error", id="tool_failure"),
+    pytest.param(
+        "Error: 12 matching log lines\nE/Foo: Error opening file", "success", id="plain_error_text"
+    ),
+]
+
+
+def _text_tool(name: str, result):
+    async def _run(keyword: str) -> str:
+        return result
+
+    return StructuredTool.from_function(coroutine=_run, name=name, description=name)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("result, expected_status", _STATUS_CASES)
+async def test_agent_loop_tool_message_status_is_structural(result, expected_status):
+    ctx = MagicMock(spec=ArtemisContext)
+    ctx.data_engine = None
+    node = LogAnalyzerNode(ctx=ctx)
+    tool = _text_tool("search_logs", result)
+
+    base_llm = MagicMock()
+    base_llm.bind_tools.return_value = base_llm
+    tool_turn = AIMessage(
+        content="", tool_calls=[{"name": "search_logs", "args": {"keyword": "Error"}, "id": "c1"}]
+    )
+    final_turn = AIMessage(content="Summary.", tool_calls=[])
+    messages = [SystemMessage(content="s"), HumanMessage(content="h")]
+
+    with patch(
+        "artemis.agents.log_analyzer.log_analyzer.acomplete",
+        new=AsyncMock(side_effect=[tool_turn, final_turn]),
+    ):
+        outcome = await node._run_agent_loop(
+            base_llm=base_llm,
+            tools_for_binding=[tool],
+            tools=[tool],
+            current_messages=messages,
+            state=MagicMock(),
+            max_iterations=5,
+            agent_name="Log Analyzer",
+        )
+
+    assert outcome == "Summary."
+    tool_msgs = [m for m in messages if isinstance(m, ToolMessage)]
+    assert len(tool_msgs) == 1
+    assert tool_msgs[0].tool_call_id == "c1"
+    assert tool_msgs[0].status == expected_status
+    assert tool_msgs[0].content == str(result)

tests/unit/agents/test_operator.py

@@ -18,6 +18,7 @@ from unittest.mock import AsyncMock, MagicMock, Mock, patch
 from artemis.agents.operator.operator import OperatorNode
 from artemis.config.agent import MemoryTranscriptConfig
 from artemis.context import ArtemisContext
+from artemis.core.tool_failure import ToolFailure
 import pytest
 
 # These tests exercise the legacy 2-message prompt path. Since M5 the
@@ -54,7 +55,11 @@ async def test_operator_node_fast_path():
     mock_response.tool_calls = [
         {
             "name": "click",
-            "args": {"target": [50, 50], "reasoning": "Test reasoning"},
+            "args": {
+                "target": [50, 50],
+                "target_description": "button",
+                "reasoning": "Test reasoning",
+            },
             "id": "call_123",
         }
     ]
@@ -121,7 +126,6 @@ async def test_perform_action_validation():
             "target_bounds": "[100,100][300,500]",
             "target_resource_id": "btn_1",
             "target_class": "android.widget.Button",
-            "target_label_source": "index",
         }
     ]
 
@@ -152,7 +156,6 @@ async def test_perform_action_validation():
         "target_bounds": "[300,400][500,600]",
         "target_resource_id": "input_1",
         "target_class": "android.widget.EditText",
-        "target_label_source": "index",
     }
 
     # Test swipe direction
@@ -181,7 +184,6 @@ async def test_perform_action_validation():
             "target_bounds": "[300,400][500,600]",
             "target_resource_id": "input_1",
             "target_class": "android.widget.EditText",
-            "target_label_source": "index",
         }
     ]
 
@@ -235,8 +237,13 @@ async def test_perform_action_validation():
 
 
 @pytest.mark.asyncio
-async def test_bare_coordinate_click_enriched_by_hit_test():
-    """A coordinate click hit tests the pre-action frame for element semantics."""
+async def test_coordinate_click_requires_and_records_the_models_description():
+    """A coordinate target carries the model's own description, never an inferred label.
+
+    Even when an indexed element sits right under the point, nothing is hit
+    tested on the model's behalf: the recorded semantics are exactly what the
+    model declared, kept apart from the observed ``target_*`` fields.
+    """
     from artemis.agents.operator.operator import OperatorNode
     from unittest.mock import MagicMock
     from artemis.graph.state import State
@@ -256,10 +263,22 @@ async def test_bare_coordinate_click_enriched_by_hit_test():
         }
     ]
 
-    # Normalized (500, 600) on the default 1080x2400 device -> pixel (540, 1440),
-    # which falls inside the "Confirm" button bounds.
+    # Coordinates without a description are refused with a retryable error.
+    for args in (
+        {"target": [500, 600]},
+        {"target": [500, 600], "target_description": "   "},
+        {"target": [500, 600], "target_description": 12},
+    ):
         actions, err = node._translate_and_validate_tool(
-        {"name": "click", "args": {"target": [500, 600]}}, mock_state
+            {"name": "click", "args": args}, mock_state
+        )
+        assert actions == []
+        assert err.startswith("Error:")
+        assert "target_description" in err
+
+    actions, err = node._translate_and_validate_tool(
+        {"name": "click", "args": {"target": [500, 600], "target_description": " play button "}},
+        mock_state,
     )
     assert err is None
     assert actions == [
@@ -270,22 +289,59 @@ async def test_bare_coordinate_click_enriched_by_hit_test():
             "normalized_coordinates": [500, 600],
             "times": 1,
             "delay_ms": 100,
-            "target_text": "Confirm",
-            "target_bounds": [500, 1400, 600, 1480],
-            "target_resource_id": "btn_confirm",
-            "target_class": "android.widget.Button",
-            "target_label_source": "hit_test",
+            "target_description": "play button",
         }
     ]
 
-    # Graceful degradation: no perception data at all -> bare coordinates, "none".
-    mock_state.indexed_elements = []
+    # The same rule applies to input_text and the description is ignored for an index.
     actions, err = node._translate_and_validate_tool(
-        {"name": "click", "args": {"target": [500, 600]}}, mock_state
+        {"name": "input_text", "args": {"text": "hi", "target": [500, 600]}}, mock_state
+    )
+    assert actions == [] and "target_description" in err
+    actions, err = node._translate_and_validate_tool(
+        {
+            "name": "input_text",
+            "args": {"text": "hi", "target": 1, "target_description": "ignored"},
+        },
+        mock_state,
     )
     assert err is None
-    assert actions[0]["target_text"] is None
-    assert actions[0]["target_label_source"] == "none"
+    assert actions[0]["target_text"] == "Confirm"
+    assert "target_description" not in actions[0]
+
+
+@pytest.mark.asyncio
+async def test_coordinate_swipe_requires_description_directional_does_not():
+    from artemis.agents.operator.operator import OperatorNode
+    from unittest.mock import MagicMock
+    from artemis.graph.state import State
+
+    node = OperatorNode(MagicMock(), transcript_config=LEGACY_TRANSCRIPT)
+    mock_state = MagicMock(spec=State)
+    mock_state.indexed_elements = []
+    mock_state.latest_ui_hierarchy = None
+
+    actions, err = node._translate_and_validate_tool(
+        {"name": "swipe", "args": {"start": [200, 600], "end": [800, 600]}}, mock_state
+    )
+    assert actions == [] and "target_description" in err
+
+    actions, err = node._translate_and_validate_tool(
+        {
+            "name": "swipe",
+            "args": {"start": [200, 600], "end": [800, 600], "target_description": "volume knob"},
+        },
+        mock_state,
+    )
+    assert err is None
+    assert actions[0]["action"] == "swipe"
+    assert actions[0]["target_description"] == "volume knob"
+
+    actions, err = node._translate_and_validate_tool(
+        {"name": "swipe", "args": {"direction": "up"}}, mock_state
+    )
+    assert err is None
+    assert "target_description" not in actions[0]
 
 
 @pytest.mark.asyncio
@@ -314,10 +370,14 @@ async def test_operator_node_multiple_actions():
     mock_response.tool_calls = [
         {
             "name": "input_text",
-            "args": {"text": "hello", "target": [50, 50]},
+            "args": {"text": "hello", "target": [50, 50], "target_description": "field"},
             "id": "call_1",
         },
-        {"name": "click", "args": {"target": [50, 50]}, "id": "call_2"},
+        {
+            "name": "click",
+            "args": {"target": [50, 50], "target_description": "button"},
+            "id": "call_2",
+        },
     ]
 
     mock_llm.ainvoke = AsyncMock(return_value=mock_response)
@@ -587,7 +647,13 @@ async def test_operator_tracks_subagent_calls():
     ]
 
     mock_response_2 = MagicMock()
-    mock_response_2.tool_calls = [{"name": "click", "args": {"target": [10, 20]}, "id": "call_2"}]
+    mock_response_2.tool_calls = [
+        {
+            "name": "click",
+            "args": {"target": [10, 20], "target_description": "button"},
+            "id": "call_2",
+        }
+    ]
 
     responses = [mock_response, mock_response_2]
     call_count = 0
@@ -659,7 +725,11 @@ async def test_operator_defer_action_for_gathering():
             "args": {"query": "check something"},
             "id": "call_diagnose",
         },
-        {"name": "click", "args": {"target": [50, 50]}, "id": "call_click"},
+        {
+            "name": "click",
+            "args": {"target": [50, 50], "target_description": "button"},
+            "id": "call_click",
+        },
     ]
 
     # Second response only calls click
@@ -667,7 +737,7 @@ async def test_operator_defer_action_for_gathering():
     mock_response_2.tool_calls = [
         {
             "name": "click",
-            "args": {"target": [50, 50]},
+            "args": {"target": [50, 50], "target_description": "button"},
             "id": "call_click_final",
         }
     ]
@@ -795,7 +865,6 @@ async def test_long_press_action_translation():
             "target_bounds": "[100,100][300,500]",
             "target_resource_id": "btn_1",
             "target_class": "android.widget.Button",
-            "target_label_source": "index",
         }
     ]
 
@@ -803,6 +872,14 @@ async def test_long_press_action_translation():
     actions, err = node._translate_and_validate_tool(
         {"name": "long_press", "args": {"target": [500, 600]}}, mock_state
     )
+    assert actions == [] and "target_description" in err
+    actions, err = node._translate_and_validate_tool(
+        {
+            "name": "long_press",
+            "args": {"target": [500, 600], "target_description": "app icon"},
+        },
+        mock_state,
+    )
     assert err is None
     # Since coordinates are in 0-1000 scale, they should be converted using device width (default 1080) and height (default 2400)
     # x = 500 * 1080 / 1000 = 540
@@ -814,11 +891,7 @@ async def test_long_press_action_translation():
             "coordinate_space": "pixel",
             "normalized_coordinates": [500, 600],
             "duration": 1000,
-            "target_text": None,
-            "target_bounds": None,
-            "target_resource_id": None,
-            "target_class": None,
-            "target_label_source": "none",
+            "target_description": "app icon",
         }
     ]
 
@@ -860,7 +933,11 @@ async def test_operator_no_defer_for_note_updating():
             "args": {"key": "progress", "content": "step 1"},
             "id": "call_save",
         },
-        {"name": "click", "args": {"target": [50, 50]}, "id": "call_click"},
+        {
+            "name": "click",
+            "args": {"target": [50, 50], "target_description": "button"},
+            "id": "call_click",
+        },
     ]
 
     # We only expect 1 LLM call (astream should only yield once)
@@ -943,7 +1020,11 @@ async def test_operator_defer_for_note_reading():
     mock_response_1 = MagicMock()
     mock_response_1.tool_calls = [
         {"name": "read_note", "args": {"key": "progress"}, "id": "call_read"},
-        {"name": "click", "args": {"target": [50, 50]}, "id": "call_click"},
+        {
+            "name": "click",
+            "args": {"target": [50, 50], "target_description": "button"},
+            "id": "call_click",
+        },
     ]
 
     # 2nd turn: LLM calls only click after receiving note content
@@ -951,7 +1032,7 @@ async def test_operator_defer_for_note_reading():
     mock_response_2.tool_calls = [
         {
             "name": "click",
-            "args": {"target": [50, 50]},
+            "args": {"target": [50, 50], "target_description": "button"},
             "id": "call_click_final",
         }
     ]
@@ -1043,7 +1124,11 @@ async def test_operator_no_defer_for_note_appending():
             "args": {"key": "progress", "content": "step 1"},
             "id": "call_append",
         },
-        {"name": "click", "args": {"target": [50, 50]}, "id": "call_click"},
+        {
+            "name": "click",
+            "args": {"target": [50, 50], "target_description": "button"},
+            "id": "call_click",
+        },
     ]
 
     responses = [mock_response_1]
@@ -1291,7 +1376,7 @@ async def test_operator_fallback_function_call_parsing():
         additional_kwargs={
             "function_call": {
                 "name": "click",
-                "arguments": '{"target": [123, 456]}',
+                "arguments": '{"target": [123, 456], "target_description": "button"}',
             }
         },
         tool_calls=[],
@@ -1339,7 +1424,12 @@ async def test_operator_swipe_translation():
     actions, err = node._translate_and_validate_tool(
         {
             "name": "swipe",
-            "args": {"start": [100, 200], "end": [500, 600], "duration": 1500},
+            "args": {
+                "start": [100, 200],
+                "end": [500, 600],
+                "duration": 1500,
+                "target_description": "list item",
+            },
         },
         mock_state,
     )
@@ -1348,12 +1438,13 @@ async def test_operator_swipe_translation():
     assert actions[0]["action"] == "swipe"
     assert actions[0]["coordinates"] == [108, 480, 540, 1440]
     assert actions[0]["duration"] == 1500
+    assert actions[0]["target_description"] == "list item"
 
     # 3. Coordinates alias
     actions, err = node._translate_and_validate_tool(
         {
             "name": "swipe",
-            "args": {"coordinates": [200, 300, 400, 500]},
+            "args": {"coordinates": [200, 300, 400, 500], "target_description": "slider"},
         },
         mock_state,
     )
@@ -1374,3 +1465,77 @@ async def test_operator_swipe_translation():
     assert len(actions) == 1
     assert actions[0]["action"] == "swipe"
     assert actions[0]["coordinates"] == [648, 720, 648, 1680]
+
+
+# --- helper tool results: status is structural, never sniffed from the words ----------
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "result, expected_status",
+    [
+        pytest.param(ToolFailure("Error: note 'progress' not found"), "error", id="tool_failure"),
+        pytest.param("Error 404 was typed into the search box", "success", id="plain_error_text"),
+    ],
+)
+async def test_operator_helper_tool_message_status_is_structural(result, expected_status):
+    from langchain_core.messages import ToolMessage
+
+    mock_ctx = MagicMock(spec=ArtemisContext)
+    mock_ctx.execution_setup = None
+    mock_ctx.data_engine = None
+
+    mock_state = MagicMock()
+    mock_state.subagent_calls = []
+    mock_state.initial_goal = "Test goal"
+
+    # Turn 1: read_note next to a click. A failed helper rejects the click; a
+    # deferring helper defers it. Either way turn 2 carries the tool message.
+    turn_1 = MagicMock()
+    turn_1.tool_calls = [
+        {"name": "read_note", "args": {"key": "progress"}, "id": "call_read"},
+        {
+            "name": "click",
+            "args": {"target": [50, 50], "target_description": "button"},
+            "id": "call_click",
+        },
+    ]
+    turn_2 = MagicMock()
+    turn_2.tool_calls = [
+        {
+            "name": "click",
+            "args": {"target": [50, 50], "target_description": "button"},
+            "id": "call_click_final",
+        }
+    ]
+    mock_llm = MagicMock()
+    mock_llm.ainvoke = AsyncMock(side_effect=[turn_1, turn_2])
+    mock_llm.bind_tools.return_value = mock_llm
+
+    read_note_tool = MagicMock()
+    read_note_tool.name = "read_note"
+    read_note_tool.args = {}
+
+    async def dummy_read_note(key: str, **kwargs):
+        return result
+
+    read_note_tool.coroutine = dummy_read_note
+    read_note_tool.func = None
+
+    with (
+        patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm),
+        patch(
+            "artemis.agents.operator.operator.trace_langchain_tool",
+            side_effect=lambda t, ctx: t,
+        ),
+    ):
+        node = OperatorNode(mock_ctx, tools=[read_note_tool], transcript_config=LEGACY_TRANSCRIPT)
+        await node(mock_state)
+
+    assert mock_llm.ainvoke.await_count == 2
+    second_turn_messages = mock_llm.ainvoke.call_args_list[1].args[0]
+    tool_msgs = {m.tool_call_id: m for m in second_turn_messages if isinstance(m, ToolMessage)}
+    assert tool_msgs["call_read"].status == expected_status
+    assert tool_msgs["call_read"].content == str(result)
+    # The accompanying click is rejected only when the helper actually failed.
+    assert tool_msgs["call_click"].status == ("error" if expected_status == "error" else "success")

tests/unit/agents/test_operator_plan_ledger.py

@@ -158,7 +158,12 @@ def _click_response(n_actions=1):
     response = MagicMock()
     response.content = ""
     response.tool_calls = [
-        {"name": "click", "args": {"target": [50, 50]}, "id": f"call_{i}"} for i in range(n_actions)
+        {
+            "name": "click",
+            "args": {"target": [50, 50], "target_description": "button"},
+            "id": f"call_{i}",
+        }
+        for i in range(n_actions)
     ]
     return response
 

tests/unit/agents/test_operator_transcript.py

@@ -53,8 +53,8 @@ from artemis.memory.transcript import (
 
 # SHA-256 snapshots of the legacy system message with the fixed inputs below.
 # Update these when an intentional template change alters the rendered prompt.
-GOLDEN_EMPTY_PLAN = "2ed35be503e3872f008ae5a12616af6f90cde65f23b63adbb6da782079ae4e7c"
-GOLDEN_SENTINEL_PLAN = "8bd6aa60cf3fad8d21cf5c284a01f586c57d209fcd1d2b4a517a791736f9b1de"
+GOLDEN_EMPTY_PLAN = "d8cf83565c766d32893b124c974060bbd98de2d862c3b9925e546ea0c946c324"
+GOLDEN_SENTINEL_PLAN = "8f871b5e5700ba5ee07ed03b80a8f82664abe68b0c170d3185bf1ee35f7821ed"
 
 SCREENSHOT_B64 = base64.b64encode(b"fake-jpeg-bytes").decode("utf-8")
 
@@ -198,7 +198,7 @@ async def test_transcript_mode_two_turns_build_four_regions(tmp_path):
             _transcript_state(
                 current_step_id="step-1",
                 structured_decisions='[{"action": "tap"}]',
-                last_execution_result={"status": "success"},
+                last_execution_result={"status": "dispatched"},
             )
         )
 
@@ -237,7 +237,7 @@ async def test_transcript_actionless_turn_commits_without_validator_message():
             _transcript_state(
                 current_step_id="step-1",
                 structured_decisions="",
-                last_execution_result={"status": "success"},
+                last_execution_result={"status": "dispatched"},
             )
         )
 

tests/unit/agents/test_outputter.py

@@ -20,6 +20,7 @@ import pytest
 from artemis.agents.outputter.outputter import outputter
 from artemis.config import LLM, OutputConfig  # noqa: E402
 from artemis.context import ArtemisContext  # noqa: E402
+from artemis.core.tool_failure import ToolFailure  # noqa: E402
 from artemis.utils.logger import get_logger  # noqa: E402
 
 logger = get_logger(__name__)
@@ -509,3 +510,52 @@ async def test_outputter_executes_save_note_tool(mock_get_llm, mock_context, moc
 
     # Assert final result
     assert result == "Verification code is 123456."
+
+
+# --- tool results: status is structural, never sniffed from the words -----------------
+
+
+# A helper tool reports failure structurally (``ToolFailure``); free-form text
+# that merely starts with "Error" is an ordinary answer.
+_STATUS_CASES = [
+    pytest.param(ToolFailure("Error: note 'progress' not found"), "error", id="tool_failure"),
+    pytest.param("Error 404 was typed into the search box", "success", id="plain_error_text"),
+]
+
+
+def _text_tool(name: str, result):  # noqa: D103
+    from langchain_core.tools import StructuredTool
+
+    async def _run(key: str) -> str:
+        return result
+
+    return StructuredTool.from_function(coroutine=_run, name=name, description=name)
+
+
+@patch("artemis.agents.outputter.outputter.get_read_note_tool_pure")
+@patch("artemis.agents.outputter.outputter.get_llm")
+@pytest.mark.asyncio
+@pytest.mark.parametrize("result, expected_status", _STATUS_CASES)
+async def test_outputter_tool_message_status_is_structural(
+    mock_get_llm, mock_get_read_note, result, expected_status, mock_context, mock_state
+):
+    from langchain_core.messages import AIMessage, ToolMessage
+
+    mock_get_read_note.return_value = _text_tool("read_note", result)
+    _, mock_llm_with_tools, _ = setup_mock_llm(mock_get_llm)
+    tool_turn = AIMessage(
+        content="", tool_calls=[{"name": "read_note", "args": {"key": "progress"}, "id": "c1"}]
+    )
+    final_turn = AIMessage(content="The answer.", tool_calls=[])
+    mock_llm_with_tools.ainvoke.side_effect = [tool_turn, final_turn]
+
+    config = OutputConfig(structured_output=None, output_description=None)
+    answer = await outputter(ctx=mock_context, output_config=config, graph_output=mock_state)
+
+    assert answer == "The answer."
+    second_turn_messages = mock_llm_with_tools.ainvoke.call_args_list[1].args[0]
+    tool_msgs = [m for m in second_turn_messages if isinstance(m, ToolMessage)]
+    assert len(tool_msgs) == 1
+    assert tool_msgs[0].tool_call_id == "c1"
+    assert tool_msgs[0].status == expected_status
+    assert tool_msgs[0].content == str(result)

tests/unit/agents/test_precondition_pixel_target.py

@@ -64,6 +64,10 @@ def test_coordinates_only_target_is_labelled_as_surface():
 
     first_text = messages[1].content[0]["text"]
     assert "Kind: coordinates only" in first_text
+    # No hit test runs any more: the block states that nothing was recorded, it
+    # does not claim the point was looked up in the UI hierarchy.
+    assert "no target metadata recorded" in first_text
+    assert "UI hierarchy" not in first_text
     assert "Label:" not in first_text
     assert "Resource ID:" not in first_text
     assert not any("Original Thinking" in t for t in _texts(messages))
@@ -77,13 +81,12 @@ def test_label_alone_counts_as_named_control():
     assert "Resource ID" not in text
 
 
-def test_unlabelled_hit_test_element_is_still_a_control():
+def test_unlabelled_indexed_element_is_still_a_control():
     item = {
         "action": "tap",
         "coordinates": [1001, 718],
         "target_class": "android.widget.ImageView",
         "target_bounds": [922, 655, 1080, 781],
-        "target_label_source": "hit_test",
     }
     text = pp._describe_target(item)
     assert "Kind: specific UI control" in text
@@ -92,9 +95,33 @@ def test_unlabelled_hit_test_element_is_still_a_control():
     assert "[922, 655, 1080, 781]" in text
 
 
+def test_described_coordinate_target_is_its_own_kind():
+    """The model's description is presented as its belief, not as observed metadata."""
+    item = {"action": "tap", "coordinates": [540, 400], "target_description": "play button"}
+    text = pp._describe_target(item)
+    assert "Kind: described target" in text
+    assert "Operator's description: play button" in text
+    assert "Kind: specific UI control" not in text
+    assert "Label:" not in text
+
+
+def test_observed_metadata_outranks_description():
+    item = {
+        "action": "tap",
+        "coordinates": [10, 10],
+        "target_text": "Submit",
+        "target_description": "ignored",
+    }
+    text = pp._describe_target(item)
+    assert "Kind: specific UI control" in text
+    assert "Label: Submit" in text
+    assert "described target" not in text
+
+
 def test_prompt_rules_are_keyed_on_target_kind():
     prompt = Path(pp.__file__).with_name("pixel_safety_net.md").read_text(encoding="utf-8")
     assert "`Kind: specific UI control`" in prompt
+    assert "`Kind: described target`" in prompt
     assert "`Kind: coordinates only`" in prompt
     assert "hidden" in prompt
     assert "Identify what appears under the red dot" in prompt

tests/unit/agents/test_summarizer.py

@@ -37,12 +37,16 @@ class DummyState:
         operator_native_thinking=None,
         last_execution_result=None,
         current_step_id=None,
+        initial_goal=None,
+        injected_instruction=None,
     ):
         self.structured_decisions = structured_decisions
         self.operator_raw_thinking = operator_raw_thinking
         self.operator_native_thinking = operator_native_thinking
         self.last_execution_result = last_execution_result
         self.current_step_id = current_step_id
+        self.initial_goal = initial_goal
+        self.injected_instruction = injected_instruction
 
 
 @pytest.fixture
@@ -58,6 +62,7 @@ def mock_context(tmp_path):
     post_path.write_bytes(b"POST_IMAGE")
 
     engine = Mock()
+    engine.base_dir = None  # no task_plan on disk: no sub-goal in the focus
     engine.get_step_number.return_value = 7
     engine.get_step_record.return_value = SimpleNamespace(
         action_taken=[
@@ -80,7 +85,7 @@ def mock_context(tmp_path):
 async def test_summarizer_dispatches_visual_lens(mock_context):
     """The node dispatches to the shared service instead of calling an LLM."""
     state = DummyState(
-        last_execution_result={"status": "success"},
+        last_execution_result={"status": "dispatched"},
         current_step_id="12345678-1234-5678-1234-567812345678",
     )
 
@@ -98,8 +103,9 @@ async def test_summarizer_dispatches_visual_lens(mock_context):
         },
         pre_img_bytes=b"PRE_IMAGE",
         post_img_bytes=b"POST_IMAGE",
-        exec_outcome="success",
+        exec_outcome="dispatched",
         data_engine_step_id="12345678-1234-5678-1234-567812345678",
+        focus=None,
     )
     # No direct capsule write from the node: the lens owns the versioned
     # summary write when its background job completes.
@@ -138,7 +144,7 @@ async def test_summarizer_falls_back_to_structured_decisions(mock_context):
             ' "coordinate_space": "pixel"},'
             ' {"action": "tap", "coordinates": [540, 1200], "coordinate_space": "pixel"}]'
         ),
-        last_execution_result={"status": "success"},
+        last_execution_result={"status": "dispatched"},
         current_step_id="12345678-1234-5678-1234-567812345678",
     )
 
@@ -153,6 +159,55 @@ async def test_summarizer_falls_back_to_structured_decisions(mock_context):
     ]
 
 
+@pytest.mark.asyncio
+async def test_summarizer_passes_operator_focus(mock_context, tmp_path):
+    """The operator's reasoning, the goal, the injected instruction and the live
+    sub-goal leaf ride along as the lens's focus context; the record's
+    reasoning is the fallback when the state carries none."""
+    notes = tmp_path / "notes"
+    notes.mkdir()
+    (notes / "task_plan.md").write_text(
+        "- [/] Enable Wi-Fi\n  - [/] Tap the Wi-Fi switch so it reads On\n- [ ] Join network\n",
+        encoding="utf-8",
+    )
+    mock_context.data_engine.base_dir = str(tmp_path)
+    mock_context.data_engine.get_step_record.return_value = SimpleNamespace(
+        action_taken=[
+            {
+                "action": "tap",
+                "coordinates": [950, 984],
+                "coordinate_space": "pixel",
+                "target_description": "Wi-Fi switch",
+            }
+        ],
+        operator_raw_thinking="Recorded reasoning: the switch reads Off; tapping it should read On.",
+    )
+    state = DummyState(
+        last_execution_result={"status": "dispatched"},
+        current_step_id="12345678-1234-5678-1234-567812345678",
+        initial_goal="Turn on Wi-Fi",
+        injected_instruction="Do not join any network yet.",
+    )
+
+    await SummarizerNode(mock_context)(state)
+
+    kwargs = mock_context.step_memory.dispatch.call_args.kwargs
+    assert kwargs["action_args"]["target_description"] == "Wi-Fi switch"
+    assert kwargs["focus"] == {
+        "goal": "Turn on Wi-Fi",
+        "subgoal": "Enable Wi-Fi > Tap the Wi-Fi switch so it reads On",
+        "injected_instruction": "Do not join any network yet.",
+        "intent": "Recorded reasoning: the switch reads Off; tapping it should read On.",
+    }
+
+    # Live state reasoning outranks the record's copy.
+    mock_context.step_memory.dispatch.reset_mock()
+    state.operator_raw_thinking = "Live reasoning wins."
+    await SummarizerNode(mock_context)(state)
+    focus = mock_context.step_memory.dispatch.call_args.kwargs["focus"]
+    assert focus["intent"] == "Live reasoning wins."
+
+
 @pytest.mark.asyncio
 async def test_summarizer_noop_without_step_id(mock_context):
     state = DummyState(current_step_id=None)

tests/unit/agents/test_validator.py

@@ -257,7 +257,7 @@ async def test_validator_burst_success_executes_every_member(
         result = await node(state)
 
     report = result["last_execution_result"]
-    assert report["status"] == "success"
+    assert report["status"] == "dispatched"
     assert report["burst"] is True
     assert report["incident"] is None
     assert result["open_incident"] is None
@@ -287,7 +287,7 @@ async def test_validator_success_closes_open_incident(mock_mcp, mock_context, te
     with patch("artemis.utils.image_diff.check_ui_change", return_value=True):
         result = await node(state)
 
-    assert result["last_execution_result"]["status"] == "success"
+    assert result["last_execution_result"]["status"] == "dispatched"
     assert result["open_incident"] is None
     # The closed record is handed over once so the Operator settles the intent.
     assert result["last_closed_incident"]["kind"] == "safety_net"
@@ -805,7 +805,7 @@ async def test_validator_pre_execution_validation_ocr_direct_to_pixel(
 
     assert "last_execution_result" in result
     report = result["last_execution_result"]
-    assert report["status"] == "success"
+    assert report["status"] == "dispatched"
 
 
 @pytest.mark.asyncio

tests/unit/agents/test_video_analyzer.py

@@ -14,10 +14,21 @@
 
 import os
 from pathlib import Path
+from types import SimpleNamespace
 from unittest.mock import AsyncMock, MagicMock, patch
 
+from artemis.agents.video_analyzer import chunk_conversation
+from artemis.agents.video_analyzer.chunk_conversation import (
+    _MISSING_SUMMARY_ERROR,
+    _NO_TOOL_CALL_ERROR,
+    SubAgentAnswerExhausted,
+    _ChunkMedia,
+    _drive_sub_agent_loop,
+    _run_native_chunk_conversation,
+)
 from artemis.agents.video_analyzer.video_analyzer import VideoAnalyzer
 from artemis.context import ArtemisContext
+from google.genai import types
 import pytest
 
 
@@ -29,6 +40,7 @@ async def test_video_analyzer_run():
     mock_ctx.llm_config = MagicMock()
     mock_ctx.llm_config.utils.video_analyzer = MagicMock()
     mock_ctx.llm_config.utils.video_analyzer.model = "gemini-3.7-flash"
+    mock_ctx.agent_config = SimpleNamespace(video_analyzer=SimpleNamespace(processing="static"))
 
     # Mock Gemini Client
     mock_client = MagicMock()
@@ -80,6 +92,7 @@ async def test_video_analyzer_preserves_thought_signature():
     mock_ctx.llm_config = MagicMock()
     mock_ctx.llm_config.utils.video_analyzer = MagicMock()
     mock_ctx.llm_config.utils.video_analyzer.model = "gemini-3.7-flash"
+    mock_ctx.agent_config = SimpleNamespace(video_analyzer=SimpleNamespace(processing="static"))
 
     # Mock MobileDeviceController
     mock_controller = MagicMock()
@@ -256,6 +269,7 @@ async def test_video_analyzer_sub_agent_confidence_validation():
     mock_ctx.llm_config = MagicMock()
     mock_ctx.llm_config.utils.video_analyzer = MagicMock()
     mock_ctx.llm_config.utils.video_analyzer.model = "gemini-3.7-flash"
+    mock_ctx.agent_config = SimpleNamespace(video_analyzer=SimpleNamespace(processing="static"))
 
     # Mock MobileDeviceController
     mock_controller = MagicMock()
@@ -454,3 +468,149 @@ async def test_video_analyzer_sub_agent_confidence_validation():
         model_turns = [c for c in sub_turn_2_contents if c.role == "model"]
         assert len(model_turns) == 1
         assert model_turns[0].parts[0].thought_signature == sub_dummy_signature
+
+
+# ------------------------------------------------- static sub-agent loop
+
+
+def _static_media(tmp_path) -> _ChunkMedia:
+    media = _ChunkMedia()
+    media.path = tmp_path / "segment.mp4"
+    media.path.write_bytes(b"video")
+    media.compressed_path = media.path
+    media.actual_start = 0.0
+    media.prompt_with_context = "Watch the toggle."
+    return media
+
+
+def _valid_static_event() -> dict:
+    return {
+        "start_time": 1.0,
+        "end_time": 2.0,
+        "transcription": "Battery Saver toggled on",
+        "confidence_score": 0.9,
+        "verification_timestamp_secs": 1.5,
+    }
+
+
+def _scripted_static_turns(*turns):
+    """Fakes ``_stream_sub_agent_turn`` with ``(text, function_calls)`` per turn."""
+    scripted = list(turns)
+
+    async def fake_turn(analyzer, sub_agent_contents, trace_id):
+        text, function_calls = scripted.pop(0)
+        return text, function_calls, []
+
+    return fake_turn
+
+
+def _static_analyzer() -> MagicMock:
+    analyzer = MagicMock()
+    analyzer.model_name = "gemini-3.7-flash"
+    analyzer.sub_system_prompt = "system"
+    analyzer.ctx.data_engine = None
+    return analyzer
+
+
+@pytest.mark.asyncio
+async def test_static_submit_answer_without_summary_is_sent_back_then_accepted(tmp_path):
+    first = types.FunctionCall(
+        name="submit_answer", args={"timeline_events": [_valid_static_event()]}
+    )
+    second = types.FunctionCall(
+        name="submit_answer",
+        args={"summary": "s2", "analysis": "a2", "timeline_events": []},
+    )
+    contents: list = []
+
+    with (
+        patch.object(
+            chunk_conversation,
+            "_stream_sub_agent_turn",
+            new=_scripted_static_turns(("", [first]), ("", [second])),
+        ),
+        patch.object(chunk_conversation, "_persist_timeline_events", new=AsyncMock()) as persist,
+    ):
+        answer = await _drive_sub_agent_loop(
+            _static_analyzer(), contents, None, _static_media(tmp_path), 0.0, 5.0, "verify"
+        )
+
+    assert answer == ("s2", "a2")
+    # Only the accepted answer reaches persistence.
+    persist.assert_awaited_once()
+    assert persist.await_args.args[1] == []
+    rejection = contents[-1]
+    assert rejection.role == "user"
+    assert rejection.parts[0].function_response.response["error"] == _MISSING_SUMMARY_ERROR
+
+
+@pytest.mark.asyncio
+async def test_static_loop_raises_when_no_turn_calls_submit_answer(tmp_path):
+    with (
+        patch.object(
+            chunk_conversation,
+            "_stream_sub_agent_turn",
+            new=_scripted_static_turns(("text only", []), ("still text", [])),
+        ),
+        patch.object(chunk_conversation, "_persist_timeline_events", new=AsyncMock()) as persist,
+        pytest.raises(SubAgentAnswerExhausted) as raised,
+    ):
+        await _drive_sub_agent_loop(
+            _static_analyzer(), [], None, _static_media(tmp_path), 0.0, 5.0, "verify"
+        )
+
+    assert raised.value.reason == _NO_TOOL_CALL_ERROR
+    persist.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_static_last_turn_timeline_failure_raises_and_persists_nothing(tmp_path):
+    bad = dict(_valid_static_event(), confidence_score="HIGH")
+    out_of_range = dict(_valid_static_event(), verification_timestamp_secs=42.0)
+    first = types.FunctionCall(
+        name="submit_answer", args={"summary": "s1", "timeline_events": [bad]}
+    )
+    second = types.FunctionCall(
+        name="submit_answer", args={"summary": "s2", "timeline_events": [out_of_range]}
+    )
+
+    with (
+        patch.object(
+            chunk_conversation,
+            "_stream_sub_agent_turn",
+            new=_scripted_static_turns(("", [first]), ("", [second])),
+        ),
+        patch.object(chunk_conversation, "_persist_timeline_events", new=AsyncMock()) as persist,
+        pytest.raises(SubAgentAnswerExhausted) as raised,
+    ):
+        await _drive_sub_agent_loop(
+            _static_analyzer(), [], None, _static_media(tmp_path), 0.0, 5.0, "verify"
+        )
+
+    assert "verification_timestamp_secs" in raised.value.reason
+    assert "42.0" in str(raised.value)
+    persist.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_exhausted_static_chunk_is_not_committed_to_the_blackboard(tmp_path):
+    analyzer = _static_analyzer()
+    uploaded = SimpleNamespace(name="files/abc", uri="https://files/abc", mime_type="video/mp4")
+    analyzer.upload_and_poll_file = AsyncMock(return_value=uploaded)
+    analyzer.client.aio.files.delete = AsyncMock()
+    analyzer.cloud_files_to_cleanup = set()
+
+    with (
+        patch.object(
+            chunk_conversation,
+            "_stream_sub_agent_turn",
+            new=_scripted_static_turns(("text only", []), ("still text", [])),
+        ),
+        pytest.raises(SubAgentAnswerExhausted),
+    ):
+        await _run_native_chunk_conversation(
+            analyzer, _static_media(tmp_path), 0.0, 5.0, 0.0, 5.0, "verify", "lease-1"
+        )
+
+    analyzer.blackboard.complete_segment.assert_not_called()
+    analyzer.client.aio.files.delete.assert_awaited_once_with(name="files/abc")

tests/unit/agents/test_video_analyzer_universal.py

@@ -235,3 +235,215 @@ async def test_run_universal_multi_turn_reasoning(mock_context):
         assert status == "success"
         assert "launched successfully at 3.5s" in outcome
         assert agent.exec_extract_segment_metadata.call_count == 1
+
+
+# --- No-usable-answer handling -------------------------------------------
+#
+# The universal engine is single-shot: when the model neither calls
+# submit_answer with a summary nor replies with text, it must raise
+# SubAgentAnswerExhausted so the caller fails the blackboard segment instead
+# of committing placeholder text as a succeeded entry.
+
+_BLANK_SUBMIT = AIMessage(
+    content="",
+    tool_calls=[
+        {
+            "name": "submit_answer",
+            "args": {"summary": "   ", "analysis": "", "timeline_events": []},
+            "id": "blank_call",
+        }
+    ],
+)
+_EMPTY_RESPONSE = AIMessage(content="", tool_calls=[])
+_TEXT_ONLY = AIMessage(content="Prose answer: the toast appeared at 2.0s.", tool_calls=[])
+
+
+def _universal_agent(mock_context) -> VideoAnalyzer:
+    agent = VideoAnalyzer(mock_context)
+    agent.use_native_gemini = False
+    return agent
+
+
+async def _run_chunk_with_response(agent, tmp_path, response):
+    """Drives the universal chunk path with a stubbed model reply.
+
+    Returns ``(result_or_exception, persist_mock)`` so tests can assert both
+    the outcome and whether the blackboard persistence step was reached.
+    """
+    raw_video = tmp_path / "raw.mp4"
+    raw_video.write_bytes(b"raw_bytes")
+    persist = AsyncMock()
+    with (
+        patch(
+            "artemis.agents.video_analyzer.video_analyzer.extract_keyframes_from_video",
+            return_value=[(0.0, b"frame0")],
+        ),
+        patch(
+            "artemis.agents.video_analyzer.video_analyzer.extract_frames_at_timestamps",
+            return_value=[],
+        ),
+        patch(
+            "artemis.agents.video_analyzer.video_analyzer.extract_audio_from_video",
+            new_callable=AsyncMock,
+            side_effect=RuntimeError("no audio track"),
+        ),
+        patch(
+            "artemis.agents.video_analyzer.universal_engine._persist_universal_events",
+            persist,
+        ),
+        patch.object(
+            agent,
+            "_invoke_universal_model",
+            new_callable=AsyncMock,
+            return_value=response,
+        ),
+    ):
+        try:
+            result = await agent._exec_single_chunk_universal(
+                compressed_path=raw_video,
+                raw_path=raw_video,
+                start_time=0.0,
+                end_time=3.0,
+                actual_start=0.0,
+                prompt_with_context="Find the toast.",
+                specific_query="toast",
+            )
+        except Exception as exc:  # noqa: BLE001 - the exception type is the assertion
+            return exc, persist
+    return result, persist
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("response", [_EMPTY_RESPONSE, _BLANK_SUBMIT])
+async def test_exec_single_chunk_universal_raises_without_usable_answer(
+    mock_context, tmp_path, response
+):
+    """No tool call + no text, or blank-summary tool call + no text -> raise, no persistence."""
+    from artemis.agents.video_analyzer.reliability import SubAgentAnswerExhausted
+
+    agent = _universal_agent(mock_context)
+    outcome, persist = await _run_chunk_with_response(agent, tmp_path, response)
+
+    assert isinstance(outcome, SubAgentAnswerExhausted)
+    assert outcome.turns == 1
+    assert "submit_answer" in outcome.reason
+    persist.assert_not_awaited()
+    assert agent.blackboard_entries == []
+
+
+@pytest.mark.asyncio
+async def test_exec_single_chunk_universal_text_only_is_degraded_answer(mock_context, tmp_path):
+    """Prose instead of the tool is a real (degraded) answer, not a placeholder."""
+    agent = _universal_agent(mock_context)
+    outcome, persist = await _run_chunk_with_response(agent, tmp_path, _TEXT_ONLY)
+
+    assert isinstance(outcome, str)
+    assert "Summary: Prose answer: the toast appeared at 2.0s." in outcome
+    assert "Analysis: Prose answer: the toast appeared at 2.0s." in outcome
+    persist.assert_awaited_once()
+    # A degraded text answer carries no timeline events.
+    assert persist.await_args.args[1] == []
+
+
+@pytest.mark.asyncio
+async def test_exec_single_chunk_universal_blank_summary_falls_through_to_text(
+    mock_context, tmp_path
+):
+    """submit_answer with a blank summary is ignored; the text content becomes the answer."""
+    response = AIMessage(
+        content="Fallback prose from the model.",
+        tool_calls=[
+            {
+                "name": "submit_answer",
+                "args": {
+                    "summary": "",
+                    "analysis": "Orphaned analysis.",
+                    "timeline_events": [{"start_time": 0.0, "end_time": 1.0}],
+                },
+                "id": "blank_call",
+            }
+        ],
+    )
+    agent = _universal_agent(mock_context)
+    outcome, persist = await _run_chunk_with_response(agent, tmp_path, response)
+
+    assert isinstance(outcome, str)
+    assert "Summary: Fallback prose from the model." in outcome
+    assert "Orphaned analysis" not in outcome
+    # The blank tool call is dropped wholesale, events included.
+    assert persist.await_args.args[1] == []
+
+
+async def _run_audio_with_response(agent, tmp_path, response):
+    """Drives the universal audio path with a stubbed model reply."""
+    fake_audio = tmp_path / "audio.aac"
+    fake_audio.write_bytes(b"audio_bytes")
+    with (
+        patch.object(
+            agent,
+            "_invoke_universal_model",
+            new_callable=AsyncMock,
+            return_value=response,
+        ),
+        patch.object(
+            agent, "_record_blackboard_entry", wraps=agent._record_blackboard_entry
+        ) as record,
+    ):
+        try:
+            result = await agent._exec_analyze_audio_universal(
+                audio_path=fake_audio,
+                start_time=1.0,
+                end_time=4.0,
+                actual_start=1.0,
+                prompt_with_context="Listen for the toast.",
+                specific_query="toast sound",
+            )
+        except Exception as exc:  # noqa: BLE001 - the exception type is the assertion
+            return exc, record
+    return result, record
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("response", [_EMPTY_RESPONSE, _BLANK_SUBMIT])
+async def test_exec_analyze_audio_universal_raises_without_usable_answer(
+    mock_context, tmp_path, response
+):
+    """Audio path: no usable answer -> raise before anything reaches the blackboard."""
+    from artemis.agents.video_analyzer.reliability import SubAgentAnswerExhausted
+
+    agent = _universal_agent(mock_context)
+    outcome, record = await _run_audio_with_response(agent, tmp_path, response)
+
+    assert isinstance(outcome, SubAgentAnswerExhausted)
+    assert outcome.turns == 1
+    record.assert_not_called()
+    assert agent.blackboard_entries == []
+
+
+@pytest.mark.asyncio
+async def test_exec_analyze_audio_universal_text_only_is_degraded_answer(mock_context, tmp_path):
+    """Audio path: prose instead of the tool is still committed as a degraded answer."""
+    agent = _universal_agent(mock_context)
+    outcome, record = await _run_audio_with_response(agent, tmp_path, _TEXT_ONLY)
+
+    assert isinstance(outcome, str)
+    assert "Audio Summary: Prose answer: the toast appeared at 2.0s." in outcome
+    record.assert_called_once()
+    assert len(agent.blackboard_entries) == 1
+    assert agent.blackboard_entries[0]["summary"] == "Prose answer: the toast appeared at 2.0s."
+
+
+def test_sub_agent_answer_exhausted_classifies_as_unknown():
+    """The raise must land in the retryable / split / fallback bucket of the callers."""
+    from artemis.agents.video_analyzer.reliability import (
+        SubAgentAnswerExhausted,
+        VideoFailureCategory,
+        classify_video_failure,
+    )
+
+    failure = classify_video_failure(SubAgentAnswerExhausted(1, "x"))
+
+    assert failure.category is VideoFailureCategory.UNKNOWN
+    assert failure.retryable is True
+    assert failure.should_split is True
+    assert failure.should_fallback is True

tests/unit/agents/test_video_audio_native.py

@@ -0,0 +1,157 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Native Gemini audio agent loop: only accepted answers reach the blackboard."""
+
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock
+
+from artemis.agents.video_analyzer.audio_native import (
+    _BAD_CONFIDENCE_ERROR,
+    _AudioMedia,
+    _drive_audio_agent_loop,
+    _run_native_audio_conversation,
+)
+from artemis.agents.video_analyzer.chunk_conversation import (
+    _MISSING_SUMMARY_ERROR,
+    _NO_TOOL_CALL_ERROR,
+    SUB_AGENT_MAX_TURNS,
+)
+from artemis.agents.video_analyzer.reliability import (
+    SubAgentAnswerExhausted,
+    VideoFailureCategory,
+    classify_video_failure,
+)
+from artemis.agents.video_analyzer.universal_tools import build_submit_answer_declaration
+from google.genai import types
+import pytest
+
+
+def _audio_response(text: str, function_calls: list) -> SimpleNamespace:
+    """Fakes a non-streamed ``generate_content`` response for one audio turn."""
+    parts = [types.Part(function_call=fc) for fc in function_calls]
+    if text:
+        parts.append(types.Part.from_text(text=text))
+    return SimpleNamespace(
+        text=text,
+        function_calls=function_calls,
+        candidates=[SimpleNamespace(content=SimpleNamespace(parts=parts))],
+    )
+
+
+def _audio_analyzer(*turns) -> MagicMock:
+    """Analyzer stub whose Gemini client answers the scripted ``(text, calls)`` turns."""
+    analyzer = MagicMock()
+    analyzer.model_name = "gemini-3.7-flash"
+    analyzer.audio_system_prompt = "system"
+    analyzer.submit_answer_tool = build_submit_answer_declaration()
+    analyzer.model_call_timeout_seconds = 5.0
+    analyzer.ctx.data_engine = None
+    analyzer.client.aio.models.generate_content = AsyncMock(
+        side_effect=[_audio_response(text, calls) for text, calls in turns]
+    )
+    return analyzer
+
+
+def _audio_media(tmp_path) -> _AudioMedia:
+    media = _AudioMedia()
+    media.audio_path = tmp_path / "segment.wav"
+    media.audio_path.write_bytes(b"audio")
+    media.actual_start = 0.0
+    media.prompt_with_context = "Listen for the chime."
+    return media
+
+
+@pytest.mark.asyncio
+async def test_audio_submit_answer_without_summary_is_sent_back_then_accepted():
+    first = types.FunctionCall(
+        name="submit_answer", args={"summary": "  ", "confidence_score": 0.9}
+    )
+    second = types.FunctionCall(
+        name="submit_answer",
+        args={"summary": "chime\nheard", "analysis": "a2", "confidence_score": 0.7},
+    )
+    analyzer = _audio_analyzer(("", [first]), ("", [second]))
+    contents: list = []
+
+    answer = await _drive_audio_agent_loop(analyzer, contents)
+
+    assert answer == ("chime heard", "a2", 0.7)
+    assert analyzer.client.aio.models.generate_content.await_count == 2
+    # The blank summary went back to the model as a function_response error.
+    rejections = [
+        c
+        for c in contents
+        if c.role == "user" and c.parts and c.parts[0].function_response is not None
+    ]
+    assert len(rejections) == 1
+    assert rejections[0].parts[0].function_response.response["error"] == _MISSING_SUMMARY_ERROR
+
+
+@pytest.mark.asyncio
+async def test_audio_loop_raises_when_no_turn_calls_submit_answer():
+    analyzer = _audio_analyzer(("text only", []), ("still text", []))
+
+    with pytest.raises(SubAgentAnswerExhausted) as raised:
+        await _drive_audio_agent_loop(analyzer, [])
+
+    assert raised.value.turns == SUB_AGENT_MAX_TURNS
+    assert raised.value.reason == _NO_TOOL_CALL_ERROR
+    assert "No summary provided" not in str(raised.value)
+
+
+@pytest.mark.asyncio
+async def test_audio_loop_raises_when_confidence_score_is_never_valid():
+    first = types.FunctionCall(
+        name="submit_answer", args={"summary": "s1", "confidence_score": "HIGH"}
+    )
+    second = types.FunctionCall(
+        name="submit_answer", args={"summary": "s2", "confidence_score": 1.5}
+    )
+    analyzer = _audio_analyzer(("", [first]), ("", [second]))
+    contents: list = []
+
+    with pytest.raises(SubAgentAnswerExhausted) as raised:
+        await _drive_audio_agent_loop(analyzer, contents)
+
+    assert raised.value.reason == _BAD_CONFIDENCE_ERROR
+    assert contents[-1].role == "user"
+    assert contents[-1].parts[0].function_response.response["error"] == _BAD_CONFIDENCE_ERROR
+
+
+@pytest.mark.asyncio
+async def test_exhausted_audio_segment_is_not_committed_and_cloud_file_is_deleted(tmp_path):
+    analyzer = _audio_analyzer(("text only", []), ("still text", []))
+    uploaded = SimpleNamespace(name="files/abc", uri="https://files/abc", mime_type="audio/wav")
+    analyzer.upload_and_poll_file = AsyncMock(return_value=uploaded)
+    analyzer.client.aio.files.delete = AsyncMock()
+    analyzer.cloud_files_to_cleanup = {"files/abc"}
+    analyzer._record_blackboard_entry = MagicMock()
+    analyzer.blackboard = MagicMock()
+
+    with pytest.raises(SubAgentAnswerExhausted):
+        await _run_native_audio_conversation(
+            analyzer, _audio_media(tmp_path), 0.0, 5.0, 0.0, 5.0, "chime", "lease-1"
+        )
+
+    analyzer.blackboard.complete_segment.assert_not_called()
+    analyzer._record_blackboard_entry.assert_not_called()
+    analyzer.client.aio.files.delete.assert_awaited_once_with(name="files/abc")
+    assert analyzer.cloud_files_to_cleanup == set()
+
+
+def test_sub_agent_exhaustion_is_classified_unknown():
+    failure = classify_video_failure(SubAgentAnswerExhausted(2, "x"))
+
+    assert failure.category is VideoFailureCategory.UNKNOWN

tests/unit/agents/test_video_reliability.py

@@ -3,6 +3,7 @@
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 
+import asyncio
 from types import SimpleNamespace
 from unittest.mock import AsyncMock, MagicMock, patch
 
@@ -10,6 +11,7 @@ from langchain_core.messages import AIMessage, HumanMessage
 import pytest
 
 from artemis.agents.video_analyzer.reliability import (
+    AgenticVideoDegraded,
     VideoCircuitBreaker,
     VideoFailureCategory,
     classify_video_failure,
@@ -290,3 +292,163 @@ async def test_invoke_with_retry_caps_unknown_failures_by_policy():
             await va._invoke_with_retry(op, "test-op", max_attempts=5)
     # UNKNOWN failures follow the shared policy, not the caller's outer cap.
     assert calls == retry_policy_for(FailureCategory.UNKNOWN).max_attempts
+
+
+# ------------------------------------------------ mid-run agentic degrade
+
+
+class _AgenticRejected(Exception):
+    status_code = 400
+
+
+@pytest.mark.asyncio
+async def test_pending_chunk_sized_for_agentic_ceiling_is_not_run_statically(tmp_path):
+    """A chunk planned at 600s whose run degraded before it started is handed back."""
+    with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
+        analyzer = VideoAnalyzer(_context())
+    analyzer.use_native_gemini = True
+    # A sibling chunk already flipped the run to static (60s ceiling).
+    analyzer.video_processing = "static"
+    analyzer.chunk_size_seconds = analyzer.static_chunk_size_seconds
+    controller = SimpleNamespace(extract_segment_metadata=AsyncMock())
+
+    with (
+        patch(
+            "artemis.agents.video_analyzer.video_analyzer.get_controller",
+            return_value=controller,
+        ),
+        patch(
+            "artemis.agents.video_analyzer.chunk_native._run_native_chunk_conversation",
+            new=AsyncMock(),
+        ) as static_path,
+        patch.object(analyzer, "_exec_single_chunk_universal", new=AsyncMock()) as universal,
+        pytest.raises(AgenticVideoDegraded) as raised,
+    ):
+        await analyzer._exec_single_chunk(0.0, 150.0, "verify toggle")
+
+    # Refused before any extraction/upload; no static or fallback analysis.
+    controller.extract_segment_metadata.assert_not_awaited()
+    static_path.assert_not_awaited()
+    universal.assert_not_awaited()
+    assert classify_video_failure(raised.value).should_split
+    # The lease is released so the re-planned pieces can be claimed.
+    assert analyzer.blackboard.missing_intervals(0.0, 150.0, "verify toggle") == [(0.0, 150.0)]
+
+
+@pytest.mark.asyncio
+async def test_sibling_chunk_degraded_mid_run_is_replanned_into_static_pieces(tmp_path):
+    """Two 600s chunks: the first is rejected while the second is still in media prep."""
+    with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
+        analyzer = VideoAnalyzer(_context())
+    analyzer.use_native_gemini = True
+    analyzer.video_processing = "agentic"
+    analyzer.chunk_size_seconds = analyzer.agentic_chunk_size_seconds
+    assert analyzer.chunk_size_seconds == 600.0
+
+    raw_video = tmp_path / "segment.mp4"
+    raw_video.write_bytes(b"video")
+    first_rejected = asyncio.Event()
+
+    async def extract(start: float, end: float):
+        if start >= 600.0 and analyzer.video_processing == "agentic":
+            # The second 600s chunk is mid-prep when the first one degrades.
+            await first_rejected.wait()
+        return SimpleNamespace(
+            success=True,
+            video_path=raw_video,
+            actual_start_relative_time=start,
+            duration_seconds=end - start,
+            warning=None,
+        )
+
+    controller = SimpleNamespace(extract_segment_metadata=AsyncMock(side_effect=extract))
+    agentic_calls: list[tuple[float, float]] = []
+
+    async def agentic(analyzer_, media, cs, ce, *rest):
+        agentic_calls.append((cs, ce))
+        first_rejected.set()
+        raise _AgenticRejected("400 INVALID_ARGUMENT: processing 'agentic' is not supported")
+
+    async def static_answer(analyzer_, media, cs, ce, *rest):
+        return f"[from {cs:.1f}s to {ce:.1f}s] Summary: static Analysis: ok"
+
+    with (
+        patch(
+            "artemis.agents.video_analyzer.video_analyzer.get_controller",
+            return_value=controller,
+        ),
+        patch(
+            "artemis.agents.video_analyzer.video_analyzer.compress_video_for_api",
+            new=AsyncMock(return_value=raw_video),
+        ),
+        patch(
+            "artemis.agents.video_analyzer.chunk_native.run_agentic_chunk_conversation",
+            new=AsyncMock(side_effect=agentic),
+        ),
+        patch(
+            "artemis.agents.video_analyzer.chunk_native._run_native_chunk_conversation",
+            new=AsyncMock(side_effect=static_answer),
+        ) as static_path,
+        patch.object(analyzer, "_exec_single_chunk_universal", new=AsyncMock()) as universal,
+        patch("artemis.agents.video_analyzer.video_analyzer._record_llm_event"),
+    ):
+        result = await analyzer.exec_spawn_sub_agent(0.0, 1200.0, "verify toggle")
+
+    # Only the first chunk ever reached the agentic API; the second never ran
+    # its 600s clip statically and neither went to the universal fallback.
+    assert agentic_calls == [(0.0, 600.0)]
+    universal.assert_not_awaited()
+    segments = sorted((call.args[2], call.args[3]) for call in static_path.await_args_list)
+    assert len(segments) == 20
+    assert segments[0] == (0.0, 60.0)
+    assert segments[-1] == (1140.0, 1200.0)
+    assert all(ce - cs <= 60.0 for cs, ce in segments)
+    assert "PARTIAL" not in result
+    assert result.count("Summary: static") == 20
+    assert analyzer.video_processing == "static"
+
+
+@pytest.mark.asyncio
+async def test_short_degraded_chunk_is_retried_statically_once():
+    """A 5s chunk (below min_chunk*2) is re-run as-is instead of failing."""
+    with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
+        analyzer = VideoAnalyzer(_context())
+    calls: list[tuple[float, float]] = []
+
+    async def analyze(start: float, end: float, query: str) -> str:
+        calls.append((start, end))
+        if len(calls) == 1:
+            analyzer.video_processing = "static"
+            analyzer.chunk_size_seconds = analyzer.static_chunk_size_seconds
+            raise AgenticVideoDegraded("agentic rejected")
+        return f"[from {start:.1f}s to {end:.1f}s] Summary: static {query}"
+
+    with patch.object(analyzer, "_exec_single_chunk", side_effect=analyze) as child:
+        result = await analyzer.exec_spawn_sub_agent(0.0, 5.0, "find toggle")
+
+    assert child.await_count == 2
+    assert calls == [(0.0, 5.0), (0.0, 5.0)]
+    assert "PARTIAL" not in result
+    assert "failed" not in result.lower()
+    assert result.count("Summary: static find toggle") == 1
+
+
+@pytest.mark.asyncio
+async def test_static_retry_of_degraded_chunk_is_bounded():
+    """A second degrade on the same interval is terminal: no loop, no bisect."""
+    with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
+        analyzer = VideoAnalyzer(_context())
+    calls: list[tuple[float, float]] = []
+
+    async def analyze(start: float, end: float, query: str) -> str:
+        calls.append((start, end))
+        raise AgenticVideoDegraded("agentic rejected again")
+
+    with patch.object(analyzer, "_exec_single_chunk", side_effect=analyze) as child:
+        # 30s is above min_chunk*2, so a bisect would otherwise be legal.
+        result = await analyzer.exec_spawn_sub_agent(0.0, 30.0, "find toggle")
+
+    assert child.await_count == 2
+    assert calls == [(0.0, 30.0), (0.0, 30.0)]
+    assert result.startswith("All sub-agent chunks failed")
+    assert "0.0s-30.0s [bad_request]" in result

tests/unit/config/test_video_processing_config.py

@@ -0,0 +1,39 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""The video-processing knob is typed once, in the Google provider module."""
+
+from typing import get_args
+
+from pydantic import ValidationError
+import pytest
+
+from artemis.config.agent import VideoAnalyzerConfig
+from artemis.llm.google import VideoProcessing
+
+
+def test_processing_field_reuses_the_provider_literal():
+    assert VideoAnalyzerConfig.model_fields["processing"].annotation is VideoProcessing
+    assert set(get_args(VideoProcessing)) == {"auto", "agentic", "static"}
+
+
+def test_processing_defaults_to_auto_and_accepts_every_mode():
+    assert VideoAnalyzerConfig().processing == "auto"
+    for mode in get_args(VideoProcessing):
+        assert VideoAnalyzerConfig(processing=mode).processing == mode
+
+
+def test_processing_rejects_unknown_modes():
+    with pytest.raises(ValidationError, match="processing"):
+        VideoAnalyzerConfig(processing="bogus")

tests/unit/data_engine/test_engine.py

@@ -615,7 +615,7 @@ def test_pro_pixel_record_is_normalized_exactly_once(tmp_path):
         action_taken=[action],
         last_execution_result={
             "status": "success",
-            "execution": [dict(action, attempts=["Success"])],
+            "execution": [dict(action, attempts=["Dispatched"])],
         },
         extra_metadata={"width": 1080, "height": 2400},
     )

tests/unit/mcp/test_action_executor_helper_tools.py

@@ -130,59 +130,41 @@ async def test_ask_explorer_exception_is_contained():
     assert result.text_summary == "Error executing ask_explorer: boom"
 
 
-# ---------------------------------------------------------------------------
-# Legacy MobileActionExecutor parity (same ask_explorer semantics)
-# ---------------------------------------------------------------------------
-
-
-def _make_legacy_executor(**kwargs):
-    from artemis.agents.validator.tool_declarations import MobileActionExecutor
-
-    ctx = Mock()
-    ctx.data_engine = None
-    ctx.device.device_width = 1080
-    ctx.device.device_height = 2400
-    actuator = Mock()
-    actuator.controller = Mock()
-    return MobileActionExecutor(ctx, actuator=actuator, **kwargs)
-
-
 @pytest.mark.asyncio
-async def test_legacy_executor_ask_explorer_status_follows_the_outcome():
-    executor = _make_legacy_executor()
+async def test_default_agent_name_reaches_the_tier_resolver_and_status_follows_the_outcome():
+    """Without an explicit agent name the executor asks the Explorer as ``validator``;
+    a clean not-found is an answer, an Explorer run failure is an error."""
+    executor = _make_executor()
     assert executor.agent_name == "validator"
 
     not_found = ExplorerOutcome(message="Not visible.")
-    with patch("artemis.tools.explorer_tool.locate", new=AsyncMock(return_value=not_found)):
-        # The pipeline functions are imported at module load in the legacy executor.
     with patch(
-            "artemis.agents.validator.tool_declarations.locate",
-            new=AsyncMock(return_value=not_found),
+        "artemis.tools.explorer_tool.locate", new=AsyncMock(return_value=not_found)
     ) as locate:
         result = await executor.execute(
             "ask_explorer", {"task_description": "gear icon"}, "tc", _state()
         )
     assert locate.await_args.args[2] == "gear icon"
     assert locate.await_args.kwargs["agent_name"] == "validator"
-    assert result.status == "success"  # a clean not-found is an answer, not a failure
+    assert result.status == "success"
 
     failed = ExplorerOutcome.failure("Explorer failed: boom")
-    with patch(
-        "artemis.agents.validator.tool_declarations.locate", new=AsyncMock(return_value=failed)
-    ):
+    with patch("artemis.tools.explorer_tool.locate", new=AsyncMock(return_value=failed)):
         result = await executor.execute("ask_explorer", {"query": "gear icon"}, "tc", _state())
     assert result.status == "error"
 
 
 @pytest.mark.asyncio
-async def test_legacy_executor_agent_name_reaches_the_tier_resolver():
-    executor = _make_legacy_executor(agent_name="flash")
+async def test_flash_agent_name_reaches_the_tier_resolver():
+    """The Flash profile resolves the Explorer tier from ``agent_name="flash"``."""
+    executor = _make_executor(agent_name="flash")
     with patch(
-        "artemis.agents.validator.tool_declarations.locate",
+        "artemis.tools.explorer_tool.locate",
         new=AsyncMock(return_value=ExplorerOutcome(message="no")),
     ) as locate:
         await executor.execute("ask_explorer", {"query": "x"}, "tc", _state())
     assert locate.await_args.kwargs["agent_name"] == "flash"
+    assert locate.await_args.args[0] is executor.ctx
 
 
 # ---------------------------------------------------------------------------
@@ -293,19 +275,21 @@ async def test_history_tool_exception_is_contained():
     tool = _history_tool_double(side_effect=RuntimeError("db gone"))
     with patch("artemis.mcp.action_executor.history_tool_by_name", return_value=tool):
         result = await executor.execute("replay_steps", {"query": "x"}, "tc", None)
-    assert result.status == "success"  # contained: the answer explains the failure
+    # Contained (no exception escapes) and reported structurally: the answer
+    # explains the failure and the status says it is one.
+    assert result.status == "error"
     assert result.text_summary == "replay_steps failed: db gone"
 
 
 @pytest.mark.asyncio
 async def test_history_tools_dispatch_to_the_real_shared_instances():
     """No per-tool branch: every history tool name resolves through the package."""
-    executor = _make_executor()  # ctx.data_engine is None -> tools degrade to text
+    executor = _make_executor()  # ctx.data_engine is None -> tools report a failure
     for name in ("search_history", "replay_steps", "get_step_screenshot"):
         result = await executor.execute(
             name, {"query": "x", "start_step": 1, "step_number": 1}, "tc", None
         )
-        assert result.status == "success"
+        assert result.status == "error"
         assert (
             "no active execution history" in result.text_summary
             or "no execution history" in result.text_summary