feat: add agentic video analysis and clarify action results

Report device actions as dispatched and preserve full failure details. Route
Flash through McpActionExecutor and use ToolFailure across helper tools and
agent loops instead of inferring failure from response text.

Require model-provided descriptions for coordinate targets. Keep observed
element metadata separate from those descriptions in safety checks, visual
summaries, history, and frontend labels.

Add Gemini Interactions API video analysis with configurable processing mode,
chunk size, and timeouts. Centralize Google provider capabilities and token
usage normalization, and update google-genai to 2.22.0.

Preserve system instructions on follow-up interactions, replan pending chunks
when falling back to static processing, and reject unanswered native video
and audio segments instead of caching placeholder results. Keep helper-tool
failure status intact through history tools and committee calls.

Validation: 1,762 Python unit tests passed (3 skipped), 40 frontend formatter
tests passed, and the production UI build completed. Ruff lint, formatting,
the quality ratchet, and git diff checks passed.

apps/showcase_ui/src/app/core/models/stream.model.ts

@@ -237,7 +237,7 @@ export interface StepReplayFrame {
   actionText?: string;
   targetText?: string;
   coords?: string;
-  status?: 'success' | 'failed' | string;
+  status?: 'dispatched' | 'failed' | string;
   isPost?: boolean;
   timestamp?: number;
   phaseId?: string;

apps/showcase_ui/src/app/utils/action-formatter.util.spec.ts

@@ -1,7 +1,11 @@
 import {
   getStepPreImageUrl,
   getStepPostImageUrl,
-  extractStepReplayFrames
+  extractStepReplayFrames,
+  getActionTargetText,
+  isActionFailed,
+  getActionErrorMessage,
+  extractActionExtraParams
 } from './action-formatter.util';
 
 describe('action-formatter.util screenshot chaining', () => {
@@ -403,3 +407,107 @@ describe('extractStepReplayFrames', () => {
     expect(frames[1].stepId).toBe('real-step-2');
   });
 });
+
+describe('getActionTargetText self-described targets', () => {
+  it('uses the model\'s target_description when no observed target_text exists', () => {
+    expect(getActionTargetText({ action: 'click', coordinates: [500, 900], target_description: 'play button' }))
+      .toBe('play button');
+    expect(getActionTargetText({ name: 'long_press', args: { target: [500, 900], target_description: 'song row' } }))
+      .toBe('song row');
+  });
+
+  it('prefers observed target_text over target_description when both are present', () => {
+    expect(getActionTargetText({
+      action: 'click',
+      target_text: 'Play',
+      target_description: 'play button'
+    })).toBe('Play');
+  });
+
+  it('chains per-point target_descriptions for click_sequence with the same arrow as coordinates', () => {
+    expect(getActionTargetText({
+      action: 'click_sequence',
+      sequence: [[500, 300], [876, 360]],
+      target_descriptions: ['play button', 'close ×']
+    })).toBe('play button → close ×');
+    expect(getActionTargetText({
+      name: 'click_sequence',
+      args: { sequence: [[500, 300], [876, 360], [100, 100]], target_descriptions: ['play button', 'close ×', 'back'] }
+    })).toBe('play button → close × → back');
+  });
+
+  it('skips blank entries and falls back when click_sequence descriptions are missing or unusable', () => {
+    expect(getActionTargetText({
+      action: 'click_sequence',
+      sequence: [[500, 300], [876, 360]],
+      target_descriptions: ['play button', '', null]
+    })).toBe('play button');
+    expect(getActionTargetText({ action: 'click_sequence', sequence: [[500, 300]], target_descriptions: [] })).toBe('');
+    expect(getActionTargetText({ action: 'click_sequence', sequence: [[500, 300]], target_descriptions: 'not-an-array' })).toBe('');
+    expect(getActionTargetText({
+      action: 'click_sequence',
+      sequence: [[500, 300]],
+      target_descriptions: [null],
+      target_description: 'single fallback'
+    })).toBe('single fallback');
+  });
+});
+
+describe('isActionFailed / getActionErrorMessage with Validator attempts', () => {
+  const stepWithAttempts = (attempts: any[]) => ({
+    step_id: 'step-attempts',
+    last_execution_result: {
+      status: 'dispatched',
+      execution: [{ attempts }]
+    }
+  });
+  const action = { action: 'click', coordinates: [500, 900] };
+
+  it('treats a retry that ended in Dispatched as not failed', () => {
+    const stepData = stepWithAttempts(['Execution error: x', 'Dispatched']);
+    expect(isActionFailed(action, stepData)).toBeFalse();
+  });
+
+  it('treats a terminal non-Dispatched attempt as failed and surfaces its text', () => {
+    const stepData = stepWithAttempts(['Execution error: y']);
+    expect(isActionFailed(action, stepData)).toBeTrue();
+    expect(getActionErrorMessage(action, stepData)).toBe('y');
+  });
+
+  it('keeps a generic error prefix verbatim and reports the last failing attempt', () => {
+    const stepData = stepWithAttempts(['Pre-execution validation failed: first', 'Error: y']);
+    expect(isActionFailed(action, stepData)).toBeTrue();
+    expect(getActionErrorMessage(action, stepData)).toBe('Error: y');
+  });
+
+  it('marks replay frames as dispatched rather than success', () => {
+    const frames = extractStepReplayFrames([
+      { step_id: 's-1', step_number: 1, pre_image_name: 'img1', action_taken: action, timestamp: 100 },
+      {
+        step_id: 's-2', step_number: 2, pre_image_name: 'img2', action_taken: action, timestamp: 200,
+        last_execution_result: { execution: [{ attempts: ['Execution error: y'] }] }
+      }
+    ]);
+    expect(frames[0].status).toBe('dispatched');
+    expect(frames[1].status).toBe('failed');
+  });
+});
+
+describe('extractActionExtraParams', () => {
+  it('hides target_description and target_descriptions from the extra parameter list', () => {
+    const single = extractActionExtraParams({
+      action: 'click',
+      coordinates: [500, 900],
+      target_description: 'play button',
+      times: 2
+    });
+    expect(single.map(p => p.key)).toEqual(['Times']);
+
+    const sequence = extractActionExtraParams({
+      name: 'click_sequence',
+      args: { sequence: [[1, 2], [3, 4]], target_descriptions: ['a', 'b'], interval_ms: 50 }
+    });
+    expect(sequence.map(p => p.key)).not.toContain('Target Descriptions');
+    expect(sequence.map(p => p.key)).toContain('Interval Ms');
+  });
+});

apps/showcase_ui/src/app/utils/action-formatter.util.ts

@@ -16,7 +16,7 @@
 
 import { ActionParam, StepReplayFrame } from '../core/models/stream.model';
 import { extractNumbersFromCoordinateValue, isPureDirectionString, parseSequenceCoordinates, unwrapTraceAction } from './image-overlay.util';
-import { cleanErrorMessage } from './tool-formatter.util';
+import { cleanErrorMessage, joinTargetDescriptions } from './tool-formatter.util';
 
 /**
  * Safely parse JSON string if it looks like an object/array, otherwise return original value
@@ -247,7 +247,16 @@ export function getActionTargetText(action: any): string {
     const ms = act.time_in_ms || act.delay_ms || act.delay_seconds || args.time_in_ms || args.delay_ms || args.delay_seconds || args.duration;
     if (ms) return `${ms}ms`;
   }
-  return act.target_text || act.target_class || act.element_id || args.target_text || args.element_id || '';
+  // Flash click_sequence carries one self-described target per point; render
+  // them in the same ' → ' chain that getActionCoords uses for the points.
+  if (name === 'click_sequence' || name === 'tap_sequence') {
+    const joined = joinTargetDescriptions(act.target_descriptions ?? args.target_descriptions);
+    if (joined) return joined;
+  }
+  // target_text is observed element text (index targets); target_description is
+  // what the model said it aimed at (coordinate targets). Same label either way.
+  return act.target_text || act.target_description || act.target_class || act.element_id
+    || args.target_text || args.target_description || args.element_id || '';
 }
 
 /**
@@ -360,6 +369,22 @@ export function getActionCoords(action: any): string {
   return '';
 }
 
+/**
+ * The Validator records "Dispatched" as the terminal attempt of an action the device
+ * accepted (kept only when a retry preceded it); any other terminal attempt is the
+ * failure text. Skipped burst members count as not dispatched.
+ */
+function attemptsFailed(attempts: any): boolean {
+  if (!Array.isArray(attempts) || attempts.length === 0) return false;
+  return String(attempts[attempts.length - 1]) !== 'Dispatched';
+}
+
+function lastFailedAttempt(attempts: any): string | null {
+  if (!Array.isArray(attempts)) return null;
+  const failing = attempts.filter((a: any) => String(a) !== 'Dispatched');
+  return failing.length > 0 ? String(failing[failing.length - 1]) : null;
+}
+
 /**
  * Check if the action execution failed or encountered an execution failure/interception
  */
@@ -384,7 +409,7 @@ export function isActionFailed(action: any, stepData?: any): boolean {
         }
         if (Array.isArray(res.execution) && res.execution.length > 0) {
           const firstExec = res.execution[0];
-          if (firstExec && (firstExec.status === 'failed' || firstExec.status === 'error' || firstExec.error || (Array.isArray(firstExec.attempts) && firstExec.attempts.length > 0))) {
+          if (firstExec && (firstExec.status === 'failed' || firstExec.status === 'error' || firstExec.error || attemptsFailed(firstExec.attempts))) {
             return true;
           }
         }
@@ -411,8 +436,9 @@ export function getActionErrorMessage(action: any, stepData?: any): string {
       if (Array.isArray(res.execution) && res.execution.length > 0) {
         const firstExec = res.execution[0];
         if (firstExec) {
-          if (Array.isArray(firstExec.attempts) && firstExec.attempts.length > 0) {
-            return cleanErrorMessage(firstExec.attempts[0]);
+          const failedAttempt = lastFailedAttempt(firstExec.attempts);
+          if (failedAttempt) {
+            return cleanErrorMessage(failedAttempt);
           }
           if (firstExec.error || firstExec.failure_reason || firstExec.message) {
             return cleanErrorMessage(firstExec.error || firstExec.failure_reason || firstExec.message);
@@ -469,7 +495,8 @@ export function extractActionExtraParams(action: any, cache?: WeakMap<any, Actio
   }
 
   const standardKeys = new Set([
-    'action', 'name', 'type', 'target_text', 'text', 'input_text', 'target',
+    'action', 'name', 'type', 'target_text', 'target_description', 'target_descriptions',
+    'text', 'input_text', 'target',
     'coordinates', 'coords', 'target_bounds', 'bounds', 'target_resource_id',
     'resource_id', 'target_class', 'class_name', 'normalized_coordinates',
     'normalized_start_coordinates', 'normalized_end_coordinates',
@@ -980,7 +1007,7 @@ export function extractStepReplayFrames(logsOrSteps: any[]): StepReplayFrame[] {
       isPost: false,
       timestamp: stepData.timestamp,
       summary: stepData.summary || '',
-      status: failed ? 'failed' : 'success'
+      status: failed ? 'failed' : 'dispatched'
     });
   }
 

apps/showcase_ui/src/app/utils/tool-formatter.util.spec.ts

@@ -1,10 +1,13 @@
 import {
   cleanErrorMessage,
+  extractToolExtraParams,
   getCompressionLabel,
   getToolDisplayLabel,
   getToolIcon,
+  getToolTargetText,
   getUniqueGenericTools,
   getVideoAnalysisView,
+  joinTargetDescriptions,
   shouldShowTool
 } from './tool-formatter.util';
 
@@ -255,3 +258,59 @@ describe('shouldShowTool (Option A Single Source of Truth)', () => {
     expect(shouldShowTool(plumbingTool)).toBeFalse();
   });
 });
+
+describe('getToolTargetText self-described targets', () => {
+  it('uses target_description for coordinate tools when no target_text is observed', () => {
+    expect(getToolTargetText({
+      name: 'click',
+      payload: { args: { coordinates: [500, 900], target_description: 'play button' } }
+    })).toBe('play button');
+    expect(getToolTargetText({
+      name: 'exec_long_press',
+      args: { coordinates: [500, 900], target_description: 'song row' }
+    })).toBe('song row');
+  });
+
+  it('prefers observed target_text over target_description', () => {
+    expect(getToolTargetText({
+      name: 'click',
+      payload: { args: { target_text: 'Play', target_description: 'play button' } }
+    })).toBe('Play');
+  });
+
+  it('chains click_sequence target_descriptions with the coordinate arrow', () => {
+    expect(getToolTargetText({
+      name: 'click_sequence',
+      payload: { args: { sequence: [[500, 300], [876, 360]], target_descriptions: ['play button', 'close ×'] } }
+    })).toBe('play button → close ×');
+  });
+
+  it('falls back gracefully when click_sequence descriptions are missing or short', () => {
+    expect(getToolTargetText({
+      name: 'click_sequence',
+      payload: { args: { sequence: [[500, 300], [876, 360]] } }
+    })).toBe('');
+    expect(getToolTargetText({
+      name: 'click_sequence',
+      payload: { args: { sequence: [[500, 300], [876, 360]], target_descriptions: ['play button'] } }
+    })).toBe('play button');
+    expect(getToolTargetText({
+      name: 'click_sequence',
+      payload: { args: { sequence: [[500, 300]], target_descriptions: [null, ''], target_description: 'fallback' } }
+    })).toBe('fallback');
+    expect(joinTargetDescriptions(undefined)).toBe('');
+    expect(joinTargetDescriptions('play button')).toBe('');
+  });
+});
+
+describe('extractToolExtraParams', () => {
+  it('hides target_description and target_descriptions from the extra parameter list', () => {
+    const keys = extractToolExtraParams({
+      name: 'click_sequence',
+      args: { sequence: [[1, 2]], target_descriptions: ['a'], target_description: 'b', interval_ms: 50 }
+    }).map(p => p.key);
+    expect(keys).not.toContain('Target Descriptions');
+    expect(keys).not.toContain('Target Description');
+    expect(keys).toContain('Interval Ms');
+  });
+});

apps/showcase_ui/src/app/utils/tool-formatter.util.ts

@@ -911,6 +911,19 @@ export function getToolTitle(tool: any): string {
   }
 }
 
+/**
+ * Join per-point self-described targets (click_sequence `target_descriptions`)
+ * into one label, chained with ' → ' like the coordinate rendering.
+ * Returns '' when the value is missing or holds no usable text.
+ */
+export function joinTargetDescriptions(descriptions: any): string {
+  if (!Array.isArray(descriptions)) return '';
+  const parts = descriptions
+    .map(d => (d === null || d === undefined) ? '' : String(d).trim())
+    .filter(d => d.length > 0);
+  return parts.join(' → ');
+}
+
 /**
  * Get target text description for tools
  */
@@ -950,6 +963,11 @@ export function getToolTargetText(tool: any): string {
   if (name === 'report_failure_analysis') {
     return args.status || args.reason || args.analysis || '';
   }
+  if (name === 'click_sequence' || name === 'tap_sequence') {
+    // One self-described target per point, chained like getToolCoords does.
+    const joined = joinTargetDescriptions(args.target_descriptions);
+    if (joined) return joined;
+  }
   if (args.target && typeof args.target === 'string') {
     return args.target;
   }
@@ -959,7 +977,7 @@ export function getToolTargetText(tool: any): string {
   if (args.index !== undefined) {
     return `Element #${args.index}`;
   }
-  return args.target_text || args.target_class || args.element || args.element_text || (name !== 'input_text' ? args.text : '') || '';
+  return args.target_text || args.target_description || args.target_class || args.element || args.element_text || (name !== 'input_text' ? args.text : '') || '';
 }
 
 /**
@@ -1162,7 +1180,8 @@ export function extractToolExtraParams(toolData: any, cache?: WeakMap<any, Actio
   }
 
   const standardKeys = new Set([
-    'action', 'name', 'type', 'target_text', 'text', 'input_text', 'target',
+    'action', 'name', 'type', 'target_text', 'target_description', 'target_descriptions',
+    'text', 'input_text', 'target',
     'coordinates', 'coords', 'target_bounds', 'bounds', 'target_resource_id',
     'resource_id', 'target_class', 'class_name', 'normalized_coordinates',
     'pre_image_name', 'post_image_name', 'pre_screenshot', 'post_screenshot',