feat: harden video analysis and runtime resilience

apps/showcase_ui/src/app/utils/stream-aggregator.util.spec.ts

@@ -0,0 +1,93 @@
+import { consolidateLogsToBlocks, getSortedStepEvents } from './stream-aggregator.util';
+
+describe('stream aggregator timeline ordering', () => {
+  it('interleaves text, tools, and actions by timestamp', () => {
+    const events = getSortedStepEvents({
+      timestamp: 10,
+      operator_native_thinking: 'Thought content',
+      operator_native_thinking_timestamp: 11,
+      operator_raw_thinking: 'Work content',
+      operator_raw_thinking_timestamp: 13,
+      action_taken: { action: 'tap', timestamp: 15 },
+      generic_tools: [
+        { trace_id: 'tool-late', name: 'save_note', timestamp: 14 },
+        { trace_id: 'tool-early', name: 'video_analyzer', timestamp: 12 }
+      ]
+    });
+
+    expect(events.map((event) => event.type)).toEqual([
+      'thinking',
+      'tool',
+      'text',
+      'tool',
+      'action'
+    ]);
+    expect(events.map((event) => event.timestamp)).toEqual([
+      11000,
+      12000,
+      13000,
+      14000,
+      15000
+    ]);
+  });
+
+  it('uses a deterministic text-first fallback when only a step timestamp exists', () => {
+    const events = getSortedStepEvents({
+      timestamp: 20,
+      operator_native_thinking: 'Thought content',
+      operator_raw_thinking: 'Work content',
+      action_taken: { action: 'tap' },
+      generic_tools: [{ trace_id: 'tool-1', name: 'save_note' }]
+    });
+
+    expect(events.map((event) => event.type)).toEqual([
+      'thinking',
+      'text',
+      'action',
+      'tool'
+    ]);
+  });
+
+  it('keeps the first live timestamp when later chunks update the same text stream', () => {
+    const blocks = consolidateLogsToBlocks([
+      {
+        type: 'llm_stream',
+        timestamp: '2026-08-25T20:00:01.000Z',
+        data: {
+          execution_id: 'exec-1',
+          step_id: 'step-1',
+          stream_type: 'text',
+          text: 'First',
+          isCompleted: false
+        }
+      },
+      {
+        type: 'trace_recorded',
+        timestamp: '2026-08-25T20:00:02.000Z',
+        data: {
+          trace_id: 'tool-1',
+          step_id: 'step-1',
+          type: 'tool',
+          name: 'save_note',
+          timestamp: '2026-08-25T20:00:02.000Z'
+        }
+      },
+      {
+        type: 'llm_stream',
+        timestamp: '2026-08-25T20:00:03.000Z',
+        data: {
+          execution_id: 'exec-1',
+          step_id: 'step-1',
+          stream_type: 'text',
+          text: 'First and second',
+          isCompleted: false
+        }
+      }
+    ]);
+
+    expect(blocks[0].data.operator_raw_thinking_timestamp)
+      .toBe('2026-08-25T20:00:01.000Z');
+    expect(getSortedStepEvents(blocks[0].data).map((event) => event.type))
+      .toEqual(['text', 'tool']);
+  });
+});

apps/showcase_ui/src/app/utils/stream-aggregator.util.ts

@@ -64,8 +64,12 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
         const updatedData = { ...existing.data };
         if (streamType === 'thinking') {
           updatedData.operator_native_thinking = text;
+          updatedData.operator_native_thinking_timestamp =
+            updatedData.operator_native_thinking_timestamp || log.timestamp;
         } else {
           updatedData.operator_raw_thinking = text;
+          updatedData.operator_raw_thinking_timestamp =
+            updatedData.operator_raw_thinking_timestamp || log.timestamp;
         }
         if (stepId && !updatedData.step_id) {
           updatedData.step_id = stepId;
@@ -89,8 +93,10 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
         };
         if (streamType === 'thinking') {
           blockData.operator_native_thinking = text;
+          blockData.operator_native_thinking_timestamp = log.timestamp;
         } else {
           blockData.operator_raw_thinking = text;
+          blockData.operator_raw_thinking_timestamp = log.timestamp;
         }
 
         blocks.push({
@@ -134,6 +140,12 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
           ...log.data,
           operator_native_thinking: log.data.operator_native_thinking || existingBlock.data.operator_native_thinking || undefined,
           operator_raw_thinking: log.data.operator_raw_thinking || existingBlock.data.operator_raw_thinking || undefined,
+          operator_native_thinking_timestamp: existingBlock.data.operator_native_thinking_timestamp
+            || log.data.operator_native_thinking_timestamp
+            || (log.data.operator_native_thinking ? log.timestamp : undefined),
+          operator_raw_thinking_timestamp: existingBlock.data.operator_raw_thinking_timestamp
+            || log.data.operator_raw_thinking_timestamp
+            || (log.data.operator_raw_thinking ? log.timestamp : undefined),
           action_taken: log.data.action_taken || existingBlock.data.action_taken || undefined,
           last_execution_result: log.data.last_execution_result || existingBlock.data.last_execution_result || undefined,
           generic_tools: mergedTools,
@@ -155,6 +167,10 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
           data: {
             ...log.data,
             isCompleted: true,
+            operator_native_thinking_timestamp: log.data.operator_native_thinking_timestamp
+              || (log.data.operator_native_thinking ? log.timestamp : undefined),
+            operator_raw_thinking_timestamp: log.data.operator_raw_thinking_timestamp
+              || (log.data.operator_raw_thinking ? log.timestamp : undefined),
             generic_tools: log.data.generic_tools || []
           }
         });
@@ -410,23 +426,58 @@ export function groupBlocksToPhases(blocks: StepBlock[], sessionStartTime: numbe
  */
 export function getSortedStepEvents(
   stepData: any, 
-  cache?: WeakMap<any, { length: number; actionTs: any; events: StepEvent[] }>
+  cache?: WeakMap<any, { signature: string; events: StepEvent[] }>
 ): StepEvent[] {
   if (!stepData || typeof stepData !== 'object') return [];
   const toolsLen = stepData.generic_tools ? stepData.generic_tools.length : 0;
   const actionTs = stepData.action_taken ? (stepData.action_taken.timestamp || stepData.action_taken.start_time || stepData.action_taken.created_at) : null;
+  const nativeText = typeof stepData.operator_native_thinking === 'string'
+    ? stepData.operator_native_thinking.trim()
+    : '';
+  const rawText = typeof stepData.operator_raw_thinking === 'string'
+    ? stepData.operator_raw_thinking.trim()
+    : '';
+  const signature = [
+    toolsLen,
+    actionTs ?? '',
+    stepData.operator_native_thinking_timestamp ?? '',
+    stepData.operator_raw_thinking_timestamp ?? '',
+    nativeText.length,
+    rawText.length
+  ].join('|');
 
   if (cache) {
     const cached = cache.get(stepData);
-    if (cached && cached.length === toolsLen && cached.actionTs === actionTs) {
+    if (cached?.signature === signature) {
       return cached.events;
     }
   }
 
-  const events: Array<{ type: 'action' | 'tool'; data: any; timestamp: number }> = [];
+  const events: Array<{ type: StepEvent['type']; data: any; timestamp: number; sequence: number }> = [];
   const fallbackTime = getItemTimestamp(stepData.timestamp, 0);
+  let sequence = 0;
 
-  // 1. Add action_taken if present and valid
+  // Text and tools share one timeline. Live streams carry their first-seen
+  // timestamps; persisted steps use the step timestamp as a stable fallback.
+  if (nativeText) {
+    events.push({
+      type: 'thinking',
+      data: { text: stepData.operator_native_thinking },
+      timestamp: getItemTimestamp(stepData.operator_native_thinking_timestamp, fallbackTime),
+      sequence: sequence++
+    });
+  }
+
+  if (rawText) {
+    events.push({
+      type: 'text',
+      data: { text: stepData.operator_raw_thinking },
+      timestamp: getItemTimestamp(stepData.operator_raw_thinking_timestamp, fallbackTime),
+      sequence: sequence++
+    });
+  }
+
+  // Add action_taken if present and valid.
   if (stepData.action_taken && (isAndroidAction(stepData.action_taken) || isReportStatusAction(stepData.action_taken))) {
     const actTime = getItemTimestamp(
       stepData.action_taken.timestamp ?? stepData.action_taken.start_time ?? stepData.action_taken.created_at,
@@ -435,11 +486,12 @@ export function getSortedStepEvents(
     events.push({
       type: 'action',
       data: stepData.action_taken,
-      timestamp: actTime
+      timestamp: actTime,
+      sequence: sequence++
     });
   }
 
-  // 2. Add generic tools if present
+  // Add generic tools if present.
   const uniqueTools = getUniqueGenericTools(stepData.generic_tools);
   uniqueTools.forEach((tool: any) => {
     const toolTime = getItemTimestamp(
@@ -449,16 +501,22 @@ export function getSortedStepEvents(
     events.push({
       type: 'tool',
       data: tool,
-      timestamp: toolTime
+      timestamp: toolTime,
+      sequence: sequence++
     });
   });
 
-  // 3. Sort chronologically by timestamp
-  events.sort((a, b) => a.timestamp - b.timestamp);
+  // Sort chronologically. The explicit sequence makes equal/missing timestamp
+  // behavior deterministic across browsers and preserves the legacy fallback.
+  events.sort((a, b) => (a.timestamp - b.timestamp) || (a.sequence - b.sequence));
 
-  const result: StepEvent[] = events.map(e => ({ type: e.type, data: e.data }));
+  const result: StepEvent[] = events.map(e => ({
+    type: e.type,
+    data: e.data,
+    timestamp: e.timestamp
+  }));
   if (cache) {
-    cache.set(stepData, { length: toolsLen, actionTs, events: result });
+    cache.set(stepData, { signature, events: result });
   }
   return result;
 }

apps/showcase_ui/src/app/utils/tool-formatter.util.spec.ts

@@ -1,4 +1,8 @@
-import { cleanErrorMessage, getUniqueGenericTools } from './tool-formatter.util';
+import {
+  cleanErrorMessage,
+  getUniqueGenericTools,
+  getVideoAnalysisView
+} from './tool-formatter.util';
 
 describe('cleanErrorMessage', () => {
   it('removes repeated LLM wrapper labels while preserving the provider reason', () => {
@@ -86,3 +90,77 @@ describe('getUniqueGenericTools retry aggregation', () => {
     expect(result.map(tool => tool.name)).toEqual(['llm_pause']);
   });
 });
+
+describe('video analysis timeline formatting', () => {
+  it('distinguishes a cache hit without presenting it as running', () => {
+    const view = getVideoAnalysisView({
+      name: 'spawn_sub_agent',
+      status: 'success',
+      payload: {
+        args: { start_time: 0, end_time: 42 },
+        result: 'CACHED VIDEO ANALYSIS: existing evidence'
+      }
+    });
+
+    expect(view?.outcome).toBe('complete');
+    expect(view?.reuse).toBe('full');
+    expect(view?.title).toBe('Reused video analysis');
+    expect(view?.requestedRange).toEqual({ start: 0, end: 42 });
+  });
+
+  it('collapses child video chunks into one stable note-style timeline item', () => {
+    const result = getUniqueGenericTools([
+      {
+        trace_id: 'chunk-1',
+        parent_trace_id: 'video-agent-1',
+        type: 'tool',
+        name: 'spawn_sub_agent',
+        status: 'success',
+        timestamp: 100,
+        payload: {
+          args: { start_time: 0, end_time: 30, specific_query: 'find the result' },
+          result: '[from 0.0s to 30.0s] Summary: first'
+        }
+      },
+      {
+        trace_id: 'chunk-2',
+        parent_trace_id: 'video-agent-1',
+        type: 'tool',
+        name: 'spawn_sub_agent',
+        status: 'success',
+        timestamp: 101,
+        payload: {
+          args: { start_time: 30, end_time: 60, specific_query: 'find the result' },
+          result: 'PARTIAL VIDEO ANALYSIS (successful chunks were persisted). Failed intervals: 45.0s-60.0s'
+        }
+      }
+    ]);
+
+    expect(result.length).toBe(1);
+    expect(result[0].name).toBe('video_analysis');
+    expect(result[0].trace_id).toBe('video-analysis-video-agent-1');
+    expect(result[0].payload.result.outcome).toBe('partial');
+    expect(result[0].payload.result.requested_range).toEqual({ start: 0, end: 60 });
+    expect(getVideoAnalysisView(result[0])?.title).toBe('Video analysis partially completed');
+  });
+
+  it('keeps video analyses from independent parent executions separate', () => {
+    const result = getUniqueGenericTools([
+      {
+        trace_id: 'chunk-1', parent_trace_id: 'video-agent-1', type: 'tool',
+        name: 'spawn_sub_agent', status: 'success',
+        payload: { args: { start_time: 0, end_time: 10 }, result: 'first' }
+      },
+      {
+        trace_id: 'chunk-2', parent_trace_id: 'video-agent-2', type: 'tool',
+        name: 'spawn_sub_agent', status: 'success',
+        payload: { args: { start_time: 10, end_time: 20 }, result: 'second' }
+      }
+    ]);
+
+    expect(result.map(tool => tool.trace_id)).toEqual([
+      'video-analysis-video-agent-1',
+      'video-analysis-video-agent-2'
+    ]);
+  });
+});

apps/showcase_ui/src/app/utils/tool-formatter.util.ts

@@ -64,7 +64,133 @@ export function isVideoTool(tool: any): boolean {
   if (!tool || !tool.name) return false;
   const cleanName = tool.name.replace(/^(_)?(self\.)?exec_/, '');
   const nameLower = cleanName.toLowerCase();
-  return ['video_analyzer', 'video_analyzer_pure'].includes(nameLower);
+  return [
+    'video_analysis',
+    'video_analyzer',
+    'video_analyzer_pure',
+    'spawn_sub_agent',
+    'analyze_audio_only'
+  ].includes(nameLower);
+}
+
+export type VideoAnalysisOutcome = 'running' | 'recovering' | 'waiting' | 'complete' | 'partial' | 'failed';
+
+export interface VideoAnalysisRange {
+  start: number;
+  end: number;
+  category?: string;
+  retryable?: boolean;
+}
+
+export interface VideoAnalysisView {
+  outcome: VideoAnalysisOutcome;
+  title: string;
+  query: string;
+  summary: string;
+  reuse: 'none' | 'partial' | 'full';
+  requestedRange: VideoAnalysisRange | null;
+  completedRanges: VideoAnalysisRange[];
+  failedRanges: VideoAnalysisRange[];
+  evidenceCount: number;
+  completedCount: number;
+  totalCount: number;
+  fallbackUsed: boolean;
+}
+
+function numericRange(value: any): VideoAnalysisRange | null {
+  if (!value || typeof value !== 'object') return null;
+  const start = Number(value.start);
+  const end = Number(value.end);
+  if (!Number.isFinite(start) || !Number.isFinite(end) || end <= start) return null;
+  return {
+    start,
+    end,
+    category: value.category ? String(value.category) : undefined,
+    retryable: value.retryable === true
+  };
+}
+
+function parseVideoResultText(text: string): Partial<VideoAnalysisView> {
+  const value = text.trim();
+  if (!value) return {};
+  if (value.startsWith('CACHED VIDEO ANALYSIS:')) {
+    return { outcome: 'complete', reuse: 'full', summary: value.replace(/^CACHED VIDEO ANALYSIS:\s*/, '') };
+  }
+  if (value.startsWith('PARTIAL VIDEO ANALYSIS')) {
+    return { outcome: 'partial', reuse: 'partial', summary: value };
+  }
+  if (value.startsWith('All sub-agent chunks failed') || value.startsWith('Error:')) {
+    return { outcome: 'failed', summary: value };
+  }
+  if (value.includes('Analysis is already in progress in another video agent')) {
+    return { outcome: 'waiting', summary: 'Another video agent is already analyzing this evidence.' };
+  }
+  return { outcome: 'complete', summary: value };
+}
+
+/** Build the user-facing, backward-compatible video analysis state. */
+export function getVideoAnalysisView(tool: any): VideoAnalysisView | null {
+  if (!isVideoTool(tool)) return null;
+  const args = getToolArgs(tool);
+  const payload = tool?.payload && typeof tool.payload === 'object' ? tool.payload : {};
+  const rawResult = payload.result ?? tool.result ?? null;
+  const structured = rawResult && typeof rawResult === 'object' ? rawResult : {};
+  const parsed = typeof rawResult === 'string' ? parseVideoResultText(rawResult) : {};
+
+  let outcome = String(structured.outcome || payload.outcome || parsed.outcome || '').toLowerCase() as VideoAnalysisOutcome;
+  if (!['running', 'recovering', 'waiting', 'complete', 'partial', 'failed'].includes(outcome)) {
+    outcome = tool?.status === 'failed' || tool?.status === 'error'
+      ? 'failed'
+      : tool?.status === 'running' ? 'running' : 'complete';
+  }
+  const recovering = structured.recovering === true || structured.fallback_used === true;
+  if (outcome === 'running' && recovering) outcome = 'recovering';
+
+  const start = Number(args.start_time ?? structured.requested_range?.start);
+  const end = Number(args.end_time ?? structured.requested_range?.end);
+  const requestedRange = Number.isFinite(start) && Number.isFinite(end) && end > start
+    ? { start, end }
+    : numericRange(structured.requested_range);
+  const completedRanges = Array.isArray(structured.completed_ranges)
+    ? structured.completed_ranges.map(numericRange).filter((range: VideoAnalysisRange | null): range is VideoAnalysisRange => Boolean(range))
+    : [];
+  const failedRanges = Array.isArray(structured.failed_ranges)
+    ? structured.failed_ranges.map(numericRange).filter((range: VideoAnalysisRange | null): range is VideoAnalysisRange => Boolean(range))
+    : [];
+  const completedCount = Number(structured.completed_count ?? completedRanges.length ?? 0);
+  const totalCount = Number(structured.total_count ?? (completedRanges.length + failedRanges.length));
+  const titleByOutcome: Record<VideoAnalysisOutcome, string> = {
+    running: 'Analyzing screen recording',
+    recovering: 'Analyzing unfinished recording segment',
+    waiting: 'Waiting for existing video analysis',
+    complete: structured.reuse === 'full' || parsed.reuse === 'full'
+      ? 'Reused video analysis'
+      : 'Analyzed screen recording',
+    partial: 'Video analysis partially completed',
+    failed: 'Video analysis returned no result'
+  };
+
+  return {
+    outcome,
+    title: titleByOutcome[outcome],
+    query: String(args.specific_query || args.query || args.prompt || structured.query || ''),
+    summary: String(structured.summary || parsed.summary || ''),
+    reuse: (structured.reuse || parsed.reuse || 'none') as 'none' | 'partial' | 'full',
+    requestedRange,
+    completedRanges,
+    failedRanges,
+    evidenceCount: Number(structured.evidence_count || 0),
+    completedCount: Number.isFinite(completedCount) ? completedCount : 0,
+    totalCount: Number.isFinite(totalCount) ? totalCount : 0,
+    fallbackUsed: structured.fallback_used === true
+  };
+}
+
+export function formatVideoTime(seconds: number): string {
+  const safe = Math.max(0, Number(seconds) || 0);
+  const minutes = Math.floor(safe / 60);
+  const secs = Math.floor(safe % 60);
+  return `${String(minutes).padStart(2, '0')}:${String(secs).padStart(2, '0')}`;
 }
 
 /**
@@ -301,7 +427,107 @@ export function getUniqueGenericTools(tools: any[] | undefined): any[] {
       }
     }
   }
-  return result;
+  return collapseVideoAnalysisTools(result);
+}
+
+/**
+ * Present one note-style timeline row for one video-analyzer execution. Child
+ * chunk, audio, and wrapper traces stay available in the raw trace but do not
+ * look like repeated user-visible analyses.
+ */
+function collapseVideoAnalysisTools(tools: any[]): any[] {
+  const groups = new Map<string, { firstIndex: number; tools: any[] }>();
+  const passthrough: Array<{ index: number; tool: any }> = [];
+
+  tools.forEach((tool, index) => {
+    if (!isVideoTool(tool)) {
+      passthrough.push({ index, tool });
+      return;
+    }
+    const cleanName = String(tool.name || '').replace(/^(_)?(self\.)?exec_/, '').toLowerCase();
+    const groupId = cleanName === 'video_analyzer' || cleanName === 'video_analyzer_pure'
+      ? String(tool.trace_id || tool.parent_trace_id || `video-${index}`)
+      : String(tool.parent_trace_id || tool.trace_id || `video-${index}`);
+    const existing = groups.get(groupId);
+    if (existing) {
+      existing.tools.push(tool);
+    } else {
+      groups.set(groupId, { firstIndex: index, tools: [tool] });
+    }
+  });
+
+  const collapsed = [...passthrough];
+  for (const [groupId, group] of groups) {
+    const views = group.tools
+      .map(getVideoAnalysisView)
+      .filter((view): view is VideoAnalysisView => Boolean(view));
+    if (!views.length) continue;
+
+    const hasActive = views.some(view => view.outcome === 'running' || view.outcome === 'recovering');
+    const completed = views.filter(view => view.outcome === 'complete');
+    const hasPartial = views.some(view => view.outcome === 'partial');
+    const failed = views.filter(view => view.outcome === 'failed');
+    const waiting = views.filter(view => view.outcome === 'waiting');
+    let outcome: VideoAnalysisOutcome;
+    if (hasActive) {
+      outcome = views.some(view => view.outcome === 'recovering') ? 'recovering' : 'running';
+    } else if (hasPartial || (completed.length > 0 && failed.length > 0)) {
+      outcome = 'partial';
+    } else if (completed.length > 0) {
+      outcome = 'complete';
+    } else if (waiting.length > 0) {
+      outcome = 'waiting';
+    } else {
+      outcome = 'failed';
+    }
+
+    const ranges = views
+      .map(view => view.requestedRange)
+      .filter((range): range is VideoAnalysisRange => Boolean(range));
+    const requestedRange = ranges.length
+      ? {
+          start: Math.min(...ranges.map(range => range.start)),
+          end: Math.max(...ranges.map(range => range.end))
+        }
+      : null;
+    const reuse = completed.length > 0 && completed.every(view => view.reuse === 'full')
+      ? 'full'
+      : views.some(view => view.reuse !== 'none') ? 'partial' : 'none';
+    const base = group.tools[0];
+    const structuredResult = {
+      outcome,
+      reuse,
+      requested_range: requestedRange,
+      completed_count: completed.length,
+      total_count: views.length,
+      evidence_count: views.reduce((sum, view) => sum + view.evidenceCount, 0),
+      query: views.find(view => view.query)?.query || '',
+      recovering: outcome === 'recovering',
+      fallback_used: views.some(view => view.fallbackUsed)
+    };
+
+    collapsed.push({
+      index: group.firstIndex,
+      tool: {
+        ...base,
+        trace_id: `video-analysis-${groupId}`,
+        name: 'video_analysis',
+        status: outcome === 'running' || outcome === 'recovering' ? 'running' : 'success',
+        payload: {
+          ...(base.payload || {}),
+          args: {
+            ...(base.payload?.args || base.args || {}),
+            start_time: requestedRange?.start,
+            end_time: requestedRange?.end
+          },
+          result: structuredResult
+        }
+      }
+    });
+  }
+
+  collapsed.sort((a, b) => a.index - b.index);
+  return collapsed.map(item => item.tool);
 }
 
 /**