feat: show history compression progress and fix action trace overlays

apps/showcase_ui/src/app/utils/action-formatter.util.ts

@@ -15,7 +15,7 @@
  */
 
 import { ActionParam, StepReplayFrame } from '../core/models/stream.model';
-import { extractNumbersFromCoordinateValue, isPureDirectionString, parseSequenceCoordinates } from './image-overlay.util';
+import { extractNumbersFromCoordinateValue, isPureDirectionString, parseSequenceCoordinates, unwrapTraceAction } from './image-overlay.util';
 import { cleanErrorMessage } from './tool-formatter.util';
 
 /**
@@ -35,35 +35,6 @@ export function safeParseJson(val: any): any {
 // WeakMap memo for parsed action objects to avoid repeated JSON.parse and object allocation across change detection cycles
 const actionObjectCache = new WeakMap<object, any>();
 
-function computeActionObject(action: any): any {
-  if (action && typeof action === 'object' && action.payload) {
-    const payload = typeof action.payload === 'string' ? safeParseJson(action.payload) : action.payload;
-    const innerAction = payload?.args?.action;
-    if (innerAction && typeof innerAction === 'object') {
-      return {
-        ...action,
-        ...innerAction,
-        name: action.name || innerAction.action || innerAction.name,
-        action: innerAction.action || action.name,
-        trace_id: action.trace_id,
-        timestamp: action.timestamp ?? innerAction.timestamp
-      };
-    }
-    const innerArgs = payload?.args;
-    if (innerArgs && typeof innerArgs === 'object') {
-      return {
-        ...action,
-        ...innerArgs,
-        name: action.name || innerArgs.action || innerArgs.name,
-        action: innerArgs.action || action.name,
-        trace_id: action.trace_id,
-        timestamp: action.timestamp ?? innerArgs.timestamp
-      };
-    }
-  }
-  return action;
-}
-
 /**
  * Safely extract action object from possible array or nested structure
  */
@@ -75,7 +46,7 @@ export function getActionObject(action: any): any {
   if (typeof action === 'object') {
     const cached = actionObjectCache.get(action);
     if (cached !== undefined) return cached;
-    const res = computeActionObject(action);
+    const res = unwrapTraceAction(action);
     actionObjectCache.set(action, res);
     return res;
   }

apps/showcase_ui/src/app/utils/image-overlay.util.spec.ts

@@ -0,0 +1,85 @@
+import { drawActionCoordinatesOnOverlay, unwrapTraceAction } from './image-overlay.util';
+import { getActionCoords, getActionTargetText } from './action-formatter.util';
+
+// Trace records as the DataEngine publishes them in `generic_tools`.
+const PRO_TAP_TRACE = {
+  trace_id: 'pro-tap',
+  type: 'action',
+  name: 'tap',
+  timestamp: 1788478482.05,
+  status: 'success',
+  payload: {
+    action: {
+      action: 'tap',
+      coordinates: [922, 1710],
+      coordinate_space: 'pixel',
+      normalized_coordinates: [854, 705],
+      target_text: 'YouTube'
+    },
+    success: true,
+    post_screenshot: null
+  }
+};
+
+const FLASH_CLICK_TRACE = {
+  trace_id: 'flash-click',
+  type: 'action',
+  name: 'click',
+  timestamp: 1788478305.12,
+  status: 'success',
+  payload: {
+    args: { target: [618, 705], target_text: '相册' },
+    result: '{"outcome": "Clicked"}'
+  }
+};
+
+function fakeImage(): HTMLImageElement {
+  return { naturalWidth: 1080, naturalHeight: 2424 } as unknown as HTMLImageElement;
+}
+
+describe('unwrapTraceAction', () => {
+  it('flattens a Pro action trace (payload.action)', () => {
+    const act = unwrapTraceAction(PRO_TAP_TRACE);
+    expect(act.action).toBe('tap');
+    expect(act.normalized_coordinates).toEqual([854, 705]);
+    expect(act.target_text).toBe('YouTube');
+    expect(act.trace_id).toBe('pro-tap');
+    expect(act.timestamp).toBe(1788478482.05);
+  });
+
+  it('flattens a Flash action trace (payload.args)', () => {
+    const act = unwrapTraceAction(FLASH_CLICK_TRACE);
+    expect(act.action).toBe('click');
+    expect(act.target).toEqual([618, 705]);
+    expect(act.args.target).toEqual([618, 705]);
+  });
+
+  it('passes plain action objects through untouched', () => {
+    const plain = { action: 'tap', normalized_coordinates: [10, 20] };
+    expect(unwrapTraceAction(plain)).toBe(plain);
+  });
+});
+
+describe('drawActionCoordinatesOnOverlay', () => {
+  it('draws the tap marker for a Pro action trace', () => {
+    const overlay = document.createElement('div');
+    drawActionCoordinatesOnOverlay(fakeImage(), overlay, PRO_TAP_TRACE);
+    const marker = overlay.querySelector('.action-point-marker') as HTMLElement;
+    expect(marker).not.toBeNull();
+    expect(marker.style.left).toBe('85.4%');
+    expect(marker.style.top).toBe('70.5%');
+  });
+
+  it('draws the tap marker for a Flash action trace', () => {
+    const overlay = document.createElement('div');
+    drawActionCoordinatesOnOverlay(fakeImage(), overlay, FLASH_CLICK_TRACE);
+    expect(overlay.querySelector('.action-point-marker')).not.toBeNull();
+  });
+});
+
+describe('action card details from trace records', () => {
+  it('reads coordinates and target text from a Pro action trace', () => {
+    expect(getActionCoords(PRO_TAP_TRACE)).toContain('854');
+    expect(getActionTargetText(PRO_TAP_TRACE)).toBe('YouTube');
+  });
+});

apps/showcase_ui/src/app/utils/image-overlay.util.ts

@@ -14,6 +14,39 @@
  * limitations under the License.
  */
 
+/**
+ * Unwrap Flash (`payload.args`) and Pro (`payload.action`) traces.
+ * Preserve trace metadata and leave plain action objects unchanged.
+ */
+export function unwrapTraceAction(record: any): any {
+  if (!record || typeof record !== 'object' || !record.payload) return record;
+
+  let payload = record.payload;
+  if (typeof payload === 'string') {
+    try {
+      payload = JSON.parse(payload);
+    } catch {
+      return record;
+    }
+  }
+  if (!payload || typeof payload !== 'object') return record;
+
+  const asObject = (v: any) => (Array.isArray(v) ? v[0] : v);
+  const candidates = [asObject(payload.args?.action), asObject(payload.action), asObject(payload.args)];
+  const inner = candidates.find(c => c && typeof c === 'object');
+  if (!inner) return record;
+
+  return {
+    ...record,
+    ...inner,
+    args: (record.args && typeof record.args === 'object') ? record.args : inner,
+    name: record.name || inner.action || inner.name,
+    action: inner.action || record.name,
+    trace_id: record.trace_id,
+    timestamp: record.timestamp ?? inner.timestamp
+  };
+}
+
 /**
  * Helper to parse any coordinate representation into an array of numbers.
  * Handles:
@@ -141,12 +174,11 @@ export function drawActionCoordinatesOnOverlay(
   const naturalWidth = img.naturalWidth || 1080;
   const naturalHeight = img.naturalHeight || 2400;
 
-  actions.forEach(act => {
+  actions.forEach(record => {
+    const act = unwrapTraceAction(record);
     if (!act || typeof act !== 'object') return;
     const actionName = (act.action || act.name || 'tap').toLowerCase();
-
-    const payload = (act.payload && typeof act.payload === 'object') ? act.payload : {};
-    const args = (payload.args && typeof payload.args === 'object') ? payload.args : (act.args || {});
+    const args = (act.args && typeof act.args === 'object') ? act.args : {};
 
     // Filter out non-touch / system actions that do not operate via touch coordinates
     const nonDrawingActions = [

apps/showcase_ui/src/app/utils/tool-formatter.util.spec.ts

@@ -1,10 +1,65 @@
 import {
   cleanErrorMessage,
+  getCompressionLabel,
+  getToolDisplayLabel,
+  getToolIcon,
   getUniqueGenericTools,
   getVideoAnalysisView,
   shouldShowTool
 } from './tool-formatter.util';
 
+describe('compress_history timeline line', () => {
+  const trace = (status: string, args: Record<string, any>) => ({
+    type: 'tool',
+    name: 'compress_history',
+    status,
+    payload: { args }
+  });
+
+  it('is shown as a plain tool line with its own icon', () => {
+    const running = trace('running', { start_step: 12, end_step: 27 });
+    expect(shouldShowTool(running)).toBeTrue();
+    expect(getToolIcon(running)).toBe('compress');
+    expect(getToolDisplayLabel(running)).toBe(getCompressionLabel(running));
+  });
+
+  it('says which steps are being condensed while running', () => {
+    expect(getCompressionLabel(trace('running', { start_step: 12, end_step: 27 })))
+      .toBe('Condensing steps 12–27 into a short memory to free up room…');
+    expect(getCompressionLabel(trace('running', { start_step: 12, end_step: 27, note: 'retrying' })))
+      .toBe('Retrying the memory summary for steps 12–27…');
+    expect(getCompressionLabel(trace('running', { start_step: 5, end_step: 5 })))
+      .toBe('Condensing step 5 into a short memory to free up room…');
+  });
+
+  it('reports the size reduction and the working memory once done', () => {
+    const done = trace('success', {
+      start_step: 12,
+      end_step: 27,
+      source_tokens: 8400,
+      summary_tokens: 600,
+      context_tokens: 54000,
+      context_budget: 80000
+    });
+    expect(getCompressionLabel(done)).toBe(
+      'Steps 12–27 condensed into a short memory · 8.4k → 600 tokens (14× smaller) · working memory ≈ 54k of 80k tokens'
+    );
+  });
+
+  it('omits the working memory figure on lines that do not carry it', () => {
+    const done = trace('success', { start_step: 1, end_step: 4, source_tokens: 3000, summary_tokens: 1500 });
+    expect(getCompressionLabel(done)).toBe('Steps 1–4 condensed into a short memory · 3k → 1.5k tokens (2× smaller)');
+  });
+
+  it('explains a forced snapshot and a failed attempt in plain words', () => {
+    const forced = trace('success', { start_step: 1, end_step: 4, forced: true, context_tokens: 70000, context_budget: 80000 });
+    expect(getCompressionLabel(forced))
+      .toBe('Steps 1–4 replaced by a brief snapshot (memory was nearly full) · working memory ≈ 70k of 80k tokens');
+    expect(getCompressionLabel(trace('failed', { start_step: 1, end_step: 4 })))
+      .toBe("Couldn't condense steps 1–4 yet; keeping the full record and retrying later");
+  });
+});
+
 describe('cleanErrorMessage', () => {
   it('removes repeated LLM wrapper labels while preserving the provider reason', () => {
     expect(cleanErrorMessage('LLM Error: LLM Request Error: 503 model overloaded'))

apps/showcase_ui/src/app/utils/tool-formatter.util.ts

@@ -699,12 +699,65 @@ export function getToolDisplayLabel(tool: any, isFirstSaveNote: boolean = false)
     }
     case 'read_url':
       return 'Fetching web page';
+    case 'compress_history':
+      return getCompressionLabel(tool);
 
     default:
       return `Executing ${cleanName.split('_').map((w: string) => w.charAt(0).toUpperCase() + w.slice(1)).join(' ')}`;
   }
 }
 
+function formatTokenFigure(value: any): string {
+  const n = Number(value);
+  if (!Number.isFinite(n) || n < 0) return '';
+  if (n < 1000) return `${Math.round(n)}`;
+  return `${(n / 1000).toFixed(1).replace(/\.0$/, '')}k`;
+}
+
+/** Format compression progress and token usage for the timeline. */
+export function getCompressionLabel(tool: any): string {
+  const args = getToolArgs(tool) || {};
+  const start = Number(args.start_step);
+  const end = Number(args.end_step);
+  const hasRange = Number.isFinite(start) && Number.isFinite(end) && start > 0 && end > 0;
+  const range = hasRange
+    ? (start === end ? `step ${start}` : `steps ${start}–${end}`)
+    : 'earlier steps';
+  const rangeCapitalized = range.charAt(0).toUpperCase() + range.slice(1);
+  const status = String(tool?.status || '').toLowerCase();
+
+  if (status === 'failed') {
+    return `Couldn't condense ${range} yet; keeping the full record and retrying later`;
+  }
+  if (status !== 'success') {
+    return String(args.note || '').toLowerCase() === 'retrying'
+      ? `Retrying the memory summary for ${range}…`
+      : `Condensing ${range} into a short memory to free up room…`;
+  }
+
+  const parts: string[] = [];
+  const source = Number(args.source_tokens);
+  const summary = Number(args.summary_tokens);
+  if (args.forced) {
+    parts.push(`${rangeCapitalized} replaced by a brief snapshot (memory was nearly full)`);
+  } else {
+    parts.push(`${rangeCapitalized} condensed into a short memory`);
+    if (source > 0 && summary > 0) {
+      const factor = source / summary;
+      const factorText = factor >= 2 ? ` (${Math.round(factor)}× smaller)` : '';
+      parts.push(`${formatTokenFigure(source)} → ${formatTokenFigure(summary)} tokens${factorText}`);
+    }
+  }
+  const context = Number(args.context_tokens);
+  const budget = Number(args.context_budget);
+  if (context > 0) {
+    parts.push(budget > 0
+      ? `working memory ≈ ${formatTokenFigure(context)} of ${formatTokenFigure(budget)} tokens`
+      : `working memory ≈ ${formatTokenFigure(context)} tokens`);
+  }
+  return parts.join(' · ');
+}
+
 /**
  * Get Material Symbol icon for tools
  */
@@ -772,6 +825,8 @@ export function getToolIcon(tool: any): string {
     case 'outputter':
     case 'output_synthesis':
       return 'assignment_turned_in';
+    case 'compress_history':
+      return 'compress';
     default:
       return 'build';
   }