feat: handle mid-stream LLM resets and deduplicate UI action stream events

- Backend (LLM Gateway):
  - Record structured 'llm_stream_reset' event with discard action, failure reason, and retry messaging on mid-stream generation errors.
  - Add unit test verifying stream reset event payload on mid-stream errors.

- Frontend (Showcase UI):
  - Handle 'llm_stream_reset' events in agent service and stream aggregator, discarding interrupted chunks while showing retry indicators.
  - Establish a single source of truth for tool and action events in the timeline to eliminate duplicated action rows while retaining ADB and explorer tools.
  - Add @angular/cdk dependency and update agent stream component styling and layout.
  - Add unit tests for stream aggregation and tool display filtering.
This commit is contained in:
somew1nd committed 2026-09-03 14:38:02 -07:00
1 parent d7c64f571d
commit d23a4f6472
15 files changed
+1862 -1285

No files matched your search

@@ -18,13 +18,66 @@ import { ActionParam, StepReplayFrame } from '../core/models/stream.model';
import { extractNumbersFromCoordinateValue, isPureDirectionString, parseSequenceCoordinates } from './image-overlay.util';
import { cleanErrorMessage } from './tool-formatter.util';
/**
* Safely parse JSON string if it looks like an object/array, otherwise return original value
*/
export function safeParseJson(val: any): any {
if (typeof val === 'string' && (val.trim().startsWith('{') || val.trim().startsWith('['))) {
try {
return JSON.parse(val);
} catch {
return val;
}
}
return val;
}
// WeakMap memo for parsed action objects to avoid repeated JSON.parse and object allocation across change detection cycles
const actionObjectCache = new WeakMap<object, any>();
function computeActionObject(action: any): any {
if (action && typeof action === 'object' && action.payload) {
const payload = typeof action.payload === 'string' ? safeParseJson(action.payload) : action.payload;
const innerAction = payload?.args?.action;
if (innerAction && typeof innerAction === 'object') {
return {
...action,
...innerAction,
name: action.name || innerAction.action || innerAction.name,
action: innerAction.action || action.name,
trace_id: action.trace_id,
timestamp: action.timestamp ?? innerAction.timestamp
};
}
const innerArgs = payload?.args;
if (innerArgs && typeof innerArgs === 'object') {
return {
...action,
...innerArgs,
name: action.name || innerArgs.action || innerArgs.name,
action: innerArgs.action || action.name,
trace_id: action.trace_id,
timestamp: action.timestamp ?? innerArgs.timestamp
};
}
}
return action;
}
/**
* Safely extract action object from possible array or nested structure
*/
export function getActionObject(action: any): any {
if (!action) return null;
if (Array.isArray(action)) {
return action.length > 0 ? action[0] : null;
return action.length > 0 ? getActionObject(action[0]) : null;
}
if (typeof action === 'object') {
const cached = actionObjectCache.get(action);
if (cached !== undefined) return cached;
const res = computeActionObject(action);
actionObjectCache.set(action, res);
return res;
}
return action;
}
@@ -46,7 +99,7 @@ export function isAndroidAction(action: any): boolean {
'tap', 'click', 'click_sequence', 'input', 'input_text', 'swipe', 'scroll', 'press_key',
'press_home', 'press_back', 'launch_app', 'manage_app', 'open_app',
'focus_and_input_text', 'focus_and_clear_text', 'long_press', 'long_press_on',
'wait_for_delay', 'wait'
'wait_for_delay', 'wait', 'drag', 'drag_and_drop'
].includes(name);
}
@@ -543,7 +596,8 @@ export function isGenericToolTrace(actionData: any, stepData?: any): boolean {
}
}
if (actionData.is_primary_step_action) return false;
return Boolean(actionData.trace_id || actionData.payload || actionData.type === 'tool' || (actionData.type === 'action' && actionData.trace_id));
if (actionData.type === 'action' || isAndroidAction(actionData)) return false;
return Boolean(actionData.trace_id || actionData.payload || actionData.type === 'tool');
}
interface ResolvedImageEntry {
@@ -554,16 +608,6 @@ interface ResolvedImageEntry {
const stepImageMapCache = new WeakMap<any, { key: string; map: Map<any, ResolvedImageEntry> }>();
function safeParseJson(val: any): any {
if (typeof val === 'string' && (val.trim().startsWith('{') || val.trim().startsWith('['))) {
try {
return JSON.parse(val);
} catch {
return val;
}
}
return val;
}
export function extractItemPreImage(item: any): string | null {
const act = getActionObject(item);
@@ -639,7 +683,10 @@ export function resolveStepImageMap(stepData: any): Map<any, ResolvedImageEntry>
if (!stepData || typeof stepData !== 'object') return emptyMap;
const toolsLen = Array.isArray(stepData.generic_tools) ? stepData.generic_tools.length : 0;
const cacheKey = `${stepData.step_id || ''}_${toolsLen}_${stepData.post_image_name || ''}_${stepData.pre_image_name || ''}`;
const toolsImageSig = Array.isArray(stepData.generic_tools)
? stepData.generic_tools.map((t: any) => `${t?.trace_id || ''}:${t?.post_image_name || t?.post_screenshot || ''}`).join(',')
: '';
const cacheKey = `${stepData.step_id || ''}_${toolsLen}_${toolsImageSig}_${stepData.post_image_name || ''}_${stepData.pre_image_name || ''}`;
const cached = stepImageMapCache.get(stepData);
if (cached && cached.key === cacheKey) {
@@ -649,7 +696,11 @@ export function resolveStepImageMap(stepData: any): Map<any, ResolvedImageEntry>
const map = new Map<any, ResolvedImageEntry>();
const events: Array<{ item: any; act: any; timestamp: number; isPrimary: boolean }> = [];
if (stepData.action_taken) {
const hasActionInTools = Array.isArray(stepData.generic_tools) && stepData.generic_tools.some(
(t: any) => t && (t.type === 'action' || isAndroidAction(t) || isReportStatusAction(t))
);
if (!hasActionInTools && stepData.action_taken) {
events.push({
item: stepData.action_taken,
act: getActionObject(stepData.action_taken),
@@ -661,11 +712,12 @@ export function resolveStepImageMap(stepData: any): Map<any, ResolvedImageEntry>
if (Array.isArray(stepData.generic_tools)) {
stepData.generic_tools.forEach((t: any) => {
if (t) {
const isAction = t.type === 'action' || isAndroidAction(t) || isReportStatusAction(t);
events.push({
item: t,
act: getActionObject(t),
timestamp: getEventTimestamp(t),
isPrimary: false
isPrimary: isAction
});
}
});
@@ -728,6 +780,19 @@ export function resolveStepImageMap(stepData: any): Map<any, ResolvedImageEntry>
map.set(e.item, entry);
if (act && act !== e.item) map.set(act, entry);
if (act?.trace_id) map.set(act.trace_id, entry);
if (e.item?.trace_id) map.set(e.item.trace_id, entry);
}
// Ensure stepData.action_taken cross-resolves to the action images
if (stepData.action_taken && lastActionItem) {
const lastEntry = map.get(lastActionItem);
if (lastEntry) {
map.set(stepData.action_taken, lastEntry);
const actObj = getActionObject(stepData.action_taken);
if (actObj && actObj !== stepData.action_taken) {
map.set(actObj, lastEntry);
}
}
}
stepImageMapCache.set(stepData, { key: cacheKey, map });
@@ -885,7 +950,9 @@ export function extractStepReplayFrames(logsOrSteps: any[]): StepReplayFrame[] {
const visualCandidates: VisualCandidate[] = [];
for (const stepData of stepsList) {
const act = stepData.action_taken;
const act = stepData.action_taken || (Array.isArray(stepData.generic_tools)
? stepData.generic_tools.find((t: any) => t && (t.type === 'action' || isAndroidAction(t)))
: null);
const preUrl = getStepPreImageUrl(stepData, act);
const postUrl = getStepPostImageUrl(stepData, act);
const primaryImg = preUrl || postUrl;
@@ -349,3 +349,139 @@ describe('stream aggregator checker history relocation', () => {
expect(checker.data.generic_tools.map((t: any) => t.trace_id)).toEqual(['tool-probe']);
});
});
describe('stream aggregator single source of truth (Option A)', () => {
it('does not duplicate action_taken when generic_tools already contains the action trace', () => {
const events = getSortedStepEvents({
timestamp: 100,
action_taken: [{ action: 'launch_app', app_name: 'Maps', timestamp: 102 }],
generic_tools: [
{ trace_id: 'tool-note', type: 'tool', name: 'update_note', timestamp: 101 },
{ trace_id: 'act-launch', type: 'action', name: 'launch_app', timestamp: 102, payload: { args: { action: { action: 'launch_app', app_name: 'Maps' } } } }
]
});
expect(events.length).toBe(2);
expect(events.map(e => e.type)).toEqual(['tool', 'action']);
expect(events.map(e => e.data.name)).toEqual(['update_note', 'launch_app']);
});
it('retains multiple adb commands and tools alongside physical actions without dropping any', () => {
const events = getSortedStepEvents({
timestamp: 100,
action_taken: [{ action: 'wait_for_delay', time_in_ms: 1000, timestamp: 105 }],
generic_tools: [
{ trace_id: 'adb-1', type: 'tool', name: 'run_adb_command', timestamp: 101, payload: { CommandLine: 'dumpsys telephony.registry' } },
{ trace_id: 'adb-2', type: 'tool', name: 'run_adb_command', timestamp: 102, payload: { CommandLine: 'am start -a ...' } },
{ trace_id: 'exp-1', type: 'tool', name: 'ask_explorer', timestamp: 103 },
{ trace_id: 'act-wait', type: 'action', name: 'wait_for_delay', timestamp: 104, payload: { args: { time_in_ms: 1000 } } }
]
});
expect(events.length).toBe(4);
expect(events.map(e => e.type)).toEqual(['tool', 'tool', 'tool', 'action']);
expect(events.map(e => e.data.name)).toEqual(['run_adb_command', 'run_adb_command', 'ask_explorer', 'wait_for_delay']);
});
it('supports multi-action bursts sequentially from generic_tools', () => {
const events = getSortedStepEvents({
timestamp: 100,
action_taken: [{ action: 'tap', coordinates: [200, 300] }],
generic_tools: [
{ trace_id: 'tap-1', type: 'action', name: 'tap', timestamp: 101, payload: { args: { action: { action: 'tap', coordinates: [100, 200] } } } },
{ trace_id: 'tap-2', type: 'action', name: 'tap', timestamp: 102, payload: { args: { action: { action: 'tap', coordinates: [200, 300] } } } }
]
});
expect(events.length).toBe(2);
expect(events.every(e => e.type === 'action')).toBeTrue();
expect(events[0].data.trace_id).toBe('tap-1');
expect(events[1].data.trace_id).toBe('tap-2');
});
it('correctly processes real trace 38fd934e Step 16 keeping ADB commands and wait_for_delay visible without duplicates', () => {
const step16 = {
step_id: '31e8c7ec-6b67-485e-94f4-355a853ccf8a',
step_number: 16,
action_taken: [{ action: 'wait_for_delay', time_in_ms: 1000 }],
operator_native_thinking: 'Let me check telephony and send SMS...',
operator_native_thinking_timestamp: 1788457630,
timestamp: 1788457630,
generic_tools: [
{ trace_id: 't-adb-1', parent_trace_id: 't-op', name: 'run_adb_command', type: 'tool', timestamp: 1788457631, status: 'success', payload: { args: { CommandLine: 'dumpsys telephony.registry' } } },
{ trace_id: 't-adb-1-sub', parent_trace_id: 't-adb-1', name: 'run_adb_command', type: 'tool', timestamp: 1788457631.5, status: 'success' },
{ trace_id: 't-adb-2', parent_trace_id: 't-op', name: 'run_adb_command', type: 'tool', timestamp: 1788457632, status: 'success', payload: { args: { CommandLine: 'am start -a android.intent.action.SENDTO ...' } } },
{ trace_id: 't-adb-2-sub', parent_trace_id: 't-adb-2', name: 'run_adb_command', type: 'tool', timestamp: 1788457632.5, status: 'success' },
{ trace_id: 't-exp', parent_trace_id: 't-op', name: 'ask_explorer', type: 'tool', timestamp: 1788457633, status: 'success', payload: { args: { query: 'any text on screen' } } },
{ trace_id: 't-exp-sub', parent_trace_id: 't-exp', name: 'ask_explorer', type: 'tool', timestamp: 1788457633.5, status: 'success' },
{ trace_id: 't-wait', parent_trace_id: 't-val', name: 'wait_for_delay', type: 'action', timestamp: 1788457635, status: 'success', payload: { args: { action: { action: 'wait_for_delay', time_in_ms: 1000 } } } },
{ trace_id: 't-safety', parent_trace_id: 't-wait', name: 'safety_net_pixel_validation', type: 'tool', timestamp: 1788457636, status: 'success' }
]
};
const events = getSortedStepEvents(step16);
// 1 native thinking + 2 adb tools + 1 explorer tool + 1 wait_for_delay action
expect(events.length).toBe(5);
expect(events.map(e => e.type)).toEqual(['thinking', 'tool', 'tool', 'tool', 'action']);
expect(events.map(e => e.data.name || e.data.action || 'thinking')).toEqual([
'thinking',
'run_adb_command',
'run_adb_command',
'ask_explorer',
'wait_for_delay'
]);
});
it('correctly handles mid-stream reset and supersedes the reset block on retry', () => {
// 1. Initial stream fails mid-stream and is marked reset
const resetLogs = [
{
type: 'llm_stream',
timestamp: '2026-09-03T10:00:01.000Z',
data: {
execution_id: 'exec-failed-1',
stream_type: 'text',
text: 'Here is the aborted plan...',
isCompleted: false,
isReset: true,
resetMessage: 'A request error occurred during output generation, typically caused by lower API priority. Retrying automatically...'
}
}
];
const initialBlocks = consolidateLogsToBlocks(resetLogs);
expect(initialBlocks.length).toBe(1);
expect(initialBlocks[0].data.isReset).toBeTrue();
expect(initialBlocks[0].data.resetMessage).toBe('A request error occurred during output generation, typically caused by lower API priority. Retrying automatically...');
// getSortedStepEvents retains the reset state and message
const resetEvents = getSortedStepEvents(initialBlocks[0].data);
expect(resetEvents.length).toBe(1);
expect(resetEvents[0].type).toBe('text');
expect(resetEvents[0].data.isReset).toBeTrue();
// 2. Retry stream arrives without step_id (e.g. pre-planning retry)
const retryLogs = [
...resetLogs,
{
type: 'llm_stream',
timestamp: '2026-09-03T10:00:03.000Z',
data: {
execution_id: 'exec-retry-2',
stream_type: 'text',
text: 'Here is the fresh completed plan.',
isCompleted: true
}
}
];
const updatedBlocks = consolidateLogsToBlocks(retryLogs);
// Should supersede the reset block rather than creating an orphaned duplicate card
expect(updatedBlocks.length).toBe(1);
expect(updatedBlocks[0].data.execution_id).toBe('exec-retry-2');
expect(updatedBlocks[0].data.isReset).toBeFalse();
expect(updatedBlocks[0].data.operator_raw_thinking).toBe('Here is the fresh completed plan.');
expect(updatedBlocks[0].data.isCompleted).toBeTrue();
});
});
@@ -14,9 +14,9 @@
* limitations under the License.
*/
import { StepBlock, PhaseBlock, StepEvent, StreamSegment } from '../core/models/stream.model';
import { isAndroidAction, isReportStatusAction, getReportStatusExplanation, getReportStatusValue } from './action-formatter.util';
import { getUniqueGenericTools } from './tool-formatter.util';
import { StepBlock, PhaseBlock, StepEvent, StreamSegment, StreamResetNotice, DEFAULT_STREAM_RESET_MESSAGE } from '../core/models/stream.model';
import { isAndroidAction, isReportStatusAction, getReportStatusExplanation, getReportStatusValue, getActionObject } from './action-formatter.util';
import { getUniqueGenericTools, isInternalPlumbingTool } from './tool-formatter.util';
/**
* Helper to safely extract milliseconds timestamp
@@ -193,6 +193,8 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
const streamType = log.data?.stream_type || 'text';
const text = log.data?.text || '';
const isCompleted = log.data?.isCompleted ?? false;
const isReset = log.data?.isReset ?? false;
const resetMessage = log.data?.resetMessage;
// The Checker's own reasoning streams under its agent trace: route it to
// the attempt block, never to the Operator step that is current.
@@ -215,7 +217,9 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
stream_type: streamType === 'thinking' ? 'thinking' : 'text',
text,
timestamp: segmentIndex > -1 ? segments[segmentIndex].timestamp : log.timestamp,
isCompleted
isCompleted,
isReset,
resetMessage
};
if (segmentIndex > -1) segments[segmentIndex] = segment; else segments.push(segment);
checkerData.stream_segments = segments;
@@ -248,10 +252,42 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
if (existingIndex === -1 && execId) {
existingIndex = blocks.findIndex(b => b.data?.execution_id === execId || b.id === `stream-${execId}`);
}
// If no stepId and not matched by execId, check if there is an unattached stream block that was reset.
// The retry stream supersedes that reset block rather than spawning an orphaned duplicate card.
if (existingIndex === -1 && !stepId) {
existingIndex = blocks.findIndex(b => b.type === 'llm_stream' && b.data?.isReset);
}
if (existingIndex > -1) {
const existing = blocks[existingIndex];
const updatedData = { ...existing.data };
const existingResets: StreamResetNotice[] =
Array.isArray(updatedData.stream_resets) ? [...updatedData.stream_resets] : [];
if (isReset) {
updatedData.isReset = true;
updatedData.resetMessage = resetMessage;
const msg = resetMessage || DEFAULT_STREAM_RESET_MESSAGE;
const resetId = execId || `reset-${log.timestamp || updatedData.step_id || 'stream'}`;
const existingResetIdx = existingResets.findIndex(r => r.id === resetId);
if (existingResetIdx > -1) {
existingResets[existingResetIdx] = { id: resetId, message: msg, isWaiting: true, streamType };
} else {
existingResets.push({ id: resetId, message: msg, isWaiting: true, streamType });
}
updatedData.stream_resets = existingResets;
} else if (execId && updatedData.execution_id !== execId) {
// New execution replacing a reset stream: mark prior resets as finished waiting
updatedData.stream_resets = existingResets.map(r => ({ ...r, isWaiting: false }));
updatedData.isReset = false;
updatedData.resetMessage = undefined;
updatedData.execution_id = execId;
} else if (!isReset && text && text.length > 0 && existingResets.some(r => r.isWaiting)) {
updatedData.stream_resets = existingResets.map(r => ({ ...r, isWaiting: false }));
updatedData.isReset = false;
updatedData.resetMessage = undefined;
}
if (streamType === 'thinking') {
updatedData.operator_native_thinking = text;
updatedData.operator_native_thinking_timestamp =
@@ -268,6 +304,9 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
updatedData.execution_id = execId;
}
updatedData.isCompleted = isCompleted;
if (isCompleted && updatedData.stream_resets) {
updatedData.stream_resets = updatedData.stream_resets.map((r: any) => ({ ...r, isWaiting: false }));
}
blocks[existingIndex] = {
...existing,
@@ -279,6 +318,14 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
execution_id: execId,
step_id: stepId,
isCompleted: isCompleted,
isReset: isReset,
resetMessage: resetMessage,
stream_resets: isReset ? [{
id: execId || `reset-${log.timestamp || stepId || 'stream'}`,
message: resetMessage || DEFAULT_STREAM_RESET_MESSAGE,
isWaiting: true,
streamType
}] : [],
generic_tools: []
};
if (streamType === 'thinking') {
@@ -338,6 +385,9 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
generic_tools: mergedTools,
isCompleted: true
};
if (mergedData.stream_resets) {
mergedData.stream_resets = mergedData.stream_resets.map((r: any) => ({ ...r, isWaiting: false }));
}
blocks[existingIndex] = {
...existingBlock,
@@ -667,6 +717,9 @@ export function getSortedStepEvents(
): StepEvent[] {
if (!stepData || typeof stepData !== 'object') return [];
const toolsLen = stepData.generic_tools ? stepData.generic_tools.length : 0;
const toolsSignature = stepData.generic_tools && Array.isArray(stepData.generic_tools)
? stepData.generic_tools.map((t: any) => `${t?.trace_id || t?.name || ''}:${t?.status || ''}`).join(',')
: '';
const actionTs = stepData.action_taken ? (stepData.action_taken.timestamp || stepData.action_taken.start_time || stepData.action_taken.created_at) : null;
const nativeText = typeof stepData.operator_native_thinking === 'string'
? stepData.operator_native_thinking.trim()
@@ -679,11 +732,15 @@ export function getSortedStepEvents(
: null;
const signature = [
toolsLen,
toolsSignature,
actionTs ?? '',
stepData.operator_native_thinking_timestamp ?? '',
stepData.operator_raw_thinking_timestamp ?? '',
nativeText.length,
rawText.length,
stepData.isReset ? '1' : '0',
stepData.resetMessage ?? '',
stepData.stream_resets ? stepData.stream_resets.map((r: any) => `${r.id}:${r.isWaiting ? 1 : 0}`).join(',') : '',
segments ? segments.map((s) => `${s.execution_id}:${s.stream_type}:${s.text.length}`).join(',') : ''
].join('|');
@@ -704,38 +761,58 @@ export function getSortedStepEvents(
// Multi-turn agent (Checker): one event per streamed turn, each at the
// time its first chunk arrived, so turns sort between the tool calls.
segments.forEach((segment) => {
if (!segment.text || !segment.text.trim()) return;
if ((!segment.text || !segment.text.trim()) && !segment.isReset) return;
events.push({
type: segment.stream_type === 'thinking' ? 'thinking' : 'text',
data: { text: segment.text, execution_id: segment.execution_id, segment: true },
data: {
text: segment.text,
execution_id: segment.execution_id,
segment: true,
isReset: segment.isReset,
resetMessage: segment.resetMessage
},
timestamp: getItemTimestamp(segment.timestamp, fallbackTime),
sequence: sequence++
});
});
} else {
if (nativeText) {
const hasThinkingResets = Array.isArray(stepData.stream_resets) && stepData.stream_resets.some((r: any) => r.streamType === 'thinking');
if (nativeText || hasThinkingResets) {
events.push({
type: 'thinking',
data: { text: stepData.operator_native_thinking },
data: { text: stepData.operator_native_thinking || '' },
timestamp: getItemTimestamp(stepData.operator_native_thinking_timestamp, fallbackTime),
sequence: sequence++
});
}
if (rawText) {
if (rawText || (stepData.isReset && stepData.resetMessage) || (Array.isArray(stepData.stream_resets) && stepData.stream_resets.length > 0)) {
events.push({
type: 'text',
data: { text: stepData.operator_raw_thinking },
data: {
text: stepData.operator_raw_thinking || '',
isReset: stepData.isReset,
resetMessage: stepData.resetMessage
},
timestamp: getItemTimestamp(stepData.operator_raw_thinking_timestamp, fallbackTime),
sequence: sequence++
});
}
}
// Add action_taken if present and valid.
if (stepData.action_taken && (isAndroidAction(stepData.action_taken) || isReportStatusAction(stepData.action_taken))) {
// 2. Single Source of Truth & Fallback Resolution:
const uniqueTools = getUniqueGenericTools(stepData.generic_tools);
const hasActionInTools = uniqueTools.some((tool: any) =>
tool && (tool.type === 'action' || isAndroidAction(tool) || isReportStatusAction(tool))
);
// Fallback: only add action_taken if generic_tools did NOT already contain an action trace.
// Pushed before uniqueTools so that when timestamps are missing/equal, the primary action
// preserves deterministic sequence ordering ahead of generic fallback tools.
if (!hasActionInTools && stepData.action_taken && (isAndroidAction(stepData.action_taken) || isReportStatusAction(stepData.action_taken))) {
const actObj = getActionObject(stepData.action_taken);
const actTime = getItemTimestamp(
stepData.action_taken.timestamp ?? stepData.action_taken.start_time ?? stepData.action_taken.created_at,
actObj?.timestamp ?? actObj?.start_time ?? actObj?.created_at,
fallbackTime
);
events.push({
@@ -746,15 +823,19 @@ export function getSortedStepEvents(
});
}
// Add generic tools if present.
const uniqueTools = getUniqueGenericTools(stepData.generic_tools);
// Add all real tool calls and actions from the event stream.
uniqueTools.forEach((tool: any) => {
if (isInternalPlumbingTool(tool) && !isReportStatusAction(tool)) {
return;
}
const isAction = tool.type === 'action' || isAndroidAction(tool) || isReportStatusAction(tool);
const toolTime = getItemTimestamp(
tool.timestamp ?? tool.start_time ?? tool.created_at,
fallbackTime
);
events.push({
type: 'tool',
type: isAction ? 'action' : 'tool',
data: tool,
timestamp: toolTime,
sequence: sequence++
@@ -1,7 +1,8 @@
import {
cleanErrorMessage,
getUniqueGenericTools,
getVideoAnalysisView
getVideoAnalysisView,
shouldShowTool
} from './tool-formatter.util';
describe('cleanErrorMessage', () => {
@@ -164,3 +165,38 @@ describe('video analysis timeline formatting', () => {
]);
});
});
describe('shouldShowTool (Option A Single Source of Truth)', () => {
it('keeps ADB commands visible even when stepData.action_taken is present', () => {
const adbTool = {
name: 'run_adb_command',
type: 'tool',
payload: { CommandLine: 'dumpsys telephony.registry' }
};
const stepData = {
action_taken: [{ action: 'wait_for_delay', time_in_ms: 1000 }]
};
expect(shouldShowTool(adbTool, stepData)).toBeTrue();
});
it('keeps explorer calls visible when stepData.action_taken is present', () => {
const explorerTool = {
name: 'ask_explorer',
type: 'tool'
};
const stepData = {
action_taken: [{ action: 'wait_for_delay', time_in_ms: 1000 }]
};
expect(shouldShowTool(explorerTool, stepData)).toBeTrue();
});
it('filters out internal plumbing tools', () => {
const plumbingTool = {
name: 'safety_net_pixel_validation',
type: 'tool'
};
expect(shouldShowTool(plumbingTool)).toBeFalse();
});
});
@@ -15,7 +15,6 @@
*/
import { ActionParam } from '../core/models/stream.model';
import { isAndroidAction, isReportStatusAction } from './action-formatter.util';
import { extractNumbersFromCoordinateValue, isPureDirectionString, parseSequenceCoordinates } from './image-overlay.util';
// Tool objects are replaced (not mutated) when a trace is updated, so a
@@ -276,24 +275,12 @@ export function isInternalPlumbingTool(tool: any): boolean {
/**
* Determines whether a tool invocation should be displayed in the timeline.
*/
export function shouldShowTool(tool: any, stepData?: any): boolean {
export function shouldShowTool(tool: any, _stepData?: any): boolean {
if (!tool || !tool.name) return false;
if (tool.type === 'llm_call') return false;
if (tool.type === 'agent') return false;
if (isInternalPlumbingTool(tool)) return false;
// If this tool is an operator/validator atomic action that duplicates stepData.action_taken, skip it
if (stepData && stepData.action_taken && isAndroidAction(stepData.action_taken) && isDeviceActionTool(tool)) {
const agentName = (tool.agent_name || tool.agent || '').toLowerCase();
const isSelfHealingOrSubAgent = agentName.includes('failure') ||
agentName.includes('analyzer') ||
agentName.includes('diagnos') ||
(agentName && agentName !== 'operator' && agentName !== 'flashrunner' && agentName !== 'validator');
if (!isSelfHealingOrSubAgent) {
return false;
}
}
return true;
}