feat: handle mid-stream LLM resets and deduplicate UI action stream events
- Backend (LLM Gateway): - Record structured 'llm_stream_reset' event with discard action, failure reason, and retry messaging on mid-stream generation errors. - Add unit test verifying stream reset event payload on mid-stream errors. - Frontend (Showcase UI): - Handle 'llm_stream_reset' events in agent service and stream aggregator, discarding interrupted chunks while showing retry indicators. - Establish a single source of truth for tool and action events in the timeline to eliminate duplicated action rows while retaining ADB and explorer tools. - Add @angular/cdk dependency and update agent stream component styling and layout. - Add unit tests for stream aggregation and tool display filtering.
This commit is contained in:
1 parent
d7c64f571d
commit
d23a4f6472
15 files changed
+1862
-1285
No files matched your search
@@ -18,13 +18,66 @@ import { ActionParam, StepReplayFrame } from '../core/models/stream.model';
|
||||
import { extractNumbersFromCoordinateValue, isPureDirectionString, parseSequenceCoordinates } from './image-overlay.util';
|
||||
import { cleanErrorMessage } from './tool-formatter.util';
|
||||
|
||||
/**
|
||||
* Safely parse JSON string if it looks like an object/array, otherwise return original value
|
||||
*/
|
||||
export function safeParseJson(val: any): any {
|
||||
if (typeof val === 'string' && (val.trim().startsWith('{') || val.trim().startsWith('['))) {
|
||||
try {
|
||||
return JSON.parse(val);
|
||||
} catch {
|
||||
return val;
|
||||
}
|
||||
}
|
||||
return val;
|
||||
}
|
||||
|
||||
// WeakMap memo for parsed action objects to avoid repeated JSON.parse and object allocation across change detection cycles
|
||||
const actionObjectCache = new WeakMap<object, any>();
|
||||
|
||||
function computeActionObject(action: any): any {
|
||||
if (action && typeof action === 'object' && action.payload) {
|
||||
const payload = typeof action.payload === 'string' ? safeParseJson(action.payload) : action.payload;
|
||||
const innerAction = payload?.args?.action;
|
||||
if (innerAction && typeof innerAction === 'object') {
|
||||
return {
|
||||
...action,
|
||||
...innerAction,
|
||||
name: action.name || innerAction.action || innerAction.name,
|
||||
action: innerAction.action || action.name,
|
||||
trace_id: action.trace_id,
|
||||
timestamp: action.timestamp ?? innerAction.timestamp
|
||||
};
|
||||
}
|
||||
const innerArgs = payload?.args;
|
||||
if (innerArgs && typeof innerArgs === 'object') {
|
||||
return {
|
||||
...action,
|
||||
...innerArgs,
|
||||
name: action.name || innerArgs.action || innerArgs.name,
|
||||
action: innerArgs.action || action.name,
|
||||
trace_id: action.trace_id,
|
||||
timestamp: action.timestamp ?? innerArgs.timestamp
|
||||
};
|
||||
}
|
||||
}
|
||||
return action;
|
||||
}
|
||||
|
||||
/**
|
||||
* Safely extract action object from possible array or nested structure
|
||||
*/
|
||||
export function getActionObject(action: any): any {
|
||||
if (!action) return null;
|
||||
if (Array.isArray(action)) {
|
||||
return action.length > 0 ? action[0] : null;
|
||||
return action.length > 0 ? getActionObject(action[0]) : null;
|
||||
}
|
||||
if (typeof action === 'object') {
|
||||
const cached = actionObjectCache.get(action);
|
||||
if (cached !== undefined) return cached;
|
||||
const res = computeActionObject(action);
|
||||
actionObjectCache.set(action, res);
|
||||
return res;
|
||||
}
|
||||
return action;
|
||||
}
|
||||
@@ -46,7 +99,7 @@ export function isAndroidAction(action: any): boolean {
|
||||
'tap', 'click', 'click_sequence', 'input', 'input_text', 'swipe', 'scroll', 'press_key',
|
||||
'press_home', 'press_back', 'launch_app', 'manage_app', 'open_app',
|
||||
'focus_and_input_text', 'focus_and_clear_text', 'long_press', 'long_press_on',
|
||||
'wait_for_delay', 'wait'
|
||||
'wait_for_delay', 'wait', 'drag', 'drag_and_drop'
|
||||
].includes(name);
|
||||
}
|
||||
|
||||
@@ -543,7 +596,8 @@ export function isGenericToolTrace(actionData: any, stepData?: any): boolean {
|
||||
}
|
||||
}
|
||||
if (actionData.is_primary_step_action) return false;
|
||||
return Boolean(actionData.trace_id || actionData.payload || actionData.type === 'tool' || (actionData.type === 'action' && actionData.trace_id));
|
||||
if (actionData.type === 'action' || isAndroidAction(actionData)) return false;
|
||||
return Boolean(actionData.trace_id || actionData.payload || actionData.type === 'tool');
|
||||
}
|
||||
|
||||
interface ResolvedImageEntry {
|
||||
@@ -554,16 +608,6 @@ interface ResolvedImageEntry {
|
||||
|
||||
const stepImageMapCache = new WeakMap<any, { key: string; map: Map<any, ResolvedImageEntry> }>();
|
||||
|
||||
function safeParseJson(val: any): any {
|
||||
if (typeof val === 'string' && (val.trim().startsWith('{') || val.trim().startsWith('['))) {
|
||||
try {
|
||||
return JSON.parse(val);
|
||||
} catch {
|
||||
return val;
|
||||
}
|
||||
}
|
||||
return val;
|
||||
}
|
||||
|
||||
export function extractItemPreImage(item: any): string | null {
|
||||
const act = getActionObject(item);
|
||||
@@ -639,7 +683,10 @@ export function resolveStepImageMap(stepData: any): Map<any, ResolvedImageEntry>
|
||||
if (!stepData || typeof stepData !== 'object') return emptyMap;
|
||||
|
||||
const toolsLen = Array.isArray(stepData.generic_tools) ? stepData.generic_tools.length : 0;
|
||||
const cacheKey = `${stepData.step_id || ''}_${toolsLen}_${stepData.post_image_name || ''}_${stepData.pre_image_name || ''}`;
|
||||
const toolsImageSig = Array.isArray(stepData.generic_tools)
|
||||
? stepData.generic_tools.map((t: any) => `${t?.trace_id || ''}:${t?.post_image_name || t?.post_screenshot || ''}`).join(',')
|
||||
: '';
|
||||
const cacheKey = `${stepData.step_id || ''}_${toolsLen}_${toolsImageSig}_${stepData.post_image_name || ''}_${stepData.pre_image_name || ''}`;
|
||||
|
||||
const cached = stepImageMapCache.get(stepData);
|
||||
if (cached && cached.key === cacheKey) {
|
||||
@@ -649,7 +696,11 @@ export function resolveStepImageMap(stepData: any): Map<any, ResolvedImageEntry>
|
||||
const map = new Map<any, ResolvedImageEntry>();
|
||||
const events: Array<{ item: any; act: any; timestamp: number; isPrimary: boolean }> = [];
|
||||
|
||||
if (stepData.action_taken) {
|
||||
const hasActionInTools = Array.isArray(stepData.generic_tools) && stepData.generic_tools.some(
|
||||
(t: any) => t && (t.type === 'action' || isAndroidAction(t) || isReportStatusAction(t))
|
||||
);
|
||||
|
||||
if (!hasActionInTools && stepData.action_taken) {
|
||||
events.push({
|
||||
item: stepData.action_taken,
|
||||
act: getActionObject(stepData.action_taken),
|
||||
@@ -661,11 +712,12 @@ export function resolveStepImageMap(stepData: any): Map<any, ResolvedImageEntry>
|
||||
if (Array.isArray(stepData.generic_tools)) {
|
||||
stepData.generic_tools.forEach((t: any) => {
|
||||
if (t) {
|
||||
const isAction = t.type === 'action' || isAndroidAction(t) || isReportStatusAction(t);
|
||||
events.push({
|
||||
item: t,
|
||||
act: getActionObject(t),
|
||||
timestamp: getEventTimestamp(t),
|
||||
isPrimary: false
|
||||
isPrimary: isAction
|
||||
});
|
||||
}
|
||||
});
|
||||
@@ -728,6 +780,19 @@ export function resolveStepImageMap(stepData: any): Map<any, ResolvedImageEntry>
|
||||
map.set(e.item, entry);
|
||||
if (act && act !== e.item) map.set(act, entry);
|
||||
if (act?.trace_id) map.set(act.trace_id, entry);
|
||||
if (e.item?.trace_id) map.set(e.item.trace_id, entry);
|
||||
}
|
||||
|
||||
// Ensure stepData.action_taken cross-resolves to the action images
|
||||
if (stepData.action_taken && lastActionItem) {
|
||||
const lastEntry = map.get(lastActionItem);
|
||||
if (lastEntry) {
|
||||
map.set(stepData.action_taken, lastEntry);
|
||||
const actObj = getActionObject(stepData.action_taken);
|
||||
if (actObj && actObj !== stepData.action_taken) {
|
||||
map.set(actObj, lastEntry);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stepImageMapCache.set(stepData, { key: cacheKey, map });
|
||||
@@ -885,7 +950,9 @@ export function extractStepReplayFrames(logsOrSteps: any[]): StepReplayFrame[] {
|
||||
const visualCandidates: VisualCandidate[] = [];
|
||||
|
||||
for (const stepData of stepsList) {
|
||||
const act = stepData.action_taken;
|
||||
const act = stepData.action_taken || (Array.isArray(stepData.generic_tools)
|
||||
? stepData.generic_tools.find((t: any) => t && (t.type === 'action' || isAndroidAction(t)))
|
||||
: null);
|
||||
const preUrl = getStepPreImageUrl(stepData, act);
|
||||
const postUrl = getStepPostImageUrl(stepData, act);
|
||||
const primaryImg = preUrl || postUrl;
|
||||
|
||||
@@ -349,3 +349,139 @@ describe('stream aggregator checker history relocation', () => {
|
||||
expect(checker.data.generic_tools.map((t: any) => t.trace_id)).toEqual(['tool-probe']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('stream aggregator single source of truth (Option A)', () => {
|
||||
it('does not duplicate action_taken when generic_tools already contains the action trace', () => {
|
||||
const events = getSortedStepEvents({
|
||||
timestamp: 100,
|
||||
action_taken: [{ action: 'launch_app', app_name: 'Maps', timestamp: 102 }],
|
||||
generic_tools: [
|
||||
{ trace_id: 'tool-note', type: 'tool', name: 'update_note', timestamp: 101 },
|
||||
{ trace_id: 'act-launch', type: 'action', name: 'launch_app', timestamp: 102, payload: { args: { action: { action: 'launch_app', app_name: 'Maps' } } } }
|
||||
]
|
||||
});
|
||||
|
||||
expect(events.length).toBe(2);
|
||||
expect(events.map(e => e.type)).toEqual(['tool', 'action']);
|
||||
expect(events.map(e => e.data.name)).toEqual(['update_note', 'launch_app']);
|
||||
});
|
||||
|
||||
it('retains multiple adb commands and tools alongside physical actions without dropping any', () => {
|
||||
const events = getSortedStepEvents({
|
||||
timestamp: 100,
|
||||
action_taken: [{ action: 'wait_for_delay', time_in_ms: 1000, timestamp: 105 }],
|
||||
generic_tools: [
|
||||
{ trace_id: 'adb-1', type: 'tool', name: 'run_adb_command', timestamp: 101, payload: { CommandLine: 'dumpsys telephony.registry' } },
|
||||
{ trace_id: 'adb-2', type: 'tool', name: 'run_adb_command', timestamp: 102, payload: { CommandLine: 'am start -a ...' } },
|
||||
{ trace_id: 'exp-1', type: 'tool', name: 'ask_explorer', timestamp: 103 },
|
||||
{ trace_id: 'act-wait', type: 'action', name: 'wait_for_delay', timestamp: 104, payload: { args: { time_in_ms: 1000 } } }
|
||||
]
|
||||
});
|
||||
|
||||
expect(events.length).toBe(4);
|
||||
expect(events.map(e => e.type)).toEqual(['tool', 'tool', 'tool', 'action']);
|
||||
expect(events.map(e => e.data.name)).toEqual(['run_adb_command', 'run_adb_command', 'ask_explorer', 'wait_for_delay']);
|
||||
});
|
||||
|
||||
it('supports multi-action bursts sequentially from generic_tools', () => {
|
||||
const events = getSortedStepEvents({
|
||||
timestamp: 100,
|
||||
action_taken: [{ action: 'tap', coordinates: [200, 300] }],
|
||||
generic_tools: [
|
||||
{ trace_id: 'tap-1', type: 'action', name: 'tap', timestamp: 101, payload: { args: { action: { action: 'tap', coordinates: [100, 200] } } } },
|
||||
{ trace_id: 'tap-2', type: 'action', name: 'tap', timestamp: 102, payload: { args: { action: { action: 'tap', coordinates: [200, 300] } } } }
|
||||
]
|
||||
});
|
||||
|
||||
expect(events.length).toBe(2);
|
||||
expect(events.every(e => e.type === 'action')).toBeTrue();
|
||||
expect(events[0].data.trace_id).toBe('tap-1');
|
||||
expect(events[1].data.trace_id).toBe('tap-2');
|
||||
});
|
||||
|
||||
it('correctly processes real trace 38fd934e Step 16 keeping ADB commands and wait_for_delay visible without duplicates', () => {
|
||||
const step16 = {
|
||||
step_id: '31e8c7ec-6b67-485e-94f4-355a853ccf8a',
|
||||
step_number: 16,
|
||||
action_taken: [{ action: 'wait_for_delay', time_in_ms: 1000 }],
|
||||
operator_native_thinking: 'Let me check telephony and send SMS...',
|
||||
operator_native_thinking_timestamp: 1788457630,
|
||||
timestamp: 1788457630,
|
||||
generic_tools: [
|
||||
{ trace_id: 't-adb-1', parent_trace_id: 't-op', name: 'run_adb_command', type: 'tool', timestamp: 1788457631, status: 'success', payload: { args: { CommandLine: 'dumpsys telephony.registry' } } },
|
||||
{ trace_id: 't-adb-1-sub', parent_trace_id: 't-adb-1', name: 'run_adb_command', type: 'tool', timestamp: 1788457631.5, status: 'success' },
|
||||
{ trace_id: 't-adb-2', parent_trace_id: 't-op', name: 'run_adb_command', type: 'tool', timestamp: 1788457632, status: 'success', payload: { args: { CommandLine: 'am start -a android.intent.action.SENDTO ...' } } },
|
||||
{ trace_id: 't-adb-2-sub', parent_trace_id: 't-adb-2', name: 'run_adb_command', type: 'tool', timestamp: 1788457632.5, status: 'success' },
|
||||
{ trace_id: 't-exp', parent_trace_id: 't-op', name: 'ask_explorer', type: 'tool', timestamp: 1788457633, status: 'success', payload: { args: { query: 'any text on screen' } } },
|
||||
{ trace_id: 't-exp-sub', parent_trace_id: 't-exp', name: 'ask_explorer', type: 'tool', timestamp: 1788457633.5, status: 'success' },
|
||||
{ trace_id: 't-wait', parent_trace_id: 't-val', name: 'wait_for_delay', type: 'action', timestamp: 1788457635, status: 'success', payload: { args: { action: { action: 'wait_for_delay', time_in_ms: 1000 } } } },
|
||||
{ trace_id: 't-safety', parent_trace_id: 't-wait', name: 'safety_net_pixel_validation', type: 'tool', timestamp: 1788457636, status: 'success' }
|
||||
]
|
||||
};
|
||||
|
||||
const events = getSortedStepEvents(step16);
|
||||
// 1 native thinking + 2 adb tools + 1 explorer tool + 1 wait_for_delay action
|
||||
expect(events.length).toBe(5);
|
||||
expect(events.map(e => e.type)).toEqual(['thinking', 'tool', 'tool', 'tool', 'action']);
|
||||
expect(events.map(e => e.data.name || e.data.action || 'thinking')).toEqual([
|
||||
'thinking',
|
||||
'run_adb_command',
|
||||
'run_adb_command',
|
||||
'ask_explorer',
|
||||
'wait_for_delay'
|
||||
]);
|
||||
});
|
||||
|
||||
it('correctly handles mid-stream reset and supersedes the reset block on retry', () => {
|
||||
// 1. Initial stream fails mid-stream and is marked reset
|
||||
const resetLogs = [
|
||||
{
|
||||
type: 'llm_stream',
|
||||
timestamp: '2026-09-03T10:00:01.000Z',
|
||||
data: {
|
||||
execution_id: 'exec-failed-1',
|
||||
stream_type: 'text',
|
||||
text: 'Here is the aborted plan...',
|
||||
isCompleted: false,
|
||||
isReset: true,
|
||||
resetMessage: 'A request error occurred during output generation, typically caused by lower API priority. Retrying automatically...'
|
||||
}
|
||||
}
|
||||
];
|
||||
|
||||
const initialBlocks = consolidateLogsToBlocks(resetLogs);
|
||||
expect(initialBlocks.length).toBe(1);
|
||||
expect(initialBlocks[0].data.isReset).toBeTrue();
|
||||
expect(initialBlocks[0].data.resetMessage).toBe('A request error occurred during output generation, typically caused by lower API priority. Retrying automatically...');
|
||||
|
||||
// getSortedStepEvents retains the reset state and message
|
||||
const resetEvents = getSortedStepEvents(initialBlocks[0].data);
|
||||
expect(resetEvents.length).toBe(1);
|
||||
expect(resetEvents[0].type).toBe('text');
|
||||
expect(resetEvents[0].data.isReset).toBeTrue();
|
||||
|
||||
// 2. Retry stream arrives without step_id (e.g. pre-planning retry)
|
||||
const retryLogs = [
|
||||
...resetLogs,
|
||||
{
|
||||
type: 'llm_stream',
|
||||
timestamp: '2026-09-03T10:00:03.000Z',
|
||||
data: {
|
||||
execution_id: 'exec-retry-2',
|
||||
stream_type: 'text',
|
||||
text: 'Here is the fresh completed plan.',
|
||||
isCompleted: true
|
||||
}
|
||||
}
|
||||
];
|
||||
|
||||
const updatedBlocks = consolidateLogsToBlocks(retryLogs);
|
||||
// Should supersede the reset block rather than creating an orphaned duplicate card
|
||||
expect(updatedBlocks.length).toBe(1);
|
||||
expect(updatedBlocks[0].data.execution_id).toBe('exec-retry-2');
|
||||
expect(updatedBlocks[0].data.isReset).toBeFalse();
|
||||
expect(updatedBlocks[0].data.operator_raw_thinking).toBe('Here is the fresh completed plan.');
|
||||
expect(updatedBlocks[0].data.isCompleted).toBeTrue();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -14,9 +14,9 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { StepBlock, PhaseBlock, StepEvent, StreamSegment } from '../core/models/stream.model';
|
||||
import { isAndroidAction, isReportStatusAction, getReportStatusExplanation, getReportStatusValue } from './action-formatter.util';
|
||||
import { getUniqueGenericTools } from './tool-formatter.util';
|
||||
import { StepBlock, PhaseBlock, StepEvent, StreamSegment, StreamResetNotice, DEFAULT_STREAM_RESET_MESSAGE } from '../core/models/stream.model';
|
||||
import { isAndroidAction, isReportStatusAction, getReportStatusExplanation, getReportStatusValue, getActionObject } from './action-formatter.util';
|
||||
import { getUniqueGenericTools, isInternalPlumbingTool } from './tool-formatter.util';
|
||||
|
||||
/**
|
||||
* Helper to safely extract milliseconds timestamp
|
||||
@@ -193,6 +193,8 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
|
||||
const streamType = log.data?.stream_type || 'text';
|
||||
const text = log.data?.text || '';
|
||||
const isCompleted = log.data?.isCompleted ?? false;
|
||||
const isReset = log.data?.isReset ?? false;
|
||||
const resetMessage = log.data?.resetMessage;
|
||||
|
||||
// The Checker's own reasoning streams under its agent trace: route it to
|
||||
// the attempt block, never to the Operator step that is current.
|
||||
@@ -215,7 +217,9 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
|
||||
stream_type: streamType === 'thinking' ? 'thinking' : 'text',
|
||||
text,
|
||||
timestamp: segmentIndex > -1 ? segments[segmentIndex].timestamp : log.timestamp,
|
||||
isCompleted
|
||||
isCompleted,
|
||||
isReset,
|
||||
resetMessage
|
||||
};
|
||||
if (segmentIndex > -1) segments[segmentIndex] = segment; else segments.push(segment);
|
||||
checkerData.stream_segments = segments;
|
||||
@@ -248,10 +252,42 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
|
||||
if (existingIndex === -1 && execId) {
|
||||
existingIndex = blocks.findIndex(b => b.data?.execution_id === execId || b.id === `stream-${execId}`);
|
||||
}
|
||||
// If no stepId and not matched by execId, check if there is an unattached stream block that was reset.
|
||||
// The retry stream supersedes that reset block rather than spawning an orphaned duplicate card.
|
||||
if (existingIndex === -1 && !stepId) {
|
||||
existingIndex = blocks.findIndex(b => b.type === 'llm_stream' && b.data?.isReset);
|
||||
}
|
||||
|
||||
if (existingIndex > -1) {
|
||||
const existing = blocks[existingIndex];
|
||||
const updatedData = { ...existing.data };
|
||||
const existingResets: StreamResetNotice[] =
|
||||
Array.isArray(updatedData.stream_resets) ? [...updatedData.stream_resets] : [];
|
||||
|
||||
if (isReset) {
|
||||
updatedData.isReset = true;
|
||||
updatedData.resetMessage = resetMessage;
|
||||
const msg = resetMessage || DEFAULT_STREAM_RESET_MESSAGE;
|
||||
const resetId = execId || `reset-${log.timestamp || updatedData.step_id || 'stream'}`;
|
||||
const existingResetIdx = existingResets.findIndex(r => r.id === resetId);
|
||||
if (existingResetIdx > -1) {
|
||||
existingResets[existingResetIdx] = { id: resetId, message: msg, isWaiting: true, streamType };
|
||||
} else {
|
||||
existingResets.push({ id: resetId, message: msg, isWaiting: true, streamType });
|
||||
}
|
||||
updatedData.stream_resets = existingResets;
|
||||
} else if (execId && updatedData.execution_id !== execId) {
|
||||
// New execution replacing a reset stream: mark prior resets as finished waiting
|
||||
updatedData.stream_resets = existingResets.map(r => ({ ...r, isWaiting: false }));
|
||||
updatedData.isReset = false;
|
||||
updatedData.resetMessage = undefined;
|
||||
updatedData.execution_id = execId;
|
||||
} else if (!isReset && text && text.length > 0 && existingResets.some(r => r.isWaiting)) {
|
||||
updatedData.stream_resets = existingResets.map(r => ({ ...r, isWaiting: false }));
|
||||
updatedData.isReset = false;
|
||||
updatedData.resetMessage = undefined;
|
||||
}
|
||||
|
||||
if (streamType === 'thinking') {
|
||||
updatedData.operator_native_thinking = text;
|
||||
updatedData.operator_native_thinking_timestamp =
|
||||
@@ -268,6 +304,9 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
|
||||
updatedData.execution_id = execId;
|
||||
}
|
||||
updatedData.isCompleted = isCompleted;
|
||||
if (isCompleted && updatedData.stream_resets) {
|
||||
updatedData.stream_resets = updatedData.stream_resets.map((r: any) => ({ ...r, isWaiting: false }));
|
||||
}
|
||||
|
||||
blocks[existingIndex] = {
|
||||
...existing,
|
||||
@@ -279,6 +318,14 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
|
||||
execution_id: execId,
|
||||
step_id: stepId,
|
||||
isCompleted: isCompleted,
|
||||
isReset: isReset,
|
||||
resetMessage: resetMessage,
|
||||
stream_resets: isReset ? [{
|
||||
id: execId || `reset-${log.timestamp || stepId || 'stream'}`,
|
||||
message: resetMessage || DEFAULT_STREAM_RESET_MESSAGE,
|
||||
isWaiting: true,
|
||||
streamType
|
||||
}] : [],
|
||||
generic_tools: []
|
||||
};
|
||||
if (streamType === 'thinking') {
|
||||
@@ -338,6 +385,9 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
|
||||
generic_tools: mergedTools,
|
||||
isCompleted: true
|
||||
};
|
||||
if (mergedData.stream_resets) {
|
||||
mergedData.stream_resets = mergedData.stream_resets.map((r: any) => ({ ...r, isWaiting: false }));
|
||||
}
|
||||
|
||||
blocks[existingIndex] = {
|
||||
...existingBlock,
|
||||
@@ -667,6 +717,9 @@ export function getSortedStepEvents(
|
||||
): StepEvent[] {
|
||||
if (!stepData || typeof stepData !== 'object') return [];
|
||||
const toolsLen = stepData.generic_tools ? stepData.generic_tools.length : 0;
|
||||
const toolsSignature = stepData.generic_tools && Array.isArray(stepData.generic_tools)
|
||||
? stepData.generic_tools.map((t: any) => `${t?.trace_id || t?.name || ''}:${t?.status || ''}`).join(',')
|
||||
: '';
|
||||
const actionTs = stepData.action_taken ? (stepData.action_taken.timestamp || stepData.action_taken.start_time || stepData.action_taken.created_at) : null;
|
||||
const nativeText = typeof stepData.operator_native_thinking === 'string'
|
||||
? stepData.operator_native_thinking.trim()
|
||||
@@ -679,11 +732,15 @@ export function getSortedStepEvents(
|
||||
: null;
|
||||
const signature = [
|
||||
toolsLen,
|
||||
toolsSignature,
|
||||
actionTs ?? '',
|
||||
stepData.operator_native_thinking_timestamp ?? '',
|
||||
stepData.operator_raw_thinking_timestamp ?? '',
|
||||
nativeText.length,
|
||||
rawText.length,
|
||||
stepData.isReset ? '1' : '0',
|
||||
stepData.resetMessage ?? '',
|
||||
stepData.stream_resets ? stepData.stream_resets.map((r: any) => `${r.id}:${r.isWaiting ? 1 : 0}`).join(',') : '',
|
||||
segments ? segments.map((s) => `${s.execution_id}:${s.stream_type}:${s.text.length}`).join(',') : ''
|
||||
].join('|');
|
||||
|
||||
@@ -704,38 +761,58 @@ export function getSortedStepEvents(
|
||||
// Multi-turn agent (Checker): one event per streamed turn, each at the
|
||||
// time its first chunk arrived, so turns sort between the tool calls.
|
||||
segments.forEach((segment) => {
|
||||
if (!segment.text || !segment.text.trim()) return;
|
||||
if ((!segment.text || !segment.text.trim()) && !segment.isReset) return;
|
||||
events.push({
|
||||
type: segment.stream_type === 'thinking' ? 'thinking' : 'text',
|
||||
data: { text: segment.text, execution_id: segment.execution_id, segment: true },
|
||||
data: {
|
||||
text: segment.text,
|
||||
execution_id: segment.execution_id,
|
||||
segment: true,
|
||||
isReset: segment.isReset,
|
||||
resetMessage: segment.resetMessage
|
||||
},
|
||||
timestamp: getItemTimestamp(segment.timestamp, fallbackTime),
|
||||
sequence: sequence++
|
||||
});
|
||||
});
|
||||
} else {
|
||||
if (nativeText) {
|
||||
const hasThinkingResets = Array.isArray(stepData.stream_resets) && stepData.stream_resets.some((r: any) => r.streamType === 'thinking');
|
||||
if (nativeText || hasThinkingResets) {
|
||||
events.push({
|
||||
type: 'thinking',
|
||||
data: { text: stepData.operator_native_thinking },
|
||||
data: { text: stepData.operator_native_thinking || '' },
|
||||
timestamp: getItemTimestamp(stepData.operator_native_thinking_timestamp, fallbackTime),
|
||||
sequence: sequence++
|
||||
});
|
||||
}
|
||||
|
||||
if (rawText) {
|
||||
if (rawText || (stepData.isReset && stepData.resetMessage) || (Array.isArray(stepData.stream_resets) && stepData.stream_resets.length > 0)) {
|
||||
events.push({
|
||||
type: 'text',
|
||||
data: { text: stepData.operator_raw_thinking },
|
||||
data: {
|
||||
text: stepData.operator_raw_thinking || '',
|
||||
isReset: stepData.isReset,
|
||||
resetMessage: stepData.resetMessage
|
||||
},
|
||||
timestamp: getItemTimestamp(stepData.operator_raw_thinking_timestamp, fallbackTime),
|
||||
sequence: sequence++
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Add action_taken if present and valid.
|
||||
if (stepData.action_taken && (isAndroidAction(stepData.action_taken) || isReportStatusAction(stepData.action_taken))) {
|
||||
// 2. Single Source of Truth & Fallback Resolution:
|
||||
const uniqueTools = getUniqueGenericTools(stepData.generic_tools);
|
||||
const hasActionInTools = uniqueTools.some((tool: any) =>
|
||||
tool && (tool.type === 'action' || isAndroidAction(tool) || isReportStatusAction(tool))
|
||||
);
|
||||
|
||||
// Fallback: only add action_taken if generic_tools did NOT already contain an action trace.
|
||||
// Pushed before uniqueTools so that when timestamps are missing/equal, the primary action
|
||||
// preserves deterministic sequence ordering ahead of generic fallback tools.
|
||||
if (!hasActionInTools && stepData.action_taken && (isAndroidAction(stepData.action_taken) || isReportStatusAction(stepData.action_taken))) {
|
||||
const actObj = getActionObject(stepData.action_taken);
|
||||
const actTime = getItemTimestamp(
|
||||
stepData.action_taken.timestamp ?? stepData.action_taken.start_time ?? stepData.action_taken.created_at,
|
||||
actObj?.timestamp ?? actObj?.start_time ?? actObj?.created_at,
|
||||
fallbackTime
|
||||
);
|
||||
events.push({
|
||||
@@ -746,15 +823,19 @@ export function getSortedStepEvents(
|
||||
});
|
||||
}
|
||||
|
||||
// Add generic tools if present.
|
||||
const uniqueTools = getUniqueGenericTools(stepData.generic_tools);
|
||||
// Add all real tool calls and actions from the event stream.
|
||||
uniqueTools.forEach((tool: any) => {
|
||||
if (isInternalPlumbingTool(tool) && !isReportStatusAction(tool)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const isAction = tool.type === 'action' || isAndroidAction(tool) || isReportStatusAction(tool);
|
||||
const toolTime = getItemTimestamp(
|
||||
tool.timestamp ?? tool.start_time ?? tool.created_at,
|
||||
fallbackTime
|
||||
);
|
||||
events.push({
|
||||
type: 'tool',
|
||||
type: isAction ? 'action' : 'tool',
|
||||
data: tool,
|
||||
timestamp: toolTime,
|
||||
sequence: sequence++
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import {
|
||||
cleanErrorMessage,
|
||||
getUniqueGenericTools,
|
||||
getVideoAnalysisView
|
||||
getVideoAnalysisView,
|
||||
shouldShowTool
|
||||
} from './tool-formatter.util';
|
||||
|
||||
describe('cleanErrorMessage', () => {
|
||||
@@ -164,3 +165,38 @@ describe('video analysis timeline formatting', () => {
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('shouldShowTool (Option A Single Source of Truth)', () => {
|
||||
it('keeps ADB commands visible even when stepData.action_taken is present', () => {
|
||||
const adbTool = {
|
||||
name: 'run_adb_command',
|
||||
type: 'tool',
|
||||
payload: { CommandLine: 'dumpsys telephony.registry' }
|
||||
};
|
||||
const stepData = {
|
||||
action_taken: [{ action: 'wait_for_delay', time_in_ms: 1000 }]
|
||||
};
|
||||
|
||||
expect(shouldShowTool(adbTool, stepData)).toBeTrue();
|
||||
});
|
||||
|
||||
it('keeps explorer calls visible when stepData.action_taken is present', () => {
|
||||
const explorerTool = {
|
||||
name: 'ask_explorer',
|
||||
type: 'tool'
|
||||
};
|
||||
const stepData = {
|
||||
action_taken: [{ action: 'wait_for_delay', time_in_ms: 1000 }]
|
||||
};
|
||||
|
||||
expect(shouldShowTool(explorerTool, stepData)).toBeTrue();
|
||||
});
|
||||
|
||||
it('filters out internal plumbing tools', () => {
|
||||
const plumbingTool = {
|
||||
name: 'safety_net_pixel_validation',
|
||||
type: 'tool'
|
||||
};
|
||||
expect(shouldShowTool(plumbingTool)).toBeFalse();
|
||||
});
|
||||
});
|
||||
@@ -15,7 +15,6 @@
|
||||
*/
|
||||
|
||||
import { ActionParam } from '../core/models/stream.model';
|
||||
import { isAndroidAction, isReportStatusAction } from './action-formatter.util';
|
||||
import { extractNumbersFromCoordinateValue, isPureDirectionString, parseSequenceCoordinates } from './image-overlay.util';
|
||||
|
||||
// Tool objects are replaced (not mutated) when a trace is updated, so a
|
||||
@@ -276,24 +275,12 @@ export function isInternalPlumbingTool(tool: any): boolean {
|
||||
/**
|
||||
* Determines whether a tool invocation should be displayed in the timeline.
|
||||
*/
|
||||
export function shouldShowTool(tool: any, stepData?: any): boolean {
|
||||
export function shouldShowTool(tool: any, _stepData?: any): boolean {
|
||||
if (!tool || !tool.name) return false;
|
||||
if (tool.type === 'llm_call') return false;
|
||||
if (tool.type === 'agent') return false;
|
||||
if (isInternalPlumbingTool(tool)) return false;
|
||||
|
||||
// If this tool is an operator/validator atomic action that duplicates stepData.action_taken, skip it
|
||||
if (stepData && stepData.action_taken && isAndroidAction(stepData.action_taken) && isDeviceActionTool(tool)) {
|
||||
const agentName = (tool.agent_name || tool.agent || '').toLowerCase();
|
||||
const isSelfHealingOrSubAgent = agentName.includes('failure') ||
|
||||
agentName.includes('analyzer') ||
|
||||
agentName.includes('diagnos') ||
|
||||
(agentName && agentName !== 'operator' && agentName !== 'flashrunner' && agentName !== 'validator');
|
||||
if (!isSelfHealingOrSubAgent) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user