feat: handle mid-stream LLM resets and deduplicate UI action stream events
- Backend (LLM Gateway): - Record structured 'llm_stream_reset' event with discard action, failure reason, and retry messaging on mid-stream generation errors. - Add unit test verifying stream reset event payload on mid-stream errors. - Frontend (Showcase UI): - Handle 'llm_stream_reset' events in agent service and stream aggregator, discarding interrupted chunks while showing retry indicators. - Establish a single source of truth for tool and action events in the timeline to eliminate duplicated action rows while retaining ADB and explorer tools. - Add @angular/cdk dependency and update agent stream component styling and layout. - Add unit tests for stream aggregation and tool display filtering.
This commit is contained in:
somew1nd committed
1 parent
d7c64f571d
commit
d23a4f6472
15 files changed
+1862
-1285
No files matched your search
@@ -19,6 +19,8 @@ limitations under the License.
|
||||
<header class="vscode-nav-header">
|
||||
<div class="vscode-tab-container">
|
||||
<button
|
||||
#taskQueueOrigin="cdkOverlayOrigin"
|
||||
cdkOverlayOrigin
|
||||
class="vscode-tab-btn"
|
||||
[class.active]="isTaskDropdownOpen()"
|
||||
(click)="toggleTaskDropdown($event)"
|
||||
@@ -29,6 +31,8 @@ limitations under the License.
|
||||
<span class="tab-label">Task Queue</span>
|
||||
</button>
|
||||
<button
|
||||
#notesOrigin="cdkOverlayOrigin"
|
||||
cdkOverlayOrigin
|
||||
class="vscode-tab-btn"
|
||||
[class.active]="isNotesDropdownOpen()"
|
||||
(click)="toggleNotesDropdown($event)"
|
||||
@@ -41,7 +45,15 @@ limitations under the License.
|
||||
</div>
|
||||
|
||||
<!-- Dropdown Popover Panel for Task Queue -->
|
||||
@if (isTaskDropdownOpen()) {
|
||||
<ng-template
|
||||
cdkConnectedOverlay
|
||||
[cdkConnectedOverlayOrigin]="taskQueueOrigin"
|
||||
[cdkConnectedOverlayOpen]="isTaskDropdownOpen()"
|
||||
[cdkConnectedOverlayHasBackdrop]="true"
|
||||
cdkConnectedOverlayBackdropClass="cdk-overlay-transparent-backdrop"
|
||||
(backdropClick)="isTaskDropdownOpen.set(false)"
|
||||
[cdkConnectedOverlayPositions]="dropdownPositions"
|
||||
>
|
||||
<div class="vscode-dropdown-menu task-dropdown">
|
||||
<div class="popover-title-header">
|
||||
<span class="material-symbols-outlined">list_alt</span>
|
||||
@@ -173,10 +185,18 @@ limitations under the License.
|
||||
}
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
</ng-template>
|
||||
|
||||
<!-- Dropdown Popover Panel for Notes & Plans -->
|
||||
@if (isNotesDropdownOpen()) {
|
||||
<ng-template
|
||||
cdkConnectedOverlay
|
||||
[cdkConnectedOverlayOrigin]="notesOrigin"
|
||||
[cdkConnectedOverlayOpen]="isNotesDropdownOpen()"
|
||||
[cdkConnectedOverlayHasBackdrop]="true"
|
||||
cdkConnectedOverlayBackdropClass="cdk-overlay-transparent-backdrop"
|
||||
(backdropClick)="isNotesDropdownOpen.set(false)"
|
||||
[cdkConnectedOverlayPositions]="dropdownPositions"
|
||||
>
|
||||
<div class="vscode-dropdown-menu notes-dropdown">
|
||||
<div class="popover-title-header">
|
||||
<span class="material-symbols-outlined">description</span>
|
||||
@@ -361,7 +381,7 @@ limitations under the License.
|
||||
</div>
|
||||
}
|
||||
</div>
|
||||
}
|
||||
</ng-template>
|
||||
</header>
|
||||
|
||||
<!-- Floating Top-Right Badges & Toolbar (Pinned overlay, hidden in empty/initial state) -->
|
||||
@@ -558,62 +578,109 @@ limitations under the License.
|
||||
@if (isCheckerBlock(block) && item.type === 'thinking') {
|
||||
<div
|
||||
class="thinking-stream-details native-thinking-details"
|
||||
[class.is-running]="isCheckerSegmentRunning(block, itemIdx)"
|
||||
[class.is-running]="isCheckerSegmentRunning(block, itemIdx) && !isCheckerSegmentResetNotice(item)"
|
||||
[class.is-resetting]="isCheckerSegmentResetNotice(item)"
|
||||
[class.collapsed]="isCheckerSegmentCollapsed(block, itemIdx)">
|
||||
<div class="details-summary" (click)="toggleCheckerSegment(block, itemIdx)">
|
||||
<span class="stream-status-dot" [class.streaming]="isCheckerSegmentRunning(block, itemIdx)"></span>
|
||||
Thought
|
||||
<span class="stream-status-dot" [class.streaming]="isCheckerSegmentRunning(block, itemIdx) && !isCheckerSegmentResetNotice(item)" [class.resetting]="isCheckerSegmentResetNotice(item)"></span>
|
||||
<span class="header-label">{{ isCheckerSegmentResetNotice(item) ? getCheckerSegmentResetMessage(item) : 'Thought' }}</span>
|
||||
</div>
|
||||
<div class="details-content-wrapper">
|
||||
<div class="details-content stream-box">
|
||||
<div class="stream-text" [innerHTML]="renderMarkdown(item.data.text, block.id + '-seg-' + itemIdx)"></div>
|
||||
@if (item.data.text && item.data.text.trim()) {
|
||||
<div class="details-content-wrapper">
|
||||
<div class="details-content stream-box">
|
||||
<div class="stream-text" [innerHTML]="renderMarkdown(item.data.text, block.id + '-seg-' + itemIdx)"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
</div>
|
||||
}
|
||||
@if (isCheckerBlock(block) && item.type === 'text' && isCheckerTextVisible(item)) {
|
||||
<div
|
||||
class="step-raw-thinking-message"
|
||||
[class.is-running]="isCheckerSegmentRunning(block, itemIdx)">
|
||||
<div class="report-header">
|
||||
<span class="stream-status-dot" [class.streaming]="isCheckerSegmentRunning(block, itemIdx)"></span>
|
||||
Work
|
||||
@if (item.data.isReset) {
|
||||
<div class="llm-retry-notes">
|
||||
<div class="llm-retry-note">
|
||||
<span class="stream-status-dot" [class.streaming]="isCheckerSegmentRunning(block, itemIdx)"></span>
|
||||
<div class="llm-retry-note-text">
|
||||
<span class="llm-retry-status-line">{{ formatResetMessage(getCheckerSegmentResetMessage(item), isCheckerSegmentRunning(block, itemIdx)) }}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="report-content stream-box">
|
||||
<div class="stream-text" [innerHTML]="renderMarkdown(item.data.text, block.id + '-seg-' + itemIdx)"></div>
|
||||
}
|
||||
@if (item.data.text && item.data.text.trim()) {
|
||||
<div
|
||||
class="step-raw-thinking-message"
|
||||
[class.is-running]="isCheckerSegmentRunning(block, itemIdx)">
|
||||
<div class="report-header">
|
||||
<span class="stream-status-dot" [class.streaming]="isCheckerSegmentRunning(block, itemIdx)"></span>
|
||||
<span class="header-label">Work</span>
|
||||
</div>
|
||||
<div class="report-content stream-box">
|
||||
<div class="stream-text" [innerHTML]="renderMarkdown(item.data.text, block.id + '-seg-' + itemIdx)"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
}
|
||||
<!-- Native Thinking (Thought) -->
|
||||
@if (!isCheckerBlock(block) && item.type === 'thinking') {
|
||||
<div
|
||||
class="thinking-stream-details native-thinking-details"
|
||||
[class.is-running]="isBlockRunning(block)"
|
||||
[class.collapsed]="collapsedStreams().has(block.data.execution_id || block.data.step_id || block.id)">
|
||||
<div class="details-summary" (click)="toggleCollapse(block.data.execution_id || block.data.step_id || block.id)">
|
||||
<span class="stream-status-dot" [class.streaming]="isBlockRunning(block)"></span>
|
||||
Thought
|
||||
@if (getBlockResets(block, true).length > 0) {
|
||||
<div class="llm-retry-notes">
|
||||
@for (reset of getBlockResets(block, true); track reset.id || $index) {
|
||||
<div class="llm-retry-note">
|
||||
<span class="stream-status-dot" [class.streaming]="reset.isWaiting"></span>
|
||||
<div class="llm-retry-note-text">
|
||||
<span class="llm-retry-status-line">{{ formatResetMessage(reset.message, reset.isWaiting) }}</span>
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
</div>
|
||||
<div class="details-content-wrapper">
|
||||
<div class="details-content stream-box">
|
||||
<div class="stream-text" [innerHTML]="renderMarkdown(typedTextsSignal()[block.id + '-native'] || item.data.text, block.id + '-native')"></div>
|
||||
}
|
||||
@if (getStreamDisplayText(block.id + '-native', item.data.text) || (isBlockRunning(block) && !isStreamRewinding(block.id + '-native') && !hasWaitingReset(block, true))) {
|
||||
<div
|
||||
class="thinking-stream-details native-thinking-details"
|
||||
[class.is-running]="isBlockRunning(block)"
|
||||
[class.collapsed]="collapsedStreams().has(block.data.execution_id || block.data.step_id || block.id)">
|
||||
<div class="details-summary" (click)="toggleCollapse(block.data.execution_id || block.data.step_id || block.id)">
|
||||
<span class="stream-status-dot" [class.streaming]="isBlockRunning(block)"></span>
|
||||
<span class="header-label">Thought</span>
|
||||
</div>
|
||||
@if (getStreamDisplayText(block.id + '-native', item.data.text)) {
|
||||
<div class="details-content-wrapper">
|
||||
<div class="details-content stream-box">
|
||||
<div class="stream-text" [innerHTML]="renderMarkdown(getStreamDisplayText(block.id + '-native', item.data.text), block.id + '-native')"></div>
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
}
|
||||
<!-- Raw Thinking / Plan / Message (Work) -->
|
||||
@if (!isCheckerBlock(block) && item.type === 'text' && getRawThinking(block)) {
|
||||
<div
|
||||
class="step-raw-thinking-message"
|
||||
[class.is-running]="isBlockRunning(block)">
|
||||
<div class="report-header">
|
||||
<span class="stream-status-dot" [class.streaming]="isBlockRunning(block)"></span>
|
||||
Work
|
||||
@if (!isCheckerBlock(block) && item.type === 'text') {
|
||||
@if (getBlockResets(block, false).length > 0) {
|
||||
<div class="llm-retry-notes">
|
||||
@for (reset of getBlockResets(block, false); track reset.id || $index) {
|
||||
<div class="llm-retry-note">
|
||||
<span class="stream-status-dot" [class.streaming]="reset.isWaiting"></span>
|
||||
<div class="llm-retry-note-text">
|
||||
<span class="llm-retry-status-line">{{ formatResetMessage(reset.message, reset.isWaiting) }}</span>
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
</div>
|
||||
<div class="report-content stream-box">
|
||||
<div class="stream-text" [innerHTML]="renderMarkdown(typedTextsSignal()[block.id] || item.data.text, block.id)"></div>
|
||||
}
|
||||
@if (getStreamDisplayText(block.id, item.data.text) || (isBlockRunning(block) && !isStreamRewinding(block.id) && !hasWaitingReset(block, false))) {
|
||||
<div
|
||||
class="step-raw-thinking-message"
|
||||
[class.is-running]="isBlockRunning(block)">
|
||||
<div class="report-header">
|
||||
<span class="stream-status-dot" [class.streaming]="isBlockRunning(block)"></span>
|
||||
<span class="header-label">Work</span>
|
||||
</div>
|
||||
@if (getStreamDisplayText(block.id, item.data.text)) {
|
||||
<div class="report-content stream-box">
|
||||
<div class="stream-text" [innerHTML]="renderMarkdown(getStreamDisplayText(block.id, item.data.text), block.id)"></div>
|
||||
</div>
|
||||
}
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
}
|
||||
<!-- Task Submission / Report Card (when item is report_task_status action or tool) -->
|
||||
@if (isReportStatusAction(item.data)) {
|
||||
@@ -689,7 +756,7 @@ limitations under the License.
|
||||
</div>
|
||||
}
|
||||
<!-- Android Action Card (when item is an action) -->
|
||||
@if (item.type === 'action' && isAndroidAction(item.data)) {
|
||||
@if (item.type === 'action' && !isReportStatusAction(item.data) && (isAndroidAction(item.data) || item.data?.type === 'action')) {
|
||||
<div class="step-android-action-card"
|
||||
[class.is-running]="isItemRunning(item, block)"
|
||||
[class.is-failed]="isActionFailed(item.data, block.data)"
|
||||
@@ -974,9 +1041,6 @@ limitations under the License.
|
||||
<div class="action-card-header" (click)="toggleActionExpanded('tool-' + block.id + '-' + itemIdx, $event)">
|
||||
<span class="material-symbols-outlined action-icon" [class.is-failed]="isToolFailed(item.data)">{{ getToolIcon(item.data) }}</span>
|
||||
<span class="action-title">{{ getToolTitle(item.data) }}</span>
|
||||
@if (getToolAgentName(item.data)) {
|
||||
<span class="action-agent-badge" [class.is-outputter]="getToolAgentName(item.data) === 'Outputter'">{{ getToolAgentName(item.data) }}</span>
|
||||
}
|
||||
@if (isToolFailed(item.data)) {
|
||||
<span class="action-status-badge is-failed">
|
||||
<span class="material-symbols-outlined status-icon">error</span>
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -22,7 +22,8 @@ import { HttpClient } from '@angular/common/http';
|
||||
import { AgentService, StartupProgressEvent } from '../../services/agent.service';
|
||||
import { Session, ModelInfo } from '../../core/models/session.model';
|
||||
import { MarkdownSegment, MarkdownLine, NoteMilestone, ParsedNote } from '../../core/models/markdown.model';
|
||||
import { StepBlock, PhaseBlock, StepEvent, ActionParam, CheckerResult } from '../../core/models/stream.model';
|
||||
import { OverlayModule, ConnectedPosition } from '@angular/cdk/overlay';
|
||||
import { StepBlock, PhaseBlock, StepEvent, ActionParam, CheckerResult, StreamResetNotice, DEFAULT_STREAM_RESET_MESSAGE } from '../../core/models/stream.model';
|
||||
|
||||
export const PLANNING_LOADER_PHRASES: string[] = [
|
||||
'Planning next step...',
|
||||
@@ -195,12 +196,12 @@ import {
|
||||
checkPlanningLoader
|
||||
} from '../../utils/stream-aggregator.util';
|
||||
|
||||
export type { MarkdownSegment, MarkdownLine, NoteMilestone, ParsedNote };
|
||||
export type { MarkdownSegment, MarkdownLine, NoteMilestone, ParsedNote, StreamResetNotice };
|
||||
|
||||
@Component({
|
||||
selector: 'app-agent-stream',
|
||||
standalone: true,
|
||||
imports: [CommonModule, FormsModule],
|
||||
imports: [CommonModule, FormsModule, OverlayModule],
|
||||
templateUrl: './agent-stream.component.html',
|
||||
styleUrl: './agent-stream.component.scss',
|
||||
changeDetection: ChangeDetectionStrategy.OnPush
|
||||
@@ -234,6 +235,31 @@ export class AgentStreamComponent implements AfterViewInit {
|
||||
public isTaskDropdownOpen = signal<boolean>(false);
|
||||
public isNotesDropdownOpen = signal<boolean>(false);
|
||||
|
||||
// CDK Connected Overlay Positioning Strategy
|
||||
public readonly dropdownPositions: ConnectedPosition[] = [
|
||||
{
|
||||
originX: 'end',
|
||||
originY: 'bottom',
|
||||
overlayX: 'end',
|
||||
overlayY: 'top',
|
||||
offsetY: 6,
|
||||
},
|
||||
{
|
||||
originX: 'end',
|
||||
originY: 'top',
|
||||
overlayX: 'end',
|
||||
overlayY: 'bottom',
|
||||
offsetY: -6,
|
||||
},
|
||||
{
|
||||
originX: 'start',
|
||||
originY: 'bottom',
|
||||
overlayX: 'start',
|
||||
overlayY: 'top',
|
||||
offsetY: 6,
|
||||
}
|
||||
];
|
||||
|
||||
// Zoom Modal State
|
||||
public selectedZoomImage = signal<string | null>(null);
|
||||
public selectedZoomAction = signal<any | null>(null);
|
||||
@@ -258,6 +284,18 @@ export class AgentStreamComponent implements AfterViewInit {
|
||||
private static readonly TYPING_CHARS_PER_SECOND = 667;
|
||||
private static readonly TYPING_MIN_FRAME_MS = 28;
|
||||
|
||||
// Active reset notices per block (displayed after typewriter rewind finishes)
|
||||
public activeBlockResets = signal<Map<string, StreamResetNotice[]>>(new Map());
|
||||
public activeResetNotices = signal<Map<string, string>>(new Map());
|
||||
private rewindTargets = new Map<string, {
|
||||
blockId: string;
|
||||
isThinking: boolean;
|
||||
execId?: string;
|
||||
message: string;
|
||||
}>();
|
||||
// Fast rewind pace (backspace speed ≈ 4x normal typing speed for snappy deletion)
|
||||
private static readonly REWIND_CHARS_PER_SECOND = 2600;
|
||||
|
||||
// Rendered-markdown memo per template slot: returning the identical string
|
||||
// instance for unchanged text lets the [innerHTML] binding skip re-sanitizing
|
||||
// and re-parsing the whole fragment on every change-detection pass.
|
||||
@@ -466,6 +504,18 @@ export class AgentStreamComponent implements AfterViewInit {
|
||||
this.scheduleAutoScroll(false);
|
||||
});
|
||||
|
||||
// Listen for stream reset events to trigger fast typewriter rewind
|
||||
effect(() => {
|
||||
const resetEvent = this.agentService.streamResetEvent();
|
||||
if (!resetEvent) return;
|
||||
untracked(() => {
|
||||
this.triggerStreamRewind(
|
||||
resetEvent.stream_execution_id || resetEvent.stream_exec_id,
|
||||
resetEvent.message || DEFAULT_STREAM_RESET_MESSAGE
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// Typewriter drive logic: triggers typing when block appears or expands
|
||||
effect(() => {
|
||||
const blocks = this.consolidatedBlocks();
|
||||
@@ -473,6 +523,39 @@ export class AgentStreamComponent implements AfterViewInit {
|
||||
// Check blocks render their per-turn segments directly (no typewriter).
|
||||
if (block.type === 'checker') return;
|
||||
const blockId = block.id;
|
||||
|
||||
// If the block is flagged as reset, ensure typewriter rewind is active or reset notice is set
|
||||
if (block.data?.isReset) {
|
||||
untracked(() => {
|
||||
const resetMsg = block.data.resetMessage || DEFAULT_STREAM_RESET_MESSAGE;
|
||||
const currentRecord = this.typedTextsSignal();
|
||||
const hasText = (currentRecord[blockId]?.length || 0) > 0 || (currentRecord[blockId + '-native']?.length || 0) > 0;
|
||||
if (hasText) {
|
||||
if (!this.rewindTargets.has(blockId) && !this.rewindTargets.has(blockId + '-native')) {
|
||||
this.triggerStreamRewind(block.data.execution_id, resetMsg);
|
||||
}
|
||||
} else {
|
||||
const activeResets = this.activeBlockResets().get(blockId) || [];
|
||||
if (activeResets.length === 0) {
|
||||
this.upsertBlockResetNotice(blockId, {
|
||||
id: block.data.execution_id || `${blockId}-text`,
|
||||
message: resetMsg,
|
||||
isWaiting: !block.data?.isCompleted,
|
||||
streamType: 'text'
|
||||
});
|
||||
}
|
||||
if (!this.activeResetNotices().has(blockId)) {
|
||||
this.activeResetNotices.update(m => {
|
||||
const next = new Map(m);
|
||||
next.set(blockId, resetMsg);
|
||||
return next;
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const rawText = this.getRawThinking(block) || '';
|
||||
const nativeText = this.getNativeThinking(block) || '';
|
||||
|
||||
@@ -541,6 +624,8 @@ export class AgentStreamComponent implements AfterViewInit {
|
||||
untracked(() => {
|
||||
this.stopTypingLoop();
|
||||
this.typedTextsSignal.set({});
|
||||
this.activeResetNotices.set(new Map());
|
||||
this.activeBlockResets.set(new Map());
|
||||
this.markdownHtmlCache.clear();
|
||||
this.scheduledCollapses.clear();
|
||||
this.collapsedStreams.set(new Set<string>());
|
||||
@@ -754,8 +839,51 @@ export class AgentStreamComponent implements AfterViewInit {
|
||||
}
|
||||
|
||||
// Typewriter & Delayed Collapse
|
||||
private upsertBlockResetNotice(blockId: string, notice: StreamResetNotice): void {
|
||||
this.activeBlockResets.update(m => {
|
||||
const next = new Map(m);
|
||||
const list = [...(next.get(blockId) || [])];
|
||||
const existingIdx = list.findIndex(r => r.id === notice.id);
|
||||
if (existingIdx > -1) {
|
||||
list[existingIdx] = notice;
|
||||
} else {
|
||||
list.push(notice);
|
||||
}
|
||||
next.set(blockId, list);
|
||||
return next;
|
||||
});
|
||||
}
|
||||
|
||||
private startTyping(key: string, targetText: string, isThinking: boolean = false, execId?: string) {
|
||||
const blockId = key.endsWith('-native') ? key.replace('-native', '') : key;
|
||||
// If there was an active rewind, cancel it, reset text to empty, and record its reset notice as completed waiting
|
||||
const activeRewind = this.rewindTargets.get(key);
|
||||
if (activeRewind) {
|
||||
this.rewindTargets.delete(key);
|
||||
this.typedTextsSignal.update(r => ({ ...r, [key]: '' }));
|
||||
this.upsertBlockResetNotice(blockId, {
|
||||
id: activeRewind.execId || `${blockId}-${isThinking ? 'thinking' : 'text'}`,
|
||||
message: activeRewind.message,
|
||||
isWaiting: false,
|
||||
streamType: isThinking ? 'thinking' : 'text'
|
||||
});
|
||||
}
|
||||
|
||||
// Cancel any active rewind on this key
|
||||
this.rewindTargets.delete(key);
|
||||
|
||||
// When retry stream starts outputting content, change waiting dot to normal dot (isWaiting: false)
|
||||
// The previous prompt sentence DOES NOT disappear.
|
||||
this.activeBlockResets.update(m => {
|
||||
const list = m.get(blockId);
|
||||
if (!list || list.length === 0) return m;
|
||||
const targetType = isThinking ? 'thinking' : 'text';
|
||||
const updated = list.map(r => r.streamType === targetType ? { ...r, isWaiting: false } : r);
|
||||
const next = new Map(m);
|
||||
next.set(blockId, updated);
|
||||
return next;
|
||||
});
|
||||
|
||||
this.typingTargets.set(key, { blockId, isThinking, execId, fallbackText: targetText });
|
||||
this.ensureTypingLoop();
|
||||
}
|
||||
@@ -771,7 +899,7 @@ export class AgentStreamComponent implements AfterViewInit {
|
||||
this.lastTypingTimestamp = now;
|
||||
this.advanceTyping(elapsed);
|
||||
}
|
||||
if (this.typingTargets.size > 0) {
|
||||
if (this.typingTargets.size > 0 || this.rewindTargets.size > 0) {
|
||||
this.typingRafId = requestAnimationFrame(step);
|
||||
}
|
||||
};
|
||||
@@ -781,6 +909,7 @@ export class AgentStreamComponent implements AfterViewInit {
|
||||
|
||||
private stopTypingLoop(): void {
|
||||
this.typingTargets.clear();
|
||||
this.rewindTargets.clear();
|
||||
if (this.typingRafId !== null) {
|
||||
cancelAnimationFrame(this.typingRafId);
|
||||
this.typingRafId = null;
|
||||
@@ -798,9 +927,14 @@ export class AgentStreamComponent implements AfterViewInit {
|
||||
1,
|
||||
Math.round(AgentStreamComponent.TYPING_CHARS_PER_SECOND * elapsedMs / 1000)
|
||||
);
|
||||
const rewindChars = Math.max(
|
||||
2,
|
||||
Math.round(AgentStreamComponent.REWIND_CHARS_PER_SECOND * elapsedMs / 1000)
|
||||
);
|
||||
const updates: Record<string, string> = {};
|
||||
let hasUpdates = false;
|
||||
|
||||
// 1. Advance forward typing targets
|
||||
for (const [key, typing] of this.typingTargets) {
|
||||
const block = blocks.find((b) => b.id === typing.blockId);
|
||||
let target = typing.fallbackText;
|
||||
@@ -821,11 +955,211 @@ export class AgentStreamComponent implements AfterViewInit {
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Advance fast rewind targets (backspacing for mid-stream resets)
|
||||
const completedRewinds: Array<{ blockId: string; execId?: string; isThinking: boolean; message: string }> = [];
|
||||
for (const [key, rewind] of this.rewindTargets) {
|
||||
const current = updates[key] !== undefined ? updates[key] : (currentRecord[key] || '');
|
||||
if (current.length > 0) {
|
||||
const nextLen = Math.max(0, current.length - rewindChars);
|
||||
updates[key] = current.slice(0, nextLen);
|
||||
hasUpdates = true;
|
||||
if (nextLen === 0) {
|
||||
this.rewindTargets.delete(key);
|
||||
completedRewinds.push({
|
||||
blockId: rewind.blockId,
|
||||
execId: rewind.execId,
|
||||
isThinking: rewind.isThinking,
|
||||
message: rewind.message
|
||||
});
|
||||
}
|
||||
} else {
|
||||
this.rewindTargets.delete(key);
|
||||
completedRewinds.push({
|
||||
blockId: rewind.blockId,
|
||||
execId: rewind.execId,
|
||||
isThinking: rewind.isThinking,
|
||||
message: rewind.message
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (completedRewinds.length > 0) {
|
||||
for (const item of completedRewinds) {
|
||||
this.upsertBlockResetNotice(item.blockId, {
|
||||
id: item.execId || `${item.blockId}-${item.isThinking ? 'thinking' : 'text'}`,
|
||||
message: item.message,
|
||||
isWaiting: true,
|
||||
streamType: item.isThinking ? 'thinking' : 'text'
|
||||
});
|
||||
}
|
||||
this.activeResetNotices.update((notices) => {
|
||||
const next = new Map(notices);
|
||||
for (const item of completedRewinds) {
|
||||
next.set(item.blockId, item.message);
|
||||
}
|
||||
return next;
|
||||
});
|
||||
}
|
||||
|
||||
if (hasUpdates) {
|
||||
this.typedTextsSignal.update((r) => ({ ...r, ...updates }));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Triggers fast typewriter rewind on a stream execution when a mid-stream reset occurs.
|
||||
* Once characters rewind to zero, the block displays the graceful retry notification.
|
||||
*/
|
||||
public triggerStreamRewind(execId: string | undefined, message: string): void {
|
||||
const blocks = untracked(() => this.consolidatedBlocks());
|
||||
const currentTexts = untracked(() => this.typedTextsSignal());
|
||||
|
||||
const targetBlock = blocks.find(b =>
|
||||
b.data?.execution_id === execId
|
||||
|| b.id === `stream-${execId}`
|
||||
|| b.data?.step_id === execId
|
||||
|| (Array.isArray(b.data?.stream_segments) && b.data.stream_segments.some((s: any) => s.execution_id === execId))
|
||||
) || blocks.find(b => b.data?.isReset);
|
||||
|
||||
if (!targetBlock) return;
|
||||
const blockId = targetBlock.id;
|
||||
|
||||
const keysToCheck: Array<{ key: string; isThinking: boolean }> = [
|
||||
{ key: blockId, isThinking: false },
|
||||
{ key: blockId + '-native', isThinking: true }
|
||||
];
|
||||
|
||||
let startedRewind = false;
|
||||
for (const { key, isThinking } of keysToCheck) {
|
||||
this.typingTargets.delete(key);
|
||||
|
||||
const currentText = currentTexts[key] || '';
|
||||
if (currentText.length > 0) {
|
||||
this.rewindTargets.set(key, {
|
||||
blockId,
|
||||
isThinking,
|
||||
execId,
|
||||
message
|
||||
});
|
||||
startedRewind = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (startedRewind) {
|
||||
this.ensureTypingLoop();
|
||||
} else {
|
||||
this.upsertBlockResetNotice(blockId, {
|
||||
id: execId || `${blockId}-text`,
|
||||
message,
|
||||
isWaiting: true,
|
||||
streamType: 'text'
|
||||
});
|
||||
this.activeResetNotices.update(notices => {
|
||||
const next = new Map(notices);
|
||||
next.set(blockId, message);
|
||||
return next;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public getBlockResets(block: any, isThinking: boolean = false): StreamResetNotice[] {
|
||||
if (!block) return [];
|
||||
const blockId = block.id;
|
||||
const targetStreamType = isThinking ? 'thinking' : 'text';
|
||||
|
||||
// 1. From activeBlockResets signal
|
||||
const activeList = (this.activeBlockResets().get(blockId) || [])
|
||||
.filter(r => r.streamType === targetStreamType);
|
||||
|
||||
// 2. From block.data.stream_resets (aggregated from logs)
|
||||
const blockDataResets = (Array.isArray(block.data?.stream_resets) ? block.data.stream_resets : [])
|
||||
.filter((r: any) => (r.streamType || 'text') === targetStreamType);
|
||||
|
||||
// Fast path: if both lists are empty, check fallback or return empty array immediately
|
||||
if (activeList.length === 0 && blockDataResets.length === 0) {
|
||||
if (!isThinking && block.data?.isReset && block.data?.resetMessage) {
|
||||
return [{
|
||||
id: block.data.execution_id || blockId,
|
||||
message: block.data.resetMessage,
|
||||
isWaiting: !block.data?.isCompleted,
|
||||
streamType: 'text'
|
||||
}];
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
// Merge by id, giving precedence to activeList
|
||||
const mergedMap = new Map<string, StreamResetNotice>();
|
||||
for (const r of blockDataResets) {
|
||||
mergedMap.set(r.id, {
|
||||
id: r.id,
|
||||
message: r.message,
|
||||
isWaiting: block.data?.isCompleted ? false : (r.isWaiting ?? false),
|
||||
streamType: r.streamType || targetStreamType
|
||||
});
|
||||
}
|
||||
for (const r of activeList) {
|
||||
mergedMap.set(r.id, {
|
||||
...r,
|
||||
isWaiting: block.data?.isCompleted ? false : r.isWaiting
|
||||
});
|
||||
}
|
||||
|
||||
return Array.from(mergedMap.values());
|
||||
}
|
||||
|
||||
public hasWaitingReset(block: any, isThinking: boolean = false): boolean {
|
||||
const resets = this.getBlockResets(block, isThinking);
|
||||
return resets.some(r => r.isWaiting);
|
||||
}
|
||||
|
||||
public hasBlockResets(block: any, isThinking: boolean = false): boolean {
|
||||
return this.getBlockResets(block, isThinking).length > 0;
|
||||
}
|
||||
|
||||
public formatResetMessage(message: string, isWaiting: boolean): string {
|
||||
if (!message) return '';
|
||||
if (isWaiting) return message;
|
||||
return message
|
||||
.replace(/[,.\s]*(?:Retrying automatically(?:\.{3}|\.\.\.)?)\s*$/i, '')
|
||||
.trim();
|
||||
}
|
||||
|
||||
public getCheckerSegmentResetMessage(item: any): string {
|
||||
return item?.data?.resetMessage || DEFAULT_STREAM_RESET_MESSAGE;
|
||||
}
|
||||
|
||||
public getResetNotice(block: any): string | null {
|
||||
if (!block) return null;
|
||||
const resets = this.getBlockResets(block, false);
|
||||
if (resets.length > 0) {
|
||||
const last = resets[resets.length - 1];
|
||||
return this.formatResetMessage(last.message, last.isWaiting);
|
||||
}
|
||||
const blockId = block.id;
|
||||
const active = this.activeResetNotices().get(blockId);
|
||||
if (active) return this.formatResetMessage(active, !block.data?.isCompleted);
|
||||
if (block.data?.isReset && block.data?.resetMessage) {
|
||||
return this.formatResetMessage(block.data.resetMessage, !block.data?.isCompleted);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public isStreamRewinding(key: string): boolean {
|
||||
return this.rewindTargets.has(key);
|
||||
}
|
||||
|
||||
public isCheckerSegmentResetNotice(item: any): boolean {
|
||||
if (!item?.data?.isReset) return false;
|
||||
const text = item.data.text;
|
||||
return !text || text.trim().length === 0;
|
||||
}
|
||||
|
||||
public getStreamDisplayText(key: string, fallback: string = ''): string {
|
||||
const typed = this.typedTextsSignal()[key];
|
||||
return typed !== undefined ? typed : fallback;
|
||||
}
|
||||
|
||||
private triggerDelayedCollapse(execId: string) {
|
||||
if (!this.scheduledCollapses.has(execId)) {
|
||||
this.scheduledCollapses.add(execId);
|
||||
@@ -1275,6 +1609,7 @@ export class AgentStreamComponent implements AfterViewInit {
|
||||
}
|
||||
|
||||
public isCheckerTextVisible(item: any): boolean {
|
||||
if (item?.data?.isReset) return true;
|
||||
const text = item?.data?.text;
|
||||
return typeof text === 'string' && text.trim().length > 0 && this.isHumanThinking(text);
|
||||
}
|
||||
@@ -1347,6 +1682,7 @@ export class AgentStreamComponent implements AfterViewInit {
|
||||
if (block.type === 'checker') return true;
|
||||
const hasNative = Boolean(this.getNativeThinking(block));
|
||||
const hasRaw = Boolean(this.getRawThinking(block));
|
||||
const hasReset = this.hasBlockResets(block, false) || this.hasBlockResets(block, true);
|
||||
const hasVisibleTools = Boolean(block.data?.generic_tools) && block.data.generic_tools.some((t: any) =>
|
||||
this.shouldShowTool(t, block.data)
|
||||
|| this.isDisplayableLLMFailure(t)
|
||||
@@ -1354,7 +1690,7 @@ export class AgentStreamComponent implements AfterViewInit {
|
||||
|| this.isReportStatusAction(t)
|
||||
);
|
||||
const hasAndroidActions = Boolean(block.data?.action_taken) && (this.isAndroidAction(block.data.action_taken) || this.isReportStatusAction(block.data.action_taken));
|
||||
return hasNative || hasRaw || hasVisibleTools || hasAndroidActions;
|
||||
return hasNative || hasRaw || hasReset || hasVisibleTools || hasAndroidActions;
|
||||
}
|
||||
|
||||
public isReportStatusAction(action: any): boolean {
|
||||
|
||||
@@ -52,6 +52,32 @@ export interface LLMStreamEventData {
|
||||
chunk?: string;
|
||||
stream_type?: 'thinking' | 'text' | string;
|
||||
isCompleted?: boolean;
|
||||
isReset?: boolean;
|
||||
resetMessage?: string;
|
||||
}
|
||||
|
||||
export const DEFAULT_STREAM_RESET_MESSAGE =
|
||||
'A request error occurred during output generation, typically caused by lower API priority. Retrying automatically...';
|
||||
|
||||
export interface StreamResetNotice {
|
||||
id: string;
|
||||
message: string;
|
||||
isWaiting: boolean;
|
||||
streamType: string;
|
||||
}
|
||||
|
||||
export interface LLMStreamResetEventData {
|
||||
stream_exec_id?: string;
|
||||
stream_execution_id?: string;
|
||||
step_id?: string;
|
||||
session_id?: string;
|
||||
action?: 'discard' | string;
|
||||
reason?: string;
|
||||
category?: string;
|
||||
error?: string;
|
||||
message?: string;
|
||||
retry_attempt?: number;
|
||||
timestamp?: number;
|
||||
}
|
||||
|
||||
export interface StepItemData {
|
||||
@@ -119,6 +145,8 @@ export interface StreamSegment {
|
||||
text: string;
|
||||
timestamp: string;
|
||||
isCompleted?: boolean;
|
||||
isReset?: boolean;
|
||||
resetMessage?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -20,9 +20,9 @@ import { Observable } from 'rxjs';
|
||||
|
||||
import { Session, ModelInfo, TaskQueueItem, AgentStatusResponse } from '../core/models/session.model';
|
||||
import { ProTuningDefaults, ProTuningOptions } from '../core/models/pro-tuning.model';
|
||||
import { StepItemData, StepReplayFrame } from '../core/models/stream.model';
|
||||
import { StepItemData, StepReplayFrame, LLMStreamResetEventData, StreamResetNotice, DEFAULT_STREAM_RESET_MESSAGE } from '../core/models/stream.model';
|
||||
import { extractStepReplayFrames } from '../utils/action-formatter.util';
|
||||
export type { Session, ModelInfo, TaskQueueItem, AgentStatusResponse, StepItemData, StepReplayFrame };
|
||||
export type { Session, ModelInfo, TaskQueueItem, AgentStatusResponse, StepItemData, StepReplayFrame, LLMStreamResetEventData, StreamResetNotice };
|
||||
|
||||
const SESSION_CACHE_KEY = 'artemis.sessions.v1';
|
||||
|
||||
@@ -240,6 +240,7 @@ export class AgentService {
|
||||
public videoSeekRequest = signal<{ seconds: number; requestId: number } | null>(null);
|
||||
public stepSeekRequest = signal<{ index: number; requestId: number } | null>(null);
|
||||
public playerMode = signal<'video' | 'steps'>('video');
|
||||
public streamResetEvent = signal<LLMStreamResetEventData | null>(null);
|
||||
private activeVideoSessionId: string | null = null;
|
||||
private videoSeekRequestId = 0;
|
||||
private stepSeekRequestId = 0;
|
||||
@@ -348,7 +349,7 @@ export class AgentService {
|
||||
|
||||
// Streaming chunks are coalesced into one signal update per short window so a
|
||||
// chatty LLM stream cannot force a full recompute pass per SSE chunk.
|
||||
private pendingStreamChunks = new Map<string, { execId: any; stepId: any; parentTraceId?: any; streamType: string; chunk: string }>();
|
||||
private pendingStreamChunks = new Map<string, { execId: any; stepId: any; sessionId?: string | null; parentTraceId?: any; streamType: string; chunk: string }>();
|
||||
private streamFlushTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private lastQueueSignature: string | null = null;
|
||||
private lastActiveTasksSignature: string | null = null;
|
||||
@@ -762,6 +763,7 @@ export class AgentService {
|
||||
|
||||
const eventTypes = [
|
||||
'llm_stream',
|
||||
'llm_stream_reset',
|
||||
'trace_recorded',
|
||||
'step_recorded',
|
||||
'step_updated',
|
||||
@@ -951,6 +953,67 @@ export class AgentService {
|
||||
this.isRetrying.set(false);
|
||||
}
|
||||
|
||||
if (eventType === 'llm_stream_reset') {
|
||||
const streamExecId = parsedData?.stream_exec_id || parsedData?.stream_execution_id;
|
||||
if (streamExecId) {
|
||||
for (const key of Array.from(this.pendingStreamChunks.keys())) {
|
||||
if (key.startsWith(`${streamExecId}|`) || key === streamExecId) {
|
||||
this.pendingStreamChunks.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
const resetMessage = parsedData?.message || DEFAULT_STREAM_RESET_MESSAGE;
|
||||
this.streamResetEvent.set({
|
||||
stream_exec_id: streamExecId,
|
||||
stream_execution_id: streamExecId,
|
||||
step_id: parsedData?.step_id,
|
||||
session_id: parsedData?.session_id,
|
||||
action: parsedData?.action || 'discard',
|
||||
reason: parsedData?.reason || 'mid_stream_failure',
|
||||
category: parsedData?.category,
|
||||
error: parsedData?.error,
|
||||
message: resetMessage,
|
||||
retry_attempt: parsedData?.retry_attempt,
|
||||
timestamp: Date.now()
|
||||
});
|
||||
|
||||
this.sessionLogs.update((logs) => {
|
||||
let updated = false;
|
||||
const next = logs.map((log) => {
|
||||
if (log.type === 'llm_stream' && log.data?.execution_id === streamExecId) {
|
||||
updated = true;
|
||||
return {
|
||||
...log,
|
||||
data: {
|
||||
...log.data,
|
||||
isReset: true,
|
||||
resetMessage
|
||||
}
|
||||
};
|
||||
}
|
||||
return log;
|
||||
});
|
||||
if (!updated && streamExecId) {
|
||||
next.push({
|
||||
type: 'llm_stream',
|
||||
timestamp: new Date().toISOString(),
|
||||
session_id: parsedData?.session_id,
|
||||
data: {
|
||||
execution_id: streamExecId,
|
||||
step_id: parsedData?.step_id,
|
||||
stream_type: 'text',
|
||||
text: '',
|
||||
isCompleted: false,
|
||||
isReset: true,
|
||||
resetMessage
|
||||
}
|
||||
});
|
||||
}
|
||||
return next;
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (eventType === 'llm_stream') {
|
||||
this.queueStreamChunk(parsedData);
|
||||
} else {
|
||||
@@ -1033,19 +1096,22 @@ export class AgentService {
|
||||
private queueStreamChunk(parsedData: any): void {
|
||||
const execId = parsedData.execution_id;
|
||||
const stepId = parsedData.step_id;
|
||||
const sessionId = parsedData.session_id || null;
|
||||
// The emitting agent's trace: lets the aggregator route a concurrent
|
||||
// agent's stream (e.g. the Checker) to its own block instead of the
|
||||
// Operator step that happens to be current.
|
||||
const parentTraceId = parsedData.parent_trace_id || null;
|
||||
const streamType = parsedData.stream_type || 'text';
|
||||
const key = `${execId}|${streamType}`;
|
||||
const chunk = parsedData.chunk !== undefined ? parsedData.chunk : (parsedData.text || '');
|
||||
const pending = this.pendingStreamChunks.get(key);
|
||||
if (pending) {
|
||||
pending.chunk += parsedData.chunk;
|
||||
pending.chunk += chunk;
|
||||
if (stepId) pending.stepId = stepId;
|
||||
if (sessionId) pending.sessionId = sessionId;
|
||||
if (parentTraceId) pending.parentTraceId = parentTraceId;
|
||||
} else {
|
||||
this.pendingStreamChunks.set(key, { execId, stepId, parentTraceId, streamType, chunk: parsedData.chunk });
|
||||
this.pendingStreamChunks.set(key, { execId, stepId, sessionId, parentTraceId, streamType, chunk });
|
||||
}
|
||||
if (!this.streamFlushTimer) {
|
||||
const delay = typeof document !== 'undefined' && document.hidden ? 500 : 80;
|
||||
@@ -1083,9 +1149,11 @@ export class AgentService {
|
||||
const existingLog = next[existingIndex];
|
||||
next[existingIndex] = {
|
||||
...existingLog,
|
||||
session_id: batch.sessionId || existingLog.session_id,
|
||||
data: {
|
||||
...existingLog.data,
|
||||
text: existingLog.data.text + batch.chunk,
|
||||
session_id: batch.sessionId || existingLog.data?.session_id,
|
||||
text: (existingLog.data.text || '') + batch.chunk,
|
||||
step_id: batch.stepId || existingLog.data.step_id,
|
||||
parent_trace_id: batch.parentTraceId || existingLog.data.parent_trace_id || null
|
||||
}
|
||||
@@ -1105,8 +1173,10 @@ export class AgentService {
|
||||
next.push({
|
||||
type: 'llm_stream',
|
||||
timestamp: new Date().toISOString(),
|
||||
session_id: batch.sessionId || undefined,
|
||||
data: {
|
||||
execution_id: batch.execId,
|
||||
session_id: batch.sessionId || undefined,
|
||||
step_id: batch.stepId,
|
||||
parent_trace_id: batch.parentTraceId || null,
|
||||
text: batch.chunk,
|
||||
|
||||
@@ -18,13 +18,66 @@ import { ActionParam, StepReplayFrame } from '../core/models/stream.model';
|
||||
import { extractNumbersFromCoordinateValue, isPureDirectionString, parseSequenceCoordinates } from './image-overlay.util';
|
||||
import { cleanErrorMessage } from './tool-formatter.util';
|
||||
|
||||
/**
|
||||
* Safely parse JSON string if it looks like an object/array, otherwise return original value
|
||||
*/
|
||||
export function safeParseJson(val: any): any {
|
||||
if (typeof val === 'string' && (val.trim().startsWith('{') || val.trim().startsWith('['))) {
|
||||
try {
|
||||
return JSON.parse(val);
|
||||
} catch {
|
||||
return val;
|
||||
}
|
||||
}
|
||||
return val;
|
||||
}
|
||||
|
||||
// WeakMap memo for parsed action objects to avoid repeated JSON.parse and object allocation across change detection cycles
|
||||
const actionObjectCache = new WeakMap<object, any>();
|
||||
|
||||
function computeActionObject(action: any): any {
|
||||
if (action && typeof action === 'object' && action.payload) {
|
||||
const payload = typeof action.payload === 'string' ? safeParseJson(action.payload) : action.payload;
|
||||
const innerAction = payload?.args?.action;
|
||||
if (innerAction && typeof innerAction === 'object') {
|
||||
return {
|
||||
...action,
|
||||
...innerAction,
|
||||
name: action.name || innerAction.action || innerAction.name,
|
||||
action: innerAction.action || action.name,
|
||||
trace_id: action.trace_id,
|
||||
timestamp: action.timestamp ?? innerAction.timestamp
|
||||
};
|
||||
}
|
||||
const innerArgs = payload?.args;
|
||||
if (innerArgs && typeof innerArgs === 'object') {
|
||||
return {
|
||||
...action,
|
||||
...innerArgs,
|
||||
name: action.name || innerArgs.action || innerArgs.name,
|
||||
action: innerArgs.action || action.name,
|
||||
trace_id: action.trace_id,
|
||||
timestamp: action.timestamp ?? innerArgs.timestamp
|
||||
};
|
||||
}
|
||||
}
|
||||
return action;
|
||||
}
|
||||
|
||||
/**
|
||||
* Safely extract action object from possible array or nested structure
|
||||
*/
|
||||
export function getActionObject(action: any): any {
|
||||
if (!action) return null;
|
||||
if (Array.isArray(action)) {
|
||||
return action.length > 0 ? action[0] : null;
|
||||
return action.length > 0 ? getActionObject(action[0]) : null;
|
||||
}
|
||||
if (typeof action === 'object') {
|
||||
const cached = actionObjectCache.get(action);
|
||||
if (cached !== undefined) return cached;
|
||||
const res = computeActionObject(action);
|
||||
actionObjectCache.set(action, res);
|
||||
return res;
|
||||
}
|
||||
return action;
|
||||
}
|
||||
@@ -46,7 +99,7 @@ export function isAndroidAction(action: any): boolean {
|
||||
'tap', 'click', 'click_sequence', 'input', 'input_text', 'swipe', 'scroll', 'press_key',
|
||||
'press_home', 'press_back', 'launch_app', 'manage_app', 'open_app',
|
||||
'focus_and_input_text', 'focus_and_clear_text', 'long_press', 'long_press_on',
|
||||
'wait_for_delay', 'wait'
|
||||
'wait_for_delay', 'wait', 'drag', 'drag_and_drop'
|
||||
].includes(name);
|
||||
}
|
||||
|
||||
@@ -543,7 +596,8 @@ export function isGenericToolTrace(actionData: any, stepData?: any): boolean {
|
||||
}
|
||||
}
|
||||
if (actionData.is_primary_step_action) return false;
|
||||
return Boolean(actionData.trace_id || actionData.payload || actionData.type === 'tool' || (actionData.type === 'action' && actionData.trace_id));
|
||||
if (actionData.type === 'action' || isAndroidAction(actionData)) return false;
|
||||
return Boolean(actionData.trace_id || actionData.payload || actionData.type === 'tool');
|
||||
}
|
||||
|
||||
interface ResolvedImageEntry {
|
||||
@@ -554,16 +608,6 @@ interface ResolvedImageEntry {
|
||||
|
||||
const stepImageMapCache = new WeakMap<any, { key: string; map: Map<any, ResolvedImageEntry> }>();
|
||||
|
||||
function safeParseJson(val: any): any {
|
||||
if (typeof val === 'string' && (val.trim().startsWith('{') || val.trim().startsWith('['))) {
|
||||
try {
|
||||
return JSON.parse(val);
|
||||
} catch {
|
||||
return val;
|
||||
}
|
||||
}
|
||||
return val;
|
||||
}
|
||||
|
||||
export function extractItemPreImage(item: any): string | null {
|
||||
const act = getActionObject(item);
|
||||
@@ -639,7 +683,10 @@ export function resolveStepImageMap(stepData: any): Map<any, ResolvedImageEntry>
|
||||
if (!stepData || typeof stepData !== 'object') return emptyMap;
|
||||
|
||||
const toolsLen = Array.isArray(stepData.generic_tools) ? stepData.generic_tools.length : 0;
|
||||
const cacheKey = `${stepData.step_id || ''}_${toolsLen}_${stepData.post_image_name || ''}_${stepData.pre_image_name || ''}`;
|
||||
const toolsImageSig = Array.isArray(stepData.generic_tools)
|
||||
? stepData.generic_tools.map((t: any) => `${t?.trace_id || ''}:${t?.post_image_name || t?.post_screenshot || ''}`).join(',')
|
||||
: '';
|
||||
const cacheKey = `${stepData.step_id || ''}_${toolsLen}_${toolsImageSig}_${stepData.post_image_name || ''}_${stepData.pre_image_name || ''}`;
|
||||
|
||||
const cached = stepImageMapCache.get(stepData);
|
||||
if (cached && cached.key === cacheKey) {
|
||||
@@ -649,7 +696,11 @@ export function resolveStepImageMap(stepData: any): Map<any, ResolvedImageEntry>
|
||||
const map = new Map<any, ResolvedImageEntry>();
|
||||
const events: Array<{ item: any; act: any; timestamp: number; isPrimary: boolean }> = [];
|
||||
|
||||
if (stepData.action_taken) {
|
||||
const hasActionInTools = Array.isArray(stepData.generic_tools) && stepData.generic_tools.some(
|
||||
(t: any) => t && (t.type === 'action' || isAndroidAction(t) || isReportStatusAction(t))
|
||||
);
|
||||
|
||||
if (!hasActionInTools && stepData.action_taken) {
|
||||
events.push({
|
||||
item: stepData.action_taken,
|
||||
act: getActionObject(stepData.action_taken),
|
||||
@@ -661,11 +712,12 @@ export function resolveStepImageMap(stepData: any): Map<any, ResolvedImageEntry>
|
||||
if (Array.isArray(stepData.generic_tools)) {
|
||||
stepData.generic_tools.forEach((t: any) => {
|
||||
if (t) {
|
||||
const isAction = t.type === 'action' || isAndroidAction(t) || isReportStatusAction(t);
|
||||
events.push({
|
||||
item: t,
|
||||
act: getActionObject(t),
|
||||
timestamp: getEventTimestamp(t),
|
||||
isPrimary: false
|
||||
isPrimary: isAction
|
||||
});
|
||||
}
|
||||
});
|
||||
@@ -728,6 +780,19 @@ export function resolveStepImageMap(stepData: any): Map<any, ResolvedImageEntry>
|
||||
map.set(e.item, entry);
|
||||
if (act && act !== e.item) map.set(act, entry);
|
||||
if (act?.trace_id) map.set(act.trace_id, entry);
|
||||
if (e.item?.trace_id) map.set(e.item.trace_id, entry);
|
||||
}
|
||||
|
||||
// Ensure stepData.action_taken cross-resolves to the action images
|
||||
if (stepData.action_taken && lastActionItem) {
|
||||
const lastEntry = map.get(lastActionItem);
|
||||
if (lastEntry) {
|
||||
map.set(stepData.action_taken, lastEntry);
|
||||
const actObj = getActionObject(stepData.action_taken);
|
||||
if (actObj && actObj !== stepData.action_taken) {
|
||||
map.set(actObj, lastEntry);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stepImageMapCache.set(stepData, { key: cacheKey, map });
|
||||
@@ -885,7 +950,9 @@ export function extractStepReplayFrames(logsOrSteps: any[]): StepReplayFrame[] {
|
||||
const visualCandidates: VisualCandidate[] = [];
|
||||
|
||||
for (const stepData of stepsList) {
|
||||
const act = stepData.action_taken;
|
||||
const act = stepData.action_taken || (Array.isArray(stepData.generic_tools)
|
||||
? stepData.generic_tools.find((t: any) => t && (t.type === 'action' || isAndroidAction(t)))
|
||||
: null);
|
||||
const preUrl = getStepPreImageUrl(stepData, act);
|
||||
const postUrl = getStepPostImageUrl(stepData, act);
|
||||
const primaryImg = preUrl || postUrl;
|
||||
|
||||
@@ -349,3 +349,139 @@ describe('stream aggregator checker history relocation', () => {
|
||||
expect(checker.data.generic_tools.map((t: any) => t.trace_id)).toEqual(['tool-probe']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('stream aggregator single source of truth (Option A)', () => {
|
||||
it('does not duplicate action_taken when generic_tools already contains the action trace', () => {
|
||||
const events = getSortedStepEvents({
|
||||
timestamp: 100,
|
||||
action_taken: [{ action: 'launch_app', app_name: 'Maps', timestamp: 102 }],
|
||||
generic_tools: [
|
||||
{ trace_id: 'tool-note', type: 'tool', name: 'update_note', timestamp: 101 },
|
||||
{ trace_id: 'act-launch', type: 'action', name: 'launch_app', timestamp: 102, payload: { args: { action: { action: 'launch_app', app_name: 'Maps' } } } }
|
||||
]
|
||||
});
|
||||
|
||||
expect(events.length).toBe(2);
|
||||
expect(events.map(e => e.type)).toEqual(['tool', 'action']);
|
||||
expect(events.map(e => e.data.name)).toEqual(['update_note', 'launch_app']);
|
||||
});
|
||||
|
||||
it('retains multiple adb commands and tools alongside physical actions without dropping any', () => {
|
||||
const events = getSortedStepEvents({
|
||||
timestamp: 100,
|
||||
action_taken: [{ action: 'wait_for_delay', time_in_ms: 1000, timestamp: 105 }],
|
||||
generic_tools: [
|
||||
{ trace_id: 'adb-1', type: 'tool', name: 'run_adb_command', timestamp: 101, payload: { CommandLine: 'dumpsys telephony.registry' } },
|
||||
{ trace_id: 'adb-2', type: 'tool', name: 'run_adb_command', timestamp: 102, payload: { CommandLine: 'am start -a ...' } },
|
||||
{ trace_id: 'exp-1', type: 'tool', name: 'ask_explorer', timestamp: 103 },
|
||||
{ trace_id: 'act-wait', type: 'action', name: 'wait_for_delay', timestamp: 104, payload: { args: { time_in_ms: 1000 } } }
|
||||
]
|
||||
});
|
||||
|
||||
expect(events.length).toBe(4);
|
||||
expect(events.map(e => e.type)).toEqual(['tool', 'tool', 'tool', 'action']);
|
||||
expect(events.map(e => e.data.name)).toEqual(['run_adb_command', 'run_adb_command', 'ask_explorer', 'wait_for_delay']);
|
||||
});
|
||||
|
||||
it('supports multi-action bursts sequentially from generic_tools', () => {
|
||||
const events = getSortedStepEvents({
|
||||
timestamp: 100,
|
||||
action_taken: [{ action: 'tap', coordinates: [200, 300] }],
|
||||
generic_tools: [
|
||||
{ trace_id: 'tap-1', type: 'action', name: 'tap', timestamp: 101, payload: { args: { action: { action: 'tap', coordinates: [100, 200] } } } },
|
||||
{ trace_id: 'tap-2', type: 'action', name: 'tap', timestamp: 102, payload: { args: { action: { action: 'tap', coordinates: [200, 300] } } } }
|
||||
]
|
||||
});
|
||||
|
||||
expect(events.length).toBe(2);
|
||||
expect(events.every(e => e.type === 'action')).toBeTrue();
|
||||
expect(events[0].data.trace_id).toBe('tap-1');
|
||||
expect(events[1].data.trace_id).toBe('tap-2');
|
||||
});
|
||||
|
||||
it('correctly processes real trace 38fd934e Step 16 keeping ADB commands and wait_for_delay visible without duplicates', () => {
|
||||
const step16 = {
|
||||
step_id: '31e8c7ec-6b67-485e-94f4-355a853ccf8a',
|
||||
step_number: 16,
|
||||
action_taken: [{ action: 'wait_for_delay', time_in_ms: 1000 }],
|
||||
operator_native_thinking: 'Let me check telephony and send SMS...',
|
||||
operator_native_thinking_timestamp: 1788457630,
|
||||
timestamp: 1788457630,
|
||||
generic_tools: [
|
||||
{ trace_id: 't-adb-1', parent_trace_id: 't-op', name: 'run_adb_command', type: 'tool', timestamp: 1788457631, status: 'success', payload: { args: { CommandLine: 'dumpsys telephony.registry' } } },
|
||||
{ trace_id: 't-adb-1-sub', parent_trace_id: 't-adb-1', name: 'run_adb_command', type: 'tool', timestamp: 1788457631.5, status: 'success' },
|
||||
{ trace_id: 't-adb-2', parent_trace_id: 't-op', name: 'run_adb_command', type: 'tool', timestamp: 1788457632, status: 'success', payload: { args: { CommandLine: 'am start -a android.intent.action.SENDTO ...' } } },
|
||||
{ trace_id: 't-adb-2-sub', parent_trace_id: 't-adb-2', name: 'run_adb_command', type: 'tool', timestamp: 1788457632.5, status: 'success' },
|
||||
{ trace_id: 't-exp', parent_trace_id: 't-op', name: 'ask_explorer', type: 'tool', timestamp: 1788457633, status: 'success', payload: { args: { query: 'any text on screen' } } },
|
||||
{ trace_id: 't-exp-sub', parent_trace_id: 't-exp', name: 'ask_explorer', type: 'tool', timestamp: 1788457633.5, status: 'success' },
|
||||
{ trace_id: 't-wait', parent_trace_id: 't-val', name: 'wait_for_delay', type: 'action', timestamp: 1788457635, status: 'success', payload: { args: { action: { action: 'wait_for_delay', time_in_ms: 1000 } } } },
|
||||
{ trace_id: 't-safety', parent_trace_id: 't-wait', name: 'safety_net_pixel_validation', type: 'tool', timestamp: 1788457636, status: 'success' }
|
||||
]
|
||||
};
|
||||
|
||||
const events = getSortedStepEvents(step16);
|
||||
// 1 native thinking + 2 adb tools + 1 explorer tool + 1 wait_for_delay action
|
||||
expect(events.length).toBe(5);
|
||||
expect(events.map(e => e.type)).toEqual(['thinking', 'tool', 'tool', 'tool', 'action']);
|
||||
expect(events.map(e => e.data.name || e.data.action || 'thinking')).toEqual([
|
||||
'thinking',
|
||||
'run_adb_command',
|
||||
'run_adb_command',
|
||||
'ask_explorer',
|
||||
'wait_for_delay'
|
||||
]);
|
||||
});
|
||||
|
||||
it('correctly handles mid-stream reset and supersedes the reset block on retry', () => {
|
||||
// 1. Initial stream fails mid-stream and is marked reset
|
||||
const resetLogs = [
|
||||
{
|
||||
type: 'llm_stream',
|
||||
timestamp: '2026-09-03T10:00:01.000Z',
|
||||
data: {
|
||||
execution_id: 'exec-failed-1',
|
||||
stream_type: 'text',
|
||||
text: 'Here is the aborted plan...',
|
||||
isCompleted: false,
|
||||
isReset: true,
|
||||
resetMessage: 'A request error occurred during output generation, typically caused by lower API priority. Retrying automatically...'
|
||||
}
|
||||
}
|
||||
];
|
||||
|
||||
const initialBlocks = consolidateLogsToBlocks(resetLogs);
|
||||
expect(initialBlocks.length).toBe(1);
|
||||
expect(initialBlocks[0].data.isReset).toBeTrue();
|
||||
expect(initialBlocks[0].data.resetMessage).toBe('A request error occurred during output generation, typically caused by lower API priority. Retrying automatically...');
|
||||
|
||||
// getSortedStepEvents retains the reset state and message
|
||||
const resetEvents = getSortedStepEvents(initialBlocks[0].data);
|
||||
expect(resetEvents.length).toBe(1);
|
||||
expect(resetEvents[0].type).toBe('text');
|
||||
expect(resetEvents[0].data.isReset).toBeTrue();
|
||||
|
||||
// 2. Retry stream arrives without step_id (e.g. pre-planning retry)
|
||||
const retryLogs = [
|
||||
...resetLogs,
|
||||
{
|
||||
type: 'llm_stream',
|
||||
timestamp: '2026-09-03T10:00:03.000Z',
|
||||
data: {
|
||||
execution_id: 'exec-retry-2',
|
||||
stream_type: 'text',
|
||||
text: 'Here is the fresh completed plan.',
|
||||
isCompleted: true
|
||||
}
|
||||
}
|
||||
];
|
||||
|
||||
const updatedBlocks = consolidateLogsToBlocks(retryLogs);
|
||||
// Should supersede the reset block rather than creating an orphaned duplicate card
|
||||
expect(updatedBlocks.length).toBe(1);
|
||||
expect(updatedBlocks[0].data.execution_id).toBe('exec-retry-2');
|
||||
expect(updatedBlocks[0].data.isReset).toBeFalse();
|
||||
expect(updatedBlocks[0].data.operator_raw_thinking).toBe('Here is the fresh completed plan.');
|
||||
expect(updatedBlocks[0].data.isCompleted).toBeTrue();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -14,9 +14,9 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { StepBlock, PhaseBlock, StepEvent, StreamSegment } from '../core/models/stream.model';
|
||||
import { isAndroidAction, isReportStatusAction, getReportStatusExplanation, getReportStatusValue } from './action-formatter.util';
|
||||
import { getUniqueGenericTools } from './tool-formatter.util';
|
||||
import { StepBlock, PhaseBlock, StepEvent, StreamSegment, StreamResetNotice, DEFAULT_STREAM_RESET_MESSAGE } from '../core/models/stream.model';
|
||||
import { isAndroidAction, isReportStatusAction, getReportStatusExplanation, getReportStatusValue, getActionObject } from './action-formatter.util';
|
||||
import { getUniqueGenericTools, isInternalPlumbingTool } from './tool-formatter.util';
|
||||
|
||||
/**
|
||||
* Helper to safely extract milliseconds timestamp
|
||||
@@ -193,6 +193,8 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
|
||||
const streamType = log.data?.stream_type || 'text';
|
||||
const text = log.data?.text || '';
|
||||
const isCompleted = log.data?.isCompleted ?? false;
|
||||
const isReset = log.data?.isReset ?? false;
|
||||
const resetMessage = log.data?.resetMessage;
|
||||
|
||||
// The Checker's own reasoning streams under its agent trace: route it to
|
||||
// the attempt block, never to the Operator step that is current.
|
||||
@@ -215,7 +217,9 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
|
||||
stream_type: streamType === 'thinking' ? 'thinking' : 'text',
|
||||
text,
|
||||
timestamp: segmentIndex > -1 ? segments[segmentIndex].timestamp : log.timestamp,
|
||||
isCompleted
|
||||
isCompleted,
|
||||
isReset,
|
||||
resetMessage
|
||||
};
|
||||
if (segmentIndex > -1) segments[segmentIndex] = segment; else segments.push(segment);
|
||||
checkerData.stream_segments = segments;
|
||||
@@ -248,10 +252,42 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
|
||||
if (existingIndex === -1 && execId) {
|
||||
existingIndex = blocks.findIndex(b => b.data?.execution_id === execId || b.id === `stream-${execId}`);
|
||||
}
|
||||
// If no stepId and not matched by execId, check if there is an unattached stream block that was reset.
|
||||
// The retry stream supersedes that reset block rather than spawning an orphaned duplicate card.
|
||||
if (existingIndex === -1 && !stepId) {
|
||||
existingIndex = blocks.findIndex(b => b.type === 'llm_stream' && b.data?.isReset);
|
||||
}
|
||||
|
||||
if (existingIndex > -1) {
|
||||
const existing = blocks[existingIndex];
|
||||
const updatedData = { ...existing.data };
|
||||
const existingResets: StreamResetNotice[] =
|
||||
Array.isArray(updatedData.stream_resets) ? [...updatedData.stream_resets] : [];
|
||||
|
||||
if (isReset) {
|
||||
updatedData.isReset = true;
|
||||
updatedData.resetMessage = resetMessage;
|
||||
const msg = resetMessage || DEFAULT_STREAM_RESET_MESSAGE;
|
||||
const resetId = execId || `reset-${log.timestamp || updatedData.step_id || 'stream'}`;
|
||||
const existingResetIdx = existingResets.findIndex(r => r.id === resetId);
|
||||
if (existingResetIdx > -1) {
|
||||
existingResets[existingResetIdx] = { id: resetId, message: msg, isWaiting: true, streamType };
|
||||
} else {
|
||||
existingResets.push({ id: resetId, message: msg, isWaiting: true, streamType });
|
||||
}
|
||||
updatedData.stream_resets = existingResets;
|
||||
} else if (execId && updatedData.execution_id !== execId) {
|
||||
// New execution replacing a reset stream: mark prior resets as finished waiting
|
||||
updatedData.stream_resets = existingResets.map(r => ({ ...r, isWaiting: false }));
|
||||
updatedData.isReset = false;
|
||||
updatedData.resetMessage = undefined;
|
||||
updatedData.execution_id = execId;
|
||||
} else if (!isReset && text && text.length > 0 && existingResets.some(r => r.isWaiting)) {
|
||||
updatedData.stream_resets = existingResets.map(r => ({ ...r, isWaiting: false }));
|
||||
updatedData.isReset = false;
|
||||
updatedData.resetMessage = undefined;
|
||||
}
|
||||
|
||||
if (streamType === 'thinking') {
|
||||
updatedData.operator_native_thinking = text;
|
||||
updatedData.operator_native_thinking_timestamp =
|
||||
@@ -268,6 +304,9 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
|
||||
updatedData.execution_id = execId;
|
||||
}
|
||||
updatedData.isCompleted = isCompleted;
|
||||
if (isCompleted && updatedData.stream_resets) {
|
||||
updatedData.stream_resets = updatedData.stream_resets.map((r: any) => ({ ...r, isWaiting: false }));
|
||||
}
|
||||
|
||||
blocks[existingIndex] = {
|
||||
...existing,
|
||||
@@ -279,6 +318,14 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
|
||||
execution_id: execId,
|
||||
step_id: stepId,
|
||||
isCompleted: isCompleted,
|
||||
isReset: isReset,
|
||||
resetMessage: resetMessage,
|
||||
stream_resets: isReset ? [{
|
||||
id: execId || `reset-${log.timestamp || stepId || 'stream'}`,
|
||||
message: resetMessage || DEFAULT_STREAM_RESET_MESSAGE,
|
||||
isWaiting: true,
|
||||
streamType
|
||||
}] : [],
|
||||
generic_tools: []
|
||||
};
|
||||
if (streamType === 'thinking') {
|
||||
@@ -338,6 +385,9 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
|
||||
generic_tools: mergedTools,
|
||||
isCompleted: true
|
||||
};
|
||||
if (mergedData.stream_resets) {
|
||||
mergedData.stream_resets = mergedData.stream_resets.map((r: any) => ({ ...r, isWaiting: false }));
|
||||
}
|
||||
|
||||
blocks[existingIndex] = {
|
||||
...existingBlock,
|
||||
@@ -667,6 +717,9 @@ export function getSortedStepEvents(
|
||||
): StepEvent[] {
|
||||
if (!stepData || typeof stepData !== 'object') return [];
|
||||
const toolsLen = stepData.generic_tools ? stepData.generic_tools.length : 0;
|
||||
const toolsSignature = stepData.generic_tools && Array.isArray(stepData.generic_tools)
|
||||
? stepData.generic_tools.map((t: any) => `${t?.trace_id || t?.name || ''}:${t?.status || ''}`).join(',')
|
||||
: '';
|
||||
const actionTs = stepData.action_taken ? (stepData.action_taken.timestamp || stepData.action_taken.start_time || stepData.action_taken.created_at) : null;
|
||||
const nativeText = typeof stepData.operator_native_thinking === 'string'
|
||||
? stepData.operator_native_thinking.trim()
|
||||
@@ -679,11 +732,15 @@ export function getSortedStepEvents(
|
||||
: null;
|
||||
const signature = [
|
||||
toolsLen,
|
||||
toolsSignature,
|
||||
actionTs ?? '',
|
||||
stepData.operator_native_thinking_timestamp ?? '',
|
||||
stepData.operator_raw_thinking_timestamp ?? '',
|
||||
nativeText.length,
|
||||
rawText.length,
|
||||
stepData.isReset ? '1' : '0',
|
||||
stepData.resetMessage ?? '',
|
||||
stepData.stream_resets ? stepData.stream_resets.map((r: any) => `${r.id}:${r.isWaiting ? 1 : 0}`).join(',') : '',
|
||||
segments ? segments.map((s) => `${s.execution_id}:${s.stream_type}:${s.text.length}`).join(',') : ''
|
||||
].join('|');
|
||||
|
||||
@@ -704,38 +761,58 @@ export function getSortedStepEvents(
|
||||
// Multi-turn agent (Checker): one event per streamed turn, each at the
|
||||
// time its first chunk arrived, so turns sort between the tool calls.
|
||||
segments.forEach((segment) => {
|
||||
if (!segment.text || !segment.text.trim()) return;
|
||||
if ((!segment.text || !segment.text.trim()) && !segment.isReset) return;
|
||||
events.push({
|
||||
type: segment.stream_type === 'thinking' ? 'thinking' : 'text',
|
||||
data: { text: segment.text, execution_id: segment.execution_id, segment: true },
|
||||
data: {
|
||||
text: segment.text,
|
||||
execution_id: segment.execution_id,
|
||||
segment: true,
|
||||
isReset: segment.isReset,
|
||||
resetMessage: segment.resetMessage
|
||||
},
|
||||
timestamp: getItemTimestamp(segment.timestamp, fallbackTime),
|
||||
sequence: sequence++
|
||||
});
|
||||
});
|
||||
} else {
|
||||
if (nativeText) {
|
||||
const hasThinkingResets = Array.isArray(stepData.stream_resets) && stepData.stream_resets.some((r: any) => r.streamType === 'thinking');
|
||||
if (nativeText || hasThinkingResets) {
|
||||
events.push({
|
||||
type: 'thinking',
|
||||
data: { text: stepData.operator_native_thinking },
|
||||
data: { text: stepData.operator_native_thinking || '' },
|
||||
timestamp: getItemTimestamp(stepData.operator_native_thinking_timestamp, fallbackTime),
|
||||
sequence: sequence++
|
||||
});
|
||||
}
|
||||
|
||||
if (rawText) {
|
||||
if (rawText || (stepData.isReset && stepData.resetMessage) || (Array.isArray(stepData.stream_resets) && stepData.stream_resets.length > 0)) {
|
||||
events.push({
|
||||
type: 'text',
|
||||
data: { text: stepData.operator_raw_thinking },
|
||||
data: {
|
||||
text: stepData.operator_raw_thinking || '',
|
||||
isReset: stepData.isReset,
|
||||
resetMessage: stepData.resetMessage
|
||||
},
|
||||
timestamp: getItemTimestamp(stepData.operator_raw_thinking_timestamp, fallbackTime),
|
||||
sequence: sequence++
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Add action_taken if present and valid.
|
||||
if (stepData.action_taken && (isAndroidAction(stepData.action_taken) || isReportStatusAction(stepData.action_taken))) {
|
||||
// 2. Single Source of Truth & Fallback Resolution:
|
||||
const uniqueTools = getUniqueGenericTools(stepData.generic_tools);
|
||||
const hasActionInTools = uniqueTools.some((tool: any) =>
|
||||
tool && (tool.type === 'action' || isAndroidAction(tool) || isReportStatusAction(tool))
|
||||
);
|
||||
|
||||
// Fallback: only add action_taken if generic_tools did NOT already contain an action trace.
|
||||
// Pushed before uniqueTools so that when timestamps are missing/equal, the primary action
|
||||
// preserves deterministic sequence ordering ahead of generic fallback tools.
|
||||
if (!hasActionInTools && stepData.action_taken && (isAndroidAction(stepData.action_taken) || isReportStatusAction(stepData.action_taken))) {
|
||||
const actObj = getActionObject(stepData.action_taken);
|
||||
const actTime = getItemTimestamp(
|
||||
stepData.action_taken.timestamp ?? stepData.action_taken.start_time ?? stepData.action_taken.created_at,
|
||||
actObj?.timestamp ?? actObj?.start_time ?? actObj?.created_at,
|
||||
fallbackTime
|
||||
);
|
||||
events.push({
|
||||
@@ -746,15 +823,19 @@ export function getSortedStepEvents(
|
||||
});
|
||||
}
|
||||
|
||||
// Add generic tools if present.
|
||||
const uniqueTools = getUniqueGenericTools(stepData.generic_tools);
|
||||
// Add all real tool calls and actions from the event stream.
|
||||
uniqueTools.forEach((tool: any) => {
|
||||
if (isInternalPlumbingTool(tool) && !isReportStatusAction(tool)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const isAction = tool.type === 'action' || isAndroidAction(tool) || isReportStatusAction(tool);
|
||||
const toolTime = getItemTimestamp(
|
||||
tool.timestamp ?? tool.start_time ?? tool.created_at,
|
||||
fallbackTime
|
||||
);
|
||||
events.push({
|
||||
type: 'tool',
|
||||
type: isAction ? 'action' : 'tool',
|
||||
data: tool,
|
||||
timestamp: toolTime,
|
||||
sequence: sequence++
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import {
|
||||
cleanErrorMessage,
|
||||
getUniqueGenericTools,
|
||||
getVideoAnalysisView
|
||||
getVideoAnalysisView,
|
||||
shouldShowTool
|
||||
} from './tool-formatter.util';
|
||||
|
||||
describe('cleanErrorMessage', () => {
|
||||
@@ -164,3 +165,38 @@ describe('video analysis timeline formatting', () => {
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('shouldShowTool (Option A Single Source of Truth)', () => {
|
||||
it('keeps ADB commands visible even when stepData.action_taken is present', () => {
|
||||
const adbTool = {
|
||||
name: 'run_adb_command',
|
||||
type: 'tool',
|
||||
payload: { CommandLine: 'dumpsys telephony.registry' }
|
||||
};
|
||||
const stepData = {
|
||||
action_taken: [{ action: 'wait_for_delay', time_in_ms: 1000 }]
|
||||
};
|
||||
|
||||
expect(shouldShowTool(adbTool, stepData)).toBeTrue();
|
||||
});
|
||||
|
||||
it('keeps explorer calls visible when stepData.action_taken is present', () => {
|
||||
const explorerTool = {
|
||||
name: 'ask_explorer',
|
||||
type: 'tool'
|
||||
};
|
||||
const stepData = {
|
||||
action_taken: [{ action: 'wait_for_delay', time_in_ms: 1000 }]
|
||||
};
|
||||
|
||||
expect(shouldShowTool(explorerTool, stepData)).toBeTrue();
|
||||
});
|
||||
|
||||
it('filters out internal plumbing tools', () => {
|
||||
const plumbingTool = {
|
||||
name: 'safety_net_pixel_validation',
|
||||
type: 'tool'
|
||||
};
|
||||
expect(shouldShowTool(plumbingTool)).toBeFalse();
|
||||
});
|
||||
});
|
||||
@@ -15,7 +15,6 @@
|
||||
*/
|
||||
|
||||
import { ActionParam } from '../core/models/stream.model';
|
||||
import { isAndroidAction, isReportStatusAction } from './action-formatter.util';
|
||||
import { extractNumbersFromCoordinateValue, isPureDirectionString, parseSequenceCoordinates } from './image-overlay.util';
|
||||
|
||||
// Tool objects are replaced (not mutated) when a trace is updated, so a
|
||||
@@ -276,24 +275,12 @@ export function isInternalPlumbingTool(tool: any): boolean {
|
||||
/**
|
||||
* Determines whether a tool invocation should be displayed in the timeline.
|
||||
*/
|
||||
export function shouldShowTool(tool: any, stepData?: any): boolean {
|
||||
export function shouldShowTool(tool: any, _stepData?: any): boolean {
|
||||
if (!tool || !tool.name) return false;
|
||||
if (tool.type === 'llm_call') return false;
|
||||
if (tool.type === 'agent') return false;
|
||||
if (isInternalPlumbingTool(tool)) return false;
|
||||
|
||||
// If this tool is an operator/validator atomic action that duplicates stepData.action_taken, skip it
|
||||
if (stepData && stepData.action_taken && isAndroidAction(stepData.action_taken) && isDeviceActionTool(tool)) {
|
||||
const agentName = (tool.agent_name || tool.agent || '').toLowerCase();
|
||||
const isSelfHealingOrSubAgent = agentName.includes('failure') ||
|
||||
agentName.includes('analyzer') ||
|
||||
agentName.includes('diagnos') ||
|
||||
(agentName && agentName !== 'operator' && agentName !== 'flashrunner' && agentName !== 'validator');
|
||||
if (!isSelfHealingOrSubAgent) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -16,6 +16,11 @@
|
||||
|
||||
@import url('https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700;800&family=Outfit:wght@300;400;500;600;700&family=Roboto:wght@400;500;700&display=swap');
|
||||
@import url('https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:opsz,wght,FILL,GRAD@20..48,100..700,0..1,-50..200');
|
||||
@import '@angular/cdk/overlay-prebuilt.css';
|
||||
|
||||
.cdk-overlay-container {
|
||||
z-index: 1200 !important;
|
||||
}
|
||||
|
||||
*, *::before, *::after {
|
||||
box-sizing: border-box;
|
||||
|
||||
Reference in new issue
Block a user