chenmingxuan/Artemis-Eapil
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 32 Wiki Activity

feat: harden video analysis and runtime resilience

Browse Source
This commit is contained in:
somew1nd committed 2026-08-25 20:58:25 -07:00
1 parent 5ada543a09
commit bb16991f2c
55 files changed
+5023 -437
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.html
+46 -38
View File
@@ -290,14 +290,19 @@
<!-- Event Logs Section -->
<main class="stream-logs-content" (scroll)="onStreamScroll($event)">
<div *ngIf="consolidatedBlocks().length === 0 && !showPlanningLoader() && startupWorkItems().length === 0" class="empty-state">
<div *ngIf="agentService.isSessionContentLoading()" class="session-loading-state" aria-live="polite">
<span class="material-symbols-outlined spin-animation">progress_activity</span>
<span>Loading run history...</span>
</div>
<div *ngIf="!agentService.isSessionContentLoading() && consolidatedBlocks().length === 0 && !showPlanningLoader() && startupWorkItems().length === 0" class="empty-state">
<div class="empty-icon"><span class="material-symbols-outlined">sensors</span></div>
<h3>No Stream Activity Selected</h3>
<p>Please select a session from the Task List on the right to view its real-time log payloads.</p>
</div>
<!-- The phase-based session logs list -->
<div class="logs-list-container" *ngIf="phases().length > 0 || showPlanningLoader() || startupWorkItems().length > 0">
<div class="logs-list-container" *ngIf="!agentService.isSessionContentLoading() && (phases().length > 0 || showPlanningLoader() || startupWorkItems().length > 0)">
<div class="phase-container startup-phase" *ngIf="startupWorkItems().length > 0" aria-live="polite">
<div class="phase-header first-phase">
<span class="phase-worked-time">Worked for {{ startupPreparationDuration() }}s</span>
@@ -343,37 +348,38 @@
>
<!-- Unified Step / Stream Body -->
<div class="step-body">
<!-- Native Thinking (Thought) -->
<div *ngIf="getNativeThinking(block) as nativeText"
class="thinking-stream-details native-thinking-details"
[class.is-running]="isBlockRunning(block)"
[class.collapsed]="collapsedStreams().has(block.data.execution_id || block.data.step_id || block.id)">
<div class="details-summary" (click)="toggleCollapse(block.data.execution_id || block.data.step_id || block.id)">
<span class="stream-status-dot" [class.streaming]="isBlockRunning(block)"></span>
Thought
</div>
<div class="details-content-wrapper">
<div class="details-content stream-box">
<div class="stream-text" [innerHTML]="renderMarkdown(typedTextsSignal()[block.id + '-native'] || nativeText)"></div>
</div>
</div>
</div>
<!-- Raw Thinking / Plan / Message (Work) -->
<div *ngIf="getRawThinking(block) as rawText" class="step-raw-thinking-message" [class.is-running]="isBlockRunning(block)">
<div class="report-header">
<span class="stream-status-dot" [class.streaming]="isBlockRunning(block)"></span>
Work
</div>
<div class="report-content stream-box">
<div class="stream-text" [innerHTML]="renderMarkdown(typedTextsSignal()[block.id] || rawText)"></div>
</div>
</div>
<!-- Tool Call / Action section (Unified Chronological Loop matching Debug UI execution order) -->
<div class="step-actions-container" *ngIf="(block.data.generic_tools && block.data.generic_tools.length > 0) || block.data.action_taken">
<!-- Text, tool calls, and actions share one chronological timeline. -->
<div class="step-actions-container" *ngIf="getSortedStepEvents(block.data).length > 0">
<ng-container *ngFor="let item of getSortedStepEvents(block.data); let itemIdx = index; trackBy: trackStepEvent">
<!-- Native Thinking (Thought) -->
<div *ngIf="item.type === 'thinking'"
class="thinking-stream-details native-thinking-details"
[class.is-running]="isBlockRunning(block)"
[class.collapsed]="collapsedStreams().has(block.data.execution_id || block.data.step_id || block.id)">
<div class="details-summary" (click)="toggleCollapse(block.data.execution_id || block.data.step_id || block.id)">
<span class="stream-status-dot" [class.streaming]="isBlockRunning(block)"></span>
Thought
</div>
<div class="details-content-wrapper">
<div class="details-content stream-box">
<div class="stream-text" [innerHTML]="renderMarkdown(typedTextsSignal()[block.id + '-native'] || item.data.text)"></div>
</div>
</div>
</div>
<!-- Raw Thinking / Plan / Message (Work) -->
<div *ngIf="item.type === 'text' && getRawThinking(block)"
class="step-raw-thinking-message"
[class.is-running]="isBlockRunning(block)">
<div class="report-header">
<span class="stream-status-dot" [class.streaming]="isBlockRunning(block)"></span>
Work
</div>
<div class="report-content stream-box">
<div class="stream-text" [innerHTML]="renderMarkdown(typedTextsSignal()[block.id] || item.data.text)"></div>
</div>
</div>
<!-- Task Submission / Report Card (when item is report_task_status action or tool) -->
<div class="step-android-action-card task-report-card"
[class.is-completed]="getReportStatusValue(item.data) === 'completed' || getReportStatusValue(item.data) === 'success'"
@@ -630,11 +636,12 @@
[class.is-running]="isItemRunning(item, block)"
[class.is-note-tool]="isNoteTool(item.data)"
[class.is-video-tool]="isVideoTool(item.data)"
[class.is-video-attention]="isVideoTool(item.data) && isVideoAnalysisAttention(item.data)"
*ngIf="shouldShowTool(item.data, block.data) && !isDeviceActionTool(item.data)">
<div class="tool-call-label">
<span class="stream-status-dot" [class.streaming]="isItemRunning(item, block)"></span>
<div class="tool-call-text">
<span>{{ getToolDisplayLabel(item.data) }}{{ ((isNoteTool(item.data) && getToolKey(item.data)) || (isVideoTool(item.data) && getVideoToolTarget(item.data))) ? ':' : '' }}</span>
<span>{{ isVideoTool(item.data) ? getVideoAnalysisLabel(item.data) : getToolDisplayLabel(item.data) }}{{ ((isNoteTool(item.data) && getToolKey(item.data)) || isVideoTool(item.data)) ? ':' : '' }}</span>
<!-- Note Tool Pill -->
<ng-container *ngIf="isNoteTool(item.data) && getToolKey(item.data) as key">
@@ -649,15 +656,16 @@
</ng-container>
<!-- Video Tool Pill -->
<ng-container *ngIf="isVideoTool(item.data) && getVideoToolTarget(item.data) as videoTarget">
<span
class="tool-key-pill video-pill"
<ng-container *ngIf="isVideoTool(item.data)">
<span
class="tool-key-pill"
(click)="onVideoToolClick(item.data)"
title="Click to open video player"
[title]="'Open recording at ' + getVideoAnalysisRangeLabel(item.data)"
>
<span class="material-symbols-outlined pill-icon">videocam</span>
{{ videoTarget }}
{{ getVideoAnalysisRangeLabel(item.data) }}
</span>
<span class="video-status-detail" *ngIf="getVideoAnalysisDetail(item.data) as detail">· {{ detail }}</span>
</ng-container>
</div>
</div>
apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.scss
+30 -10
View File
@@ -20,6 +20,7 @@ $text-secondary: #1f1f1f;
$text-muted: #1f1f1f;
$bg-main: #f9fafb;
$uniform-gap: 8px;
$stream-row-height: 25px;
:host {
display: flex;
@@ -957,6 +958,20 @@ $uniform-gap: 8px;
overflow-y: auto;
padding: 72px 24px 100px 24px;
position: relative;
.session-loading-state {
min-height: 180px;
display: flex;
align-items: center;
justify-content: center;
gap: 9px;
color: #64748b;
font-size: 13px;
.material-symbols-outlined {
font-size: 18px;
}
}
display: flex;
flex-direction: column;
@@ -1879,7 +1894,8 @@ $uniform-gap: 8px;
padding: 0;
.llm-retry-note {
min-height: 20px;
min-height: $stream-row-height;
box-sizing: border-box;
display: flex;
align-items: center;
gap: 8px;
@@ -1924,6 +1940,14 @@ $uniform-gap: 8px;
.step-generic-tool {
border: 1px solid transparent;
&.is-video-attention:not(.is-running) {
.tool-call-label .stream-status-dot {
background: #f59e0b;
box-shadow: none;
animation: none;
}
}
&.is-llm-error {
.tool-call-label {
display: flex;
@@ -1994,7 +2018,7 @@ $uniform-gap: 8px;
font-family: 'Google Sans', 'Google Sans Text', 'Inter', Arial, sans-serif;
letter-spacing: 0.5px;
line-height: 1.5;
min-height: 20px;
min-height: 22px;
box-sizing: border-box;
display: inline-flex;
align-items: center;
@@ -2065,15 +2089,11 @@ $uniform-gap: 8px;
color: #1557b0;
background: rgba(26, 115, 232, 0.08);
}
}
&.video-pill {
color: #0284c7;
&:hover {
color: #0369a1;
background: rgba(2, 132, 199, 0.08);
}
}
.video-status-detail {
color: inherit;
margin-left: 2px;
}
}
}
apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.ts
+35 -3
View File
@@ -153,6 +153,8 @@ import {
getToolArgs,
isNoteTool,
isVideoTool,
getVideoAnalysisView,
formatVideoTime,
getVideoToolTarget,
isDeviceActionTool,
isFailureAnalyzerActionTool,
@@ -240,7 +242,7 @@ export class AgentStreamComponent implements AfterViewInit {
// Performance caches for parameter and event extractions
private actionParamsCache = new WeakMap<any, ActionParam[]>();
private toolParamsCache = new WeakMap<any, ActionParam[]>();
private sortedEventsCache = new WeakMap<any, { length: number; actionTs: any; events: StepEvent[] }>();
private sortedEventsCache = new WeakMap<any, { signature: string; events: StepEvent[] }>();
// Top Nav Task Queue Computed Properties
public activeQueue = computed(() => {
@@ -897,10 +899,40 @@ export class AgentStreamComponent implements AfterViewInit {
return getVideoToolTarget(tool);
}
public getVideoAnalysisLabel(tool: any): string {
return getVideoAnalysisView(tool)?.title || 'Analyzing screen recording';
}
public getVideoAnalysisRangeLabel(tool: any): string {
const range = getVideoAnalysisView(tool)?.requestedRange;
if (!range) return 'Screen recording';
return `${formatVideoTime(range.start)}–${formatVideoTime(range.end)}`;
}
public getVideoAnalysisDetail(tool: any): string {
const view = getVideoAnalysisView(tool);
if (!view || view.totalCount <= 1) return '';
if (view.outcome === 'running' || view.outcome === 'recovering') {
return view.completedCount > 0
? `${view.completedCount}/${view.totalCount} segments saved`
: '';
}
if (view.outcome === 'partial') {
return `${view.completedCount}/${view.totalCount} segments saved`;
}
return '';
}
public isVideoAnalysisAttention(tool: any): boolean {
const outcome = getVideoAnalysisView(tool)?.outcome;
return outcome === 'partial' || outcome === 'failed';
}
public onVideoToolClick(toolData: any): void {
const curSessionId = this.agentService.currentSessionId();
if (curSessionId) {
this.agentService.openVideoPlayer(curSessionId);
const start = getVideoAnalysisView(toolData)?.requestedRange?.start;
this.agentService.openVideoPlayer(curSessionId, undefined, undefined, start);
}
}
@@ -1231,7 +1263,7 @@ export class AgentStreamComponent implements AfterViewInit {
public trackStepEvent(index: number, item: { type: string; data: any }): string {
if (!item || !item.data) return index.toString();
return item.type + '-' + (item.data.timestamp || item.data.start_time || item.data.created_at || item.data.id || item.data.action || item.data.name || index);
return item.type + '-' + (item.data.timestamp || item.data.start_time || item.data.created_at || item.data.id || item.data.action || item.data.name || item.data.text?.length || index);
}
public trackParam(index: number, param: { key: string; value: string }): string {
apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.ts
+22
View File
@@ -58,6 +58,7 @@ export class FloatingVideoPlayerComponent {
});
private pendingLocalTime: number | null = null;
private pendingAutoplay = false;
private pendingAbsoluteSeek: number | null = null;
// Available speed options
public speedOptions = [0.5, 1.0, 1.5, 2.0, 4.0];
@@ -90,6 +91,19 @@ export class FloatingVideoPlayerComponent {
},
{ allowSignalWrites: true }
);
effect(() => {
const request = this.agentService.videoSeekRequest();
if (!request) return;
this.pendingAbsoluteSeek = request.seconds;
if (this.agentService.recordingPlaybackStatus() !== 'ready') return;
const video = this.videoRef?.nativeElement;
if (video && video.readyState >= 1) {
const target = this.pendingAbsoluteSeek;
this.pendingAbsoluteSeek = null;
this.seek(target, true);
}
});
}
public onLiveStreamError(): void {
@@ -194,6 +208,14 @@ export class FloatingVideoPlayerComponent {
v.playbackRate = this.playbackRate();
v.loop = this.isLooping() && !segments.length;
v.muted = this.isMuted();
if (this.pendingAbsoluteSeek !== null) {
const target = this.pendingAbsoluteSeek;
this.pendingAbsoluteSeek = null;
this.seek(target, true);
// A seek into another finalized segment changes the video source. Let
// that segment's metadata event apply pendingLocalTime.
if (this.pendingLocalTime !== null) return;
}
if (this.pendingLocalTime !== null) {
v.currentTime = Math.max(0, Math.min(v.duration || this.pendingLocalTime, this.pendingLocalTime));
this.pendingLocalTime = null;
apps/showcase_ui/src/app/core/models/stream.model.ts
+1 -1
View File
@@ -39,7 +39,7 @@ export interface ActionExecution {
}
export interface StepEvent {
type: 'action' | 'tool';
type: 'thinking' | 'text' | 'action' | 'tool';
data: any;
timestamp?: number;
}
apps/showcase_ui/src/app/pages/home/home.component.html
+4 -4
View File
@@ -109,7 +109,7 @@
<div class="guide-card-body">
<!-- One-Click Dependency Installer Bar when any env dep is missing -->
@if (!isEnvironmentReady()) {
@if (hasReadinessReport() && !isEnvironmentReady()) {
<div class="one-click-install-bar">
<div class="install-bar-left">
<div class="install-bar-icon-box">
@@ -154,7 +154,7 @@
</div>
<div class="item-title-group">
<div class="item-name">Python Runtime</div>
<div class="item-subtitle">{{ pythonProbe()?.summary || 'Python 3.12+' }}</div>
<div class="item-subtitle">{{ pythonProbe()?.summary || 'Checking…' }}</div>
</div>
<div class="item-status">
<span class="status-dot" [class.dot-pass]="pythonProbe()?.status === 'pass'" [class.dot-fail]="pythonProbe()?.status === 'fail'"></span>
@@ -187,7 +187,7 @@
</div>
<div class="item-title-group">
<div class="item-name">Android Platform-Tools</div>
<div class="item-subtitle">{{ adbProbe()?.metadata?.['installed'] ? 'ADB Installed' : 'ADB Missing' }}</div>
<div class="item-subtitle">{{ !adbProbe() ? 'Checking…' : (adbProbe()?.metadata?.['installed'] ? 'ADB Installed' : 'ADB Missing') }}</div>
</div>
<div class="item-status">
<span class="status-dot" [class.dot-pass]="adbProbe()?.metadata?.['installed']" [class.dot-fail]="adbProbe()?.status === 'fail'"></span>
@@ -1497,7 +1497,7 @@
<section class="task-launcher-view">
<!-- Prerequisites Drop Alert if any blocker probe is not passing -->
@if (!isReady()) {
@if (hasReadinessReport() && !isReady()) {
<div class="launcher-warning-bar" (click)="setTab('diagnostics')">
<span class="material-symbols-outlined warn-icon">{{ adbProbe()?.summary === 'Device Locked' ? 'lock' : 'warning' }}</span>
<span>
apps/showcase_ui/src/app/pages/home/home.component.ts
+2 -1
View File
@@ -140,6 +140,7 @@ export class HomeComponent implements OnInit, OnDestroy {
// Computed helper states delegating to SystemService
public isReady = computed(() => this.systemService.isReady());
public hasReadinessReport = computed(() => this.systemService.hasReadinessReport());
public isLoading = computed(() => this.systemService.isLoading());
public isRestartingAdb = computed(() => this.systemService.isRestartingAdb());
public launchingAvd = computed(() => this.systemService.launchingAvd());
@@ -591,7 +592,7 @@ export class HomeComponent implements OnInit, OnDestroy {
public refreshReadiness(): void {
this.isRefreshingDiagnostics.set(true);
this.systemService.fetchReadiness().subscribe({
this.systemService.fetchReadiness(false, true).subscribe({
next: () => {
this.systemService.fetchModelConfigEnv().subscribe({
next: () => {
apps/showcase_ui/src/app/services/agent.service.spec.ts
+24
View File
@@ -7,8 +7,13 @@ describe('AgentService live LLM retry timeline', () => {
function createServiceWithoutPolling(): AgentService {
const service = Object.create(AgentService.prototype) as AgentService;
service.sessionLogs = signal<any[]>([]);
service.isSessionContentLoading = signal(false);
service.startupProgressBySession = signal({});
(service as any).pendingStartupProgress = signal<any[]>([]);
(service as any).sessionLoadGeneration = 0;
(service as any).sessionSnapshotRequestId = 0;
(service as any).sessionSnapshotAppliedId = 0;
(service as any).pendingSnapshotRequests = new Set<number>();
return service;
}
@@ -108,6 +113,7 @@ describe('AgentService live LLM retry timeline', () => {
{ type: 'step_updated', history_snapshot: true, data: { step_id: 'old-step' } },
{ type: 'trace_recorded', data: { trace_id: 'live-retry', name: 'llm_retry' } }
]);
service.isSessionContentLoading.set(true);
(service as any).backfillSessionSteps('session-1');
@@ -116,6 +122,7 @@ describe('AgentService live LLM retry timeline', () => {
expect(logs[0].history_snapshot).toBeTrue();
expect(logs[0].data.step_id).toBe('step-1');
expect(logs[1].data.trace_id).toBe('live-retry');
expect(service.isSessionContentLoading()).toBeFalse();
});
it('follows a just-started task even when status polling saw it first', () => {
@@ -248,3 +255,20 @@ describe('AgentService recording finalization lifecycle', () => {
expect(service.recordingPlaybackMessage()).toBe('ffmpeg failed');
});
});
describe('AgentService video analysis seeking', () => {
it('publishes repeatable, clamped seek requests for the floating player', () => {
const service = Object.create(AgentService.prototype) as AgentService;
service.videoSeekRequest = signal<{ seconds: number; requestId: number } | null>(null);
(service as any).videoSeekRequestId = 0;
service.requestVideoSeek(-5);
const first = service.videoSeekRequest();
service.requestVideoSeek(0);
const second = service.videoSeekRequest();
expect(first?.seconds).toBe(0);
expect(second?.seconds).toBe(0);
expect(second?.requestId).toBeGreaterThan(first?.requestId || 0);
});
});
apps/showcase_ui/src/app/services/agent.service.ts
+96 -8
View File
@@ -22,6 +22,8 @@ import { Session, ModelInfo, TaskQueueItem, AgentStatusResponse } from '../core/
import { StepItemData } from '../core/models/stream.model';
export type { Session, ModelInfo, TaskQueueItem, AgentStatusResponse, StepItemData };
const SESSION_CACHE_KEY = 'artemis.sessions.v1';
export interface VideoSegment {
url: string;
start: number;
@@ -118,6 +120,7 @@ export class AgentService {
public currentSessionId = signal<string | null>(null);
public sessionLogs = signal<any[]>([]); // Dynamic array of all raw events received
public isSessionContentLoading = signal<boolean>(false);
public agentStatus = signal<string>('idle'); // Status of the agent runner process
public runningSessionId = signal<string | null>(null);
public runningGoal = signal<string | null>(null);
@@ -154,7 +157,9 @@ export class AgentService {
public recordingPlaybackStatus = signal<RecordingPlaybackStatus>('idle');
public recordingPlaybackMessage = signal<string>('');
public shouldAutoplayVideo = signal<boolean>(false);
public videoSeekRequest = signal<{ seconds: number; requestId: number } | null>(null);
private activeVideoSessionId: string | null = null;
private videoSeekRequestId = 0;
private videoRequestGeneration = 0;
private videoRetryTimer: ReturnType<typeof setTimeout> | null = null;
private videoWaitStartedAt = 0;
@@ -204,8 +209,13 @@ export class AgentService {
private eventSource: EventSource | null = null;
private statusInterval: any = null;
private sessionLoadGeneration = 0;
private sessionSnapshotRequestId = 0;
private sessionSnapshotAppliedId = 0;
private pendingSnapshotRequests = new Set<number>();
constructor() {
this.restoreSessionsCache();
this.fetchSessions();
this.startStatusPolling();
}
@@ -359,6 +369,7 @@ export class AgentService {
this.http.get<Session[]>('/api/sessions').subscribe({
next: (data) => {
this.rawSessions.set(data);
this.persistSessionsCache(data);
this.updateModelForCurrentView(data);
// On initial load, if nothing is selected, not pinned, not running, and sessions exist, select latest
if (!this.currentSessionId() && !this.userPinnedSessionId() && this.agentStatus() !== 'running' && data.length > 0) {
@@ -377,6 +388,7 @@ export class AgentService {
public deleteSession(sessionId: string): Observable<any> {
// 1. Optimistically update local session state immediately
this.rawSessions.update((list) => list.filter((s) => s.session_id !== sessionId));
this.persistSessionsCache(this.rawSessions());
this.pendingQueue.update((list) => list.filter((s) => s.session_id !== sessionId));
if (this.userPinnedSessionId() === sessionId) {
@@ -420,6 +432,7 @@ export class AgentService {
// 1. Optimistically clear all local session state immediately
this.userPinnedSessionId.set(null);
this.rawSessions.set([]);
this.clearSessionsCache();
this.pendingQueue.set([]);
this.selectSession('', false);
if (this.isVideoWindowOpen()) {
@@ -464,6 +477,9 @@ export class AgentService {
}
if (!sessionId) {
this.sessionLoadGeneration++;
this.pendingSnapshotRequests.clear();
this.isSessionContentLoading.set(false);
this.currentSessionId.set(null);
this.sessionLogs.set([]);
this.currentNotes.set({});
@@ -478,6 +494,10 @@ export class AgentService {
return; // Already selected
}
const loadGeneration = ++this.sessionLoadGeneration;
this.pendingSnapshotRequests.clear();
this.sessionSnapshotAppliedId = 0;
this.isSessionContentLoading.set(true);
this.currentSessionId.set(sessionId);
this.sessionLogs.set([]); // Reset logs for new session selection
this.activePauseCardKey = null;
@@ -498,14 +518,22 @@ export class AgentService {
}
console.log(`Connecting to session: ${sessionId}`);
// Start reading the persisted snapshot immediately. Previously this waited
// for the SSE `info` event, adding connection latency to every history click.
this.backfillSessionSteps(sessionId, loadGeneration);
this.eventSource = new EventSource(`/api/stream/${sessionId}`);
// The server emits info only after this client has been registered as an
// SSE subscriber. Fetching the persisted snapshot after that acknowledgement
// closes the snapshot-to-stream gap: earlier events are in history and later
// events are already queued for this connection.
// Active runs get one reconciliation snapshot after the stream subscribes,
// closing the snapshot-to-stream gap without delaying the first paint.
const selectedStatus = this.sessions().find((session) => session.session_id === sessionId)?.status;
const shouldReconcileAfterSubscribe = selectedStatus === 'running'
|| selectedStatus === 'paused'
|| selectedStatus === 'pending';
this.eventSource.addEventListener('info', () => {
this.backfillSessionSteps(sessionId);
if (shouldReconcileAfterSubscribe) {
this.backfillSessionSteps(sessionId, loadGeneration);
}
});
// Listen to standard SSE keep-alive
@@ -804,10 +832,18 @@ export class AgentService {
* from SSE. Snapshot logs are replaceable, while live logs remain append-only;
* the stream aggregator merges matching steps and trace IDs within each step.
*/
private backfillSessionSteps(sessionId: string): void {
private backfillSessionSteps(sessionId: string, loadGeneration = this.sessionLoadGeneration): void {
const requestId = ++this.sessionSnapshotRequestId;
this.pendingSnapshotRequests.add(requestId);
this.http.get<StepItemData[]>(`/api/sessions/${sessionId}/steps`).subscribe({
next: (steps) => {
if (this.currentSessionId() !== sessionId) return;
this.pendingSnapshotRequests.delete(requestId);
if (
this.currentSessionId() !== sessionId
|| loadGeneration !== this.sessionLoadGeneration
|| requestId < this.sessionSnapshotAppliedId
) return;
this.sessionSnapshotAppliedId = requestId;
const historicalLogs = steps.map((step) => ({
type: 'step_updated',
session_id: sessionId,
@@ -820,13 +856,51 @@ export class AgentService {
...historicalLogs,
...logs.filter((log) => !log.history_snapshot)
]);
this.isSessionContentLoading.set(false);
},
error: (err) => {
this.pendingSnapshotRequests.delete(requestId);
console.error('Failed to backfill session steps:', err);
if (
this.currentSessionId() === sessionId
&& loadGeneration === this.sessionLoadGeneration
&& this.pendingSnapshotRequests.size === 0
) {
this.isSessionContentLoading.set(false);
}
}
});
}
private restoreSessionsCache(): void {
try {
const cached = localStorage.getItem(SESSION_CACHE_KEY);
if (!cached) return;
const sessions = JSON.parse(cached);
if (Array.isArray(sessions)) {
this.rawSessions.set(sessions);
}
} catch {
this.clearSessionsCache();
}
}
private persistSessionsCache(sessions: Session[]): void {
try {
localStorage.setItem(SESSION_CACHE_KEY, JSON.stringify(sessions));
} catch {
// Storage can be unavailable in private browsing or embedded contexts.
}
}
private clearSessionsCache(): void {
try {
localStorage.removeItem(SESSION_CACHE_KEY);
} catch {
// No-op when browser storage is unavailable.
}
}
/**
* Convert the dedicated live retry event into the same trace shape used by
* history APIs. This keeps live streaming and post-refresh rendering on one
@@ -1169,7 +1243,12 @@ export class AgentService {
/**
* Open the video player for a given session or the current session
*/
public openVideoPlayer(sessionId?: string, videoUrl?: string, title?: string): void {
public openVideoPlayer(
sessionId?: string,
videoUrl?: string,
title?: string,
seekSeconds?: number
): void {
const targetSessionId = sessionId || this.currentSessionId();
const session = this.sessions().find(s => s.session_id === targetSessionId);
const targetUrl = videoUrl || session?.video_url || null;
@@ -1184,6 +1263,7 @@ export class AgentService {
this.shouldAutoplayVideo.set(false);
this.recordingPlaybackMessage.set('');
this.activeVideoSegments.set([]);
if (Number.isFinite(seekSeconds)) this.requestVideoSeek(Number(seekSeconds));
if (!targetSessionId) {
this.activeVideoUrl.set(null);
this.isVideoLoading.set(false);
@@ -1347,6 +1427,14 @@ export class AgentService {
return shouldAutoplay;
}
public requestVideoSeek(seconds: number): void {
if (!Number.isFinite(seconds)) return;
this.videoSeekRequest.set({
seconds: Math.max(0, seconds),
requestId: ++this.videoSeekRequestId
});
}
/**
* Toggle the video player for the current session
*/
apps/showcase_ui/src/app/services/system.service.spec.ts
+72
View File
@@ -0,0 +1,72 @@
import { provideHttpClient } from '@angular/common/http';
import { HttpTestingController, provideHttpClientTesting } from '@angular/common/http/testing';
import { TestBed } from '@angular/core/testing';
import { SystemReadinessReport } from '../core/models/system.model';
import { SystemService } from './system.service';
describe('SystemService readiness polling', () => {
let service: SystemService;
let http: HttpTestingController;
const report = (timestamp: number): SystemReadinessReport => ({
overall_ready: true,
blocker_count: 3,
passed_blocker_count: 3,
probes: [],
active_device: null,
os_type: 'windows',
timestamp
});
beforeEach(() => {
TestBed.configureTestingModule({
providers: [
SystemService,
provideHttpClient(),
provideHttpClientTesting()
]
});
service = TestBed.inject(SystemService);
http = TestBed.inject(HttpTestingController);
service.stopAutoPolling();
});
afterEach(() => http.verify());
it('shares one HTTP request across overlapping readiness callers', () => {
let firstTimestamp = 0;
let secondTimestamp = 0;
service.fetchReadiness().subscribe(value => firstTimestamp = value.timestamp);
service.fetchReadiness(true).subscribe(value => secondTimestamp = value.timestamp);
const request = http.expectOne('/api/system/readiness');
request.flush(report(10));
expect(firstTimestamp).toBe(10);
expect(secondTimestamp).toBe(10);
expect(service.readinessReport()?.timestamp).toBe(10);
expect(service.isLoading()).toBeFalse();
});
it('does not let an older snapshot replace a newer report', () => {
service.fetchReadiness().subscribe();
http.expectOne('/api/system/readiness').flush(report(20));
service.fetchReadiness().subscribe();
http.expectOne('/api/system/readiness').flush(report(10));
expect(service.readinessReport()?.timestamp).toBe(20);
});
it('requests a forced backend refresh for a manual re-check', () => {
service.fetchReadiness(false, true).subscribe();
const request = http.expectOne(req =>
req.url === '/api/system/readiness' && req.params.get('force') === 'true'
);
expect(request.request.params.get('force')).toBe('true');
request.flush(report(30));
});
});
apps/showcase_ui/src/app/services/system.service.ts
+41 -20
View File
@@ -16,7 +16,7 @@
import { Injectable, signal, computed, inject } from '@angular/core';
import { HttpClient } from '@angular/common/http';
import { Observable, tap } from 'rxjs';
import { Observable, finalize, shareReplay, tap } from 'rxjs';
import { SystemReadinessReport, DeviceInfo, ProbeResult, EmulatorLaunchState, EmulatorLaunchStage } from '../core/models/system.model';
@Injectable({
@@ -27,6 +27,7 @@ export class SystemService {
// Core reactive signals
public readinessReport = signal<SystemReadinessReport | null>(null);
public hasReadinessReport = computed(() => this.readinessReport() !== null);
public isLoading = signal<boolean>(false);
public isRestartingAdb = signal<boolean>(false);
public launchingAvd = signal<string | null>(null);
@@ -136,9 +137,9 @@ export class SystemService {
});
private pollingTimer: any = null;
private readinessRequest$: Observable<SystemReadinessReport> | null = null;
constructor() {
this.fetchReadiness();
this.startAutoPolling(3000);
}
@@ -171,27 +172,50 @@ export class SystemService {
* Fetch latest system readiness report from backend
* @param silent If true, updates signals silently without triggering global isLoading spinner
*/
public fetchReadiness(silent: boolean = false): Observable<SystemReadinessReport> {
public fetchReadiness(
silent: boolean = false,
forceRefresh: boolean = false
): Observable<SystemReadinessReport> {
if (!silent) {
this.isLoading.set(true);
}
return this.http.get<SystemReadinessReport>('/api/system/readiness').pipe(
// Share one request across initialization, focus events, polling, and manual
// refreshes. This prevents a slow probe from creating an unbounded queue.
if (this.readinessRequest$) {
return this.readinessRequest$;
}
const request$ = this.http.get<SystemReadinessReport>('/api/system/readiness', {
params: forceRefresh ? { force: true } : {}
}).pipe(
tap({
next: (report) => {
this.readinessReport.set(report);
this.lastCheckedTime.set(new Date());
if (!silent) {
this.isLoading.set(false);
}
this.applyReadinessReport(report);
},
error: (err) => {
console.error('Failed to fetch system readiness:', err);
if (!silent) {
this.isLoading.set(false);
}
}
})
}),
finalize(() => {
if (this.readinessRequest$ === request$) {
this.readinessRequest$ = null;
}
this.isLoading.set(false);
}),
shareReplay({ bufferSize: 1, refCount: false })
);
this.readinessRequest$ = request$;
return request$;
}
private applyReadinessReport(report: SystemReadinessReport): void {
const current = this.readinessReport();
if (current && report.timestamp < current.timestamp) {
return;
}
this.readinessReport.set(report);
this.lastCheckedTime.set(new Date(report.timestamp * 1000));
}
/**
@@ -337,8 +361,7 @@ export class SystemService {
tap({
next: (res) => {
if (res?.report) {
this.readinessReport.set(res.report);
this.lastCheckedTime.set(new Date());
this.applyReadinessReport(res.report);
}
this.isRestartingAdb.set(false);
},
@@ -358,8 +381,7 @@ export class SystemService {
tap({
next: (res) => {
if (res?.report) {
this.readinessReport.set(res.report);
this.lastCheckedTime.set(new Date());
this.applyReadinessReport(res.report);
}
}
})
@@ -375,7 +397,7 @@ export class SystemService {
tap({
next: (res) => {
if (res?.report) {
this.readinessReport.set(res.report);
this.applyReadinessReport(res.report);
}
this.isLoading.set(false);
},
@@ -436,8 +458,7 @@ export class SystemService {
tap({
next: (res) => {
if (res?.report) {
this.readinessReport.set(res.report);
this.lastCheckedTime.set(new Date());
this.applyReadinessReport(res.report);
}
// Refresh model config & env after updating key
this.fetchModelConfigEnv().subscribe();
apps/showcase_ui/src/app/utils/stream-aggregator.util.spec.ts
+93
View File
@@ -0,0 +1,93 @@
import { consolidateLogsToBlocks, getSortedStepEvents } from './stream-aggregator.util';
describe('stream aggregator timeline ordering', () => {
it('interleaves text, tools, and actions by timestamp', () => {
const events = getSortedStepEvents({
timestamp: 10,
operator_native_thinking: 'Thought content',
operator_native_thinking_timestamp: 11,
operator_raw_thinking: 'Work content',
operator_raw_thinking_timestamp: 13,
action_taken: { action: 'tap', timestamp: 15 },
generic_tools: [
{ trace_id: 'tool-late', name: 'save_note', timestamp: 14 },
{ trace_id: 'tool-early', name: 'video_analyzer', timestamp: 12 }
]
});
expect(events.map((event) => event.type)).toEqual([
'thinking',
'tool',
'text',
'tool',
'action'
]);
expect(events.map((event) => event.timestamp)).toEqual([
11000,
12000,
13000,
14000,
15000
]);
});
it('uses a deterministic text-first fallback when only a step timestamp exists', () => {
const events = getSortedStepEvents({
timestamp: 20,
operator_native_thinking: 'Thought content',
operator_raw_thinking: 'Work content',
action_taken: { action: 'tap' },
generic_tools: [{ trace_id: 'tool-1', name: 'save_note' }]
});
expect(events.map((event) => event.type)).toEqual([
'thinking',
'text',
'action',
'tool'
]);
});
it('keeps the first live timestamp when later chunks update the same text stream', () => {
const blocks = consolidateLogsToBlocks([
{
type: 'llm_stream',
timestamp: '2026-08-25T20:00:01.000Z',
data: {
execution_id: 'exec-1',
step_id: 'step-1',
stream_type: 'text',
text: 'First',
isCompleted: false
}
},
{
type: 'trace_recorded',
timestamp: '2026-08-25T20:00:02.000Z',
data: {
trace_id: 'tool-1',
step_id: 'step-1',
type: 'tool',
name: 'save_note',
timestamp: '2026-08-25T20:00:02.000Z'
}
},
{
type: 'llm_stream',
timestamp: '2026-08-25T20:00:03.000Z',
data: {
execution_id: 'exec-1',
step_id: 'step-1',
stream_type: 'text',
text: 'First and second',
isCompleted: false
}
}
]);
expect(blocks[0].data.operator_raw_thinking_timestamp)
.toBe('2026-08-25T20:00:01.000Z');
expect(getSortedStepEvents(blocks[0].data).map((event) => event.type))
.toEqual(['text', 'tool']);
});
});
apps/showcase_ui/src/app/utils/stream-aggregator.util.ts
+69 -11
View File
@@ -64,8 +64,12 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
const updatedData = { ...existing.data };
if (streamType === 'thinking') {
updatedData.operator_native_thinking = text;
updatedData.operator_native_thinking_timestamp =
updatedData.operator_native_thinking_timestamp || log.timestamp;
} else {
updatedData.operator_raw_thinking = text;
updatedData.operator_raw_thinking_timestamp =
updatedData.operator_raw_thinking_timestamp || log.timestamp;
}
if (stepId && !updatedData.step_id) {
updatedData.step_id = stepId;
@@ -89,8 +93,10 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
};
if (streamType === 'thinking') {
blockData.operator_native_thinking = text;
blockData.operator_native_thinking_timestamp = log.timestamp;
} else {
blockData.operator_raw_thinking = text;
blockData.operator_raw_thinking_timestamp = log.timestamp;
}
blocks.push({
@@ -134,6 +140,12 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
...log.data,
operator_native_thinking: log.data.operator_native_thinking || existingBlock.data.operator_native_thinking || undefined,
operator_raw_thinking: log.data.operator_raw_thinking || existingBlock.data.operator_raw_thinking || undefined,
operator_native_thinking_timestamp: existingBlock.data.operator_native_thinking_timestamp
|| log.data.operator_native_thinking_timestamp
|| (log.data.operator_native_thinking ? log.timestamp : undefined),
operator_raw_thinking_timestamp: existingBlock.data.operator_raw_thinking_timestamp
|| log.data.operator_raw_thinking_timestamp
|| (log.data.operator_raw_thinking ? log.timestamp : undefined),
action_taken: log.data.action_taken || existingBlock.data.action_taken || undefined,
last_execution_result: log.data.last_execution_result || existingBlock.data.last_execution_result || undefined,
generic_tools: mergedTools,
@@ -155,6 +167,10 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
data: {
...log.data,
isCompleted: true,
operator_native_thinking_timestamp: log.data.operator_native_thinking_timestamp
|| (log.data.operator_native_thinking ? log.timestamp : undefined),
operator_raw_thinking_timestamp: log.data.operator_raw_thinking_timestamp
|| (log.data.operator_raw_thinking ? log.timestamp : undefined),
generic_tools: log.data.generic_tools || []
}
});
@@ -410,23 +426,58 @@ export function groupBlocksToPhases(blocks: StepBlock[], sessionStartTime: numbe
*/
export function getSortedStepEvents(
stepData: any,
cache?: WeakMap<any, { length: number; actionTs: any; events: StepEvent[] }>
cache?: WeakMap<any, { signature: string; events: StepEvent[] }>
): StepEvent[] {
if (!stepData || typeof stepData !== 'object') return [];
const toolsLen = stepData.generic_tools ? stepData.generic_tools.length : 0;
const actionTs = stepData.action_taken ? (stepData.action_taken.timestamp || stepData.action_taken.start_time || stepData.action_taken.created_at) : null;
const nativeText = typeof stepData.operator_native_thinking === 'string'
? stepData.operator_native_thinking.trim()
: '';
const rawText = typeof stepData.operator_raw_thinking === 'string'
? stepData.operator_raw_thinking.trim()
: '';
const signature = [
toolsLen,
actionTs ?? '',
stepData.operator_native_thinking_timestamp ?? '',
stepData.operator_raw_thinking_timestamp ?? '',
nativeText.length,
rawText.length
].join('|');
if (cache) {
const cached = cache.get(stepData);
if (cached && cached.length === toolsLen && cached.actionTs === actionTs) {
if (cached?.signature === signature) {
return cached.events;
}
}
const events: Array<{ type: 'action' | 'tool'; data: any; timestamp: number }> = [];
const events: Array<{ type: StepEvent['type']; data: any; timestamp: number; sequence: number }> = [];
const fallbackTime = getItemTimestamp(stepData.timestamp, 0);
let sequence = 0;
// 1. Add action_taken if present and valid
// Text and tools share one timeline. Live streams carry their first-seen
// timestamps; persisted steps use the step timestamp as a stable fallback.
if (nativeText) {
events.push({
type: 'thinking',
data: { text: stepData.operator_native_thinking },
timestamp: getItemTimestamp(stepData.operator_native_thinking_timestamp, fallbackTime),
sequence: sequence++
});
}
if (rawText) {
events.push({
type: 'text',
data: { text: stepData.operator_raw_thinking },
timestamp: getItemTimestamp(stepData.operator_raw_thinking_timestamp, fallbackTime),
sequence: sequence++
});
}
// Add action_taken if present and valid.
if (stepData.action_taken && (isAndroidAction(stepData.action_taken) || isReportStatusAction(stepData.action_taken))) {
const actTime = getItemTimestamp(
stepData.action_taken.timestamp ?? stepData.action_taken.start_time ?? stepData.action_taken.created_at,
@@ -435,11 +486,12 @@ export function getSortedStepEvents(
events.push({
type: 'action',
data: stepData.action_taken,
timestamp: actTime
timestamp: actTime,
sequence: sequence++
});
}
// 2. Add generic tools if present
// Add generic tools if present.
const uniqueTools = getUniqueGenericTools(stepData.generic_tools);
uniqueTools.forEach((tool: any) => {
const toolTime = getItemTimestamp(
@@ -449,16 +501,22 @@ export function getSortedStepEvents(
events.push({
type: 'tool',
data: tool,
timestamp: toolTime
timestamp: toolTime,
sequence: sequence++
});
});
// 3. Sort chronologically by timestamp
events.sort((a, b) => a.timestamp - b.timestamp);
// Sort chronologically. The explicit sequence makes equal/missing timestamp
// behavior deterministic across browsers and preserves the legacy fallback.
events.sort((a, b) => (a.timestamp - b.timestamp) || (a.sequence - b.sequence));
const result: StepEvent[] = events.map(e => ({ type: e.type, data: e.data }));
const result: StepEvent[] = events.map(e => ({
type: e.type,
data: e.data,
timestamp: e.timestamp
}));
if (cache) {
cache.set(stepData, { length: toolsLen, actionTs, events: result });
cache.set(stepData, { signature, events: result });
}
return result;
}
apps/showcase_ui/src/app/utils/tool-formatter.util.spec.ts
+79 -1
View File
@@ -1,4 +1,8 @@
import { cleanErrorMessage, getUniqueGenericTools } from './tool-formatter.util';
import {
cleanErrorMessage,
getUniqueGenericTools,
getVideoAnalysisView
} from './tool-formatter.util';
describe('cleanErrorMessage', () => {
it('removes repeated LLM wrapper labels while preserving the provider reason', () => {
@@ -86,3 +90,77 @@ describe('getUniqueGenericTools retry aggregation', () => {
expect(result.map(tool => tool.name)).toEqual(['llm_pause']);
});
});
describe('video analysis timeline formatting', () => {
it('distinguishes a cache hit without presenting it as running', () => {
const view = getVideoAnalysisView({
name: 'spawn_sub_agent',
status: 'success',
payload: {
args: { start_time: 0, end_time: 42 },
result: 'CACHED VIDEO ANALYSIS: existing evidence'
}
});
expect(view?.outcome).toBe('complete');
expect(view?.reuse).toBe('full');
expect(view?.title).toBe('Reused video analysis');
expect(view?.requestedRange).toEqual({ start: 0, end: 42 });
});
it('collapses child video chunks into one stable note-style timeline item', () => {
const result = getUniqueGenericTools([
{
trace_id: 'chunk-1',
parent_trace_id: 'video-agent-1',
type: 'tool',
name: 'spawn_sub_agent',
status: 'success',
timestamp: 100,
payload: {
args: { start_time: 0, end_time: 30, specific_query: 'find the result' },
result: '[from 0.0s to 30.0s] Summary: first'
}
},
{
trace_id: 'chunk-2',
parent_trace_id: 'video-agent-1',
type: 'tool',
name: 'spawn_sub_agent',
status: 'success',
timestamp: 101,
payload: {
args: { start_time: 30, end_time: 60, specific_query: 'find the result' },
result: 'PARTIAL VIDEO ANALYSIS (successful chunks were persisted). Failed intervals: 45.0s-60.0s'
}
}
]);
expect(result.length).toBe(1);
expect(result[0].name).toBe('video_analysis');
expect(result[0].trace_id).toBe('video-analysis-video-agent-1');
expect(result[0].payload.result.outcome).toBe('partial');
expect(result[0].payload.result.requested_range).toEqual({ start: 0, end: 60 });
expect(getVideoAnalysisView(result[0])?.title).toBe('Video analysis partially completed');
});
it('keeps video analyses from independent parent executions separate', () => {
const result = getUniqueGenericTools([
{
trace_id: 'chunk-1', parent_trace_id: 'video-agent-1', type: 'tool',
name: 'spawn_sub_agent', status: 'success',
payload: { args: { start_time: 0, end_time: 10 }, result: 'first' }
},
{
trace_id: 'chunk-2', parent_trace_id: 'video-agent-2', type: 'tool',
name: 'spawn_sub_agent', status: 'success',
payload: { args: { start_time: 10, end_time: 20 }, result: 'second' }
}
]);
expect(result.map(tool => tool.trace_id)).toEqual([
'video-analysis-video-agent-1',
'video-analysis-video-agent-2'
]);
});
});
apps/showcase_ui/src/app/utils/tool-formatter.util.ts
+228 -2
View File
@@ -64,7 +64,133 @@ export function isVideoTool(tool: any): boolean {
if (!tool || !tool.name) return false;
const cleanName = tool.name.replace(/^(_)?(self\.)?exec_/, '');
const nameLower = cleanName.toLowerCase();
return ['video_analyzer', 'video_analyzer_pure'].includes(nameLower);
return [
'video_analysis',
'video_analyzer',
'video_analyzer_pure',
'spawn_sub_agent',
'analyze_audio_only'
].includes(nameLower);
}
export type VideoAnalysisOutcome = 'running' | 'recovering' | 'waiting' | 'complete' | 'partial' | 'failed';
export interface VideoAnalysisRange {
start: number;
end: number;
category?: string;
retryable?: boolean;
}
export interface VideoAnalysisView {
outcome: VideoAnalysisOutcome;
title: string;
query: string;
summary: string;
reuse: 'none' | 'partial' | 'full';
requestedRange: VideoAnalysisRange | null;
completedRanges: VideoAnalysisRange[];
failedRanges: VideoAnalysisRange[];
evidenceCount: number;
completedCount: number;
totalCount: number;
fallbackUsed: boolean;
}
function numericRange(value: any): VideoAnalysisRange | null {
if (!value || typeof value !== 'object') return null;
const start = Number(value.start);
const end = Number(value.end);
if (!Number.isFinite(start) || !Number.isFinite(end) || end <= start) return null;
return {
start,
end,
category: value.category ? String(value.category) : undefined,
retryable: value.retryable === true
};
}
function parseVideoResultText(text: string): Partial<VideoAnalysisView> {
const value = text.trim();
if (!value) return {};
if (value.startsWith('CACHED VIDEO ANALYSIS:')) {
return { outcome: 'complete', reuse: 'full', summary: value.replace(/^CACHED VIDEO ANALYSIS:\s*/, '') };
}
if (value.startsWith('PARTIAL VIDEO ANALYSIS')) {
return { outcome: 'partial', reuse: 'partial', summary: value };
}
if (value.startsWith('All sub-agent chunks failed') || value.startsWith('Error:')) {
return { outcome: 'failed', summary: value };
}
if (value.includes('Analysis is already in progress in another video agent')) {
return { outcome: 'waiting', summary: 'Another video agent is already analyzing this evidence.' };
}
return { outcome: 'complete', summary: value };
}
/** Build the user-facing, backward-compatible video analysis state. */
export function getVideoAnalysisView(tool: any): VideoAnalysisView | null {
if (!isVideoTool(tool)) return null;
const args = getToolArgs(tool);
const payload = tool?.payload && typeof tool.payload === 'object' ? tool.payload : {};
const rawResult = payload.result ?? tool.result ?? null;
const structured = rawResult && typeof rawResult === 'object' ? rawResult : {};
const parsed = typeof rawResult === 'string' ? parseVideoResultText(rawResult) : {};
let outcome = String(structured.outcome || payload.outcome || parsed.outcome || '').toLowerCase() as VideoAnalysisOutcome;
if (!['running', 'recovering', 'waiting', 'complete', 'partial', 'failed'].includes(outcome)) {
outcome = tool?.status === 'failed' || tool?.status === 'error'
? 'failed'
: tool?.status === 'running' ? 'running' : 'complete';
}
const recovering = structured.recovering === true || structured.fallback_used === true;
if (outcome === 'running' && recovering) outcome = 'recovering';
const start = Number(args.start_time ?? structured.requested_range?.start);
const end = Number(args.end_time ?? structured.requested_range?.end);
const requestedRange = Number.isFinite(start) && Number.isFinite(end) && end > start
? { start, end }
: numericRange(structured.requested_range);
const completedRanges = Array.isArray(structured.completed_ranges)
? structured.completed_ranges.map(numericRange).filter((range: VideoAnalysisRange | null): range is VideoAnalysisRange => Boolean(range))
: [];
const failedRanges = Array.isArray(structured.failed_ranges)
? structured.failed_ranges.map(numericRange).filter((range: VideoAnalysisRange | null): range is VideoAnalysisRange => Boolean(range))
: [];
const completedCount = Number(structured.completed_count ?? completedRanges.length ?? 0);
const totalCount = Number(structured.total_count ?? (completedRanges.length + failedRanges.length));
const titleByOutcome: Record<VideoAnalysisOutcome, string> = {
running: 'Analyzing screen recording',
recovering: 'Analyzing unfinished recording segment',
waiting: 'Waiting for existing video analysis',
complete: structured.reuse === 'full' || parsed.reuse === 'full'
? 'Reused video analysis'
: 'Analyzed screen recording',
partial: 'Video analysis partially completed',
failed: 'Video analysis returned no result'
};
return {
outcome,
title: titleByOutcome[outcome],
query: String(args.specific_query || args.query || args.prompt || structured.query || ''),
summary: String(structured.summary || parsed.summary || ''),
reuse: (structured.reuse || parsed.reuse || 'none') as 'none' | 'partial' | 'full',
requestedRange,
completedRanges,
failedRanges,
evidenceCount: Number(structured.evidence_count || 0),
completedCount: Number.isFinite(completedCount) ? completedCount : 0,
totalCount: Number.isFinite(totalCount) ? totalCount : 0,
fallbackUsed: structured.fallback_used === true
};
}
export function formatVideoTime(seconds: number): string {
const safe = Math.max(0, Number(seconds) || 0);
const minutes = Math.floor(safe / 60);
const secs = Math.floor(safe % 60);
return `${String(minutes).padStart(2, '0')}:${String(secs).padStart(2, '0')}`;
}
/**
@@ -301,7 +427,107 @@ export function getUniqueGenericTools(tools: any[] | undefined): any[] {
}
}
}
return result;
return collapseVideoAnalysisTools(result);
}
/**
* Present one note-style timeline row for one video-analyzer execution. Child
* chunk, audio, and wrapper traces stay available in the raw trace but do not
* look like repeated user-visible analyses.
*/
function collapseVideoAnalysisTools(tools: any[]): any[] {
const groups = new Map<string, { firstIndex: number; tools: any[] }>();
const passthrough: Array<{ index: number; tool: any }> = [];
tools.forEach((tool, index) => {
if (!isVideoTool(tool)) {
passthrough.push({ index, tool });
return;
}
const cleanName = String(tool.name || '').replace(/^(_)?(self\.)?exec_/, '').toLowerCase();
const groupId = cleanName === 'video_analyzer' || cleanName === 'video_analyzer_pure'
? String(tool.trace_id || tool.parent_trace_id || `video-${index}`)
: String(tool.parent_trace_id || tool.trace_id || `video-${index}`);
const existing = groups.get(groupId);
if (existing) {
existing.tools.push(tool);
} else {
groups.set(groupId, { firstIndex: index, tools: [tool] });
}
});
const collapsed = [...passthrough];
for (const [groupId, group] of groups) {
const views = group.tools
.map(getVideoAnalysisView)
.filter((view): view is VideoAnalysisView => Boolean(view));
if (!views.length) continue;
const hasActive = views.some(view => view.outcome === 'running' || view.outcome === 'recovering');
const completed = views.filter(view => view.outcome === 'complete');
const hasPartial = views.some(view => view.outcome === 'partial');
const failed = views.filter(view => view.outcome === 'failed');
const waiting = views.filter(view => view.outcome === 'waiting');
let outcome: VideoAnalysisOutcome;
if (hasActive) {
outcome = views.some(view => view.outcome === 'recovering') ? 'recovering' : 'running';
} else if (hasPartial || (completed.length > 0 && failed.length > 0)) {
outcome = 'partial';
} else if (completed.length > 0) {
outcome = 'complete';
} else if (waiting.length > 0) {
outcome = 'waiting';
} else {
outcome = 'failed';
}
const ranges = views
.map(view => view.requestedRange)
.filter((range): range is VideoAnalysisRange => Boolean(range));
const requestedRange = ranges.length
? {
start: Math.min(...ranges.map(range => range.start)),
end: Math.max(...ranges.map(range => range.end))
}
: null;
const reuse = completed.length > 0 && completed.every(view => view.reuse === 'full')
? 'full'
: views.some(view => view.reuse !== 'none') ? 'partial' : 'none';
const base = group.tools[0];
const structuredResult = {
outcome,
reuse,
requested_range: requestedRange,
completed_count: completed.length,
total_count: views.length,
evidence_count: views.reduce((sum, view) => sum + view.evidenceCount, 0),
query: views.find(view => view.query)?.query || '',
recovering: outcome === 'recovering',
fallback_used: views.some(view => view.fallbackUsed)
};
collapsed.push({
index: group.firstIndex,
tool: {
...base,
trace_id: `video-analysis-${groupId}`,
name: 'video_analysis',
status: outcome === 'running' || outcome === 'recovering' ? 'running' : 'success',
payload: {
...(base.payload || {}),
args: {
...(base.payload?.args || base.args || {}),
start_time: requestedRange?.start,
end_time: requestedRange?.end
},
result: structuredResult
}
}
});
}
collapsed.sort((a, b) => a.index - b.index);
return collapsed.map(item => item.tool);
}
/**