chenmingxuan/Artemis-Eapil
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 32 Wiki Activity

feat: harden video analysis and runtime resilience

Browse Source
This commit is contained in:
somew1nd committed 2026-08-25 20:58:25 -07:00
1 parent 5ada543a09
commit bb16991f2c
55 files changed
+5023 -437
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.html
+46 -38
View File
@@ -290,14 +290,19 @@
<!-- Event Logs Section -->
<main class="stream-logs-content" (scroll)="onStreamScroll($event)">
<div *ngIf="consolidatedBlocks().length === 0 && !showPlanningLoader() && startupWorkItems().length === 0" class="empty-state">
<div *ngIf="agentService.isSessionContentLoading()" class="session-loading-state" aria-live="polite">
<span class="material-symbols-outlined spin-animation">progress_activity</span>
<span>Loading run history...</span>
</div>
<div *ngIf="!agentService.isSessionContentLoading() && consolidatedBlocks().length === 0 && !showPlanningLoader() && startupWorkItems().length === 0" class="empty-state">
<div class="empty-icon"><span class="material-symbols-outlined">sensors</span></div>
<h3>No Stream Activity Selected</h3>
<p>Please select a session from the Task List on the right to view its real-time log payloads.</p>
</div>
<!-- The phase-based session logs list -->
<div class="logs-list-container" *ngIf="phases().length > 0 || showPlanningLoader() || startupWorkItems().length > 0">
<div class="logs-list-container" *ngIf="!agentService.isSessionContentLoading() && (phases().length > 0 || showPlanningLoader() || startupWorkItems().length > 0)">
<div class="phase-container startup-phase" *ngIf="startupWorkItems().length > 0" aria-live="polite">
<div class="phase-header first-phase">
<span class="phase-worked-time">Worked for {{ startupPreparationDuration() }}s</span>
@@ -343,37 +348,38 @@
>
<!-- Unified Step / Stream Body -->
<div class="step-body">
<!-- Native Thinking (Thought) -->
<div *ngIf="getNativeThinking(block) as nativeText"
class="thinking-stream-details native-thinking-details"
[class.is-running]="isBlockRunning(block)"
[class.collapsed]="collapsedStreams().has(block.data.execution_id || block.data.step_id || block.id)">
<div class="details-summary" (click)="toggleCollapse(block.data.execution_id || block.data.step_id || block.id)">
<span class="stream-status-dot" [class.streaming]="isBlockRunning(block)"></span>
Thought
</div>
<div class="details-content-wrapper">
<div class="details-content stream-box">
<div class="stream-text" [innerHTML]="renderMarkdown(typedTextsSignal()[block.id + '-native'] || nativeText)"></div>
</div>
</div>
</div>
<!-- Raw Thinking / Plan / Message (Work) -->
<div *ngIf="getRawThinking(block) as rawText" class="step-raw-thinking-message" [class.is-running]="isBlockRunning(block)">
<div class="report-header">
<span class="stream-status-dot" [class.streaming]="isBlockRunning(block)"></span>
Work
</div>
<div class="report-content stream-box">
<div class="stream-text" [innerHTML]="renderMarkdown(typedTextsSignal()[block.id] || rawText)"></div>
</div>
</div>
<!-- Tool Call / Action section (Unified Chronological Loop matching Debug UI execution order) -->
<div class="step-actions-container" *ngIf="(block.data.generic_tools && block.data.generic_tools.length > 0) || block.data.action_taken">
<!-- Text, tool calls, and actions share one chronological timeline. -->
<div class="step-actions-container" *ngIf="getSortedStepEvents(block.data).length > 0">
<ng-container *ngFor="let item of getSortedStepEvents(block.data); let itemIdx = index; trackBy: trackStepEvent">
<!-- Native Thinking (Thought) -->
<div *ngIf="item.type === 'thinking'"
class="thinking-stream-details native-thinking-details"
[class.is-running]="isBlockRunning(block)"
[class.collapsed]="collapsedStreams().has(block.data.execution_id || block.data.step_id || block.id)">
<div class="details-summary" (click)="toggleCollapse(block.data.execution_id || block.data.step_id || block.id)">
<span class="stream-status-dot" [class.streaming]="isBlockRunning(block)"></span>
Thought
</div>
<div class="details-content-wrapper">
<div class="details-content stream-box">
<div class="stream-text" [innerHTML]="renderMarkdown(typedTextsSignal()[block.id + '-native'] || item.data.text)"></div>
</div>
</div>
</div>
<!-- Raw Thinking / Plan / Message (Work) -->
<div *ngIf="item.type === 'text' && getRawThinking(block)"
class="step-raw-thinking-message"
[class.is-running]="isBlockRunning(block)">
<div class="report-header">
<span class="stream-status-dot" [class.streaming]="isBlockRunning(block)"></span>
Work
</div>
<div class="report-content stream-box">
<div class="stream-text" [innerHTML]="renderMarkdown(typedTextsSignal()[block.id] || item.data.text)"></div>
</div>
</div>
<!-- Task Submission / Report Card (when item is report_task_status action or tool) -->
<div class="step-android-action-card task-report-card"
[class.is-completed]="getReportStatusValue(item.data) === 'completed' || getReportStatusValue(item.data) === 'success'"
@@ -630,11 +636,12 @@
[class.is-running]="isItemRunning(item, block)"
[class.is-note-tool]="isNoteTool(item.data)"
[class.is-video-tool]="isVideoTool(item.data)"
[class.is-video-attention]="isVideoTool(item.data) && isVideoAnalysisAttention(item.data)"
*ngIf="shouldShowTool(item.data, block.data) && !isDeviceActionTool(item.data)">
<div class="tool-call-label">
<span class="stream-status-dot" [class.streaming]="isItemRunning(item, block)"></span>
<div class="tool-call-text">
<span>{{ getToolDisplayLabel(item.data) }}{{ ((isNoteTool(item.data) && getToolKey(item.data)) || (isVideoTool(item.data) && getVideoToolTarget(item.data))) ? ':' : '' }}</span>
<span>{{ isVideoTool(item.data) ? getVideoAnalysisLabel(item.data) : getToolDisplayLabel(item.data) }}{{ ((isNoteTool(item.data) && getToolKey(item.data)) || isVideoTool(item.data)) ? ':' : '' }}</span>
<!-- Note Tool Pill -->
<ng-container *ngIf="isNoteTool(item.data) && getToolKey(item.data) as key">
@@ -649,15 +656,16 @@
</ng-container>
<!-- Video Tool Pill -->
<ng-container *ngIf="isVideoTool(item.data) && getVideoToolTarget(item.data) as videoTarget">
<span
class="tool-key-pill video-pill"
<ng-container *ngIf="isVideoTool(item.data)">
<span
class="tool-key-pill"
(click)="onVideoToolClick(item.data)"
title="Click to open video player"
[title]="'Open recording at ' + getVideoAnalysisRangeLabel(item.data)"
>
<span class="material-symbols-outlined pill-icon">videocam</span>
{{ videoTarget }}
{{ getVideoAnalysisRangeLabel(item.data) }}
</span>
<span class="video-status-detail" *ngIf="getVideoAnalysisDetail(item.data) as detail">· {{ detail }}</span>
</ng-container>
</div>
</div>
apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.scss
+30 -10
View File
@@ -20,6 +20,7 @@ $text-secondary: #1f1f1f;
$text-muted: #1f1f1f;
$bg-main: #f9fafb;
$uniform-gap: 8px;
$stream-row-height: 25px;
:host {
display: flex;
@@ -957,6 +958,20 @@ $uniform-gap: 8px;
overflow-y: auto;
padding: 72px 24px 100px 24px;
position: relative;
.session-loading-state {
min-height: 180px;
display: flex;
align-items: center;
justify-content: center;
gap: 9px;
color: #64748b;
font-size: 13px;
.material-symbols-outlined {
font-size: 18px;
}
}
display: flex;
flex-direction: column;
@@ -1879,7 +1894,8 @@ $uniform-gap: 8px;
padding: 0;
.llm-retry-note {
min-height: 20px;
min-height: $stream-row-height;
box-sizing: border-box;
display: flex;
align-items: center;
gap: 8px;
@@ -1924,6 +1940,14 @@ $uniform-gap: 8px;
.step-generic-tool {
border: 1px solid transparent;
&.is-video-attention:not(.is-running) {
.tool-call-label .stream-status-dot {
background: #f59e0b;
box-shadow: none;
animation: none;
}
}
&.is-llm-error {
.tool-call-label {
display: flex;
@@ -1994,7 +2018,7 @@ $uniform-gap: 8px;
font-family: 'Google Sans', 'Google Sans Text', 'Inter', Arial, sans-serif;
letter-spacing: 0.5px;
line-height: 1.5;
min-height: 20px;
min-height: 22px;
box-sizing: border-box;
display: inline-flex;
align-items: center;
@@ -2065,15 +2089,11 @@ $uniform-gap: 8px;
color: #1557b0;
background: rgba(26, 115, 232, 0.08);
}
}
&.video-pill {
color: #0284c7;
&:hover {
color: #0369a1;
background: rgba(2, 132, 199, 0.08);
}
}
.video-status-detail {
color: inherit;
margin-left: 2px;
}
}
}
apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.ts
+35 -3
View File
@@ -153,6 +153,8 @@ import {
getToolArgs,
isNoteTool,
isVideoTool,
getVideoAnalysisView,
formatVideoTime,
getVideoToolTarget,
isDeviceActionTool,
isFailureAnalyzerActionTool,
@@ -240,7 +242,7 @@ export class AgentStreamComponent implements AfterViewInit {
// Performance caches for parameter and event extractions
private actionParamsCache = new WeakMap<any, ActionParam[]>();
private toolParamsCache = new WeakMap<any, ActionParam[]>();
private sortedEventsCache = new WeakMap<any, { length: number; actionTs: any; events: StepEvent[] }>();
private sortedEventsCache = new WeakMap<any, { signature: string; events: StepEvent[] }>();
// Top Nav Task Queue Computed Properties
public activeQueue = computed(() => {
@@ -897,10 +899,40 @@ export class AgentStreamComponent implements AfterViewInit {
return getVideoToolTarget(tool);
}
public getVideoAnalysisLabel(tool: any): string {
return getVideoAnalysisView(tool)?.title || 'Analyzing screen recording';
}
public getVideoAnalysisRangeLabel(tool: any): string {
const range = getVideoAnalysisView(tool)?.requestedRange;
if (!range) return 'Screen recording';
return `${formatVideoTime(range.start)}–${formatVideoTime(range.end)}`;
}
public getVideoAnalysisDetail(tool: any): string {
const view = getVideoAnalysisView(tool);
if (!view || view.totalCount <= 1) return '';
if (view.outcome === 'running' || view.outcome === 'recovering') {
return view.completedCount > 0
? `${view.completedCount}/${view.totalCount} segments saved`
: '';
}
if (view.outcome === 'partial') {
return `${view.completedCount}/${view.totalCount} segments saved`;
}
return '';
}
public isVideoAnalysisAttention(tool: any): boolean {
const outcome = getVideoAnalysisView(tool)?.outcome;
return outcome === 'partial' || outcome === 'failed';
}
public onVideoToolClick(toolData: any): void {
const curSessionId = this.agentService.currentSessionId();
if (curSessionId) {
this.agentService.openVideoPlayer(curSessionId);
const start = getVideoAnalysisView(toolData)?.requestedRange?.start;
this.agentService.openVideoPlayer(curSessionId, undefined, undefined, start);
}
}
@@ -1231,7 +1263,7 @@ export class AgentStreamComponent implements AfterViewInit {
public trackStepEvent(index: number, item: { type: string; data: any }): string {
if (!item || !item.data) return index.toString();
return item.type + '-' + (item.data.timestamp || item.data.start_time || item.data.created_at || item.data.id || item.data.action || item.data.name || index);
return item.type + '-' + (item.data.timestamp || item.data.start_time || item.data.created_at || item.data.id || item.data.action || item.data.name || item.data.text?.length || index);
}
public trackParam(index: number, param: { key: string; value: string }): string {
apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.ts
+22
View File
@@ -58,6 +58,7 @@ export class FloatingVideoPlayerComponent {
});
private pendingLocalTime: number | null = null;
private pendingAutoplay = false;
private pendingAbsoluteSeek: number | null = null;
// Available speed options
public speedOptions = [0.5, 1.0, 1.5, 2.0, 4.0];
@@ -90,6 +91,19 @@ export class FloatingVideoPlayerComponent {
},
{ allowSignalWrites: true }
);
effect(() => {
const request = this.agentService.videoSeekRequest();
if (!request) return;
this.pendingAbsoluteSeek = request.seconds;
if (this.agentService.recordingPlaybackStatus() !== 'ready') return;
const video = this.videoRef?.nativeElement;
if (video && video.readyState >= 1) {
const target = this.pendingAbsoluteSeek;
this.pendingAbsoluteSeek = null;
this.seek(target, true);
}
});
}
public onLiveStreamError(): void {
@@ -194,6 +208,14 @@ export class FloatingVideoPlayerComponent {
v.playbackRate = this.playbackRate();
v.loop = this.isLooping() && !segments.length;
v.muted = this.isMuted();
if (this.pendingAbsoluteSeek !== null) {
const target = this.pendingAbsoluteSeek;
this.pendingAbsoluteSeek = null;
this.seek(target, true);
// A seek into another finalized segment changes the video source. Let
// that segment's metadata event apply pendingLocalTime.
if (this.pendingLocalTime !== null) return;
}
if (this.pendingLocalTime !== null) {
v.currentTime = Math.max(0, Math.min(v.duration || this.pendingLocalTime, this.pendingLocalTime));
this.pendingLocalTime = null;