feat: add step-by-step visual replay and harden process lifecycle
- UI: Implement dual-mode playback in FloatingVideoPlayer supporting video and step-by-step frame replay with scrub controls and auto-fallback - Runtime: Prevent asyncio event loop hangs on Darwin and Linux by avoiding psutil.wait_procs on direct child processes - Task Queue: Add worker exit watchdog and automatic recording recovery for completed or interrupted sessions - Recording: Add ffmpeg ultrafast transcode fallback, shield recording finalization against cancellation, and handle on-the-fly MP4 conversion - Docs: Polish README_CN styling and streamline flash runner agent prompt
This commit is contained in:
somew1nd committed
1 parent
06643e955b
commit
a36a66ce6d
26 files changed
+1152
-117
No files matched your search
@@ -263,28 +263,41 @@
|
||||
[class.recording]="agentService.isCurrentSessionRunning()"
|
||||
[class.processing]="!agentService.isCurrentSessionRunning() && agentService.currentSessionRecordingStatus() === 'processing'"
|
||||
[class.has-video]="!agentService.isCurrentSessionRunning() && !!agentService.currentSessionVideoUrl()"
|
||||
[class.has-steps]="!agentService.isCurrentSessionRunning() && !agentService.currentSessionVideoUrl() && agentService.hasCurrentSessionStepFrames()"
|
||||
[class.failed]="!agentService.isCurrentSessionRunning() && !agentService.currentSessionVideoUrl() && !agentService.hasCurrentSessionStepFrames() && agentService.currentSessionRecordingStatus() === 'failed'"
|
||||
(click)="agentService.toggleVideoPlayer()"
|
||||
[title]="agentService.isCurrentSessionRunning() ? 'Task is currently running (Recording screen)' : (agentService.currentSessionRecordingStatus() === 'processing' ? 'Preparing screen recording' : (agentService.currentSessionVideoUrl() ? 'Play Screen Recording' : 'Check Device Screen Recording'))"
|
||||
[title]="getScreenRecordingButtonTitle()"
|
||||
>
|
||||
<span class="material-symbols-outlined video-icon" *ngIf="agentService.isCurrentSessionRunning()">
|
||||
videocam
|
||||
</span>
|
||||
<span class="material-symbols-outlined video-icon" *ngIf="!agentService.isCurrentSessionRunning() && agentService.currentSessionRecordingStatus() === 'processing'">
|
||||
<span class="material-symbols-outlined video-icon spin-animation" *ngIf="!agentService.isCurrentSessionRunning() && agentService.currentSessionRecordingStatus() === 'processing'">
|
||||
progress_activity
|
||||
</span>
|
||||
<span class="material-symbols-outlined video-icon play-icon" *ngIf="!agentService.isCurrentSessionRunning() && agentService.currentSessionRecordingStatus() !== 'processing' && agentService.currentSessionVideoUrl()">
|
||||
smart_display
|
||||
</span>
|
||||
<span class="material-symbols-outlined video-icon" *ngIf="!agentService.isCurrentSessionRunning() && agentService.currentSessionRecordingStatus() !== 'processing' && !agentService.currentSessionVideoUrl()">
|
||||
<span class="material-symbols-outlined video-icon steps-icon" *ngIf="!agentService.isCurrentSessionRunning() && agentService.currentSessionRecordingStatus() !== 'processing' && !agentService.currentSessionVideoUrl() && agentService.hasCurrentSessionStepFrames()">
|
||||
slideshow
|
||||
</span>
|
||||
<span class="material-symbols-outlined video-icon error-icon" *ngIf="!agentService.isCurrentSessionRunning() && agentService.currentSessionRecordingStatus() !== 'processing' && !agentService.currentSessionVideoUrl() && !agentService.hasCurrentSessionStepFrames() && agentService.currentSessionRecordingStatus() === 'failed'">
|
||||
videocam_off
|
||||
</span>
|
||||
<span class="material-symbols-outlined video-icon" *ngIf="!agentService.isCurrentSessionRunning() && agentService.currentSessionRecordingStatus() !== 'processing' && !agentService.currentSessionVideoUrl() && !agentService.hasCurrentSessionStepFrames() && agentService.currentSessionRecordingStatus() !== 'failed'">
|
||||
videocam
|
||||
</span>
|
||||
|
||||
<span class="btn-text" *ngIf="agentService.isCurrentSessionRunning()">Recording...</span>
|
||||
<span class="btn-text" *ngIf="!agentService.isCurrentSessionRunning() && agentService.currentSessionRecordingStatus() === 'processing'">Preparing...</span>
|
||||
<span class="btn-text" *ngIf="!agentService.isCurrentSessionRunning() && agentService.currentSessionRecordingStatus() !== 'processing' && agentService.currentSessionVideoUrl()">Play Recording</span>
|
||||
<span class="btn-text" *ngIf="!agentService.isCurrentSessionRunning() && agentService.currentSessionRecordingStatus() !== 'processing' && !agentService.currentSessionVideoUrl()">Screen Recording</span>
|
||||
<span class="btn-text" *ngIf="!agentService.isCurrentSessionRunning() && agentService.currentSessionRecordingStatus() !== 'processing' && !agentService.currentSessionVideoUrl() && agentService.hasCurrentSessionStepFrames()">Step Replay</span>
|
||||
<span class="btn-text" *ngIf="!agentService.isCurrentSessionRunning() && agentService.currentSessionRecordingStatus() !== 'processing' && !agentService.currentSessionVideoUrl() && !agentService.hasCurrentSessionStepFrames() && agentService.currentSessionRecordingStatus() === 'failed'">Recording Failed</span>
|
||||
<span class="btn-text" *ngIf="!agentService.isCurrentSessionRunning() && agentService.currentSessionRecordingStatus() !== 'processing' && !agentService.currentSessionVideoUrl() && !agentService.hasCurrentSessionStepFrames() && agentService.currentSessionRecordingStatus() !== 'failed'">Screen Recording</span>
|
||||
|
||||
<span class="video-pulse-dot red" *ngIf="agentService.isCurrentSessionRunning()"></span>
|
||||
<span class="steps-count-badge" *ngIf="!agentService.isCurrentSessionRunning() && !agentService.currentSessionVideoUrl() && agentService.hasCurrentSessionStepFrames()">
|
||||
{{ agentService.currentSessionStepFrames().length }}
|
||||
</span>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -936,6 +936,56 @@ $stream-row-height: 25px;
|
||||
box-shadow: 0 1px 3px rgba(15, 23, 42, 0.08);
|
||||
}
|
||||
|
||||
&.has-steps {
|
||||
background: rgba(240, 253, 250, 0.9);
|
||||
border-color: rgba(94, 234, 212, 0.7);
|
||||
box-shadow: 0 2px 6px -1px rgba(13, 148, 136, 0.12), inset 0 1px 1px rgba(255, 255, 255, 0.9);
|
||||
|
||||
.steps-icon {
|
||||
color: #0d9488;
|
||||
}
|
||||
|
||||
.btn-text {
|
||||
color: #0f766e;
|
||||
}
|
||||
|
||||
&:hover:not(.active) {
|
||||
background: rgba(204, 251, 241, 0.95);
|
||||
border-color: rgba(45, 212, 191, 0.85);
|
||||
|
||||
.steps-icon {
|
||||
color: #0f766e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
&.failed {
|
||||
background: rgba(254, 242, 242, 0.8);
|
||||
border-color: rgba(254, 202, 202, 0.9);
|
||||
|
||||
.error-icon {
|
||||
color: #ef4444;
|
||||
}
|
||||
|
||||
.btn-text {
|
||||
color: #dc2626;
|
||||
}
|
||||
}
|
||||
|
||||
.steps-count-badge {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
padding: 1px 5px;
|
||||
border-radius: 10px;
|
||||
background: #0d9488;
|
||||
color: #ffffff;
|
||||
font-size: 10px;
|
||||
font-weight: 700;
|
||||
line-height: 1;
|
||||
margin-left: 2px;
|
||||
}
|
||||
|
||||
&.active {
|
||||
background: rgba(26, 115, 232, 0.95);
|
||||
border-color: rgba(26, 115, 232, 0.95);
|
||||
@@ -949,6 +999,11 @@ $stream-row-height: 25px;
|
||||
background: #ffffff;
|
||||
box-shadow: 0 0 6px rgba(255, 255, 255, 0.9);
|
||||
}
|
||||
|
||||
.steps-count-badge {
|
||||
background: #ffffff;
|
||||
color: #1a73e8;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -936,6 +936,27 @@ export class AgentStreamComponent implements AfterViewInit {
|
||||
}
|
||||
}
|
||||
|
||||
public getScreenRecordingButtonTitle(): string {
|
||||
if (this.agentService.isCurrentSessionRunning()) {
|
||||
return 'Task is currently running (Recording screen)';
|
||||
}
|
||||
if (this.agentService.currentSessionRecordingStatus() === 'processing') {
|
||||
return 'Preparing screen recording...';
|
||||
}
|
||||
if (this.agentService.currentSessionVideoUrl()) {
|
||||
return 'Play Screen Recording Video';
|
||||
}
|
||||
if (this.agentService.hasCurrentSessionStepFrames()) {
|
||||
return this.agentService.currentSessionRecordingStatus() === 'failed'
|
||||
? 'Video recording failed — Click to replay step-by-step screenshots'
|
||||
: 'Play step-by-step screenshots replay';
|
||||
}
|
||||
if (this.agentService.currentSessionRecordingStatus() === 'failed') {
|
||||
return 'Screen recording generation failed';
|
||||
}
|
||||
return 'Screen Recording';
|
||||
}
|
||||
|
||||
public isDeviceActionTool(tool: any): boolean {
|
||||
return isDeviceActionTool(tool);
|
||||
}
|
||||
|
||||
+125
-12
@@ -33,16 +33,38 @@
|
||||
<header class="window-header" (mousedown)="startDrag($event)">
|
||||
<div class="header-left">
|
||||
<span class="material-symbols-outlined header-icon">
|
||||
{{ agentService.recordingPlaybackStatus() === 'live' ? 'sensors' : (agentService.recordingPlaybackStatus() === 'processing' ? 'progress_activity' : 'smart_display') }}
|
||||
{{ agentService.recordingPlaybackStatus() === 'live' ? 'sensors' : (agentService.playerMode() === 'steps' ? 'slideshow' : (agentService.recordingPlaybackStatus() === 'processing' ? 'progress_activity' : 'smart_display')) }}
|
||||
</span>
|
||||
<span class="header-title" [title]="agentService.activeVideoTitle()">
|
||||
{{ agentService.recordingPlaybackStatus() === 'live' ? 'Live Device Screen' : (agentService.recordingPlaybackStatus() === 'processing' ? 'Preparing Recording' : (agentService.activeVideoTitle() || 'Screen Recording')) }}
|
||||
{{ agentService.recordingPlaybackStatus() === 'live' ? 'Live Device Screen' : (agentService.playerMode() === 'steps' ? ('Step Replay (' + (totalStepFrames() > 0 ? (activeStepIndex() + 1) + '/' + totalStepFrames() : '0') + ')') : (agentService.recordingPlaybackStatus() === 'processing' ? 'Preparing Recording' : (agentService.activeVideoTitle() || 'Screen Recording'))) }}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div class="header-actions" (mousedown)="$event.stopPropagation()">
|
||||
<!-- Playback Speed Pills (Shown only when playback video is active) -->
|
||||
<div class="speed-pill-group" *ngIf="agentService.recordingPlaybackStatus() === 'ready' && !agentService.isVideoMinimized() && !videoLoadError() && agentService.activeVideoUrl()">
|
||||
<!-- Mode Toggle Group: Video vs. Step Replay -->
|
||||
<div class="mode-toggle-group" *ngIf="agentService.recordingPlaybackStatus() === 'ready' && agentService.hasCurrentSessionStepFrames() && !agentService.isVideoMinimized()">
|
||||
<button
|
||||
class="mode-btn"
|
||||
[class.active]="agentService.playerMode() === 'video'"
|
||||
(click)="agentService.setPlayerMode('video')"
|
||||
title="Video Recording"
|
||||
>
|
||||
<span class="material-symbols-outlined mode-icon">videocam</span>
|
||||
<span>Video</span>
|
||||
</button>
|
||||
<button
|
||||
class="mode-btn"
|
||||
[class.active]="agentService.playerMode() === 'steps'"
|
||||
(click)="agentService.setPlayerMode('steps')"
|
||||
title="Step Screenshot Replay"
|
||||
>
|
||||
<span class="material-symbols-outlined mode-icon">slideshow</span>
|
||||
<span>Steps</span>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<!-- Playback Speed Pills (Shown when video is active OR when step replay is active) -->
|
||||
<div class="speed-pill-group" *ngIf="!agentService.isVideoMinimized() && (agentService.recordingPlaybackStatus() === 'ready' || agentService.playerMode() === 'steps') && (agentService.activeVideoUrl() || totalStepFrames() > 0)">
|
||||
<button
|
||||
*ngFor="let speed of speedOptions"
|
||||
class="speed-btn"
|
||||
@@ -121,16 +143,107 @@
|
||||
</div>
|
||||
</ng-container>
|
||||
|
||||
<!-- 2. RECORDING PLAYBACK (AUTOMATICALLY ACTIVE WHEN TEST FINISHED) -->
|
||||
<!-- 2. RECORDING PLAYBACK / STEP REPLAY (AUTOMATICALLY ACTIVE WHEN TEST FINISHED) -->
|
||||
<ng-container *ngIf="agentService.recordingPlaybackStatus() !== 'live'">
|
||||
<!-- Loading State -->
|
||||
<div class="video-loading-state" *ngIf="agentService.recordingPlaybackStatus() === 'processing' || agentService.isVideoLoading()">
|
||||
<!-- Loading State (when in video mode and processing) -->
|
||||
<div class="video-loading-state" *ngIf="agentService.playerMode() === 'video' && (agentService.recordingPlaybackStatus() === 'processing' || agentService.isVideoLoading())">
|
||||
<span class="material-symbols-outlined spinning-icon">progress_activity</span>
|
||||
<span>{{ agentService.recordingPlaybackMessage() || 'Preparing screen recording...' }}</span>
|
||||
</div>
|
||||
|
||||
<!-- Active Video Playback View -->
|
||||
<ng-container *ngIf="agentService.recordingPlaybackStatus() === 'ready' && !agentService.isVideoLoading() && agentService.activeVideoUrl() as videoUrl">
|
||||
<!-- Option A: STEP REPLAY VIEW (ACTIVE IN STEP MODE) -->
|
||||
<ng-container *ngIf="agentService.playerMode() === 'steps' && totalStepFrames() > 0">
|
||||
<div class="step-replay-container">
|
||||
<!-- Fallback info banner if recording had failed -->
|
||||
<div class="step-replay-notice" *ngIf="agentService.recordingPlaybackStatus() === 'failed' || videoLoadError()">
|
||||
<span class="material-symbols-outlined notice-icon">info</span>
|
||||
<span>Screen recording unavailable. Replaying step screenshots.</span>
|
||||
</div>
|
||||
|
||||
<div class="step-image-viewport">
|
||||
<img
|
||||
class="step-frame-img"
|
||||
[src]="currentStepFrame()?.imageUrl"
|
||||
[alt]="currentStepFrame()?.title"
|
||||
(click)="openImageInNewTab()"
|
||||
title="Click to view image in new tab"
|
||||
/>
|
||||
|
||||
<!-- Step Overlay Badge -->
|
||||
<div class="step-frame-badge" *ngIf="currentStepFrame() as frame">
|
||||
<span class="badge-step-no">Step {{ frame.stepNumber }}</span>
|
||||
<span class="badge-action-desc">{{ frame.actionText || frame.title }}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Step Replay Controls Footer -->
|
||||
<div class="video-controls-footer">
|
||||
<!-- Timeline Scrubber Bar -->
|
||||
<div class="scrubber-bar-container">
|
||||
<input
|
||||
type="range"
|
||||
class="scrubber-range"
|
||||
[min]="0"
|
||||
[max]="totalStepFrames() - 1"
|
||||
[step]="1"
|
||||
[value]="activeStepIndex()"
|
||||
(input)="onStepScrub($event)"
|
||||
/>
|
||||
<div class="scrubber-progress-fill" [style.width.%]="stepProgressPercent()"></div>
|
||||
</div>
|
||||
|
||||
<!-- Bottom Controls Row -->
|
||||
<div class="controls-row">
|
||||
<div class="controls-left">
|
||||
<!-- Play / Pause Button -->
|
||||
<button class="ctrl-btn play-btn" (click)="toggleStepPlay()" [title]="isStepPlaying() ? 'Pause' : 'Play Slideshow'">
|
||||
<span class="material-symbols-outlined">{{ isStepPlaying() ? 'pause' : 'play_arrow' }}</span>
|
||||
</button>
|
||||
|
||||
<!-- Previous Frame Button -->
|
||||
<button class="ctrl-btn" (click)="prevStepFrame()" [disabled]="activeStepIndex() <= 0" title="Previous Frame">
|
||||
<span class="material-symbols-outlined">skip_previous</span>
|
||||
</button>
|
||||
|
||||
<!-- Next Frame Button -->
|
||||
<button class="ctrl-btn" (click)="nextStepFrame()" [disabled]="activeStepIndex() >= totalStepFrames() - 1" title="Next Frame">
|
||||
<span class="material-symbols-outlined">skip_next</span>
|
||||
</button>
|
||||
|
||||
<!-- Step Replay / Reset -->
|
||||
<button class="ctrl-btn" (click)="restartStep()" title="Replay from start">
|
||||
<span class="material-symbols-outlined">replay</span>
|
||||
</button>
|
||||
|
||||
<!-- Frame Counter Display -->
|
||||
<span class="time-display">
|
||||
Frame {{ activeStepIndex() + 1 }} / {{ totalStepFrames() }}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div class="controls-right">
|
||||
<!-- Loop Toggle -->
|
||||
<button
|
||||
class="ctrl-btn"
|
||||
[class.active]="isLooping()"
|
||||
(click)="toggleLoop()"
|
||||
[title]="isLooping() ? 'Loop enabled' : 'Enable loop'"
|
||||
>
|
||||
<span class="material-symbols-outlined">repeat</span>
|
||||
</button>
|
||||
|
||||
<!-- Open in new tab -->
|
||||
<button class="ctrl-btn" (click)="openImageInNewTab()" title="Open image in new tab">
|
||||
<span class="material-symbols-outlined">open_in_new</span>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</ng-container>
|
||||
|
||||
<!-- Option B: ACTIVE VIDEO PLAYBACK VIEW -->
|
||||
<ng-container *ngIf="agentService.playerMode() === 'video' && agentService.recordingPlaybackStatus() === 'ready' && !agentService.isVideoLoading() && agentService.activeVideoUrl() as videoUrl">
|
||||
<div class="video-container" *ngIf="!videoLoadError()">
|
||||
<video
|
||||
#videoRef
|
||||
@@ -146,7 +259,7 @@
|
||||
></video>
|
||||
</div>
|
||||
|
||||
<!-- Dedicated Bottom Controls Bar (Outside Video: ZERO Obscurity / Zero Overlap) -->
|
||||
<!-- Dedicated Bottom Controls Bar -->
|
||||
<div class="video-controls-footer" *ngIf="!videoLoadError()">
|
||||
<!-- Timeline Scrubber Bar -->
|
||||
<div class="scrubber-bar-container">
|
||||
@@ -206,8 +319,8 @@
|
||||
</div>
|
||||
</ng-container>
|
||||
|
||||
<!-- Clean, High-Contrast Minimal Empty / Fallback State -->
|
||||
<div class="empty-video-fallback" *ngIf="!agentService.isVideoLoading() && (agentService.recordingPlaybackStatus() === 'failed' || agentService.recordingPlaybackStatus() === 'unavailable' || videoLoadError())">
|
||||
<!-- Option C: EMPTY / FALLBACK STATE (Only when no video AND no step frames) -->
|
||||
<div class="empty-video-fallback" *ngIf="!agentService.isVideoLoading() && (agentService.playerMode() === 'video' || totalStepFrames() === 0) && (agentService.recordingPlaybackStatus() === 'failed' || agentService.recordingPlaybackStatus() === 'unavailable' || videoLoadError()) && totalStepFrames() === 0">
|
||||
<div class="fallback-icon-wrap">
|
||||
<span class="material-symbols-outlined fallback-icon">videocam_off</span>
|
||||
</div>
|
||||
|
||||
+134
-2
@@ -176,6 +176,48 @@
|
||||
gap: 4px;
|
||||
flex-shrink: 0;
|
||||
|
||||
.mode-toggle-group {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
background: rgba(241, 245, 249, 0.7);
|
||||
backdrop-filter: blur(12px);
|
||||
border: 1px solid rgba(226, 232, 240, 0.85);
|
||||
border-radius: 8px;
|
||||
padding: 2px;
|
||||
margin-right: 4px;
|
||||
gap: 2px;
|
||||
|
||||
.mode-btn {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 3px;
|
||||
border: none;
|
||||
background: transparent;
|
||||
color: #475569;
|
||||
font-size: 11px;
|
||||
font-weight: 600;
|
||||
padding: 2px 7px;
|
||||
border-radius: 6px;
|
||||
cursor: pointer;
|
||||
transition: all 0.15s ease;
|
||||
|
||||
.mode-icon {
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
&:hover:not(.active) {
|
||||
color: #0f172a;
|
||||
background: rgba(255, 255, 255, 0.8);
|
||||
}
|
||||
|
||||
&.active {
|
||||
background: #0284c7;
|
||||
color: #ffffff;
|
||||
box-shadow: 0 1px 3px rgba(2, 132, 199, 0.3);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.speed-pill-group {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
@@ -185,8 +227,6 @@
|
||||
border-radius: 8px;
|
||||
padding: 2px;
|
||||
margin-right: 4px;
|
||||
padding: 2px;
|
||||
margin-right: 4px;
|
||||
|
||||
.speed-btn {
|
||||
border: none;
|
||||
@@ -370,6 +410,98 @@
|
||||
}
|
||||
}
|
||||
|
||||
.step-replay-container {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
background: #000000;
|
||||
position: relative;
|
||||
width: 100%;
|
||||
overflow: hidden;
|
||||
|
||||
.step-replay-notice {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
background: rgba(254, 243, 199, 0.95);
|
||||
color: #92400e;
|
||||
padding: 6px 12px;
|
||||
font-size: 11.5px;
|
||||
font-weight: 500;
|
||||
border-bottom: 1px solid rgba(245, 158, 11, 0.3);
|
||||
|
||||
.notice-icon {
|
||||
font-size: 14px;
|
||||
color: #d97706;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
}
|
||||
|
||||
.step-image-viewport {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
position: relative;
|
||||
background: #0a0a0c;
|
||||
min-height: 240px;
|
||||
max-height: 520px;
|
||||
cursor: pointer;
|
||||
overflow: hidden;
|
||||
|
||||
.step-frame-img {
|
||||
max-width: 100%;
|
||||
max-height: 520px;
|
||||
width: auto;
|
||||
height: auto;
|
||||
object-fit: contain;
|
||||
display: block;
|
||||
user-select: none;
|
||||
transition: opacity 0.15s ease;
|
||||
|
||||
&:hover {
|
||||
opacity: 0.96;
|
||||
}
|
||||
}
|
||||
|
||||
.step-frame-badge {
|
||||
position: absolute;
|
||||
bottom: 12px;
|
||||
left: 12px;
|
||||
right: 12px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
background: rgba(15, 23, 42, 0.84);
|
||||
backdrop-filter: blur(10px);
|
||||
border: 1px solid rgba(255, 255, 255, 0.18);
|
||||
border-radius: 8px;
|
||||
padding: 6px 10px;
|
||||
color: #ffffff;
|
||||
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.45);
|
||||
pointer-events: none;
|
||||
animation: fadeIn 0.2s ease;
|
||||
|
||||
.badge-step-no {
|
||||
font-size: 11px;
|
||||
font-weight: 700;
|
||||
background: #0284c7;
|
||||
padding: 1px 6px;
|
||||
border-radius: 4px;
|
||||
letter-spacing: 0.2px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.badge-action-desc {
|
||||
font-size: 11.5px;
|
||||
font-weight: 500;
|
||||
color: #f8fafc;
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.video-container {
|
||||
position: relative;
|
||||
width: 100%;
|
||||
|
||||
+120
-2
@@ -22,11 +22,13 @@ import {
|
||||
ElementRef,
|
||||
ViewChild,
|
||||
HostListener,
|
||||
effect
|
||||
effect,
|
||||
OnDestroy
|
||||
} from '@angular/core';
|
||||
import { CommonModule } from '@angular/common';
|
||||
import { FormsModule } from '@angular/forms';
|
||||
import { AgentService } from '../../services/agent.service';
|
||||
import { StepReplayFrame } from '../../core/models/stream.model';
|
||||
|
||||
@Component({
|
||||
selector: 'app-floating-video-player',
|
||||
@@ -35,7 +37,7 @@ import { AgentService } from '../../services/agent.service';
|
||||
templateUrl: './floating-video-player.component.html',
|
||||
styleUrl: './floating-video-player.component.scss'
|
||||
})
|
||||
export class FloatingVideoPlayerComponent {
|
||||
export class FloatingVideoPlayerComponent implements OnDestroy {
|
||||
public agentService = inject(AgentService);
|
||||
|
||||
@ViewChild('videoRef') videoRef?: ElementRef<HTMLVideoElement>;
|
||||
@@ -60,6 +62,24 @@ export class FloatingVideoPlayerComponent {
|
||||
private pendingAutoplay = false;
|
||||
private pendingAbsoluteSeek: number | null = null;
|
||||
|
||||
// Step Replay state signals
|
||||
public activeStepIndex = signal<number>(0);
|
||||
public isStepPlaying = signal<boolean>(false);
|
||||
private stepTimer: any = null;
|
||||
|
||||
public stepFrames = computed(() => this.agentService.currentSessionStepFrames());
|
||||
public totalStepFrames = computed(() => this.stepFrames().length);
|
||||
public currentStepFrame = computed<StepReplayFrame | null>(() => {
|
||||
const frames = this.stepFrames();
|
||||
const idx = this.activeStepIndex();
|
||||
return frames[idx] || null;
|
||||
});
|
||||
public stepProgressPercent = computed(() => {
|
||||
const total = this.totalStepFrames();
|
||||
if (total <= 1) return 0;
|
||||
return (this.activeStepIndex() / (total - 1)) * 100;
|
||||
});
|
||||
|
||||
// Available speed options
|
||||
public speedOptions = [0.5, 1.0, 1.5, 2.0, 4.0];
|
||||
|
||||
@@ -92,6 +112,16 @@ export class FloatingVideoPlayerComponent {
|
||||
{ allowSignalWrites: true }
|
||||
);
|
||||
|
||||
// Reset step playback state when session changes
|
||||
effect(
|
||||
() => {
|
||||
const _ = this.stepFrames();
|
||||
this.activeStepIndex.set(0);
|
||||
this.pauseStepPlay();
|
||||
},
|
||||
{ allowSignalWrites: true }
|
||||
);
|
||||
|
||||
effect(() => {
|
||||
const request = this.agentService.videoSeekRequest();
|
||||
if (!request) return;
|
||||
@@ -327,6 +357,9 @@ export class FloatingVideoPlayerComponent {
|
||||
if (v) {
|
||||
v.playbackRate = rate;
|
||||
}
|
||||
if (this.isStepPlaying()) {
|
||||
this.scheduleNextStepFrame();
|
||||
}
|
||||
}
|
||||
|
||||
public toggleLoop(): void {
|
||||
@@ -361,4 +394,89 @@ export class FloatingVideoPlayerComponent {
|
||||
window.open(url, '_blank');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Step Replay Controls
|
||||
*/
|
||||
public toggleStepPlay(): void {
|
||||
if (this.isStepPlaying()) {
|
||||
this.pauseStepPlay();
|
||||
} else {
|
||||
this.startStepPlay();
|
||||
}
|
||||
}
|
||||
|
||||
public startStepPlay(): void {
|
||||
if (this.totalStepFrames() <= 0) return;
|
||||
if (this.activeStepIndex() >= this.totalStepFrames() - 1) {
|
||||
this.activeStepIndex.set(0);
|
||||
}
|
||||
this.isStepPlaying.set(true);
|
||||
this.scheduleNextStepFrame();
|
||||
}
|
||||
|
||||
public pauseStepPlay(): void {
|
||||
this.isStepPlaying.set(false);
|
||||
if (this.stepTimer) {
|
||||
clearTimeout(this.stepTimer);
|
||||
this.stepTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
private scheduleNextStepFrame(): void {
|
||||
if (this.stepTimer) clearTimeout(this.stepTimer);
|
||||
const delay = Math.max(300, 1500 / this.playbackRate());
|
||||
this.stepTimer = setTimeout(() => {
|
||||
if (!this.isStepPlaying()) return;
|
||||
if (this.activeStepIndex() < this.totalStepFrames() - 1) {
|
||||
this.activeStepIndex.update((i) => i + 1);
|
||||
this.scheduleNextStepFrame();
|
||||
} else {
|
||||
if (this.isLooping()) {
|
||||
this.activeStepIndex.set(0);
|
||||
this.scheduleNextStepFrame();
|
||||
} else {
|
||||
this.pauseStepPlay();
|
||||
}
|
||||
}
|
||||
}, delay);
|
||||
}
|
||||
|
||||
public prevStepFrame(): void {
|
||||
this.pauseStepPlay();
|
||||
this.activeStepIndex.update((i) => Math.max(0, i - 1));
|
||||
}
|
||||
|
||||
public nextStepFrame(): void {
|
||||
this.pauseStepPlay();
|
||||
this.activeStepIndex.update((i) => Math.min(this.totalStepFrames() - 1, i + 1));
|
||||
}
|
||||
|
||||
public seekStepFrame(index: number): void {
|
||||
this.activeStepIndex.set(Math.max(0, Math.min(this.totalStepFrames() - 1, index)));
|
||||
}
|
||||
|
||||
public onStepScrub(event: Event): void {
|
||||
const input = event.target as HTMLInputElement;
|
||||
this.seekStepFrame(parseInt(input.value, 10));
|
||||
}
|
||||
|
||||
public restartStep(): void {
|
||||
this.seekStepFrame(0);
|
||||
this.startStepPlay();
|
||||
}
|
||||
|
||||
public openImageInNewTab(): void {
|
||||
const url = this.currentStepFrame()?.imageUrl;
|
||||
if (url) {
|
||||
window.open(url, '_blank');
|
||||
}
|
||||
}
|
||||
|
||||
public ngOnDestroy(): void {
|
||||
if (this.stepTimer) {
|
||||
clearTimeout(this.stepTimer);
|
||||
this.stepTimer = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -98,3 +98,14 @@ export interface CheckerResult {
|
||||
reason: string;
|
||||
}
|
||||
|
||||
export interface StepReplayFrame {
|
||||
index: number;
|
||||
stepNumber: number;
|
||||
title: string;
|
||||
imageUrl: string;
|
||||
actionText?: string;
|
||||
isPost?: boolean;
|
||||
timestamp?: number;
|
||||
phaseId?: string;
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { signal } from '@angular/core';
|
||||
import { signal, computed } from '@angular/core';
|
||||
import { of } from 'rxjs';
|
||||
|
||||
import { AgentService } from './agent.service';
|
||||
@@ -198,6 +198,9 @@ describe('AgentService recording finalization lifecycle', () => {
|
||||
service.recordingPlaybackStatus = signal('idle');
|
||||
service.recordingPlaybackMessage = signal('');
|
||||
service.shouldAutoplayVideo = signal(true);
|
||||
service.playerMode = signal<'video' | 'steps'>('video');
|
||||
(service as any).hasCurrentSessionStepFrames = computed(() => false);
|
||||
(service as any).currentSessionStepFrames = computed(() => []);
|
||||
return service;
|
||||
}
|
||||
|
||||
|
||||
@@ -19,8 +19,9 @@ import { HttpClient } from '@angular/common/http';
|
||||
import { Observable } from 'rxjs';
|
||||
|
||||
import { Session, ModelInfo, TaskQueueItem, AgentStatusResponse } from '../core/models/session.model';
|
||||
import { StepItemData } from '../core/models/stream.model';
|
||||
export type { Session, ModelInfo, TaskQueueItem, AgentStatusResponse, StepItemData };
|
||||
import { StepItemData, StepReplayFrame } from '../core/models/stream.model';
|
||||
import { extractStepReplayFrames } from '../utils/action-formatter.util';
|
||||
export type { Session, ModelInfo, TaskQueueItem, AgentStatusResponse, StepItemData, StepReplayFrame };
|
||||
|
||||
const SESSION_CACHE_KEY = 'artemis.sessions.v1';
|
||||
|
||||
@@ -158,12 +159,32 @@ export class AgentService {
|
||||
public recordingPlaybackMessage = signal<string>('');
|
||||
public shouldAutoplayVideo = signal<boolean>(false);
|
||||
public videoSeekRequest = signal<{ seconds: number; requestId: number } | null>(null);
|
||||
public playerMode = signal<'video' | 'steps'>('video');
|
||||
private activeVideoSessionId: string | null = null;
|
||||
private videoSeekRequestId = 0;
|
||||
private videoRequestGeneration = 0;
|
||||
private videoRetryTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private videoWaitStartedAt = 0;
|
||||
|
||||
/**
|
||||
* Computed step replay frames from current session logs
|
||||
*/
|
||||
public currentSessionStepFrames = computed<StepReplayFrame[]>(() => {
|
||||
return extractStepReplayFrames(this.sessionLogs());
|
||||
});
|
||||
|
||||
public hasCurrentSessionStepFrames = computed<boolean>(() => {
|
||||
return this.currentSessionStepFrames().length > 0;
|
||||
});
|
||||
|
||||
public setPlayerMode(mode: 'video' | 'steps'): void {
|
||||
this.playerMode.set(mode);
|
||||
}
|
||||
|
||||
public togglePlayerMode(): void {
|
||||
this.playerMode.update((mode) => (mode === 'video' ? 'steps' : 'video'));
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear user-pinned selection so subsequent runs automatically follow active runner
|
||||
*/
|
||||
@@ -1284,6 +1305,13 @@ export class AgentService {
|
||||
this.isVideoLoading.set(true);
|
||||
this.recordingPlaybackStatus.set('processing');
|
||||
this.recordingPlaybackMessage.set('Loading screen recording...');
|
||||
if (targetUrl) {
|
||||
this.playerMode?.set('video');
|
||||
} else if (this.hasCurrentSessionStepFrames?.()) {
|
||||
this.playerMode?.set('steps');
|
||||
} else {
|
||||
this.playerMode?.set('video');
|
||||
}
|
||||
this.requestSessionVideo(targetSessionId, this.videoRequestGeneration);
|
||||
}
|
||||
|
||||
@@ -1341,6 +1369,7 @@ export class AgentService {
|
||||
this.recordingPlaybackMessage.set('');
|
||||
this.activeVideoUrl.set(res.video_url);
|
||||
this.activeVideoSegments.set(res.video_segments || []);
|
||||
this.playerMode?.set('video');
|
||||
this.rawSessions.update((list) =>
|
||||
list.map((s) => s.session_id === sessionId
|
||||
? { ...s, video_url: res.video_url || undefined, recording_status: 'ready' }
|
||||
@@ -1368,6 +1397,9 @@ export class AgentService {
|
||||
? 'Recording finalization failed.'
|
||||
: 'No screen recording is available for this task.')
|
||||
);
|
||||
if (this.hasCurrentSessionStepFrames?.()) {
|
||||
this.playerMode?.set('steps');
|
||||
}
|
||||
},
|
||||
error: () => {
|
||||
if (generation !== this.videoRequestGeneration || sessionId !== this.activeVideoSessionId) {
|
||||
@@ -1380,6 +1412,9 @@ export class AgentService {
|
||||
this.isVideoLoading.set(false);
|
||||
this.recordingPlaybackStatus.set('failed');
|
||||
this.recordingPlaybackMessage.set('Unable to load the screen recording.');
|
||||
if (this.hasCurrentSessionStepFrames?.()) {
|
||||
this.playerMode?.set('steps');
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import {
|
||||
getStepPreImageUrl,
|
||||
getStepPostImageUrl
|
||||
getStepPostImageUrl,
|
||||
extractStepReplayFrames
|
||||
} from './action-formatter.util';
|
||||
|
||||
describe('action-formatter.util screenshot chaining', () => {
|
||||
@@ -109,4 +110,100 @@ describe('action-formatter.util screenshot chaining', () => {
|
||||
expect(preUrl).toBe('/images/pre123');
|
||||
expect(postUrl).toBe('/images/post456');
|
||||
});
|
||||
|
||||
it('should resolve post screenshot for primary action when generic_tools contains action traces', () => {
|
||||
const stepData = {
|
||||
step_id: 'step-02',
|
||||
pre_image_name: 'pre_screen_hash',
|
||||
post_image_name: 'post_screen_hash',
|
||||
action_taken: {
|
||||
action: 'click',
|
||||
coordinates: [500, 928],
|
||||
args: { target: [500, 928] },
|
||||
timestamp: 1000
|
||||
},
|
||||
generic_tools: [
|
||||
{
|
||||
trace_id: 'trace-click-internal',
|
||||
type: 'action',
|
||||
name: 'click',
|
||||
timestamp: 1001,
|
||||
payload: {
|
||||
args: { target: '[500, 928]', times: '1', delay_ms: '100' },
|
||||
result: {
|
||||
outcome: 'Clicked successfully.',
|
||||
post_image_name: 'post_screen_hash',
|
||||
status: 'success'
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
};
|
||||
|
||||
const preUrl = getStepPreImageUrl(stepData, stepData.action_taken);
|
||||
const postUrl = getStepPostImageUrl(stepData, stepData.action_taken);
|
||||
|
||||
expect(preUrl).toBe('/images/pre_screen_hash');
|
||||
expect(postUrl).toBe('/images/post_screen_hash');
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractStepReplayFrames', () => {
|
||||
it('should return an empty array for empty or invalid logs', () => {
|
||||
expect(extractStepReplayFrames([])).toEqual([]);
|
||||
expect(extractStepReplayFrames(null as any)).toEqual([]);
|
||||
});
|
||||
|
||||
it('should extract ordered StepReplayFrames and deduplicate consecutive identical images', () => {
|
||||
const logs = [
|
||||
{
|
||||
type: 'step_updated',
|
||||
data: {
|
||||
step_id: 's-1',
|
||||
step_number: 1,
|
||||
pre_image_name: 'img_screen_1',
|
||||
post_image_name: 'img_screen_2',
|
||||
action_taken: { action: 'tap', coordinates: [500, 1000] },
|
||||
timestamp: 100
|
||||
}
|
||||
},
|
||||
{
|
||||
type: 'step_updated',
|
||||
data: {
|
||||
step_id: 's-2',
|
||||
step_number: 2,
|
||||
pre_image_name: 'img_screen_2',
|
||||
post_image_name: 'img_screen_3',
|
||||
action_taken: { action: 'input_text', text: 'hello' },
|
||||
timestamp: 200
|
||||
}
|
||||
}
|
||||
];
|
||||
|
||||
const frames = extractStepReplayFrames(logs);
|
||||
expect(frames.length).toBe(3);
|
||||
expect(frames[0].title).toBe('Step 1 (Before Action)');
|
||||
expect(frames[0].imageUrl).toBe('/images/img_screen_1');
|
||||
expect(frames[1].title).toContain('Step 1');
|
||||
expect(frames[1].imageUrl).toBe('/images/img_screen_2');
|
||||
expect(frames[2].title).toContain('Step 2');
|
||||
expect(frames[2].imageUrl).toBe('/images/img_screen_3');
|
||||
});
|
||||
|
||||
it('should handle terminal step with only pre_image (e.g. cancelled / failed task)', () => {
|
||||
const logs = [
|
||||
{
|
||||
step_number: 1,
|
||||
pre_image_name: 'img_start',
|
||||
post_image_name: null,
|
||||
action_taken: null,
|
||||
timestamp: 100
|
||||
}
|
||||
];
|
||||
|
||||
const frames = extractStepReplayFrames(logs);
|
||||
expect(frames.length).toBe(1);
|
||||
expect(frames[0].imageUrl).toBe('/images/img_start');
|
||||
expect(frames[0].stepNumber).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -14,7 +14,7 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { ActionParam } from '../core/models/stream.model';
|
||||
import { ActionParam, StepReplayFrame } from '../core/models/stream.model';
|
||||
import { extractNumbersFromCoordinateValue, isPureDirectionString, parseSequenceCoordinates } from './image-overlay.util';
|
||||
import { cleanErrorMessage } from './tool-formatter.util';
|
||||
|
||||
@@ -698,7 +698,7 @@ export function resolveStepImageMap(stepData: any): Map<any, ResolvedImageEntry>
|
||||
if (explicitPost) {
|
||||
resolvedPost = explicitPost;
|
||||
hasExplicitPost = true;
|
||||
} else if (e.isPrimary && !hadPrimaryFailure && !hasSubsequentActions) {
|
||||
} else if (e.isPrimary && !hadPrimaryFailure) {
|
||||
resolvedPost = stepData.post_image_name || stepData.post_screenshot || null;
|
||||
} else if (e.item === lastActionItem && !hadPrimaryFailure) {
|
||||
resolvedPost = stepData.post_image_name || stepData.post_screenshot || null;
|
||||
@@ -765,3 +765,86 @@ export function getStepPostImageUrl(stepData: any, actionData?: any): string | n
|
||||
}
|
||||
return postUrl;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract an ordered array of visual StepReplayFrames from session logs or step blocks
|
||||
*/
|
||||
export function extractStepReplayFrames(logsOrSteps: any[]): StepReplayFrame[] {
|
||||
if (!Array.isArray(logsOrSteps) || logsOrSteps.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
// Extract raw step data objects
|
||||
const rawSteps: any[] = [];
|
||||
for (const item of logsOrSteps) {
|
||||
if (!item) continue;
|
||||
if (item.type === 'step_updated' && item.data) {
|
||||
rawSteps.push(item.data);
|
||||
} else if (item.type === 'step' && item.data) {
|
||||
rawSteps.push(item.data);
|
||||
} else if (item.step_number !== undefined || item.step_id !== undefined) {
|
||||
rawSteps.push(item);
|
||||
}
|
||||
}
|
||||
|
||||
// Deduplicate by step_id or step_number, keeping latest
|
||||
const stepMap = new Map<any, any>();
|
||||
for (const step of rawSteps) {
|
||||
const key = step.step_id || step.step_number;
|
||||
if (key !== undefined) {
|
||||
stepMap.set(key, step);
|
||||
}
|
||||
}
|
||||
|
||||
const steps = Array.from(stepMap.values()).sort((a, b) => {
|
||||
const numA = Number(a.step_number ?? 0);
|
||||
const numB = Number(b.step_number ?? 0);
|
||||
if (numA !== numB) return numA - numB;
|
||||
return Number(a.timestamp ?? 0) - Number(b.timestamp ?? 0);
|
||||
});
|
||||
|
||||
const frames: StepReplayFrame[] = [];
|
||||
let lastImageUrl: string | null = null;
|
||||
|
||||
for (const stepData of steps) {
|
||||
const stepNum = Number(stepData.step_number ?? (frames.length + 1));
|
||||
const act = stepData.action_taken;
|
||||
const title = act ? (getActionTitle(act) || 'Action') : 'Step';
|
||||
const coords = act ? getActionCoords(act) : '';
|
||||
const actionDesc = coords ? `${title} (${coords})` : title;
|
||||
|
||||
const preUrl = getStepPreImageUrl(stepData, act);
|
||||
const postUrl = getStepPostImageUrl(stepData, act);
|
||||
|
||||
// If pre-image exists and is new, add it
|
||||
if (preUrl && preUrl !== lastImageUrl) {
|
||||
frames.push({
|
||||
index: frames.length,
|
||||
stepNumber: stepNum,
|
||||
title: `Step ${stepNum} (Before Action)`,
|
||||
imageUrl: preUrl,
|
||||
actionText: `Preparing: ${actionDesc}`,
|
||||
isPost: false,
|
||||
timestamp: stepData.timestamp
|
||||
});
|
||||
lastImageUrl = preUrl;
|
||||
}
|
||||
|
||||
// If post-image exists and is different from current screen, add it
|
||||
if (postUrl && postUrl !== lastImageUrl) {
|
||||
frames.push({
|
||||
index: frames.length,
|
||||
stepNumber: stepNum,
|
||||
title: `Step ${stepNum}: ${actionDesc}`,
|
||||
imageUrl: postUrl,
|
||||
actionText: actionDesc,
|
||||
isPost: true,
|
||||
timestamp: stepData.timestamp
|
||||
});
|
||||
lastImageUrl = postUrl;
|
||||
}
|
||||
}
|
||||
|
||||
return frames;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user