/** * Copyright 2026 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import { Component, ChangeDetectionStrategy, NgZone, inject, signal, computed, ElementRef, ViewChild, HostListener, effect, OnDestroy } from '@angular/core'; import { FormsModule } from '@angular/forms'; import { AgentService } from '../../services/agent.service'; import { StepReplayFrame } from '../../core/models/stream.model'; import { drawActionCoordinatesOnOverlay } from '../../utils/image-overlay.util'; import { getActionIcon } from '../../utils/action-formatter.util'; import { locateTimelineTime, sessionTimeToTimelineTime } from '../../utils/recording-timeline.util'; @Component({ selector: 'app-floating-video-player', standalone: true, imports: [FormsModule], templateUrl: './floating-video-player.component.html', styleUrl: './floating-video-player.component.scss', changeDetection: ChangeDetectionStrategy.OnPush }) export class FloatingVideoPlayerComponent implements OnDestroy { public agentService = inject(AgentService); private zone = inject(NgZone); @ViewChild('videoRef') videoRef?: ElementRef; @ViewChild('stepImgRef') stepImgRef?: ElementRef; @ViewChild('stepOverlayRef') stepOverlayRef?: ElementRef; // Playback state signals public isPlaying = signal(false); public playbackRate = signal(1.0); public isLooping = signal(false); public currentTime = signal(0); public duration = signal(0); public isMuted = signal(false); public isTheaterMode = signal(false); public videoLoadError = signal(false); public liveStreamUrl = signal('/api/stream/device-live'); public liveStreamError = signal(false); public activeSegmentIndex = signal(0); public currentVideoUrl = computed(() => { const segments = this.agentService.activeVideoSegments(); return segments[this.activeSegmentIndex()]?.url || this.agentService.activeVideoUrl(); }); private pendingLocalTime: number | null = null; private pendingAutoplay = false; private pendingAbsoluteSeek: number | null = null; // Step Replay state signals public activeStepIndex = signal(0); public isStepPlaying = signal(false); public viewMode = signal<'pre' | 'post'>('pre'); // 'pre' = action preview with coords, 'post' = action result public isCardHovered = signal(false); public isCardPinned = signal(false); public isCardVisible = computed(() => this.isCardHovered() || this.isCardPinned()); public isStepImageLoading = signal(false); public stepImageError = signal(false); private stepTimer: any = null; public stepFrames = computed(() => this.agentService.currentSessionStepFrames()); public totalStepFrames = computed(() => this.stepFrames().length); public currentStepFrame = computed(() => { const frames = this.stepFrames(); const idx = this.activeStepIndex(); return frames[idx] || null; }); public currentStepImageUrl = computed(() => { const frame = this.currentStepFrame(); if (!frame) return ''; if (this.viewMode() === 'post' && frame.postImageUrl) { return frame.postImageUrl; } return frame.preImageUrl || frame.imageUrl || ''; }); public stepProgressPercent = computed(() => { const total = this.totalStepFrames(); if (total <= 1) return 0; return (this.activeStepIndex() / (total - 1)) * 100; }); // Available speed options public speedOptions = [0.5, 1.0, 1.5, 2.0, 4.0]; // Dragging and window positioning state public posX = signal( typeof window !== 'undefined' ? Math.max(20, window.innerWidth - 420) : 100 ); public posY = signal( typeof window !== 'undefined' ? Math.max(60, window.innerHeight - 700) : 100 ); private isDragging = false; private dragStartX = 0; private dragStartY = 0; private initialPosX = 0; private initialPosY = 0; constructor() { // Reset video load error whenever active video URL changes effect( () => { const url = this.agentService.activeVideoUrl(); const segments = this.agentService.activeVideoSegments(); this.videoLoadError.set(false); this.activeSegmentIndex.set(0); this.currentTime.set(0); this.duration.set(segments.reduce((sum, segment) => sum + segment.duration, 0)); this.isPlaying.set(false); }, { allowSignalWrites: true } ); // Reset step playback state only when session actually changes let lastTrackedSessionId: string | null = null; effect( () => { const curSessionId = this.agentService.currentSessionId(); const total = this.totalStepFrames(); if (curSessionId !== lastTrackedSessionId) { lastTrackedSessionId = curSessionId; this.activeStepIndex.set(0); this.viewMode.set('pre'); this.pauseStepPlay(); } else if (total > 0 && this.activeStepIndex() >= total) { this.activeStepIndex.set(total - 1); } }, { allowSignalWrites: true } ); // Track step screenshot URL changes and reset loading/error state effect(() => { const url = this.currentStepImageUrl(); if (url) { this.isStepImageLoading.set(true); this.stepImageError.set(false); } else { this.isStepImageLoading.set(false); } }, { allowSignalWrites: true }); // Re-draw overlay whenever activeStepIndex or viewMode changes effect(() => { this.activeStepIndex(); this.viewMode(); setTimeout(() => this.drawStepOverlay(), 40); }); effect(() => { const request = this.agentService.videoSeekRequest(); if (!request) return; this.pendingAbsoluteSeek = request.seconds; if (this.agentService.recordingPlaybackStatus() !== 'ready') return; const video = this.videoRef?.nativeElement; if (video && video.readyState >= 1) { const target = this.pendingAbsoluteSeek; this.pendingAbsoluteSeek = null; this.seekToSessionTime(target, true); } }); effect(() => { const request = this.agentService.stepSeekRequest(); if (!request) return; this.seekStepFrame(request.index); }); } public onLiveStreamError(): void { this.liveStreamError.set(true); } public retryLiveStream(): void { this.liveStreamError.set(false); this.liveStreamUrl.set(`/api/stream/device-live?t=${Date.now()}`); } /** * Format seconds into mm:ss format */ public formatTime(seconds: number): string { if (isNaN(seconds) || seconds < 0) return '00:00'; const mins = Math.floor(seconds / 60); const secs = Math.floor(seconds % 60); const mStr = mins < 10 ? '0' + mins : String(mins); const sStr = secs < 10 ? '0' + secs : String(secs); return `${mStr}:${sStr}`; } /** * Progress percentage for timeline scrubber (0 - 100) */ public progressPercent = computed(() => { const dur = this.duration(); if (dur <= 0) return 0; return Math.min(100, Math.max(0, (this.currentTime() / dur) * 100)); }); /** * Start dragging the window from header. The move/up listeners are attached * only for the duration of the drag and live outside the Angular zone, so * ordinary mouse movement over the page never touches change detection. */ public startDrag(event: MouseEvent): void { if (this.isTheaterMode()) return; this.isDragging = true; this.dragStartX = event.clientX; this.dragStartY = event.clientY; this.initialPosX = this.posX(); this.initialPosY = this.posY(); event.preventDefault(); this.zone.runOutsideAngular(() => { document.addEventListener('mousemove', this.onDrag); document.addEventListener('mouseup', this.stopDrag); }); } private onDrag = (event: MouseEvent): void => { if (!this.isDragging) return; const deltaX = event.clientX - this.dragStartX; const deltaY = event.clientY - this.dragStartY; const minX = 10; const minY = 10; const maxX = Math.max(10, window.innerWidth - 240); const maxY = Math.max(10, window.innerHeight - 80); const targetX = Math.max(minX, Math.min(maxX, this.initialPosX + deltaX)); const targetY = Math.max(minY, Math.min(maxY, this.initialPosY + deltaY)); this.posX.set(targetX); this.posY.set(targetY); }; private stopDrag = (): void => { this.isDragging = false; document.removeEventListener('mousemove', this.onDrag); document.removeEventListener('mouseup', this.stopDrag); }; @HostListener('window:resize') public onResize(): void { // Keep window within viewport bounds if window resized const maxX = Math.max(10, window.innerWidth - 240); const maxY = Math.max(10, window.innerHeight - 80); if (this.posX() > maxX) this.posX.set(maxX); if (this.posY() > maxY) this.posY.set(maxY); } @HostListener('window:keydown', ['$event']) public onKeyDown(event: KeyboardEvent): void { if (!this.agentService.isVideoWindowOpen()) return; if (event.key === 'Escape') { if (this.isTheaterMode()) { this.isTheaterMode.set(false); } else { this.agentService.closeVideoPlayer(); } } } /** * Video element event handlers */ public onLoadedMetadata(): void { const v = this.videoRef?.nativeElement; if (v) { const segments = this.agentService.activeVideoSegments(); if (!segments.length) { this.duration.set(v.duration || 0); } this.videoLoadError.set(false); v.playbackRate = this.playbackRate(); v.loop = this.isLooping() && !segments.length; v.muted = this.isMuted(); if (this.pendingAbsoluteSeek !== null) { const target = this.pendingAbsoluteSeek; this.pendingAbsoluteSeek = null; this.seekToSessionTime(target, true); // A seek into another finalized segment changes the video source. Let // that segment's metadata event apply pendingLocalTime. if (this.pendingLocalTime !== null) return; } if (this.pendingLocalTime !== null) { v.currentTime = Math.max(0, Math.min(v.duration || this.pendingLocalTime, this.pendingLocalTime)); this.pendingLocalTime = null; } if (this.pendingAutoplay) { this.pendingAutoplay = false; v.play().catch(() => {}); } if (this.agentService.consumeVideoAutoplay()) { this.isMuted.set(true); v.muted = true; v.play().catch(() => { this.isPlaying.set(false); }); } } } public onTimeUpdate(): void { const v = this.videoRef?.nativeElement; if (v) { const segments = this.agentService.activeVideoSegments(); const segment = segments[this.activeSegmentIndex()]; this.currentTime.set((segment?.start || 0) + v.currentTime); if (!segments.length && v.duration && v.duration !== this.duration()) { this.duration.set(v.duration); } } } public onPlay(): void { this.isPlaying.set(true); } public onPause(): void { this.isPlaying.set(false); } public onEnded(): void { const segments = this.agentService.activeVideoSegments(); if (segments.length && this.activeSegmentIndex() < segments.length - 1) { this.pendingLocalTime = 0; this.pendingAutoplay = true; this.activeSegmentIndex.update(index => index + 1); return; } if (segments.length && this.isLooping()) { this.seek(0, true); return; } this.isPlaying.set(false); } public onError(): void { this.videoLoadError.set(true); this.isPlaying.set(false); } /** * Playback actions */ public togglePlay(): void { const v = this.videoRef?.nativeElement; if (!v) return; if (v.paused) { v.play().catch(() => {}); } else { v.pause(); } } public restart(): void { this.seek(0, true); } /** * Seek to a session-relative time (seconds since the task session started, * the axis step timestamps and analyzer ranges use). Manifest v2 segments * carry real session offsets, so the time lands in the right segment even * across scrcpy restart gaps; legacy playlists fall back to the timeline axis. */ public seekToSessionTime(sessionSeconds: number, autoplay = false): void { const segments = this.agentService.activeVideoSegments(); this.seek(sessionTimeToTimelineTime(segments, sessionSeconds), autoplay); } /** Seek on the back-to-back playlist axis used by the scrubber. */ public seek(seconds: number, autoplay = false): void { const v = this.videoRef?.nativeElement; if (!v) return; const target = Math.max(0, Math.min(this.duration(), seconds)); const segments = this.agentService.activeVideoSegments(); const location = locateTimelineTime(segments, target); if (!location) { v.currentTime = target; if (autoplay) v.play().catch(() => {}); return; } const { index, localTime } = location; if (index === this.activeSegmentIndex()) { v.currentTime = localTime; if (autoplay) v.play().catch(() => {}); } else { this.pendingLocalTime = localTime; this.pendingAutoplay = autoplay || !v.paused; this.activeSegmentIndex.set(index); } this.currentTime.set(target); } public onScrub(event: Event): void { const input = event.target as HTMLInputElement; const targetTime = parseFloat(input.value); this.seek(targetTime); } public setSpeed(rate: number): void { this.playbackRate.set(rate); const v = this.videoRef?.nativeElement; if (v) { v.playbackRate = rate; } if (this.isStepPlaying()) { this.scheduleNextStepFrame(); } } public toggleLoop(): void { const next = !this.isLooping(); this.isLooping.set(next); const v = this.videoRef?.nativeElement; if (v) { v.loop = next && !this.agentService.activeVideoSegments().length; } } public toggleMute(): void { const next = !this.isMuted(); this.isMuted.set(next); const v = this.videoRef?.nativeElement; if (v) { v.muted = next; } } public toggleTheater(): void { this.isTheaterMode.set(!this.isTheaterMode()); } public toggleMinimize(): void { this.agentService.isVideoMinimized.set(!this.agentService.isVideoMinimized()); } public openInNewTab(): void { const url = this.agentService.activeVideoUrl(); if (url) { window.open(url, '_blank'); } } /** * Step Replay Controls */ public toggleStepPlay(): void { if (this.isStepPlaying()) { this.pauseStepPlay(); } else { this.startStepPlay(); } } public startStepPlay(): void { if (this.totalStepFrames() <= 0) return; if (this.activeStepIndex() >= this.totalStepFrames() - 1) { this.activeStepIndex.set(0); } this.isStepPlaying.set(true); this.scheduleNextStepFrame(); } public pauseStepPlay(): void { this.isStepPlaying.set(false); if (this.stepTimer) { clearTimeout(this.stepTimer); this.stepTimer = null; } } private scheduleNextStepFrame(): void { if (this.stepTimer) clearTimeout(this.stepTimer); const delay = Math.max(300, 1500 / this.playbackRate()); this.stepTimer = setTimeout(() => { if (!this.isStepPlaying()) return; if (this.activeStepIndex() < this.totalStepFrames() - 1) { this.activeStepIndex.update((i) => i + 1); this.scheduleNextStepFrame(); } else { if (this.isLooping()) { this.activeStepIndex.set(0); this.scheduleNextStepFrame(); } else { this.pauseStepPlay(); } } }, delay); } public prevStepFrame(): void { this.pauseStepPlay(); this.viewMode.set('pre'); this.activeStepIndex.update((i) => Math.max(0, i - 1)); } public nextStepFrame(): void { this.pauseStepPlay(); this.viewMode.set('pre'); this.activeStepIndex.update((i) => Math.min(this.totalStepFrames() - 1, i + 1)); } public seekStepFrame(index: number): void { this.viewMode.set('pre'); this.activeStepIndex.set(Math.max(0, Math.min(this.totalStepFrames() - 1, index))); } public onStepScrub(event: Event): void { const input = event.target as HTMLInputElement; this.seekStepFrame(parseInt(input.value, 10)); } public restartStep(): void { this.seekStepFrame(0); this.startStepPlay(); } public setStepViewMode(mode: 'pre' | 'post', event?: Event): void { if (event) event.stopPropagation(); this.viewMode.set(mode); setTimeout(() => this.drawStepOverlay(), 40); } public toggleStepViewMode(event?: Event): void { if (event) event.stopPropagation(); this.setStepViewMode(this.viewMode() === 'pre' ? 'post' : 'pre'); } public onViewportMouseEnter(): void { this.isCardHovered.set(true); } public onViewportMouseLeave(): void { this.isCardHovered.set(false); } public toggleCardPin(event?: Event): void { if (event) event.stopPropagation(); this.isCardPinned.update((v) => !v); } public cycleSpeed(): void { const current = this.playbackRate(); const idx = this.speedOptions.indexOf(current); const nextIdx = (idx + 1) % this.speedOptions.length; this.setSpeed(this.speedOptions[nextIdx]); } public onStepImageLoad(): void { this.isStepImageLoading.set(false); this.stepImageError.set(false); this.drawStepOverlay(); } public onStepImageError(): void { this.isStepImageLoading.set(false); this.stepImageError.set(true); } public drawStepOverlay(): void { const img = this.stepImgRef?.nativeElement; const overlay = this.stepOverlayRef?.nativeElement; if (!img || !overlay) return; overlay.innerHTML = ''; // Only draw coordinate overlays when viewing the pre-action screenshot if (this.viewMode() === 'post') { return; } const frame = this.currentStepFrame(); if (!frame || !frame.action) return; try { drawActionCoordinatesOnOverlay(img, overlay, frame.action); } catch (err) { console.warn('Failed to draw step coordinates overlay:', err); } } public getStepActionIcon(action: any): string { return getActionIcon(action); } public openImageInNewTab(): void { const url = this.currentStepImageUrl() || this.currentStepFrame()?.imageUrl; if (url) { window.open(url, '_blank'); } } public trackStepFrame(index: number, frame: StepReplayFrame): string { return frame.stepId || String(index); } public ngOnDestroy(): void { if (this.stepTimer) { clearTimeout(this.stepTimer); this.stepTimer = null; } document.removeEventListener('mousemove', this.onDrag); document.removeEventListener('mouseup', this.stopDrag); } }