chenmingxuan/Artemis-Eapil
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 32 Wiki Activity

feat(plugins): add source-aware evidence and recording timelines

Browse Source
This commit is contained in:
chenmingxuan committed 2026-10-16 10:20:11 +08:00
1 parent 72ff342605
commit ef7af9988e
61 files changed
+3847 -108
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.ts
+113 -26
View File
@@ -25,6 +25,7 @@ import {
ViewChild,
HostListener,
effect,
untracked,
OnDestroy
} from '@angular/core';
@@ -33,7 +34,7 @@ import { AgentService } from '../../services/agent.service';
import { StepReplayFrame } from '../../core/models/stream.model';
import { drawActionCoordinatesOnOverlay } from '../../utils/image-overlay.util';
import { getActionIcon } from '../../utils/action-formatter.util';
import { locateTimelineTime, sessionTimeToTimelineTime } from '../../utils/recording-timeline.util';
import { locateCoveredSessionTime, locateTimelineTime, sessionTimeToTimelineTime } from '../../utils/recording-timeline.util';
import { buildVideoStepMarkers, VideoStepMarker } from '../../utils/video-step-markers.util';
@Component({
@@ -61,6 +62,11 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
public isMuted = signal<boolean>(false);
public isTheaterMode = signal<boolean>(false);
public videoLoadError = signal<boolean>(false);
public gapMessage = signal('');
private sourceSwitch: { time: number; play: boolean } | null = null;
private lastHandledVideoSeekRequestId: number | null = null;
private gapCanAdvance = true;
public sources = computed(() => this.agentService.recordingSources?.() || []);
public liveStreamUrl = signal<string>('/api/stream/device-live');
public liveStreamError = signal<boolean>(false);
public activeSegmentIndex = signal<number>(0);
@@ -71,13 +77,14 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
private pendingLocalTime: number | null = null;
private pendingAutoplay = false;
private pendingAbsoluteSeek: number | null = null;
private waitingForMetadata = true;
private seekPlaybackIntent: boolean | null = null;
private playRequestId = 0;
// Step Replay state signals
public activeStepIndex = signal<number>(0);
public isStepPlaying = signal<boolean>(false);
public viewMode = signal<'pre' | 'post'>('pre'); // 'pre' = action preview with coords, 'post' = action result
public viewMode = signal<'pre' | 'post'>('post'); // default to the observed result
public isCardHovered = signal<boolean>(false);
public isCardPinned = signal<boolean>(false);
public isCardVisible = computed(() => this.isCardHovered() || this.isCardPinned());
@@ -87,10 +94,14 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
public stepFrames = computed(() => this.agentService.currentSessionStepFrames());
public videoStepMarkers = computed(() => buildVideoStepMarkers(
this.stepFrames(),
this.stepFrames().filter(frame => {
const source = this.sources().find(s => s.id === this.agentService.selectedRecordingSource?.());
return !source || source.target_ids.includes(frame.sourceId || 'android');
}),
this.agentService.currentSession()?.start_time,
this.agentService.activeVideoSegments(),
this.duration()
this.duration(),
this.agentService.usesSessionTimeline?.() || false
));
public totalStepFrames = computed(() => this.stepFrames().length);
public currentStepFrame = computed<StepReplayFrame | null>(() => {
@@ -136,12 +147,23 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
const url = this.agentService.activeVideoUrl();
const segments = this.agentService.activeVideoSegments();
this.videoLoadError.set(false);
this.waitingForMetadata = true;
this.activeSegmentIndex.set(0);
this.currentTime.set(0);
this.duration.set(segments.reduce((sum, segment) => sum + segment.duration, 0));
const switching = this.sourceSwitch;
this.sourceSwitch = null;
this.currentTime.set(switching?.time || 0);
this.gapMessage.set('');
this.gapCanAdvance = true;
this.duration.set(this.agentService.usesSessionTimeline?.()
? Math.max(this.agentService.recordingSessionDuration?.() || 0, ...segments.map(s => s.start + s.duration))
: segments.reduce((sum, segment) => sum + segment.duration, 0));
this.isPlaying.set(false);
this.pendingLocalTime = null;
this.pendingAutoplay = false;
if (switching) {
this.pendingAbsoluteSeek = switching.time;
this.pendingAutoplay = switching.play;
}
this.seekPlaybackIntent = null;
this.playRequestId++;
},
@@ -157,7 +179,7 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
if (curSessionId !== lastTrackedSessionId) {
lastTrackedSessionId = curSessionId;
this.activeStepIndex.set(0);
this.viewMode.set('pre');
this.viewMode.set('post');
this.pauseStepPlay();
} else if (total > 0 && this.activeStepIndex() >= total) {
this.activeStepIndex.set(total - 1);
@@ -186,15 +208,42 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
effect(() => {
const request = this.agentService.videoSeekRequest();
if (!request) return;
this.pendingAbsoluteSeek = request.seconds;
if (this.agentService.recordingPlaybackStatus() !== 'ready') return;
const video = this.videoRef?.nativeElement;
if (video && video.readyState >= 1) {
const target = this.pendingAbsoluteSeek;
this.pendingAbsoluteSeek = null;
this.seekToSessionTime(target, true);
}
const status = this.agentService.recordingPlaybackStatus();
const sources = this.sources();
if (!request || status !== 'ready' || request.requestId === this.lastHandledVideoSeekRequestId) return;
// A seek is a command, not a binding to playback/source state. Reading
// those signals in this effect would replay the command after play/pause
// or a manual source selection.
untracked(() => {
this.lastHandledVideoSeekRequestId = request.requestId;
this.pendingAbsoluteSeek = request.seconds;
if (request.sourceId && sources.length) {
const source = sources.find(s => s.id === request.sourceId || s.target_ids.includes(request.sourceId!));
if (!source) {
this.pendingAbsoluteSeek = null;
this.pendingLocalTime = null;
this.pendingAutoplay = false;
this.seekPlaybackIntent = null;
this.playRequestId++;
this.gapCanAdvance = false;
this.currentTime.set(request.seconds);
this.gapMessage.set('此步骤的来源没有录像，请查看步骤截图或日志。');
this.videoRef?.nativeElement.pause();
this.isPlaying.set(false);
return;
}
if (source.id !== this.agentService.selectedRecordingSource?.()) {
this.sourceSwitch = { time: request.seconds, play: this.isPlaying() };
this.agentService.selectRecordingSource(source.id);
return;
}
}
const video = this.videoRef?.nativeElement;
if (video && video.readyState >= 1 && !this.waitingForMetadata) {
this.pendingAbsoluteSeek = null;
this.seekToSessionTime(request.seconds, this.isPlaying());
}
});
});
effect(() => {
@@ -305,6 +354,7 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
public onLoadedMetadata(): void {
const v = this.videoRef?.nativeElement;
if (v) {
this.waitingForMetadata = false;
const segments = this.agentService.activeVideoSegments();
if (!segments.length) {
this.duration.set(v.duration || 0);
@@ -313,14 +363,6 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
v.playbackRate = this.playbackRate();
v.loop = this.isLooping() && !segments.length;
v.muted = this.isMuted();
if (this.pendingAbsoluteSeek !== null) {
const target = this.pendingAbsoluteSeek;
this.pendingAbsoluteSeek = null;
this.seekToSessionTime(target, true);
// A seek into another finalized segment changes the video source. Let
// that segment's metadata event apply pendingLocalTime.
if (this.pendingLocalTime !== null) return;
}
const autoplay = this.agentService.consumeVideoAutoplay();
if (autoplay) {
this.isMuted.set(true);
@@ -328,6 +370,14 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
}
const shouldPlay = this.pendingAutoplay || autoplay;
this.pendingAutoplay = false;
if (this.pendingAbsoluteSeek !== null) {
const target = this.pendingAbsoluteSeek;
this.pendingAbsoluteSeek = null;
this.seekToSessionTime(target, shouldPlay);
// Same-source seeks resume on seeked; cross-segment seeks resume after
// the destination metadata and seeked. Never play the old frame here.
return;
}
if (this.pendingLocalTime !== null) {
const localTime = Math.max(0, Math.min(v.duration || this.pendingLocalTime, this.pendingLocalTime));
this.pendingLocalTime = null;
@@ -339,6 +389,7 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
}
public onTimeUpdate(): void {
if (this.gapMessage()) return;
const v = this.videoRef?.nativeElement;
if (v) {
const segments = this.agentService.activeVideoSegments();
@@ -375,6 +426,14 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
this.isPlaying.set(false);
const segments = this.agentService.activeVideoSegments();
if (segments.length && this.activeSegmentIndex() < segments.length - 1) {
const current = segments[this.activeSegmentIndex()];
const next = segments[this.activeSegmentIndex() + 1];
if (this.agentService.usesSessionTimeline?.() && next.start - current.start - current.duration > 0.05) {
this.currentTime.set(current.start + current.duration);
this.gapCanAdvance = true;
this.gapMessage.set('此处录像存在中断。点击播放可前往下一段，或选择其他步骤。');
return;
}
this.pendingLocalTime = 0;
this.pendingAutoplay = true;
this.activeSegmentIndex.update(index => index + 1);
@@ -400,6 +459,12 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
* Playback actions
*/
public togglePlay(): void {
if (this.gapMessage()) {
if (!this.gapCanAdvance) return;
const next = this.agentService.activeVideoSegments().find(s => s.start > this.currentTime());
if (next) this.seek(next.start, true);
return;
}
const v = this.videoRef?.nativeElement;
if (!v) return;
if (this.pendingLocalTime !== null) {
@@ -462,7 +527,13 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
*/
public seekToSessionTime(sessionSeconds: number, autoplay = false): void {
const segments = this.agentService.activeVideoSegments();
this.seek(sessionTimeToTimelineTime(segments, sessionSeconds), autoplay);
this.seek(this.agentService.usesSessionTimeline?.() ? sessionSeconds : sessionTimeToTimelineTime(segments, sessionSeconds), autoplay);
}
public selectSource(id: string): void {
if (id === this.agentService.selectedRecordingSource?.()) return;
this.sourceSwitch = { time: this.currentTime(), play: this.isPlaying() };
if (!this.agentService.selectRecordingSource(id)) this.sourceSwitch = null;
}
public seekToVideoMarker(marker: VideoStepMarker): void {
@@ -484,7 +555,23 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
if (!v) return;
const target = Math.max(0, Math.min(this.duration(), seconds));
const segments = this.agentService.activeVideoSegments();
const location = locateTimelineTime(segments, target);
// URL-only legacy recordings have no coverage manifest. Preserve their
// native video seeking instead of treating every timestamp as a gap.
const sessionTimeline = this.agentService.usesSessionTimeline?.() && segments.length > 0;
const location = sessionTimeline ? locateCoveredSessionTime(segments, target) : locateTimelineTime(segments, target);
if (sessionTimeline && !location) {
this.playRequestId++;
this.pendingLocalTime = null;
this.pendingAutoplay = false;
this.seekPlaybackIntent = null;
v.pause();
this.isPlaying.set(false);
this.currentTime.set(target);
this.gapCanAdvance = true;
this.gapMessage.set('此时间段未录制（尚未开始、暂停或录制中断），可查看步骤截图与日志。');
return;
}
this.gapMessage.set('');
const shouldPlay = autoplay || this.pendingAutoplay || (this.seekPlaybackIntent ?? !v.paused);
if (!location) {
this.seekWithinVideo(target, shouldPlay);