From 868f611c36e227ec70fe2626e38242754e41695a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E9=93=AD=E8=BD=A9?= Date: Tue, 29 Sep 2026 22:41:44 +0800 Subject: [PATCH] feat: add step markers to recording timeline --- .../floating-video-player.component.html | 17 ++++++ .../floating-video-player.component.scss | 32 +++++++++++ .../floating-video-player.component.spec.ts | 46 ++++++++++++++++ .../floating-video-player.component.ts | 20 +++++++ .../app/utils/video-step-markers.util.spec.ts | 53 +++++++++++++++++++ .../src/app/utils/video-step-markers.util.ts | 52 ++++++++++++++++++ 6 files changed, 220 insertions(+) create mode 100644 apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.spec.ts create mode 100644 apps/showcase_ui/src/app/utils/video-step-markers.util.spec.ts create mode 100644 apps/showcase_ui/src/app/utils/video-step-markers.util.ts diff --git a/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.html b/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.html index 6114453..8cd6ca3 100644 --- a/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.html +++ b/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.html @@ -354,6 +354,7 @@ limitations under the License.
+ @if (videoStepMarkers().length > 0) { +
+ @for (marker of videoStepMarkers(); track marker.time) { + + } +
+ }
diff --git a/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.scss b/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.scss index ff7f82c..b9a1fbb 100644 --- a/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.scss +++ b/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.scss @@ -724,6 +724,38 @@ box-shadow: 0 0 8px rgba(2, 132, 199, 0.4); } + .video-markers { + position: absolute; + inset: 0; + z-index: 6; + pointer-events: none; + + .video-marker { + position: absolute; + top: 50%; + width: 12px; + height: 12px; + padding: 0; + border: 2px solid #fff; + border-radius: 50%; + background: #64748b; + box-shadow: 0 1px 4px rgba(15, 23, 42, 0.25); + transform: translate(-50%, -50%); + cursor: pointer; + pointer-events: auto; + transition: transform 0.15s ease, background 0.15s ease; + + &.passed { background: #0284c7; } + &.failed { background: #ef4444; } + &:hover, &:focus-visible { + z-index: 1; + transform: translate(-50%, -50%) scale(1.35); + outline: 2px solid #38bdf8; + outline-offset: 2px; + } + } + } + .step-ticks-wrapper { position: absolute; left: 0; diff --git a/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.spec.ts b/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.spec.ts new file mode 100644 index 0000000..f970b48 --- /dev/null +++ b/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.spec.ts @@ -0,0 +1,46 @@ +import { TestBed } from '@angular/core/testing'; +import { signal } from '@angular/core'; +import { AgentService } from '../../services/agent.service'; +import { StepReplayFrame } from '../../core/models/stream.model'; +import { FloatingVideoPlayerComponent } from './floating-video-player.component'; + +describe('FloatingVideoPlayerComponent video markers', () => { + it('shows a recorded step on the video timeline and seeks when clicked', () => { + const start = 1_800_000_000; + const frame: StepReplayFrame = { + index: 0, stepNumber: 1, title: '步骤 1: 点击设置', imageUrl: 'step.png', + actionText: '点击设置', timestamp: start + 5 + }; + const agentService = { + isVideoWindowOpen: signal(true), + isVideoMinimized: signal(false), + recordingPlaybackStatus: signal('ready'), + playerMode: signal('video'), + hasCurrentSessionStepFrames: signal(true), + activeVideoTitle: signal('测试录像'), + isVideoLoading: signal(false), + activeVideoUrl: signal('/test.mp4'), + activeVideoSegments: signal([{ url: '/test.mp4', start: 0, duration: 20, width: 400, height: 800 }]), + currentSessionStepFrames: signal([frame]), + currentSession: signal({ session_id: 'test', start_time: start }), + currentSessionId: signal('test'), + videoSeekRequest: signal(null), + stepSeekRequest: signal(null) + }; + TestBed.configureTestingModule({ + imports: [FloatingVideoPlayerComponent], + providers: [{ provide: AgentService, useValue: agentService }] + }); + + const fixture = TestBed.createComponent(FloatingVideoPlayerComponent); + fixture.detectChanges(); + const marker = fixture.nativeElement.querySelector('.video-marker') as HTMLButtonElement; + expect(marker).not.toBeNull(); + expect(marker.getAttribute('aria-label')).toContain('步骤 1'); + expect(marker.style.left).toBe('25%'); + + const seek = spyOn(fixture.componentInstance, 'seek'); + marker.click(); + expect(seek).toHaveBeenCalledOnceWith(5); + }); +}); diff --git a/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.ts b/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.ts index 6b140ce..557df63 100644 --- a/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.ts +++ b/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.ts @@ -34,6 +34,7 @@ import { StepReplayFrame } from '../../core/models/stream.model'; import { drawActionCoordinatesOnOverlay } from '../../utils/image-overlay.util'; import { getActionIcon } from '../../utils/action-formatter.util'; import { locateTimelineTime, sessionTimeToTimelineTime } from '../../utils/recording-timeline.util'; +import { buildVideoStepMarkers, VideoStepMarker } from '../../utils/video-step-markers.util'; @Component({ selector: 'app-floating-video-player', @@ -83,6 +84,12 @@ export class FloatingVideoPlayerComponent implements OnDestroy { private stepTimer: any = null; public stepFrames = computed(() => this.agentService.currentSessionStepFrames()); + public videoStepMarkers = computed(() => buildVideoStepMarkers( + this.stepFrames(), + this.agentService.currentSession()?.start_time, + this.agentService.activeVideoSegments(), + this.duration() + )); public totalStepFrames = computed(() => this.stepFrames().length); public currentStepFrame = computed(() => { const frames = this.stepFrames(); @@ -394,6 +401,19 @@ export class FloatingVideoPlayerComponent implements OnDestroy { this.seek(sessionTimeToTimelineTime(segments, sessionSeconds), autoplay); } + public seekToVideoMarker(marker: VideoStepMarker): void { + this.seek(marker.time); + } + + public videoMarkerLabel(marker: VideoStepMarker): string { + const first = marker.steps[0]; + const count = marker.steps.length; + const action = first.actionText || first.title; + return count > 1 + ? `步骤 ${first.stepNumber} 等 ${count} 步 · ${this.formatTime(marker.time)} · ${action}` + : `步骤 ${first.stepNumber} · ${this.formatTime(marker.time)} · ${action}`; + } + /** Seek on the back-to-back playlist axis used by the scrubber. */ public seek(seconds: number, autoplay = false): void { const v = this.videoRef?.nativeElement; diff --git a/apps/showcase_ui/src/app/utils/video-step-markers.util.spec.ts b/apps/showcase_ui/src/app/utils/video-step-markers.util.spec.ts new file mode 100644 index 0000000..55b0c0a --- /dev/null +++ b/apps/showcase_ui/src/app/utils/video-step-markers.util.spec.ts @@ -0,0 +1,53 @@ +import { StepReplayFrame } from '../core/models/stream.model'; +import { TimelineSegment } from './recording-timeline.util'; +import { buildVideoStepMarkers } from './video-step-markers.util'; + +describe('buildVideoStepMarkers', () => { + const start = 1_800_000_000; + const segments: TimelineSegment[] = [ + { start: 0, duration: 10, offset_ms: 1000, duration_ms: 10000 }, + { start: 10, duration: 10, offset_ms: 13000, duration_ms: 10000 } + ]; + const frame = (stepNumber: number, seconds: number, status = 'dispatched'): StepReplayFrame => ({ + index: stepNumber - 1, + stepNumber, + title: `步骤 ${stepNumber}`, + imageUrl: 'step.png', + timestamp: start + seconds, + status + }); + + it('places markers on the playable timeline across recording restart gaps', () => { + const markers = buildVideoStepMarkers([frame(1, 4), frame(2, 15)], start, segments, 20); + expect(markers.map((marker) => marker.time)).toEqual([3, 12]); + expect(markers.map((marker) => marker.percent)).toEqual([15, 60]); + }); + + it('snaps a step in a recording gap to the following segment', () => { + const markers = buildVideoStepMarkers([frame(1, 12)], start, segments, 20); + expect(markers[0].time).toBe(10); + }); + + it('omits steps outside the actual recording and without timestamps', () => { + const missing = { ...frame(4, 5), timestamp: undefined }; + const markers = buildVideoStepMarkers( + [frame(1, 0), frame(2, 2), frame(3, 24), missing], start, segments, 20 + ); + expect(markers.map((marker) => marker.steps[0].stepNumber)).toEqual([2]); + }); + + it('groups crowded steps and keeps failure information', () => { + const markers = buildVideoStepMarkers( + [frame(1, 4), frame(2, 4.1, 'failed'), frame(3, 8)], start, segments, 20 + ); + expect(markers.length).toBe(2); + expect(markers[0].steps.map((step) => step.stepNumber)).toEqual([1, 2]); + expect(markers[0].failed).toBeTrue(); + }); + + it('uses session seconds directly for a legacy playlist or single video', () => { + expect(buildVideoStepMarkers([frame(1, 6)], start, [{ start: 0, duration: 20 }], 20)[0].time).toBe(6); + expect(buildVideoStepMarkers([frame(1, 6)], start, [], 20)[0].time).toBe(6); + expect(buildVideoStepMarkers([frame(1, 6)], undefined, [], 20)).toEqual([]); + }); +}); diff --git a/apps/showcase_ui/src/app/utils/video-step-markers.util.ts b/apps/showcase_ui/src/app/utils/video-step-markers.util.ts new file mode 100644 index 0000000..8d4657e --- /dev/null +++ b/apps/showcase_ui/src/app/utils/video-step-markers.util.ts @@ -0,0 +1,52 @@ +import { StepReplayFrame } from '../core/models/stream.model'; +import { hasSessionOffsets, sessionTimeToTimelineTime, TimelineSegment } from './recording-timeline.util'; + +export interface VideoStepMarker { + time: number; + percent: number; + steps: StepReplayFrame[]; + failed: boolean; +} + +/** Place recorded action steps on the video's gap-free playback timeline. */ +export function buildVideoStepMarkers( + frames: readonly StepReplayFrame[], + sessionStart: number | undefined, + segments: readonly TimelineSegment[], + duration: number +): VideoStepMarker[] { + const start = Number(sessionStart); + if (!Number.isFinite(start) || !Number.isFinite(duration) || duration <= 0) return []; + + const hasOffsets = hasSessionOffsets(segments); + const firstOffset = hasOffsets ? (segments[0].offset_ms as number) / 1000 : 0; + const lastSegment = segments[segments.length - 1]; + const lastOffset = hasOffsets + ? ((lastSegment.offset_ms as number) + (lastSegment.duration_ms as number)) / 1000 + : duration; + + const positioned = frames.flatMap((frame) => { + const timestamp = Number(frame.timestamp); + if (frame.timestamp == null || !Number.isFinite(timestamp)) return []; + const sessionTime = timestamp - start; + // Do not create misleading ticks at the start/end for steps outside the recording. + if (sessionTime < firstOffset || sessionTime > lastOffset) return []; + const time = sessionTimeToTimelineTime(segments, sessionTime); + if (time < 0 || time > duration) return []; + return [{ time, frame }]; + }).sort((a, b) => a.time - b.time); + + const markers: VideoStepMarker[] = []; + // Nearby actions share one visible tick on a narrow player timeline. + const minGap = duration * 0.02; + for (const { time, frame } of positioned) { + const previous = markers[markers.length - 1]; + if (previous && time - previous.time < minGap) { + previous.steps.push(frame); + previous.failed ||= frame.status === 'failed'; + } else { + markers.push({ time, percent: (time / duration) * 100, steps: [frame], failed: frame.status === 'failed' }); + } + } + return markers; +}