feat: support orientation-aware recording and device wakefulness

Roll scrcpy recordings by device rotation and duration, and publish timeline manifests for playback and analysis.

Add seamless segmented playback in the showcase UI and configurable stay-awake strategies for Android automation.

apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.html

@@ -134,7 +134,7 @@
         <div class="video-container" *ngIf="!videoLoadError()">
           <video 
             #videoRef
-            [src]="videoUrl" 
+            [src]="currentVideoUrl()"
             playsinline
             (loadedmetadata)="onLoadedMetadata()"
             (timeupdate)="onTimeUpdate()"

apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.scss

@@ -373,16 +373,15 @@
     .video-container {
       position: relative;
       width: 100%;
-      height: 100%;
       display: flex;
       flex-direction: column;
       align-items: center;
       justify-content: center;
       background: #ffffff;
-      min-height: 220px;
 
       video {
         width: 100%;
+        height: auto;
         max-height: 520px;
         display: block;
         object-fit: contain;

apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.ts

@@ -51,6 +51,13 @@ export class FloatingVideoPlayerComponent {
   public videoLoadError = signal<boolean>(false);
   public liveStreamUrl = signal<string>('/api/stream/device-live');
   public liveStreamError = signal<boolean>(false);
+  public activeSegmentIndex = signal<number>(0);
+  public currentVideoUrl = computed(() => {
+    const segments = this.agentService.activeVideoSegments();
+    return segments[this.activeSegmentIndex()]?.url || this.agentService.activeVideoUrl();
+  });
+  private pendingLocalTime: number | null = null;
+  private pendingAutoplay = false;
 
   // Available speed options
   public speedOptions = [0.5, 1.0, 1.5, 2.0, 4.0];
@@ -74,9 +81,11 @@ export class FloatingVideoPlayerComponent {
     effect(
       () => {
         const url = this.agentService.activeVideoUrl();
+        const segments = this.agentService.activeVideoSegments();
         this.videoLoadError.set(false);
+        this.activeSegmentIndex.set(0);
         this.currentTime.set(0);
-        this.duration.set(0);
+        this.duration.set(segments.reduce((sum, segment) => sum + segment.duration, 0));
         this.isPlaying.set(false);
       },
       { allowSignalWrites: true }
@@ -177,18 +186,32 @@ export class FloatingVideoPlayerComponent {
   public onLoadedMetadata(): void {
     const v = this.videoRef?.nativeElement;
     if (v) {
-      this.duration.set(v.duration || 0);
+      const segments = this.agentService.activeVideoSegments();
+      if (!segments.length) {
+        this.duration.set(v.duration || 0);
+      }
       this.videoLoadError.set(false);
       v.playbackRate = this.playbackRate();
-      v.loop = this.isLooping();
+      v.loop = this.isLooping() && !segments.length;
+      v.muted = this.isMuted();
+      if (this.pendingLocalTime !== null) {
+        v.currentTime = Math.max(0, Math.min(v.duration || this.pendingLocalTime, this.pendingLocalTime));
+        this.pendingLocalTime = null;
+      }
+      if (this.pendingAutoplay) {
+        this.pendingAutoplay = false;
+        v.play().catch(() => {});
+      }
     }
   }
 
   public onTimeUpdate(): void {
     const v = this.videoRef?.nativeElement;
     if (v) {
-      this.currentTime.set(v.currentTime);
-      if (v.duration && v.duration !== this.duration()) {
+      const segments = this.agentService.activeVideoSegments();
+      const segment = segments[this.activeSegmentIndex()];
+      this.currentTime.set((segment?.start || 0) + v.currentTime);
+      if (!segments.length && v.duration && v.duration !== this.duration()) {
         this.duration.set(v.duration);
       }
     }
@@ -203,6 +226,17 @@ export class FloatingVideoPlayerComponent {
   }
 
   public onEnded(): void {
+    const segments = this.agentService.activeVideoSegments();
+    if (segments.length && this.activeSegmentIndex() < segments.length - 1) {
+      this.pendingLocalTime = 0;
+      this.pendingAutoplay = true;
+      this.activeSegmentIndex.update(index => index + 1);
+      return;
+    }
+    if (segments.length && this.isLooping()) {
+      this.seek(0, true);
+      return;
+    }
     this.isPlaying.set(false);
   }
 
@@ -225,16 +259,31 @@ export class FloatingVideoPlayerComponent {
   }
 
   public restart(): void {
-    const v = this.videoRef?.nativeElement;
-    if (!v) return;
-    v.currentTime = 0;
-    v.play().catch(() => {});
+    this.seek(0, true);
   }
 
-  public seek(seconds: number): void {
+  public seek(seconds: number, autoplay = false): void {
     const v = this.videoRef?.nativeElement;
     if (!v) return;
-    v.currentTime = Math.max(0, Math.min(this.duration(), seconds));
+    const target = Math.max(0, Math.min(this.duration(), seconds));
+    const segments = this.agentService.activeVideoSegments();
+    if (!segments.length) {
+      v.currentTime = target;
+      if (autoplay) v.play().catch(() => {});
+      return;
+    }
+    let index = segments.findIndex(segment => target < segment.start + segment.duration);
+    if (index < 0) index = segments.length - 1;
+    const localTime = Math.max(0, target - segments[index].start);
+    if (index === this.activeSegmentIndex()) {
+      v.currentTime = localTime;
+      if (autoplay) v.play().catch(() => {});
+    } else {
+      this.pendingLocalTime = localTime;
+      this.pendingAutoplay = autoplay || !v.paused;
+      this.activeSegmentIndex.set(index);
+    }
+    this.currentTime.set(target);
   }
 
   public onScrub(event: Event): void {
@@ -256,7 +305,7 @@ export class FloatingVideoPlayerComponent {
     this.isLooping.set(next);
     const v = this.videoRef?.nativeElement;
     if (v) {
-      v.loop = next;
+      v.loop = next && !this.agentService.activeVideoSegments().length;
     }
   }
 

apps/showcase_ui/src/app/services/agent.service.ts

@@ -22,6 +22,14 @@ import { Session, ModelInfo, TaskQueueItem, AgentStatusResponse } from '../core/
 import { StepItemData } from '../core/models/stream.model';
 export type { Session, ModelInfo, TaskQueueItem, AgentStatusResponse, StepItemData };
 
+export interface VideoSegment {
+  url: string;
+  start: number;
+  duration: number;
+  width: number;
+  height: number;
+}
+
 @Injectable({
   providedIn: 'root'
 })
@@ -104,6 +112,7 @@ export class AgentService {
   public isVideoWindowOpen = signal<boolean>(false);
   public isVideoMinimized = signal<boolean>(false);
   public activeVideoUrl = signal<string | null>(null);
+  public activeVideoSegments = signal<VideoSegment[]>([]);
   public activeVideoTitle = signal<string>('');
   public isVideoLoading = signal<boolean>(false);
 
@@ -1000,29 +1009,33 @@ export class AgentService {
     this.isVideoWindowOpen.set(true);
     this.isVideoMinimized.set(false);
 
-    if (targetUrl) {
-      this.activeVideoUrl.set(targetUrl);
-    } else if (targetSessionId) {
+    this.activeVideoUrl.set(targetUrl);
+    this.activeVideoSegments.set([]);
+    if (targetSessionId) {
       this.isVideoLoading.set(true);
-      this.http.get<{ session_id: string; has_video: boolean; video_url: string | null }>(`/api/sessions/${targetSessionId}/video`).subscribe({
+      this.http.get<{ session_id: string; has_video: boolean; video_url: string | null; video_segments?: VideoSegment[] }>(`/api/sessions/${targetSessionId}/video`).subscribe({
         next: (res) => {
           this.isVideoLoading.set(false);
           if (res && res.has_video && res.video_url) {
             this.activeVideoUrl.set(res.video_url);
+            this.activeVideoSegments.set(res.video_segments || []);
             this.rawSessions.update((list) =>
               list.map((s) => s.session_id === targetSessionId ? { ...s, video_url: res.video_url || undefined } : s)
             );
           } else {
             this.activeVideoUrl.set(null);
+            this.activeVideoSegments.set([]);
           }
         },
         error: () => {
           this.isVideoLoading.set(false);
           this.activeVideoUrl.set(null);
+          this.activeVideoSegments.set([]);
         }
       });
     } else {
       this.activeVideoUrl.set(null);
+      this.activeVideoSegments.set([]);
     }
   }