feat: support orientation-aware recording and device wakefulness

Roll scrcpy recordings by device rotation and duration, and publish timeline manifests for playback and analysis.

Add seamless segmented playback in the showcase UI and configurable stay-awake strategies for Android automation.

apps/admin_console/routers/media.py

@@ -92,10 +92,12 @@ async def get_session_video(session_id: str):
     row = session_repo.get_session_by_id(session_id)
     row_dict = dict(row) if row else {"session_id": session_id}
     v_url = media_service.resolve_video_url(row_dict, video_rec_map, video_idx)
+    video_segments = media_service.resolve_video_segments(v_url)
     return {
         "session_id": session_id,
         "has_video": bool(v_url),
         "video_url": v_url,
+        "video_segments": video_segments,
     }
 
 

apps/admin_console/services/media_service.py

@@ -127,6 +127,36 @@ class MediaService:
 
         return v_url
 
+    @classmethod
+    def resolve_video_segments(cls, video_url: str | None) -> list[dict[str, Any]]:
+        """Resolve a recording sidecar manifest into browser-safe segment URLs."""
+        if not video_url or not video_url.startswith("/videos/"):
+            return []
+        relative = urllib.parse.unquote(video_url.removeprefix("/videos/"))
+        video_path = (WORKSPACE_ROOT / relative).resolve()
+        manifest_path = video_path.parent / "recording.json"
+        if not manifest_path.is_file():
+            return []
+        try:
+            payload = json.loads(manifest_path.read_text(encoding="utf-8"))
+            segments = []
+            for item in payload.get("segments", []):
+                segment_path = (manifest_path.parent / str(item["file"])).resolve()
+                if segment_path.parent != manifest_path.parent.resolve() or not segment_path.is_file():
+                    continue
+                segments.append(
+                    {
+                        "url": cls.path_to_video_url(segment_path),
+                        "start": float(item.get("start", 0)),
+                        "duration": float(item.get("duration", 0)),
+                        "width": int(item.get("width", 0)),
+                        "height": int(item.get("height", 0)),
+                    }
+                )
+            return segments
+        except (OSError, KeyError, TypeError, ValueError, json.JSONDecodeError):
+            return []
+
     @staticmethod
     def unwrap_payload(obj: Any) -> Any:
         """Unwraps trace payloads, storing base64 images as disk files and

apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.html

@@ -134,7 +134,7 @@
         <div class="video-container" *ngIf="!videoLoadError()">
           <video 
             #videoRef
-            [src]="videoUrl" 
+            [src]="currentVideoUrl()"
             playsinline
             (loadedmetadata)="onLoadedMetadata()"
             (timeupdate)="onTimeUpdate()"

apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.scss

@@ -373,16 +373,15 @@
     .video-container {
       position: relative;
       width: 100%;
-      height: 100%;
       display: flex;
       flex-direction: column;
       align-items: center;
       justify-content: center;
       background: #ffffff;
-      min-height: 220px;
 
       video {
         width: 100%;
+        height: auto;
         max-height: 520px;
         display: block;
         object-fit: contain;

apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.ts

@@ -51,6 +51,13 @@ export class FloatingVideoPlayerComponent {
   public videoLoadError = signal<boolean>(false);
   public liveStreamUrl = signal<string>('/api/stream/device-live');
   public liveStreamError = signal<boolean>(false);
+  public activeSegmentIndex = signal<number>(0);
+  public currentVideoUrl = computed(() => {
+    const segments = this.agentService.activeVideoSegments();
+    return segments[this.activeSegmentIndex()]?.url || this.agentService.activeVideoUrl();
+  });
+  private pendingLocalTime: number | null = null;
+  private pendingAutoplay = false;
 
   // Available speed options
   public speedOptions = [0.5, 1.0, 1.5, 2.0, 4.0];
@@ -74,9 +81,11 @@ export class FloatingVideoPlayerComponent {
     effect(
       () => {
         const url = this.agentService.activeVideoUrl();
+        const segments = this.agentService.activeVideoSegments();
         this.videoLoadError.set(false);
+        this.activeSegmentIndex.set(0);
         this.currentTime.set(0);
-        this.duration.set(0);
+        this.duration.set(segments.reduce((sum, segment) => sum + segment.duration, 0));
         this.isPlaying.set(false);
       },
       { allowSignalWrites: true }
@@ -177,18 +186,32 @@ export class FloatingVideoPlayerComponent {
   public onLoadedMetadata(): void {
     const v = this.videoRef?.nativeElement;
     if (v) {
-      this.duration.set(v.duration || 0);
+      const segments = this.agentService.activeVideoSegments();
+      if (!segments.length) {
+        this.duration.set(v.duration || 0);
+      }
       this.videoLoadError.set(false);
       v.playbackRate = this.playbackRate();
-      v.loop = this.isLooping();
+      v.loop = this.isLooping() && !segments.length;
+      v.muted = this.isMuted();
+      if (this.pendingLocalTime !== null) {
+        v.currentTime = Math.max(0, Math.min(v.duration || this.pendingLocalTime, this.pendingLocalTime));
+        this.pendingLocalTime = null;
+      }
+      if (this.pendingAutoplay) {
+        this.pendingAutoplay = false;
+        v.play().catch(() => {});
+      }
     }
   }
 
   public onTimeUpdate(): void {
     const v = this.videoRef?.nativeElement;
     if (v) {
-      this.currentTime.set(v.currentTime);
-      if (v.duration && v.duration !== this.duration()) {
+      const segments = this.agentService.activeVideoSegments();
+      const segment = segments[this.activeSegmentIndex()];
+      this.currentTime.set((segment?.start || 0) + v.currentTime);
+      if (!segments.length && v.duration && v.duration !== this.duration()) {
         this.duration.set(v.duration);
       }
     }
@@ -203,6 +226,17 @@ export class FloatingVideoPlayerComponent {
   }
 
   public onEnded(): void {
+    const segments = this.agentService.activeVideoSegments();
+    if (segments.length && this.activeSegmentIndex() < segments.length - 1) {
+      this.pendingLocalTime = 0;
+      this.pendingAutoplay = true;
+      this.activeSegmentIndex.update(index => index + 1);
+      return;
+    }
+    if (segments.length && this.isLooping()) {
+      this.seek(0, true);
+      return;
+    }
     this.isPlaying.set(false);
   }
 
@@ -225,16 +259,31 @@ export class FloatingVideoPlayerComponent {
   }
 
   public restart(): void {
-    const v = this.videoRef?.nativeElement;
-    if (!v) return;
-    v.currentTime = 0;
-    v.play().catch(() => {});
+    this.seek(0, true);
   }
 
-  public seek(seconds: number): void {
+  public seek(seconds: number, autoplay = false): void {
     const v = this.videoRef?.nativeElement;
     if (!v) return;
-    v.currentTime = Math.max(0, Math.min(this.duration(), seconds));
+    const target = Math.max(0, Math.min(this.duration(), seconds));
+    const segments = this.agentService.activeVideoSegments();
+    if (!segments.length) {
+      v.currentTime = target;
+      if (autoplay) v.play().catch(() => {});
+      return;
+    }
+    let index = segments.findIndex(segment => target < segment.start + segment.duration);
+    if (index < 0) index = segments.length - 1;
+    const localTime = Math.max(0, target - segments[index].start);
+    if (index === this.activeSegmentIndex()) {
+      v.currentTime = localTime;
+      if (autoplay) v.play().catch(() => {});
+    } else {
+      this.pendingLocalTime = localTime;
+      this.pendingAutoplay = autoplay || !v.paused;
+      this.activeSegmentIndex.set(index);
+    }
+    this.currentTime.set(target);
   }
 
   public onScrub(event: Event): void {
@@ -256,7 +305,7 @@ export class FloatingVideoPlayerComponent {
     this.isLooping.set(next);
     const v = this.videoRef?.nativeElement;
     if (v) {
-      v.loop = next;
+      v.loop = next && !this.agentService.activeVideoSegments().length;
     }
   }
 

apps/showcase_ui/src/app/services/agent.service.ts

@@ -22,6 +22,14 @@ import { Session, ModelInfo, TaskQueueItem, AgentStatusResponse } from '../core/
 import { StepItemData } from '../core/models/stream.model';
 export type { Session, ModelInfo, TaskQueueItem, AgentStatusResponse, StepItemData };
 
+export interface VideoSegment {
+  url: string;
+  start: number;
+  duration: number;
+  width: number;
+  height: number;
+}
+
 @Injectable({
   providedIn: 'root'
 })
@@ -104,6 +112,7 @@ export class AgentService {
   public isVideoWindowOpen = signal<boolean>(false);
   public isVideoMinimized = signal<boolean>(false);
   public activeVideoUrl = signal<string | null>(null);
+  public activeVideoSegments = signal<VideoSegment[]>([]);
   public activeVideoTitle = signal<string>('');
   public isVideoLoading = signal<boolean>(false);
 
@@ -1000,29 +1009,33 @@ export class AgentService {
     this.isVideoWindowOpen.set(true);
     this.isVideoMinimized.set(false);
 
-    if (targetUrl) {
-      this.activeVideoUrl.set(targetUrl);
-    } else if (targetSessionId) {
+    this.activeVideoUrl.set(targetUrl);
+    this.activeVideoSegments.set([]);
+    if (targetSessionId) {
       this.isVideoLoading.set(true);
-      this.http.get<{ session_id: string; has_video: boolean; video_url: string | null }>(`/api/sessions/${targetSessionId}/video`).subscribe({
+      this.http.get<{ session_id: string; has_video: boolean; video_url: string | null; video_segments?: VideoSegment[] }>(`/api/sessions/${targetSessionId}/video`).subscribe({
         next: (res) => {
           this.isVideoLoading.set(false);
           if (res && res.has_video && res.video_url) {
             this.activeVideoUrl.set(res.video_url);
+            this.activeVideoSegments.set(res.video_segments || []);
             this.rawSessions.update((list) =>
               list.map((s) => s.session_id === targetSessionId ? { ...s, video_url: res.video_url || undefined } : s)
             );
           } else {
             this.activeVideoUrl.set(null);
+            this.activeVideoSegments.set([]);
           }
         },
         error: () => {
           this.isVideoLoading.set(false);
           this.activeVideoUrl.set(null);
+          this.activeVideoSegments.set([]);
         }
       });
     } else {
       this.activeVideoUrl.set(null);
+      this.activeVideoSegments.set([]);
     }
   }