feat: add multi-device concurrency, server lifecycle CLI, and unified execution stream

- Implement DevicePool and DeviceExecutionLock concurrency modes (per-device FIFO vs global serialization)
- Extend ArtemisClient SDK and CLI with explicit device targeting and concurrency strategy configuration
- Add server lifecycle management (artemis restart, stop, status) with cross-platform PID discovery
- Update MCP server tools to support multi-device dispatch, trace tagging, and per-device execution
- Enhance Showcase UI with step-by-step visual replay, playback speed controls, and device serial badges
- Add comprehensive unit test coverage and edge-case verification scripts

apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.html

@@ -68,6 +68,11 @@
             {{ getTaskStatus(session) | uppercase }}
           </span>
           <div class="item-header-actions">
+            <span class="item-device" *ngIf="getDeviceSerial(session)" [title]="'Device: ' + getDeviceSerial(session)">
+              <span class="material-symbols-outlined device-icon">phone_android</span>
+              <span class="device-name">{{ getDeviceSerial(session) }}</span>
+            </span>
+            <span class="item-time-divider" *ngIf="getDeviceSerial(session)">·</span>
             <span class="item-time">{{ session.start_time * 1000 | date:'shortTime' }}</span>
             <button
               *ngIf="getTaskStatus(session) !== 'running' && getTaskStatus(session) !== 'paused'"
@@ -113,6 +118,11 @@
               {{ getTaskStatus(session) | uppercase }}
             </span>
             <div class="item-header-actions">
+              <span class="item-device" *ngIf="getDeviceSerial(session)" [title]="'Device: ' + getDeviceSerial(session)">
+                <span class="material-symbols-outlined device-icon">phone_android</span>
+                <span class="device-name">{{ getDeviceSerial(session) }}</span>
+              </span>
+              <span class="item-time-divider" *ngIf="getDeviceSerial(session)">·</span>
               <span class="item-time">{{ session.start_time * 1000 | date:'shortTime' }}</span>
             <button
               *ngIf="getTaskStatus(session) !== 'running' && getTaskStatus(session) !== 'paused'"

apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.scss

@@ -550,7 +550,8 @@ $stream-row-height: 25px;
           transform: translateY(-1px);
 
           .item-header .item-header-actions {
-            &:has(.btn-delete-card) .item-time {
+            &:has(.btn-delete-card) .item-time,
+            &:has(.btn-delete-card) .item-time-divider {
               opacity: 0;
             }
 
@@ -580,8 +581,11 @@ $stream-row-height: 25px;
           display: flex;
           align-items: center;
           justify-content: space-between;
+          gap: 8px;
+          min-width: 0;
 
           .task-badge {
+            flex-shrink: 0;
             display: inline-flex;
             align-items: center;
             font-size: 10.5px;
@@ -658,11 +662,67 @@ $stream-row-height: 25px;
           .item-header-actions {
             display: flex;
             align-items: center;
+            gap: 4px;
             position: relative;
+            min-width: 0;
+            flex-shrink: 1;
+            justify-content: flex-end;
+
+            .item-device {
+              display: inline-flex;
+              align-items: center;
+              gap: 2.5px;
+              min-width: 0;
+              flex-shrink: 1;
+              font-size: 10.5px;
+              color: #64748b;
+              line-height: 1;
+              cursor: default;
+              transition: color 0.15s ease, opacity 0.15s ease;
+
+              .device-icon {
+                font-size: 12px;
+                width: 12px;
+                height: 12px;
+                line-height: 12px;
+                color: #64748b;
+                opacity: 0.75;
+                flex-shrink: 0;
+              }
+
+              .device-name {
+                overflow: hidden;
+                text-overflow: ellipsis;
+                white-space: nowrap;
+                min-width: 0;
+                font-family: inherit;
+                letter-spacing: -0.15px;
+              }
+
+              &:hover {
+                color: #334155;
+                .device-icon {
+                  color: #334155;
+                  opacity: 1;
+                }
+              }
+            }
+
+            .item-time-divider {
+              font-size: 10px;
+              color: #94a3b8;
+              line-height: 1;
+              user-select: none;
+              flex-shrink: 0;
+              transition: opacity 0.15s ease;
+            }
 
             .item-time {
               font-size: 11px;
               color: #64748b;
+              white-space: nowrap;
+              flex-shrink: 0;
+              line-height: 1;
               transition: opacity 0.15s ease;
             }
 

apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.ts

@@ -253,9 +253,11 @@ export class AgentStreamComponent implements AfterViewInit {
     return list.sort((a, b) => {
       const statusA = this.getTaskStatus(a);
       const statusB = this.getTaskStatus(b);
-      if (statusA === 'running' || statusA === 'paused') return -1;
-      if (statusB === 'running' || statusB === 'paused') return 1;
-      return a.start_time - b.start_time;
+      const isRunA = statusA === 'running' || statusA === 'paused';
+      const isRunB = statusB === 'running' || statusB === 'paused';
+      if (isRunA && !isRunB) return -1;
+      if (!isRunA && isRunB) return 1;
+      return (a.start_time || 0) - (b.start_time || 0);
     });
   });
 
@@ -598,18 +600,43 @@ export class AgentStreamComponent implements AfterViewInit {
   }
 
   public getTaskStatus(session: Session): 'running' | 'paused' | 'completed' | 'pending' | 'failed' | 'cancelled' {
-    if (session.session_id === this.agentService.runningSessionId() && (this.agentService.agentStatus() === 'running' || this.agentService.agentStatus() === 'paused')) {
-      return this.agentService.agentStatus() as 'running' | 'paused';
-    }
     if (session.status) {
       const s = session.status.toLowerCase();
-      if (s === 'running' || s === 'paused' || s === 'completed' || s === 'pending' || s === 'failed' || s === 'cancelled') {
+      if (s === 'completed' || s === 'success' || s === 'failed' || s === 'cancelled') {
+        return (s === 'success' ? 'completed' : s) as any;
+      }
+      if (s === 'running' || s === 'paused' || s === 'pending') {
         return s as any;
       }
     }
+    if (session.session_id === this.agentService.runningSessionId() && (this.agentService.agentStatus() === 'running' || this.agentService.agentStatus() === 'paused')) {
+      return this.agentService.agentStatus() as 'running' | 'paused';
+    }
     return 'completed';
   }
 
+  /**
+   * Determine the device serial number for the session
+   */
+  public getDeviceSerial(session: Session): string | null {
+    const serial = session.device_serial || session.device_id;
+    if (serial && serial !== 'pending' && serial !== 'null' && serial !== 'undefined') {
+      return serial;
+    }
+    if (session.device_info) {
+      try {
+        const info = typeof session.device_info === 'string' ? JSON.parse(session.device_info) : session.device_info;
+        const s = info?.device_id || info?.device_serial;
+        if (s && s !== 'pending' && s !== 'null' && s !== 'undefined') {
+          return s;
+        }
+      } catch {
+        // ignore
+      }
+    }
+    return null;
+  }
+
   public selectTask(sessionId: string, event?: Event): void {
     if (event) event.stopPropagation();
     this.agentService.selectSession(sessionId, true);

apps/showcase_ui/src/app/components/chat-interface/chat-interface.component.html

@@ -77,6 +77,11 @@
             {{ getTaskStatus(session) | uppercase }}
           </span>
           <div class="task-card-header-actions">
+            <span class="task-device" *ngIf="getDeviceSerial(session)" [title]="'Device: ' + getDeviceSerial(session)">
+              <span class="material-symbols-outlined device-icon">phone_android</span>
+              <span class="device-name">{{ getDeviceSerial(session) }}</span>
+            </span>
+            <span class="task-time-divider" *ngIf="getDeviceSerial(session)">·</span>
             <span class="task-time">{{ session.start_time * 1000 | date:'shortTime' }}</span>
             <button
               *ngIf="getTaskStatus(session) !== 'running' && getTaskStatus(session) !== 'paused'"
@@ -126,6 +131,11 @@
                 {{ getTaskStatus(session) | uppercase }}
               </span>
               <div class="task-card-header-actions">
+                <span class="task-device" *ngIf="getDeviceSerial(session)" [title]="'Device: ' + getDeviceSerial(session)">
+                  <span class="material-symbols-outlined device-icon">phone_android</span>
+                  <span class="device-name">{{ getDeviceSerial(session) }}</span>
+                </span>
+                <span class="task-time-divider" *ngIf="getDeviceSerial(session)">·</span>
                 <span class="task-time">{{ session.start_time * 1000 | date:'shortTime' }}</span>
               <button
                 *ngIf="getTaskStatus(session) !== 'running' && getTaskStatus(session) !== 'paused'"

apps/showcase_ui/src/app/components/chat-interface/chat-interface.component.scss

@@ -296,7 +296,8 @@ $text-muted: #a1a1aa;
         box-shadow: 0 3px 8px rgba(148, 163, 184, 0.08);
 
         .task-card-header .task-card-header-actions {
-          &:has(.btn-delete-card) .task-time {
+          &:has(.btn-delete-card) .task-time,
+          &:has(.btn-delete-card) .task-time-divider {
             opacity: 0;
           }
 
@@ -327,8 +328,11 @@ $text-muted: #a1a1aa;
         display: flex;
         justify-content: space-between;
         align-items: center;
+        gap: 8px;
+        min-width: 0;
 
         .task-badge {
+          flex-shrink: 0;
           display: inline-flex;
           align-items: center;
           font-size: 10.5px;
@@ -405,11 +409,67 @@ $text-muted: #a1a1aa;
         .task-card-header-actions {
           display: flex;
           align-items: center;
+          gap: 4px;
           position: relative;
+          min-width: 0;
+          flex-shrink: 1;
+          justify-content: flex-end;
+
+          .task-device {
+            display: inline-flex;
+            align-items: center;
+            gap: 2.5px;
+            min-width: 0;
+            flex-shrink: 1;
+            font-size: 11px;
+            color: #94a3b8;
+            line-height: 1;
+            cursor: default;
+            transition: color 0.15s ease, opacity 0.15s ease;
+
+            .device-icon {
+              font-size: 12.5px;
+              width: 12.5px;
+              height: 12.5px;
+              line-height: 12.5px;
+              color: #94a3b8;
+              opacity: 0.75;
+              flex-shrink: 0;
+            }
+
+            .device-name {
+              overflow: hidden;
+              text-overflow: ellipsis;
+              white-space: nowrap;
+              min-width: 0;
+              font-family: inherit;
+              letter-spacing: -0.15px;
+            }
+
+            &:hover {
+              color: #64748b;
+              .device-icon {
+                color: #64748b;
+                opacity: 1;
+              }
+            }
+          }
+
+          .task-time-divider {
+            font-size: 10px;
+            color: #cbd5e1;
+            line-height: 1;
+            user-select: none;
+            flex-shrink: 0;
+            transition: opacity 0.15s ease;
+          }
 
           .task-time {
             font-size: 11.5px;
             color: #94a3b8;
+            white-space: nowrap;
+            flex-shrink: 0;
+            line-height: 1;
             transition: opacity 0.15s ease;
           }
 

apps/showcase_ui/src/app/components/chat-interface/chat-interface.component.ts

@@ -49,9 +49,11 @@ export class ChatInterfaceComponent {
     return list.sort((a, b) => {
       const statusA = this.getTaskStatus(a);
       const statusB = this.getTaskStatus(b);
-      if (statusA === 'running' || statusA === 'paused') return -1;
-      if (statusB === 'running' || statusB === 'paused') return 1;
-      return a.start_time - b.start_time; // FIFO order
+      const isRunA = statusA === 'running' || statusA === 'paused';
+      const isRunB = statusB === 'running' || statusB === 'paused';
+      if (isRunA && !isRunB) return -1;
+      if (!isRunA && isRunB) return 1;
+      return (a.start_time || 0) - (b.start_time || 0); // stable FIFO order
     });
   });
 
@@ -155,18 +157,43 @@ export class ChatInterfaceComponent {
    * Determine the current task execution status
    */
   public getTaskStatus(session: Session): 'running' | 'paused' | 'completed' | 'pending' | 'failed' | 'cancelled' {
-    if (session.session_id === this.agentService.runningSessionId() && (this.agentService.agentStatus() === 'running' || this.agentService.agentStatus() === 'paused')) {
-      return this.agentService.agentStatus() as 'running' | 'paused';
-    }
     if (session.status) {
       const s = session.status.toLowerCase();
-      if (s === 'running' || s === 'paused' || s === 'completed' || s === 'pending' || s === 'failed' || s === 'cancelled') {
+      if (s === 'completed' || s === 'success' || s === 'failed' || s === 'cancelled') {
+        return (s === 'success' ? 'completed' : s) as any;
+      }
+      if (s === 'running' || s === 'paused' || s === 'pending') {
         return s as any;
       }
     }
+    if (session.session_id === this.agentService.runningSessionId() && (this.agentService.agentStatus() === 'running' || this.agentService.agentStatus() === 'paused')) {
+      return this.agentService.agentStatus() as 'running' | 'paused';
+    }
     return 'completed';
   }
 
+  /**
+   * Determine the device serial number for the session
+   */
+  public getDeviceSerial(session: Session): string | null {
+    const serial = session.device_serial || session.device_id;
+    if (serial && serial !== 'pending' && serial !== 'null' && serial !== 'undefined') {
+      return serial;
+    }
+    if (session.device_info) {
+      try {
+        const info = typeof session.device_info === 'string' ? JSON.parse(session.device_info) : session.device_info;
+        const s = info?.device_id || info?.device_serial;
+        if (s && s !== 'pending' && s !== 'null' && s !== 'undefined') {
+          return s;
+        }
+      } catch {
+        // ignore
+      }
+    }
+    return null;
+  }
+
   /**
    * Select a session in the UI to monitor its steps
    */

apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.html

@@ -15,13 +15,13 @@
 -->
 
 <!-- Backdrop overlay only when in theater / modal mode -->
-<div 
-  class="theater-backdrop" 
+<div
+  class="theater-backdrop"
   *ngIf="agentService.isVideoWindowOpen() && isTheaterMode()"
   (click)="isTheaterMode.set(false)"
 ></div>
 
-<div 
+<div
   class="floating-video-wrapper"
   *ngIf="agentService.isVideoWindowOpen()"
   [class.minimized]="agentService.isVideoMinimized()"
@@ -43,18 +43,18 @@
     <div class="header-actions" (mousedown)="$event.stopPropagation()">
       <!-- Mode Toggle Group: Video vs. Step Replay -->
       <div class="mode-toggle-group" *ngIf="agentService.recordingPlaybackStatus() === 'ready' && agentService.hasCurrentSessionStepFrames() && !agentService.isVideoMinimized()">
-        <button 
-          class="mode-btn" 
-          [class.active]="agentService.playerMode() === 'video'" 
+        <button
+          class="mode-btn"
+          [class.active]="agentService.playerMode() === 'video'"
           (click)="agentService.setPlayerMode('video')"
           title="Video Recording"
         >
           <span class="material-symbols-outlined mode-icon">videocam</span>
           <span>Video</span>
         </button>
-        <button 
-          class="mode-btn" 
-          [class.active]="agentService.playerMode() === 'steps'" 
+        <button
+          class="mode-btn"
+          [class.active]="agentService.playerMode() === 'steps'"
           (click)="agentService.setPlayerMode('steps')"
           title="Step Screenshot Replay"
         >
@@ -63,22 +63,9 @@
         </button>
       </div>
 
-      <!-- Playback Speed Pills (Shown when video is active OR when step replay is active) -->
-      <div class="speed-pill-group" *ngIf="!agentService.isVideoMinimized() && (agentService.recordingPlaybackStatus() === 'ready' || agentService.playerMode() === 'steps') && (agentService.activeVideoUrl() || totalStepFrames() > 0)">
-        <button 
-          *ngFor="let speed of speedOptions"
-          class="speed-btn"
-          [class.active]="playbackRate() === speed"
-          (click)="setSpeed(speed)"
-          [title]="'Speed: ' + speed + 'x'"
-        >
-          {{ speed }}x
-        </button>
-      </div>
-
       <!-- Minimize / Restore Toggle -->
-      <button 
-        class="header-btn" 
+      <button
+        class="header-btn"
         (click)="toggleMinimize()"
         [title]="agentService.isVideoMinimized() ? 'Restore' : 'Minimize'"
         aria-label="Minimize"
@@ -87,8 +74,8 @@
       </button>
 
       <!-- Theater / Fullscreen Toggle -->
-      <button 
-        class="header-btn" 
+      <button
+        class="header-btn"
         (click)="toggleTheater()"
         [title]="isTheaterMode() ? 'Exit Theater' : 'Theater Mode'"
         *ngIf="!agentService.isVideoMinimized()"
@@ -98,9 +85,9 @@
       </button>
 
       <!-- Close Window -->
-      <button 
-        class="header-btn close-btn" 
-        (click)="agentService.closeVideoPlayer()" 
+      <button
+        class="header-btn close-btn"
+        (click)="agentService.closeVideoPlayer()"
         title="Close"
         aria-label="Close"
       >
@@ -114,11 +101,11 @@
     <!-- 1. LIVE SCREEN STREAM (AUTOMATICALLY ACTIVE DURING RUNNING TEST) -->
     <ng-container *ngIf="agentService.recordingPlaybackStatus() === 'live'">
       <div class="live-stream-container" *ngIf="!liveStreamError()">
-        <img 
-          class="live-screen-img" 
-          [src]="liveStreamUrl()" 
-          (error)="onLiveStreamError()" 
-          alt="Live Device Screen" 
+        <img
+          class="live-screen-img"
+          [src]="liveStreamUrl()"
+          (error)="onLiveStreamError()"
+          alt="Live Device Screen"
         />
         <div class="live-status-overlay">
           <div class="live-badge">
@@ -157,39 +144,125 @@
           <!-- Fallback info banner if recording had failed -->
           <div class="step-replay-notice" *ngIf="agentService.recordingPlaybackStatus() === 'failed' || videoLoadError()">
             <span class="material-symbols-outlined notice-icon">info</span>
-            <span>Screen recording unavailable. Replaying step screenshots.</span>
+            <span>Screen recording unavailable. Replaying step-by-step screenshots.</span>
           </div>
 
-          <div class="step-image-viewport">
-            <img 
-              class="step-frame-img" 
-              [src]="currentStepFrame()?.imageUrl" 
-              [alt]="currentStepFrame()?.title"
-              (click)="openImageInNewTab()"
-              title="Click to view image in new tab"
-            />
+          <!-- Step Image Viewport with Action Coordinates Overlay -->
+          <div
+            class="step-image-viewport"
+            (mouseenter)="onViewportMouseEnter()"
+            (mouseleave)="onViewportMouseLeave()"
+          >
+            <!-- Loading Skeleton (Clean Light Spinner) -->
+            <div class="step-image-loading-state" *ngIf="isStepImageLoading()">
+              <span class="material-symbols-outlined spinning-icon">progress_activity</span>
+              <span>Loading screenshot...</span>
+            </div>
 
-            <!-- Step Overlay Badge -->
-            <div class="step-frame-badge" *ngIf="currentStepFrame() as frame">
-              <span class="badge-step-no">Step {{ frame.stepNumber }}</span>
-              <span class="badge-action-desc">{{ frame.actionText || frame.title }}</span>
+            <!-- Error / Missing Image Fallback -->
+            <div class="step-image-error-state" *ngIf="stepImageError() && !isStepImageLoading()">
+              <span class="material-symbols-outlined broken-icon">broken_image</span>
+              <span>Screenshot preview unavailable for this step</span>
+            </div>
+
+            <!-- Device Screen Frame & Action Coordinates Overlay -->
+            <div class="step-image-frame-wrap" *ngIf="!stepImageError()">
+              <img
+                #stepImgRef
+                class="step-frame-img"
+                [src]="currentStepImageUrl()"
+                [alt]="currentStepFrame()?.title"
+                (load)="onStepImageLoad()"
+                (error)="onStepImageError()"
+                (click)="openImageInNewTab()"
+                title="Click to view image in new tab"
+              />
+
+              <!-- Action Visual Coordinates Overlay (Touch point, swipe arrow, sequence path) -->
+              <div
+                #stepOverlayRef
+                class="overlay-layer"
+                [style.display]="viewMode() === 'post' ? 'none' : 'block'"
+              ></div>
+            </div>
+
+            <!-- Hideable Floating Detail Card (Appears on hover or when pinned) -->
+            <div
+              class="step-frame-badge"
+              *ngIf="currentStepFrame() as frame"
+              [class.visible]="isCardVisible()"
+              (click)="$event.stopPropagation()"
+            >
+              <div class="badge-content-left">
+                <span class="badge-step-no">Step {{ frame.stepNumber }}</span>
+                <span class="badge-action-desc" [title]="frame.actionText || frame.title">
+                  <span class="material-symbols-outlined action-type-icon" *ngIf="frame.action">
+                    {{ getStepActionIcon(frame.action) }}
+                  </span>
+                  {{ frame.actionText || frame.title }}
+                </span>
+              </div>
+
+              <div class="badge-actions-right">
+                <!-- Pre/Post Screenshot Switcher (Only if post-screenshot is available) -->
+                <div class="pre-post-toggle" *ngIf="frame.postImageUrl">
+                  <button
+                    class="toggle-btn"
+                    [class.active]="viewMode() === 'pre'"
+                    (click)="setStepViewMode('pre', $event)"
+                    title="View screen before action (with touch coordinates)"
+                  >
+                    Pre
+                  </button>
+                  <button
+                    class="toggle-btn"
+                    [class.active]="viewMode() === 'post'"
+                    (click)="setStepViewMode('post', $event)"
+                    title="View resulting screen after action executed"
+                  >
+                    Post
+                  </button>
+                </div>
+
+                <!-- Pin Card Button -->
+                <button
+                  class="pin-card-btn"
+                  [class.pinned]="isCardPinned()"
+                  (click)="toggleCardPin($event)"
+                  [title]="isCardPinned() ? 'Unpin details card (auto-hide on mouse leave)' : 'Pin details card (keep always visible)'"
+                >
+                  <span class="material-symbols-outlined">{{ isCardPinned() ? 'push_pin' : 'keep' }}</span>
+                </button>
+              </div>
             </div>
           </div>
 
           <!-- Step Replay Controls Footer -->
           <div class="video-controls-footer">
-            <!-- Timeline Scrubber Bar -->
+            <!-- Timeline Scrubber Bar with Step Ticks -->
             <div class="scrubber-bar-container">
-              <input 
-                type="range" 
-                class="scrubber-range" 
-                [min]="0" 
-                [max]="totalStepFrames() - 1" 
-                [step]="1" 
-                [value]="activeStepIndex()" 
-                (input)="onStepScrub($event)" 
+              <input
+                type="range"
+                class="scrubber-range"
+                [min]="0"
+                [max]="totalStepFrames() - 1"
+                [step]="1"
+                [value]="activeStepIndex()"
+                (input)="onStepScrub($event)"
               />
               <div class="scrubber-progress-fill" [style.width.%]="stepProgressPercent()"></div>
+              <!-- Step Tick Markers -->
+              <div class="step-ticks-wrapper" *ngIf="totalStepFrames() > 1">
+                <span
+                  *ngFor="let f of stepFrames(); let i = index"
+                  class="step-tick"
+                  [class.active]="i === activeStepIndex()"
+                  [class.passed]="i < activeStepIndex()"
+                  [style.left.%]="(i / (totalStepFrames() - 1)) * 100"
+                  (click)="seekStepFrame(i); $event.stopPropagation()"
+                  [title]="'Step ' + (i + 1) + ': ' + (f.actionText || f.title)"
+                ></span>
+              </div>
             </div>
 
             <!-- Bottom Controls Row -->
@@ -201,12 +274,12 @@
                 </button>
 
                 <!-- Previous Frame Button -->
-                <button class="ctrl-btn" (click)="prevStepFrame()" [disabled]="activeStepIndex() <= 0" title="Previous Frame">
+                <button class="ctrl-btn" (click)="prevStepFrame()" [disabled]="activeStepIndex() <= 0" title="Previous Step">
                   <span class="material-symbols-outlined">skip_previous</span>
                 </button>
 
                 <!-- Next Frame Button -->
-                <button class="ctrl-btn" (click)="nextStepFrame()" [disabled]="activeStepIndex() >= totalStepFrames() - 1" title="Next Frame">
+                <button class="ctrl-btn" (click)="nextStepFrame()" [disabled]="activeStepIndex() >= totalStepFrames() - 1" title="Next Step">
                   <span class="material-symbols-outlined">skip_next</span>
                 </button>
 
@@ -215,18 +288,27 @@
                   <span class="material-symbols-outlined">replay</span>
                 </button>
 
-                <!-- Frame Counter Display -->
+                <!-- Step Counter Display -->
                 <span class="time-display">
-                  Frame {{ activeStepIndex() + 1 }} / {{ totalStepFrames() }}
+                  Step {{ activeStepIndex() + 1 }} / {{ totalStepFrames() }}
                 </span>
               </div>
 
               <div class="controls-right">
+                <!-- Speed Button (Cycle 0.5x, 1x, 1.5x, 2x, 4x) -->
+                <button
+                  class="ctrl-btn speed-pill-btn"
+                  (click)="cycleSpeed()"
+                  [title]="'Speed: ' + playbackRate() + 'x (Click to cycle)'"
+                >
+                  <span class="speed-label">{{ playbackRate() }}x</span>
+                </button>
+
                 <!-- Loop Toggle -->
-                <button 
-                  class="ctrl-btn" 
-                  [class.active]="isLooping()" 
-                  (click)="toggleLoop()" 
+                <button
+                  class="ctrl-btn"
+                  [class.active]="isLooping()"
+                  (click)="toggleLoop()"
                   [title]="isLooping() ? 'Loop enabled' : 'Enable loop'"
                 >
                   <span class="material-symbols-outlined">repeat</span>
@@ -245,7 +327,7 @@
       <!-- Option B: ACTIVE VIDEO PLAYBACK VIEW -->
       <ng-container *ngIf="agentService.playerMode() === 'video' && agentService.recordingPlaybackStatus() === 'ready' && !agentService.isVideoLoading() && agentService.activeVideoUrl() as videoUrl">
         <div class="video-container" *ngIf="!videoLoadError()">
-          <video 
+          <video
             #videoRef
             [src]="currentVideoUrl()"
             playsinline
@@ -259,18 +341,18 @@
           ></video>
         </div>
 
-        <!-- Dedicated Bottom Controls Bar -->
+        <!-- Dedicated Bottom Controls Bar for Video -->
         <div class="video-controls-footer" *ngIf="!videoLoadError()">
           <!-- Timeline Scrubber Bar -->
           <div class="scrubber-bar-container">
-            <input 
-              type="range" 
-              class="scrubber-range" 
-              [min]="0" 
-              [max]="duration() || 100" 
-              [step]="0.1" 
-              [value]="currentTime()" 
-              (input)="onScrub($event)" 
+            <input
+              type="range"
+              class="scrubber-range"
+              [min]="0"
+              [max]="duration() || 100"
+              [step]="0.1"
+              [value]="currentTime()"
+              (input)="onScrub($event)"
             />
             <div class="scrubber-progress-fill" [style.width.%]="progressPercent()"></div>
           </div>
@@ -283,7 +365,7 @@
                 <span class="material-symbols-outlined">{{ isPlaying() ? 'pause' : 'play_arrow' }}</span>
               </button>
 
-              <!-- Step Replay / Reset -->
+              <!-- Replay / Reset -->
               <button class="ctrl-btn" (click)="restart()" title="Replay from start">
                 <span class="material-symbols-outlined">replay</span>
               </button>
@@ -295,11 +377,20 @@
             </div>
 
             <div class="controls-right">
+              <!-- Speed Button (Cycle 0.5x, 1x, 1.5x, 2x, 4x) -->
+              <button
+                class="ctrl-btn speed-pill-btn"
+                (click)="cycleSpeed()"
+                [title]="'Speed: ' + playbackRate() + 'x (Click to cycle)'"
+              >
+                <span class="speed-label">{{ playbackRate() }}x</span>
+              </button>
+
               <!-- Loop Toggle -->
-              <button 
-                class="ctrl-btn" 
-                [class.active]="isLooping()" 
-                (click)="toggleLoop()" 
+              <button
+                class="ctrl-btn"
+                [class.active]="isLooping()"
+                (click)="toggleLoop()"
                 [title]="isLooping() ? 'Loop enabled' : 'Enable loop'"
               >
                 <span class="material-symbols-outlined">repeat</span>

apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.scss

@@ -29,7 +29,7 @@
 
 .floating-video-wrapper {
   position: fixed;
-  width: 360px;
+  width: 400px;
   max-width: calc(100vw - 32px);
   background: rgba(255, 255, 255, 0.84);
   backdrop-filter: blur(24px) saturate(180%);
@@ -413,7 +413,7 @@
     .step-replay-container {
       display: flex;
       flex-direction: column;
-      background: #000000;
+      background: #ffffff;
       position: relative;
       width: 100%;
       overflow: hidden;
@@ -422,12 +422,12 @@
         display: flex;
         align-items: center;
         gap: 6px;
-        background: rgba(254, 243, 199, 0.95);
-        color: #92400e;
+        background: #fffbeb;
+        color: #b45309;
         padding: 6px 12px;
         font-size: 11.5px;
         font-weight: 500;
-        border-bottom: 1px solid rgba(245, 158, 11, 0.3);
+        border-bottom: 1px solid #fde68a;
 
         .notice-icon {
           font-size: 14px;
@@ -441,62 +441,214 @@
         align-items: center;
         justify-content: center;
         position: relative;
-        background: #0a0a0c;
-        min-height: 240px;
+        background: #f8fafc;
+        min-height: 280px;
         max-height: 520px;
-        cursor: pointer;
         overflow: hidden;
+        padding: 10px 0;
 
-        .step-frame-img {
-          max-width: 100%;
-          max-height: 520px;
-          width: auto;
-          height: auto;
-          object-fit: contain;
-          display: block;
-          user-select: none;
-          transition: opacity 0.15s ease;
+        .step-image-loading-state,
+        .step-image-error-state {
+          position: absolute;
+          inset: 0;
+          display: flex;
+          flex-direction: column;
+          align-items: center;
+          justify-content: center;
+          gap: 8px;
+          background: #f8fafc;
+          color: #64748b;
+          font-size: 12px;
+          font-weight: 500;
+          z-index: 5;
 
-          &:hover {
-            opacity: 0.96;
+          .spinning-icon {
+            font-size: 24px;
+            color: #0284c7;
+            animation: spin 1s linear infinite;
           }
+
+          .broken-icon {
+            font-size: 28px;
+            color: #94a3b8;
+          }
+        }
+
+        .step-image-frame-wrap {
+          position: relative;
+          display: inline-flex;
+          align-items: center;
+          justify-content: center;
+          max-width: 92%;
+          max-height: 500px;
+
+          .step-frame-img {
+            max-width: 100%;
+            max-height: 500px;
+            width: auto;
+            height: auto;
+            object-fit: contain;
+            display: block;
+            user-select: none;
+            cursor: pointer;
+            border-radius: 8px;
+            box-shadow: 0 4px 20px -2px rgba(15, 23, 42, 0.14), 0 0 0 1px rgba(226, 232, 240, 0.9);
+            background: #ffffff;
+            transition: opacity 0.15s ease;
+
+            &:hover {
+              opacity: 0.98;
+            }
+          }
+
+          .overlay-layer {
+            position: absolute;
+            top: 0;
+            left: 0;
+            width: 100%;
+            height: 100%;
+            pointer-events: none;
+            z-index: 8;
+          }
+        }
+
+        &:hover .step-frame-badge {
+          opacity: 1;
+          transform: translateY(0);
+          pointer-events: auto;
         }
 
         .step-frame-badge {
           position: absolute;
           bottom: 12px;
-          left: 12px;
-          right: 12px;
+          left: 10px;
+          right: 10px;
           display: flex;
           align-items: center;
-          gap: 6px;
-          background: rgba(15, 23, 42, 0.84);
-          backdrop-filter: blur(10px);
-          border: 1px solid rgba(255, 255, 255, 0.18);
-          border-radius: 8px;
+          justify-content: space-between;
+          gap: 8px;
+          background: rgba(15, 23, 42, 0.88);
+          backdrop-filter: blur(14px) saturate(180%);
+          -webkit-backdrop-filter: blur(14px) saturate(180%);
+          border: 1px solid rgba(255, 255, 255, 0.2);
+          border-radius: 10px;
           padding: 6px 10px;
           color: #ffffff;
-          box-shadow: 0 4px 12px rgba(0, 0, 0, 0.45);
+          box-shadow: 0 8px 24px rgba(0, 0, 0, 0.5);
+          opacity: 0;
+          transform: translateY(8px);
           pointer-events: none;
-          animation: fadeIn 0.2s ease;
+          transition: opacity 0.22s cubic-bezier(0.16, 1, 0.3, 1), transform 0.22s cubic-bezier(0.16, 1, 0.3, 1);
+          z-index: 15;
 
-          .badge-step-no {
-            font-size: 11px;
-            font-weight: 700;
-            background: #0284c7;
-            padding: 1px 6px;
-            border-radius: 4px;
-            letter-spacing: 0.2px;
-            flex-shrink: 0;
+          &.visible {
+            opacity: 1;
+            transform: translateY(0);
+            pointer-events: auto;
           }
 
-          .badge-action-desc {
-            font-size: 11.5px;
-            font-weight: 500;
-            color: #f8fafc;
-            white-space: nowrap;
+          .badge-content-left {
+            display: flex;
+            align-items: center;
+            gap: 7px;
             overflow: hidden;
-            text-overflow: ellipsis;
+            flex: 1;
+
+            .badge-step-no {
+              font-size: 10.5px;
+              font-weight: 700;
+              background: #0284c7;
+              padding: 2px 7px;
+              border-radius: 5px;
+              letter-spacing: 0.3px;
+              flex-shrink: 0;
+              box-shadow: 0 1px 3px rgba(2, 132, 199, 0.4);
+            }
+
+            .badge-action-desc {
+              font-size: 11.5px;
+              font-weight: 500;
+              color: #f8fafc;
+              white-space: nowrap;
+              overflow: hidden;
+              text-overflow: ellipsis;
+              display: flex;
+              align-items: center;
+              gap: 4px;
+
+              .action-type-icon {
+                font-size: 15px;
+                color: #38bdf8;
+                flex-shrink: 0;
+              }
+            }
+          }
+
+          .badge-actions-right {
+            display: flex;
+            align-items: center;
+            gap: 6px;
+            flex-shrink: 0;
+
+            .pre-post-toggle {
+              display: inline-flex;
+              align-items: center;
+              background: rgba(255, 255, 255, 0.12);
+              border: 1px solid rgba(255, 255, 255, 0.15);
+              border-radius: 6px;
+              padding: 2px;
+              gap: 2px;
+
+              .toggle-btn {
+                border: none;
+                background: transparent;
+                color: rgba(255, 255, 255, 0.7);
+                font-size: 10.5px;
+                font-weight: 700;
+                padding: 2px 7px;
+                border-radius: 4px;
+                cursor: pointer;
+                transition: all 0.15s ease;
+
+                &:hover {
+                  color: #ffffff;
+                  background: rgba(255, 255, 255, 0.2);
+                }
+
+                &.active {
+                  background: #0284c7;
+                  color: #ffffff;
+                  box-shadow: 0 1px 3px rgba(0, 0, 0, 0.3);
+                }
+              }
+            }
+
+            .pin-card-btn {
+              background: transparent;
+              border: none;
+              color: rgba(255, 255, 255, 0.6);
+              cursor: pointer;
+              padding: 3px;
+              display: flex;
+              align-items: center;
+              justify-content: center;
+              border-radius: 4px;
+              transition: all 0.15s ease;
+
+              &:hover {
+                color: #ffffff;
+                background: rgba(255, 255, 255, 0.15);
+              }
+
+              &.pinned {
+                color: #38bdf8;
+                background: rgba(56, 189, 248, 0.2);
+              }
+
+              .material-symbols-outlined {
+                font-size: 14px;
+              }
+            }
           }
         }
       }
@@ -571,6 +723,48 @@
           pointer-events: none;
           box-shadow: 0 0 8px rgba(2, 132, 199, 0.4);
         }
+
+        .step-ticks-wrapper {
+          position: absolute;
+          left: 0;
+          right: 0;
+          height: 100%;
+          pointer-events: none;
+          display: flex;
+          align-items: center;
+          z-index: 6;
+
+          .step-tick {
+            position: absolute;
+            width: 7px;
+            height: 7px;
+            border-radius: 50%;
+            background: rgba(255, 255, 255, 0.9);
+            border: 1.5px solid #94a3b8;
+            transform: translate(-50%, 0);
+            pointer-events: auto;
+            cursor: pointer;
+            transition: all 0.15s ease;
+
+            &:hover {
+              transform: translate(-50%, 0) scale(1.4);
+              border-color: #0284c7;
+              background: #ffffff;
+            }
+
+            &.passed {
+              background: #0284c7;
+              border-color: #ffffff;
+            }
+
+            &.active {
+              background: #38bdf8;
+              border-color: #ffffff;
+              transform: translate(-50%, 0) scale(1.3);
+              box-shadow: 0 0 6px rgba(56, 189, 248, 0.8);
+            }
+          }
+        }
       }
 
       .controls-row {
@@ -646,6 +840,22 @@
             }
           }
 
+          &.speed-pill-btn {
+            width: auto;
+            min-width: 34px;
+            padding: 0 7px;
+            font-size: 11px;
+            font-weight: 700;
+            font-family: 'JetBrains Mono', 'Roboto Mono', monospace;
+            color: #334155;
+            letter-spacing: -0.2px;
+
+            &:hover {
+              color: #0284c7;
+              border-color: rgba(191, 219, 254, 0.9);
+            }
+          }
+
           .material-symbols-outlined {
             font-size: 18px;
             color: #1e293b;
@@ -758,3 +968,20 @@
   0%, 100% { opacity: 1; }
   50% { opacity: 0.5; }
 }
+
+::ng-deep {
+  .action-point-marker {
+    z-index: 10;
+    pointer-events: none;
+  }
+  @keyframes pulse-animation {
+    0% {
+      transform: scale(0.6);
+      opacity: 1;
+    }
+    100% {
+      transform: scale(2.2);
+      opacity: 0;
+    }
+  }
+}

apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.ts

@@ -29,6 +29,8 @@ import { CommonModule } from '@angular/common';
 import { FormsModule } from '@angular/forms';
 import { AgentService } from '../../services/agent.service';
 import { StepReplayFrame } from '../../core/models/stream.model';
+import { drawActionCoordinatesOnOverlay } from '../../utils/image-overlay.util';
+import { getActionIcon } from '../../utils/action-formatter.util';
 
 @Component({
   selector: 'app-floating-video-player',
@@ -41,6 +43,8 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
   public agentService = inject(AgentService);
 
   @ViewChild('videoRef') videoRef?: ElementRef<HTMLVideoElement>;
+  @ViewChild('stepImgRef') stepImgRef?: ElementRef<HTMLImageElement>;
+  @ViewChild('stepOverlayRef') stepOverlayRef?: ElementRef<HTMLElement>;
 
   // Playback state signals
   public isPlaying = signal<boolean>(false);
@@ -65,6 +69,12 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
   // Step Replay state signals
   public activeStepIndex = signal<number>(0);
   public isStepPlaying = signal<boolean>(false);
+  public viewMode = signal<'pre' | 'post'>('pre'); // 'pre' = action preview with coords, 'post' = action result
+  public isCardHovered = signal<boolean>(false);
+  public isCardPinned = signal<boolean>(false);
+  public isCardVisible = computed(() => this.isCardHovered() || this.isCardPinned());
+  public isStepImageLoading = signal<boolean>(false);
+  public stepImageError = signal<boolean>(false);
   private stepTimer: any = null;
 
   public stepFrames = computed(() => this.agentService.currentSessionStepFrames());
@@ -74,6 +84,14 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
     const idx = this.activeStepIndex();
     return frames[idx] || null;
   });
+  public currentStepImageUrl = computed<string>(() => {
+    const frame = this.currentStepFrame();
+    if (!frame) return '';
+    if (this.viewMode() === 'post' && frame.postImageUrl) {
+      return frame.postImageUrl;
+    }
+    return frame.preImageUrl || frame.imageUrl || '';
+  });
   public stepProgressPercent = computed(() => {
     const total = this.totalStepFrames();
     if (total <= 1) return 0;
@@ -85,10 +103,10 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
 
   // Dragging and window positioning state
   public posX = signal<number>(
-    typeof window !== 'undefined' ? Math.max(20, window.innerWidth - 380) : 100
+    typeof window !== 'undefined' ? Math.max(20, window.innerWidth - 420) : 100
   );
   public posY = signal<number>(
-    typeof window !== 'undefined' ? Math.max(60, window.innerHeight - 660) : 100
+    typeof window !== 'undefined' ? Math.max(60, window.innerHeight - 700) : 100
   );
 
   private isDragging = false;
@@ -117,11 +135,30 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
       () => {
         const _ = this.stepFrames();
         this.activeStepIndex.set(0);
+        this.viewMode.set('pre');
         this.pauseStepPlay();
       },
       { allowSignalWrites: true }
     );
 
+    // Track step screenshot URL changes and reset loading/error state
+    effect(() => {
+      const url = this.currentStepImageUrl();
+      if (url) {
+        this.isStepImageLoading.set(true);
+        this.stepImageError.set(false);
+      } else {
+        this.isStepImageLoading.set(false);
+      }
+    }, { allowSignalWrites: true });
+
+    // Re-draw overlay whenever activeStepIndex or viewMode changes
+    effect(() => {
+      this.activeStepIndex();
+      this.viewMode();
+      setTimeout(() => this.drawStepOverlay(), 40);
+    });
+
     effect(() => {
       const request = this.agentService.videoSeekRequest();
       if (!request) return;
@@ -444,15 +481,18 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
 
   public prevStepFrame(): void {
     this.pauseStepPlay();
+    this.viewMode.set('pre');
     this.activeStepIndex.update((i) => Math.max(0, i - 1));
   }
 
   public nextStepFrame(): void {
     this.pauseStepPlay();
+    this.viewMode.set('pre');
     this.activeStepIndex.update((i) => Math.min(this.totalStepFrames() - 1, i + 1));
   }
 
   public seekStepFrame(index: number): void {
+    this.viewMode.set('pre');
     this.activeStepIndex.set(Math.max(0, Math.min(this.totalStepFrames() - 1, index)));
   }
 
@@ -466,8 +506,75 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
     this.startStepPlay();
   }
 
+  public setStepViewMode(mode: 'pre' | 'post', event?: Event): void {
+    if (event) event.stopPropagation();
+    this.viewMode.set(mode);
+    setTimeout(() => this.drawStepOverlay(), 40);
+  }
+
+  public toggleStepViewMode(event?: Event): void {
+    if (event) event.stopPropagation();
+    this.setStepViewMode(this.viewMode() === 'pre' ? 'post' : 'pre');
+  }
+
+  public onViewportMouseEnter(): void {
+    this.isCardHovered.set(true);
+  }
+
+  public onViewportMouseLeave(): void {
+    this.isCardHovered.set(false);
+  }
+
+  public toggleCardPin(event?: Event): void {
+    if (event) event.stopPropagation();
+    this.isCardPinned.update((v) => !v);
+  }
+
+  public cycleSpeed(): void {
+    const current = this.playbackRate();
+    const idx = this.speedOptions.indexOf(current);
+    const nextIdx = (idx + 1) % this.speedOptions.length;
+    this.setSpeed(this.speedOptions[nextIdx]);
+  }
+
+  public onStepImageLoad(): void {
+    this.isStepImageLoading.set(false);
+    this.stepImageError.set(false);
+    this.drawStepOverlay();
+  }
+
+  public onStepImageError(): void {
+    this.isStepImageLoading.set(false);
+    this.stepImageError.set(true);
+  }
+
+  public drawStepOverlay(): void {
+    const img = this.stepImgRef?.nativeElement;
+    const overlay = this.stepOverlayRef?.nativeElement;
+    if (!img || !overlay) return;
+    overlay.innerHTML = '';
+
+    // Only draw coordinate overlays when viewing the pre-action screenshot
+    if (this.viewMode() === 'post') {
+      return;
+    }
+
+    const frame = this.currentStepFrame();
+    if (!frame || !frame.action) return;
+
+    try {
+      drawActionCoordinatesOnOverlay(img, overlay, frame.action);
+    } catch (err) {
+      console.warn('Failed to draw step coordinates overlay:', err);
+    }
+  }
+
+  public getStepActionIcon(action: any): string {
+    return getActionIcon(action);
+  }
+
   public openImageInNewTab(): void {
-    const url = this.currentStepFrame()?.imageUrl;
+    const url = this.currentStepImageUrl() || this.currentStepFrame()?.imageUrl;
     if (url) {
       window.open(url, '_blank');
     }

apps/showcase_ui/src/app/components/legacy-agent-stream/legacy-agent-stream.component.html

@@ -35,7 +35,7 @@
           *ngFor="let session of agentService.sessions()"
           [value]="session.session_id"
         >
-          {{ session.initial_goal || 'Unnamed Goal' }} ({{ session.start_time * 1000 | date:'short' }})
+          {{ session.initial_goal || 'Unnamed Goal' }} ({{ session.device_serial ? session.device_serial + ' · ' : '' }}{{ session.start_time * 1000 | date:'short' }})
         </option>
       </select>
       <button class="btn-refresh" (click)="agentService.fetchSessions()">🔄 Refresh</button>

apps/showcase_ui/src/app/core/data/smart-tasks.data.ts

@@ -247,12 +247,12 @@ export const SMART_TASK_LIBRARY: SmartSuggestion[] = [
     priority: 82
   },
 
-  // 9. WeChat (微信)
+  // 9. WeChat
   {
     id: 'wechat_browse',
     title: 'Check WeChat Messages',
     description: 'Open WeChat and view top recent chat conversations',
-    goal: 'Open WeChat (微信) and view the top recent chat messages.',
+    goal: 'Open WeChat and view the top recent chat messages.',
     profile: 'flash',
     category: 'flash',
     tag: 'WeChat',
@@ -264,7 +264,7 @@ export const SMART_TASK_LIBRARY: SmartSuggestion[] = [
     id: 'pro_wechat_to_calendar',
     title: 'WeChat Notice to Calendar',
     description: 'Extract meeting notice from WeChat chat and add to Calendar',
-    goal: 'Open WeChat (微信), locate the latest meeting announcement or event message in the top chat, extract the time and topic, then open Calendar and schedule an event.',
+    goal: 'Open WeChat, locate the latest meeting announcement or event message in the top chat, extract the time and topic, then open Calendar and schedule an event.',
     profile: 'pro',
     category: 'cross_app',
     tag: 'WeChat + Calendar',
@@ -276,12 +276,12 @@ export const SMART_TASK_LIBRARY: SmartSuggestion[] = [
     priority: 93
   },
 
-  // 10. Xiaohongshu (小红书)
+  // 10. Xiaohongshu
   {
     id: 'xhs_coffee_guide',
     title: 'RED Cafe Guide Search',
     description: 'Search trending specialty cafe reviews on Xiaohongshu',
-    goal: 'Open Xiaohongshu (小红书), search for top-rated specialty coffee shops, and view the top post.',
+    goal: 'Open Xiaohongshu, search for top-rated specialty coffee shops, and view the top post.',
     profile: 'flash',
     category: 'flash',
     tag: 'Xiaohongshu',
@@ -290,12 +290,12 @@ export const SMART_TASK_LIBRARY: SmartSuggestion[] = [
     priority: 87
   },
 
-  // 11. Meituan / Dianping (美团 / 大众点评)
+  // 11. Meituan / Dianping
   {
     id: 'meituan_ramen_search',
     title: 'Meituan Food Search',
     description: 'Search top-rated Ramen nearby on Meituan or Dianping',
-    goal: 'Open Meituan (美团) or Dianping (大众点评), search for top-rated Ramen nearby, and view top restaurant rating.',
+    goal: 'Open Meituan or Dianping, search for top-rated Ramen nearby, and view top restaurant rating.',
     profile: 'flash',
     category: 'flash',
     tag: 'Meituan',
@@ -304,12 +304,12 @@ export const SMART_TASK_LIBRARY: SmartSuggestion[] = [
     priority: 86
   },
 
-  // 12. Bilibili (哔哩哔哩)
+  // 12. Bilibili
   {
     id: 'bilibili_stream',
     title: 'Bilibili Tech Video',
     description: 'Search and play an AI Agent tutorial video on Bilibili',
-    goal: 'Open Bilibili (哔哩哔哩), search for "AI Agent 架构实战", and play the top matching video.',
+    goal: 'Open Bilibili, search for "AI Agent Architecture", and play the top matching video.',
     profile: 'flash',
     category: 'flash',
     tag: 'Bilibili',

apps/showcase_ui/src/app/core/models/session.model.ts

@@ -28,6 +28,8 @@ export interface TaskQueueItem {
   status: 'pending' | 'running' | 'paused' | 'completed' | 'failed' | 'cancelled';
   created_at?: number;
   start_time?: number;
+  device_serial?: string | null;
+  device_id?: string | null;
 }
 
 export interface Session {
@@ -39,6 +41,9 @@ export interface Session {
   video_url?: string;
   recording_status?: 'recording' | 'finalizing' | 'processing' | 'ready' | 'failed' | 'unavailable';
   model_info?: ModelInfo;
+  device_serial?: string | null;
+  device_id?: string | null;
+  device_info?: any;
 }
 
 export interface AgentStatusResponse {

apps/showcase_ui/src/app/core/models/stream.model.ts

@@ -101,9 +101,17 @@ export interface CheckerResult {
 export interface StepReplayFrame {
   index: number;
   stepNumber: number;
+  stepId?: string;
   title: string;
   imageUrl: string;
+  preImageUrl?: string | null;
+  postImageUrl?: string | null;
+  action?: any;
+  actionType?: string;
   actionText?: string;
+  targetText?: string;
+  coords?: string;
+  status?: 'success' | 'failed' | string;
   isPost?: boolean;
   timestamp?: number;
   phaseId?: string;

apps/showcase_ui/src/app/services/agent.service.ts

@@ -68,6 +68,9 @@ export class AgentService {
   // Signals to expose state to components
   private rawSessions = signal<Session[]>([]);
   private pendingQueue = signal<Session[]>([]);
+  public activeTasks = signal<any[]>([]);
+  // Persistent tracking of active/pending sessions across polling boundaries
+  private activeSessionTracking = new Map<string, Session>();
 
   public sessions = computed(() => {
     const raw = this.rawSessions();
@@ -75,47 +78,111 @@ export class AgentService {
     const status = this.agentStatus();
     const goal = this.runningGoal();
     const runId = this.runningSessionId();
+    const activeList = this.activeTasks();
 
     const sessionMap = new Map<string, Session>();
 
     // 1. First add raw sessions from DB
     raw.forEach((s) => {
-      const isCurrentActive = (status === 'running' || status === 'paused') && runId === s.session_id;
-      const isPending = this.pendingQueue().some(p => p.session_id === s.session_id) && !isCurrentActive;
-      let sStatus = isCurrentActive ? status : (isPending ? 'pending' : s.status);
-      if ((sStatus === 'running' || sStatus === 'paused') && !isCurrentActive) {
-        const activeRun = raw.find(r => r.session_id === runId);
-        sStatus = (activeRun && (s.start_time || 0) > (activeRun.start_time || 0)) ? 'pending' : 'completed';
+      const isCurrentActive = ((status === 'running' || status === 'paused') && runId === s.session_id)
+        || activeList.some(at => at.session_id === s.session_id);
+      const isTerminal = s.status === 'completed' || s.status === 'success' || s.status === 'failed' || s.status === 'cancelled';
+      let sStatus = s.status;
+      if (!isTerminal) {
+        const isPending = this.pendingQueue().some(p => p.session_id === s.session_id) && !isCurrentActive;
+        sStatus = isCurrentActive ? (status === 'paused' ? 'paused' : 'running') : (isPending ? 'pending' : s.status);
+      } else {
+        sStatus = (s.status === 'success') ? 'completed' : s.status;
       }
-      sessionMap.set(s.session_id, {
+      const activeMatch = activeList.find(at => at.session_id === s.session_id);
+      const pendingMatch = this.pendingQueue().find(p => p.session_id === s.session_id);
+      let serial = s.device_serial || s.device_id || activeMatch?.device_id || pendingMatch?.device_serial || null;
+      if (!serial && s.device_info) {
+        try {
+          const info = typeof s.device_info === 'string' ? JSON.parse(s.device_info) : s.device_info;
+          serial = info?.device_id || info?.device_serial || null;
+        } catch {
+          // ignore
+        }
+      }
+      const finalSession: Session = {
         ...s,
         status: sStatus,
+        device_serial: serial,
         model_info: isCurrentActive && this.activeModel() ? this.activeModel()! : s.model_info
-      });
+      };
+      sessionMap.set(s.session_id, finalSession);
+      if (isTerminal) {
+        this.activeSessionTracking.delete(s.session_id);
+      } else if (sStatus === 'running' || sStatus === 'paused' || sStatus === 'pending') {
+        this.activeSessionTracking.set(s.session_id, finalSession);
+      }
     });
 
     // 2. Add pending queue sessions if not yet in raw sessions
     pending.forEach((p) => {
       if (!sessionMap.has(p.session_id)) {
-        const isCurrentRunning = (status === 'running' || status === 'paused') && runId === p.session_id;
-        sessionMap.set(p.session_id, {
+        const isCurrentRunning = ((status === 'running' || status === 'paused') && runId === p.session_id)
+          || activeList.some(at => at.session_id === p.session_id);
+        const activeMatch = activeList.find(at => at.session_id === p.session_id);
+        const finalPending: Session = {
           ...p,
-          status: isCurrentRunning ? status : (p.status || 'pending')
-        });
+          status: isCurrentRunning ? (status === 'paused' ? 'paused' : 'running') : (p.status || 'pending'),
+          device_serial: p.device_serial || p.device_id || activeMatch?.device_id || null
+        };
+        sessionMap.set(p.session_id, finalPending);
+        this.activeSessionTracking.set(p.session_id, finalPending);
       }
     });
 
-    // 3. Ensure currently active running session is present
-    if ((status === 'running' || status === 'paused') && runId && !sessionMap.has(runId)) {
-      sessionMap.set(runId, {
+    // 3. Ensure all currently active running sessions are present (multi-device & external runs)
+    if (activeList.length > 0) {
+      activeList.forEach((at) => {
+        const sid = at.session_id || `active-${at.device_id}`;
+        const existing = sessionMap.get(sid);
+        if (!existing) {
+          const newSession: Session = {
+            session_id: sid,
+            initial_goal: at.goal || goal || '',
+            start_time: at.acquired_at ? (new Date(at.acquired_at).getTime() / 1000) : (Date.now() / 1000),
+            status: 'running',
+            model_info: this.activeModel() || undefined,
+            device_serial: at.device_id || null
+          };
+          sessionMap.set(sid, newSession);
+          this.activeSessionTracking.set(sid, newSession);
+        } else if (existing.status === 'pending') {
+          const updated: Session = {
+            ...existing,
+            status: 'running',
+            device_serial: at.device_id || existing.device_serial
+          };
+          sessionMap.set(sid, updated);
+          this.activeSessionTracking.set(sid, updated);
+        }
+      });
+    } else if ((status === 'running' || status === 'paused') && runId && !sessionMap.has(runId)) {
+      const activeSession: Session = {
         session_id: runId,
         initial_goal: goal || '',
         start_time: Date.now() / 1000,
         status,
         model_info: this.activeModel() || undefined
-      });
+      };
+      sessionMap.set(runId, activeSession);
+      this.activeSessionTracking.set(runId, activeSession);
     }
 
+    // 4. Bridge transient queue-to-running transition gaps
+    this.activeSessionTracking.forEach((ts, sid) => {
+      if (!sessionMap.has(sid)) {
+        sessionMap.set(sid, {
+          ...ts,
+          status: 'running'
+        });
+      }
+    });
+
     return Array.from(sessionMap.values()).sort((a, b) => b.start_time - a.start_time);
   });
 
@@ -137,8 +204,14 @@ export class AgentService {
   public currentStartupProgress = computed(() => {
     const sessionId = this.currentSessionId();
     if (sessionId) {
-      const sessionEvents = this.startupProgressBySession()[sessionId];
-      if (sessionEvents?.length) return sessionEvents;
+      const normalized = String(sessionId).trim().toLowerCase();
+      const bySession = this.startupProgressBySession();
+      const direct = bySession[sessionId];
+      if (direct?.length) return direct;
+      const matched = Object.entries(bySession).find(
+        ([key]) => key.trim().toLowerCase() === normalized
+      );
+      if (matched?.[1]?.length) return matched[1];
     }
     return this.pendingStartupProgress();
   });
@@ -609,7 +682,10 @@ export class AgentService {
             return;
           }
 
-          if (evtSessionId && String(evtSessionId) !== String(sessionId)) {
+          const isDifferentSession = evtSessionId
+            && String(evtSessionId).trim().toLowerCase() !== String(sessionId).trim().toLowerCase();
+
+          if (isDifferentSession) {
             if (eventType === 'session_started') {
               this.agentStatus.set('running');
               this.runningSessionId.set(parsedData.session_id);
@@ -617,13 +693,30 @@ export class AgentService {
                 this.runningGoal.set(parsedData.initial_goal);
               }
               this.fetchSessions();
-              // Only auto-select if user hasn't explicitly chosen to inspect another task
-              const pinnedId = this.userPinnedSessionId();
-              if (!pinnedId && parsedData?.session_id) {
+              // Only auto-select if user has no current session at all
+              const currentId = this.currentSessionId();
+              if (!currentId && parsedData?.session_id) {
                 this.selectSession(parsedData.session_id, false);
               }
             } else if (eventType === 'session_ended') {
               this.applySessionEndedStatus(evtSessionId, parsedData);
+              this.activeTasks.update(list => list.filter(at => at.session_id !== evtSessionId));
+              const remaining = this.activeTasks();
+              if (remaining.length > 0) {
+                if (this.runningSessionId() === evtSessionId) {
+                  this.runningSessionId.set(remaining[0].session_id);
+                  this.runningGoal.set(remaining[0].goal || null);
+                }
+              } else if (this.pendingQueue().length > 0) {
+                const nextPending = this.pendingQueue()[0];
+                this.agentStatus.set('running');
+                this.runningSessionId.set(nextPending.session_id);
+                this.runningGoal.set(nextPending.initial_goal || null);
+              } else {
+                this.agentStatus.set('idle');
+                this.runningSessionId.set(null);
+                this.runningGoal.set(null);
+              }
               this.fetchStatus();
               this.fetchSessions();
               this.updateModelForCurrentView();
@@ -650,12 +743,26 @@ export class AgentService {
 
           if (eventType === 'session_ended') {
             this.applySessionEndedStatus(sessionId, parsedData);
-            this.agentStatus.set('idle');
-            this.runningSessionId.set(null);
-            this.runningGoal.set(null);
+            this.activeTasks.update(list => list.filter(at => at.session_id !== sessionId));
+            const remaining = this.activeTasks();
+            if (remaining.length > 0) {
+              this.agentStatus.set('running');
+              this.runningSessionId.set(remaining[0].session_id);
+              this.runningGoal.set(remaining[0].goal || null);
+            } else if (this.pendingQueue().length > 0) {
+              const nextPending = this.pendingQueue()[0];
+              this.agentStatus.set('running');
+              this.runningSessionId.set(nextPending.session_id);
+              this.runningGoal.set(nextPending.initial_goal || null);
+            } else {
+              this.agentStatus.set('idle');
+              this.runningSessionId.set(null);
+              this.runningGoal.set(null);
+            }
             this.isRetrying.set(false);
             this.isPaused.set(false);
             this.fetchSessions();
+            this.fetchStatus();
             this.fetchNotes(sessionId);
             this.updateModelForCurrentView();
             if (
@@ -1072,7 +1179,9 @@ export class AgentService {
           : null;
 
     if (status) {
+      this.activeSessionTracking.delete(String(sessionId));
       this.setSessionStatus(String(sessionId), status);
+      this.activeTasks.update((list) => list.filter((at) => at.session_id !== String(sessionId)));
     }
   }
 
@@ -1100,34 +1209,15 @@ export class AgentService {
         if (data && data.status) {
           const oldStatus = this.agentStatus();
           const oldRunningSessionId = this.runningSessionId();
-          
-          this.agentStatus.set(data.status);
           const isActive = data.status === 'running' || data.status === 'paused';
-          if (isActive) {
-            this.runningSessionId.set(data.session_id || null);
-            this.runningGoal.set(data.goal || null);
-            if (data.model_info) {
-              this.activeModel.set(data.model_info);
-            }
-          } else {
-            this.runningSessionId.set(null);
-            this.runningGoal.set(null);
-            
-            // If viewing a selected history session, use its model_info
-            const curId = this.currentSessionId();
-            if (curId) {
-              const selected = this.sessions().find(s => s.session_id === curId);
-              if (selected && selected.model_info) {
-                this.activeModel.set(selected.model_info);
-              } else if (data.model_info) {
-                this.activeModel.set(data.model_info);
-              }
-            } else if (data.model_info) {
-              this.activeModel.set(data.model_info);
-            }
+          this.agentStatus.set(data.status);
+          this.runningSessionId.set(data.session_id || null);
+          this.runningGoal.set(data.goal || null);
+          if (data.model_info) {
+            this.activeModel.set(data.model_info);
           }
-
           this.isPaused.set(data.status === 'paused');
+
           if (data.status === 'paused') {
             this.isRetrying.set(false);
             const pauseError = data.paused_error
@@ -1149,7 +1239,8 @@ export class AgentService {
                 session_id: item.session_id || `pending-task-${index}`,
                 initial_goal: item.goal || '',
                 start_time: item.start_time || item.created_at || (Date.now() / 1000 + index),
-                status: item.status || 'pending'
+                status: item.status || 'pending',
+                device_serial: item.device_serial || item.device_id || null
               };
             }
             return {
@@ -1160,6 +1251,7 @@ export class AgentService {
             };
           });
           this.pendingQueue.set(pending);
+          this.activeTasks.set(data.active_tasks || []);
           
           if (oldStatus !== data.status || oldRunningSessionId !== data.session_id) {
             this.fetchSessions();
@@ -1176,16 +1268,11 @@ export class AgentService {
             this.beginRecordingFinalization(oldRunningSessionId);
           }
 
-          // Auto-select the session if running and user hasn't explicitly chosen to inspect another task
+          // Auto-select the session ONLY if nothing is currently selected
           if (isActive && data.session_id) {
             const currentId = this.currentSessionId();
-            const pinnedId = this.userPinnedSessionId();
             if (!currentId) {
               this.selectSession(data.session_id, false);
-            } else if (currentId !== data.session_id) {
-              if (!pinnedId) {
-                this.selectSession(data.session_id, false);
-              }
             }
           }
         }
@@ -1328,8 +1415,9 @@ export class AgentService {
       timestamp
     };
 
+    const rawKey = String(data.session_id || sessionId).trim();
     this.startupProgressBySession.update((allEvents) => {
-      const current = [...(allEvents[sessionId] || [])];
+      const current = [...(allEvents[rawKey] || allEvents[sessionId] || [])];
       const existingIndex = current.findIndex((item) => item.stage === event.stage);
       if (existingIndex >= 0) {
         current[existingIndex] = event;
@@ -1337,7 +1425,11 @@ export class AgentService {
         current.push(event);
       }
       current.sort((a, b) => a.timestamp - b.timestamp);
-      return { ...allEvents, [sessionId]: current };
+      const updated = { ...allEvents, [rawKey]: current };
+      if (sessionId && sessionId !== rawKey) {
+        updated[sessionId] = current;
+      }
+      return updated;
     });
   }
 

apps/showcase_ui/src/app/utils/action-formatter.util.spec.ts

@@ -85,7 +85,7 @@ describe('action-formatter.util screenshot chaining', () => {
     expect(act1Pre).toBe('/images/fbdf17fa7a970d2c37aa52ae4336119e76a7d13ea457811cf8aaab19b3f53843');
     expect(act1Post).toBe('/images/18bd2ac49e64976f1e74e828ddae76e54da1587044b1d3fb01317dd202ff7c67');
 
-    // Action 2: "下一次的决策前是上一次的动作后" -> Pre MUST be Action 1's post (18bd2ac4...), Post should be Action 2's post (484361b1...)
+    // Action 2: "The next pre-action state is the previous post-action state" -> Pre MUST be Action 1's post (18bd2ac4...), Post should be Action 2's post (484361b1...)
     const act2Pre = getStepPreImageUrl(stepData, action2);
     const act2Post = getStepPostImageUrl(stepData, action2);
     expect(act2Pre).toBe('/images/18bd2ac49e64976f1e74e828ddae76e54da1587044b1d3fb01317dd202ff7c67');
@@ -154,7 +154,7 @@ describe('extractStepReplayFrames', () => {
     expect(extractStepReplayFrames(null as any)).toEqual([]);
   });
 
-  it('should extract ordered StepReplayFrames and deduplicate consecutive identical images', () => {
+  it('should extract 1:1 ordered StepReplayFrames with action objects and pre/post images', () => {
     const logs = [
       {
         type: 'step_updated',
@@ -181,13 +181,18 @@ describe('extractStepReplayFrames', () => {
     ];
 
     const frames = extractStepReplayFrames(logs);
-    expect(frames.length).toBe(3);
-    expect(frames[0].title).toBe('Step 1 (Before Action)');
+    expect(frames.length).toBe(2);
+    expect(frames[0].stepNumber).toBe(1);
     expect(frames[0].imageUrl).toBe('/images/img_screen_1');
-    expect(frames[1].title).toContain('Step 1');
+    expect(frames[0].preImageUrl).toBe('/images/img_screen_1');
+    expect(frames[0].postImageUrl).toBe('/images/img_screen_2');
+    expect(frames[0].action).toEqual({ action: 'tap', coordinates: [500, 1000] });
+
+    expect(frames[1].stepNumber).toBe(2);
     expect(frames[1].imageUrl).toBe('/images/img_screen_2');
-    expect(frames[2].title).toContain('Step 2');
-    expect(frames[2].imageUrl).toBe('/images/img_screen_3');
+    expect(frames[1].preImageUrl).toBe('/images/img_screen_2');
+    expect(frames[1].postImageUrl).toBe('/images/img_screen_3');
+    expect(frames[1].action).toEqual({ action: 'input_text', text: 'hello' });
   });
 
   it('should handle terminal step with only pre_image (e.g. cancelled / failed task)', () => {

apps/showcase_ui/src/app/utils/action-formatter.util.ts

@@ -706,7 +706,7 @@ export function resolveStepImageMap(stepData: any): Map<any, ResolvedImageEntry>
       resolvedPost = stepData.post_image_name || stepData.post_screenshot || null;
     }
 
-    // "下一次的决策前是上一次的动作后"
+    // "The next pre-action state is the previous post-action state"
     if (resolvedPost) {
       currentPreImage = resolvedPost;
     }
@@ -778,24 +778,37 @@ export function extractStepReplayFrames(logsOrSteps: any[]): StepReplayFrame[] {
   const rawSteps: any[] = [];
   for (const item of logsOrSteps) {
     if (!item) continue;
-    if (item.type === 'step_updated' && item.data) {
-      rawSteps.push(item.data);
-    } else if (item.type === 'step' && item.data) {
+    if ((item.type === 'step_updated' || item.type === 'step_recorded' || item.type === 'step') && item.data) {
       rawSteps.push(item.data);
     } else if (item.step_number !== undefined || item.step_id !== undefined) {
       rawSteps.push(item);
     }
   }
 
-  // Deduplicate by step_id or step_number, keeping latest
+  // Deduplicate and merge by step_id or step_number
   const stepMap = new Map<any, any>();
   for (const step of rawSteps) {
     const key = step.step_id || step.step_number;
     if (key !== undefined) {
-      stepMap.set(key, step);
+      const existing = stepMap.get(key);
+      if (existing) {
+        stepMap.set(key, {
+          ...existing,
+          ...step,
+          step_number: step.step_number !== undefined ? step.step_number : existing.step_number,
+          pre_image_name: step.pre_image_name || existing.pre_image_name,
+          post_image_name: step.post_image_name || existing.post_image_name,
+          action_taken: step.action_taken || existing.action_taken,
+          last_execution_result: step.last_execution_result || existing.last_execution_result,
+          generic_tools: step.generic_tools || existing.generic_tools
+        });
+      } else {
+        stepMap.set(key, { ...step });
+      }
     }
   }
 
+  // Sort strictly by step_number ascending
   const steps = Array.from(stepMap.values()).sort((a, b) => {
     const numA = Number(a.step_number ?? 0);
     const numB = Number(b.step_number ?? 0);
@@ -804,47 +817,45 @@ export function extractStepReplayFrames(logsOrSteps: any[]): StepReplayFrame[] {
   });
 
   const frames: StepReplayFrame[] = [];
-  let lastImageUrl: string | null = null;
 
-  for (const stepData of steps) {
-    const stepNum = Number(stepData.step_number ?? (frames.length + 1));
+  for (let i = 0; i < steps.length; i++) {
+    const stepData = steps[i];
+    const stepNum = Number(stepData.step_number ?? (i + 1));
     const act = stepData.action_taken;
-    const title = act ? (getActionTitle(act) || 'Action') : 'Step';
+    const title = act ? (getActionTitle(act) || 'Action') : `Step ${stepNum}`;
     const coords = act ? getActionCoords(act) : '';
-    const actionDesc = coords ? `${title} (${coords})` : title;
+    const targetText = act ? getActionTargetText(act) : '';
+    const actionDesc = coords ? `${title} (${coords})` : (targetText ? `${title} (${targetText})` : title);
 
     const preUrl = getStepPreImageUrl(stepData, act);
     const postUrl = getStepPostImageUrl(stepData, act);
 
-    // If pre-image exists and is new, add it
-    if (preUrl && preUrl !== lastImageUrl) {
-      frames.push({
-        index: frames.length,
-        stepNumber: stepNum,
-        title: `Step ${stepNum} (Before Action)`,
-        imageUrl: preUrl,
-        actionText: `Preparing: ${actionDesc}`,
-        isPost: false,
-        timestamp: stepData.timestamp
-      });
-      lastImageUrl = preUrl;
+    // Each step gets exactly one primary frame (Pre-action screenshot where the action is taken)
+    const primaryImg = preUrl || postUrl;
+    if (!primaryImg) {
+      continue;
     }
 
-    // If post-image exists and is different from current screen, add it
-    if (postUrl && postUrl !== lastImageUrl) {
-      frames.push({
-        index: frames.length,
-        stepNumber: stepNum,
-        title: `Step ${stepNum}: ${actionDesc}`,
-        imageUrl: postUrl,
-        actionText: actionDesc,
-        isPost: true,
-        timestamp: stepData.timestamp
-      });
-      lastImageUrl = postUrl;
-    }
+    const failed = isActionFailed(act, stepData);
+
+    frames.push({
+      index: frames.length,
+      stepNumber: stepNum,
+      stepId: String(stepData.step_id || `step-${stepNum}`),
+      title: `Step ${stepNum}: ${title}`,
+      actionText: actionDesc,
+      action: act,
+      actionType: act?.action || act?.name || 'action',
+      targetText,
+      coords,
+      imageUrl: primaryImg,
+      preImageUrl: preUrl,
+      postImageUrl: postUrl,
+      isPost: false,
+      timestamp: stepData.timestamp,
+      status: failed ? 'failed' : 'success'
+    });
   }
 
   return frames;
 }
-