From a36a66ce6d7aedb1141dc88b5208e2d59af75a97 Mon Sep 17 00:00:00 2001 From: somew1nd Date: Wed, 26 Aug 2026 00:53:24 -0700 Subject: [PATCH] feat: add step-by-step visual replay and harden process lifecycle - UI: Implement dual-mode playback in FloatingVideoPlayer supporting video and step-by-step frame replay with scrub controls and auto-fallback - Runtime: Prevent asyncio event loop hangs on Darwin and Linux by avoiding psutil.wait_procs on direct child processes - Task Queue: Add worker exit watchdog and automatic recording recovery for completed or interrupted sessions - Recording: Add ffmpeg ultrafast transcode fallback, shield recording finalization against cancellation, and handle on-the-fly MP4 conversion - Docs: Polish README_CN styling and streamline flash runner agent prompt --- README_CN.md | 36 ++--- apps/admin_console/core/state.py | 6 + .../repositories/session_repository.py | 16 ++ apps/admin_console/routers/media.py | 20 ++- apps/admin_console/routers/sessions.py | 11 +- apps/admin_console/services/media_service.py | 75 +++++++++- .../services/task_queue_service.py | 62 +++++++- .../agent-stream/agent-stream.component.html | 21 ++- .../agent-stream/agent-stream.component.scss | 55 +++++++ .../agent-stream/agent-stream.component.ts | 21 +++ .../floating-video-player.component.html | 137 ++++++++++++++++-- .../floating-video-player.component.scss | 136 ++++++++++++++++- .../floating-video-player.component.ts | 122 +++++++++++++++- .../src/app/core/models/stream.model.ts | 11 ++ .../src/app/services/agent.service.spec.ts | 5 +- .../src/app/services/agent.service.ts | 39 ++++- .../app/utils/action-formatter.util.spec.ts | 99 ++++++++++++- .../src/app/utils/action-formatter.util.ts | 87 ++++++++++- artemis/agents/flash/flash_runner.md | 2 +- artemis/controllers/unified_controller.py | 48 ++++-- artemis/interfaces/cli/commands/run.py | 15 ++ artemis/platform/darwin.py | 48 +++++- artemis/platform/linux.py | 48 +++++- artemis/sdk/agent.py | 43 ++++-- artemis/utils/video.py | 55 +++++-- .../admin_console/test_task_queue_service.py | 51 +++++++ 26 files changed, 1152 insertions(+), 117 deletions(-) diff --git a/README_CN.md b/README_CN.md index 4716a7a..1927ce2 100644 --- a/README_CN.md +++ b/README_CN.md @@ -28,21 +28,21 @@

Artemis 演示效果
- 实机演示：在 Google Maps 中设置驾车路线并计算总耗时，随后打开 YouTube 播放 Coldplay 的歌曲。 + 实机演示：在 Google Maps 中设置驾车路线并计算总耗时，随后打开 YouTube 播放 Coldplay 的歌曲。

-## ✨ 核心亮点 +## 核心亮点 -* 🤖 **跨 App 复杂自动化与 AI 个人助理**：不仅能执行严格的自动化测试用例，更具备接管真机的自主决策与长流程操作能力，一句话处理跨应用复杂业务与任务； -* 🧪 **零脚本维护自动化测试**：基于“动态优先、坐标兜底”的多模态语义定位，彻底告别传统 XPath / 控件 ID 频繁失效的痛点，无惧 App 改版、分辨率差异与系统升级； -* 🐞 **IDE 内一键 Bug 复现与 Logcat 诊断**：原生支持 **MCP 协议**，可在 **Antigravity、Claude Code、Windsurf** 中直接用自然语言驱动真机复现缺陷，自动抓取关键帧截图与 **Logcat 崩溃堆栈**，完成研发测试闭环； -* ⚡ **极速执行吞吐（单步 3–5 秒）**：首创**全链路乐观异步流水线 (Optimistic Async Pipeline)**，解耦重度推理与页面操作，在 Flash 模式下实现丝滑的高频交互与快速回归验证； -* 🛡️ **干扰弹窗自愈与超长程巡检**：独创 **Safety Net** 执行前校验机制，自动识别并清除系统权限弹窗、通知遮挡等异常；Pro 模式支持连续 **10+ 小时** 无人值守稳定性巡检与探索性测试 (Exploratory / Monkey-plus Testing)； +* **跨 App 复杂自动化与 AI 个人助理**：不仅能执行严格的自动化测试用例，更具备接管真机的自主决策与长流程操作能力，一句话处理跨应用复杂业务与任务； +* **零脚本维护自动化测试**：基于“动态优先、坐标兜底”的多模态语义定位，彻底告别传统 XPath / 控件 ID 频繁失效的痛点，无惧 App 改版、分辨率差异与系统升级； +* **IDE 内一键 Bug 复现与 Logcat 诊断**：原生支持 **MCP 协议**，可在 **Antigravity、Claude Code、Windsurf** 中直接用自然语言驱动真机复现缺陷，自动抓取关键帧截图与 **Logcat 崩溃堆栈**，完成研发测试闭环； +* **极速执行吞吐（单步 3–5 秒）**：首创**全链路乐观异步流水线 (Optimistic Async Pipeline)**，解耦重度推理与页面操作，在 Flash 模式下实现丝滑的高频交互与快速回归验证； +* **干扰弹窗自愈与超长程巡检**：独创 **Safety Net** 执行前校验机制，自动识别并清除系统权限弹窗、通知遮挡等异常；Pro 模式支持连续 **10+ 小时** 无人值守稳定性巡检与探索性测试 (Exploratory / Monkey-plus Testing)； * **业界顶尖 SOTA**：在 Google Research **AndroidWorld** 基准评测（100+ 复杂长程系统与应用交互任务）中取得 **99%+ 任务完成率**。 -## 🤝 Antigravity × ARTEMIS：全流程自主测试演示 +## Antigravity × ARTEMIS：全流程自主测试演示 通过原生 MCP 协议，**Antigravity** 与 **ARTEMIS** 深度协同——只需一句自然语言指令，即可自动完成从需求理解、测试规划、真机执行到深度报告输出的完整闭环： @@ -62,7 +62,7 @@ 3. 自主执行测试 (Autonomous Test Execution)
- 接管真机自动化操作，跳过广告并实时采集性能指标

+ 驱动真机操作、界面导航并实时分析性能指标

步骤三：自主执行测试 @@ -78,8 +78,8 @@ ## 快速上手 确保电脑已连接 Android 实体机（已开启 **USB 调试**）或 Android 模拟器。一键启动脚本将会自动完成以下配置： -- 🛠️ **安装系统环境依赖**：自动检测并安装 ADB、scrcpy、FFmpeg 与 Python（`uv`）运行时及项目依赖。 -- 🔌 **全局挂载 MCP 服务与测试准则 (Rules)**：主动引导并自动将全局 MCP 服务与 **Artemis 移动端测试思维准则 (`rules.md`)** 挂载至你使用的 AI IDE（支持 **Antigravity**、**Cursor**、**Claude Code**、**Codex**、**Windsurf**、**VS Code**、**Cline/Roo**、**OpenClaw**）。 +- **安装系统环境依赖**：自动检测并安装 ADB、scrcpy、FFmpeg 与 Python（`uv`）运行时及项目依赖。 +- **全局挂载 MCP 服务与测试准则 (Rules)**：主动引导并自动将全局 MCP 服务与 **Artemis 移动端测试思维准则 (`rules.md`)** 挂载至你使用的 AI IDE（支持 **Antigravity**、**Cursor**、**Claude Code**、**Codex**、**Windsurf**、**VS Code**、**Cline/Roo**、**OpenClaw**）。 ### macOS 与 Linux @@ -109,7 +109,7 @@ cd artemis
-🔌 接入 Codex / Antigravity / Claude Code / Windsurf (MCP)（点击展开） +Codex / Antigravity / Claude Code / Windsurf MCP 配置（点击展开）
@@ -192,17 +192,17 @@ PYTHONPATH = "/path/to/artemis" * **Codex**：将内容添加至 `~/.codex/AGENTS.md`（或当前生效的 `AGENTS.override.md`）。 * **Windsurf / OpenClaw**：将内容添加到工作区规则或全局 System Prompt 中。 -> 💡 更多规范设计细节与 MCP 架构说明，请参阅 [MCP Server 文档](./mcp_server/README.md)。 +> 更多规范设计细节与 MCP 架构说明，请参阅 [MCP Server 文档](./mcp_server/README.md)。 ### 4. 在 IDE 中体验真机协同 在 Codex / Antigravity / Claude Code 对话框中直接输入： -> 💬 *"请帮我把刚刚修改的代码编译成 APK 并安装到手机上，打开登录页面输入测试账号，验证登录后是否有异常弹窗，并把最终页面截图回传。"* +> *"请帮我把刚刚修改的代码编译成 APK 并安装到手机上，打开登录页面输入测试账号，验证登录后是否有异常弹窗，并把最终页面截图回传。"*
-🐍 Python SDK 极简集成（点击展开） +Python SDK 集成（点击展开）
@@ -238,12 +238,12 @@ if __name__ == "__main__":

Artemis 可视化控制台
- 控制台功能布局：① 顶栏视图切换（主页与工作区） · ② 运行模式与录屏回放（Flash/Pro 状态与视频回放） · ③ 实时感知推理流（动作分解、点击坐标与结构化总结） · ④ 自然语言下发胶囊（自然语言驱动真机） · ⑤ 任务队列看板（状态流转与历史回溯） + 控制台功能概览：① 视图切换（主页与工作区） · ② 运行模式与回放（Flash/Pro 状态与视频回放） · ③ 实时感知推理流（动作感知、目标坐标与结构化总结） · ④ 提示词输入坞 (Prompt Dock)（自然语言下发） · ⑤ 任务队列看板（生命周期与历史回溯）

* **Web 可视化测试控制台 (`uv run artemis ui`)**：集成设备实时投屏与交互面板，支持通过自然语言下发测试用例，实时观测推理步骤、操作轨迹、截图留存与异常状态回放； -* **MCP 协议集成 (AI IDE 协同)**：作为标准 MCP 服务器无缝接入 **Antigravity、Claude Code、Windsurf** 等开发环境，在 IDE 中直接驱动真机完成自动化测试与 Bug 复现验证； -* **命令行工具 (`uv run artemis run`)**：支持通过 CLI 直接执行自动化用例、稳定性巡检或 AndroidWorld 基准评测，提供高保真结构化终端输出； +* **原生 MCP 协议 (IDE 协同)**：作为标准 MCP 服务器无缝接入 **Antigravity、Claude Code、Windsurf** 等开发环境，在 IDE 中直接驱动真机完成自动化测试与 Bug 复现验证； +* **开发者命令行 CLI (`uv run artemis run`)**：支持通过终端直接执行自动化测试用例、探索性稳定性巡检或 AndroidWorld 基准评测，提供高保真结构化终端输出； * **Python SDK**：作为标准 Python 库无缝集成至现有自动化测试框架（如 pytest）或 CI/CD 流水线，提供基于 Pydantic 的强类型结构化结果与断言支持。 diff --git a/apps/admin_console/core/state.py b/apps/admin_console/core/state.py index 452b485..9aafe2f 100644 --- a/apps/admin_console/core/state.py +++ b/apps/admin_console/core/state.py @@ -204,3 +204,9 @@ class ServerState: # Global shared instance state = ServerState() + +import sys +if __name__ == "admin_console.core.state": + sys.modules["apps.admin_console.core.state"] = sys.modules[__name__] +elif __name__ == "apps.admin_console.core.state": + sys.modules["admin_console.core.state"] = sys.modules[__name__] diff --git a/apps/admin_console/database/repositories/session_repository.py b/apps/admin_console/database/repositories/session_repository.py index 906c0ac..2ff4988 100644 --- a/apps/admin_console/database/repositories/session_repository.py +++ b/apps/admin_console/database/repositories/session_repository.py @@ -141,6 +141,22 @@ class SessionRepository: except Exception: return False + def mark_recording_ready(self, session_id: str, local_video_path: str) -> bool: + """Mark a recording as ready with its finalized video path.""" + try: + with db_session(self.db_path) as conn: + cursor = conn.cursor() + cursor.execute( + "UPDATE video_recordings SET status = 'ready', error = NULL, " + "local_video_path = ?, end_time = COALESCE(end_time, ?) " + "WHERE session_id = ?", + (str(local_video_path), time.time(), str(session_id)), + ) + conn.commit() + return cursor.rowcount > 0 + except Exception: + return False + def get_llm_traces_for_profile(self, session_id: str, limit: int = 3) -> list[str]: with db_session(self.db_path) as conn: cursor = conn.cursor() diff --git a/apps/admin_console/routers/media.py b/apps/admin_console/routers/media.py index fb8dc33..285941c 100644 --- a/apps/admin_console/routers/media.py +++ b/apps/admin_console/routers/media.py @@ -102,14 +102,18 @@ async def get_session_video(session_id: str): } if recording_status == "failed": - return { - "session_id": session_id, - "status": "failed", - "has_video": False, - "video_url": None, - "video_segments": [], - "message": recording.get("error") or "Recording finalization failed", - } + v_url = media_service.resolve_video_url(row_dict, video_rec_map, video_idx) + if not v_url: + return { + "session_id": session_id, + "status": "failed", + "has_video": False, + "video_url": None, + "video_segments": [], + "message": recording.get("error") or "Recording finalization failed", + } + # If a video was recovered/found, update DB to ready and proceed to serve it + session_repo.mark_recording_ready(session_id, v_url) v_url = media_service.resolve_video_url(row_dict, video_rec_map, video_idx) video_segments = media_service.resolve_video_segments(v_url) diff --git a/apps/admin_console/routers/sessions.py b/apps/admin_console/routers/sessions.py index 79d5ae1..91c71f9 100644 --- a/apps/admin_console/routers/sessions.py +++ b/apps/admin_console/routers/sessions.py @@ -63,12 +63,17 @@ async def list_sessions(): orphaned_ids.append(s_id) recording_status = str((recording or {}).get("status") or "unavailable") - row_dict["recording_status"] = recording_status - row_dict["video_url"] = ( + resolved_v_url = ( media_service.resolve_video_url(row_dict, video_rec_map, video_idx) - if recording_status in ("ready", "unavailable") + if recording_status != "recording" else None ) + if resolved_v_url: + recording_status = "ready" + row_dict["video_url"] = resolved_v_url + else: + row_dict["video_url"] = None + row_dict["recording_status"] = recording_status agent_names = agent_names_by_session.get(s_id, []) sess_profile = model_service.resolve_session_profile( diff --git a/apps/admin_console/services/media_service.py b/apps/admin_console/services/media_service.py index 807c91e..6e2883a 100644 --- a/apps/admin_console/services/media_service.py +++ b/apps/admin_console/services/media_service.py @@ -17,6 +17,7 @@ import hashlib import json import os from pathlib import Path +import subprocess from typing import Any import urllib.parse from fastapi import HTTPException @@ -30,6 +31,72 @@ except ImportError: class MediaService: """Service handling media indexing, local file security, payload unwrapping and notes/plans.""" + @classmethod + def ensure_browser_playable_video(cls, p: Path) -> Path: + """Ensure the video file is in a browser-supported container (MP4). + + If given an MKV, check if a converted MP4 already exists or convert it. + """ + try: + if not p.exists(): + return p + suffix = p.suffix.lower() + if suffix == ".mp4": + return p + if suffix in (".mkv", ".webm"): + mp4_cand = p.with_suffix(".mp4") + if mp4_cand.exists() and mp4_cand.stat().st_size > 0: + return mp4_cand + from artemis.utils.video import get_ffmpeg_path + ffmpeg = get_ffmpeg_path() + # 1. Attempt fast copy remux + subprocess.run( + [ + ffmpeg, + "-y", + "-fflags", + "+genpts", + "-i", + str(p), + "-c", + "copy", + "-movflags", + "+faststart", + str(mp4_cand), + ], + capture_output=True, + timeout=15, + ) + if mp4_cand.exists() and mp4_cand.stat().st_size > 0: + return mp4_cand + # 2. If copy remux failed, fallback to ultrafast re-encode + subprocess.run( + [ + ffmpeg, + "-y", + "-fflags", + "+genpts+discardcorrupt", + "-i", + str(p), + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + str(mp4_cand), + ], + capture_output=True, + timeout=30, + ) + if mp4_cand.exists() and mp4_cand.stat().st_size > 0: + return mp4_cand + except Exception: + pass + return p + @staticmethod def path_to_video_url(p: Path) -> str: resolved = p.resolve() @@ -56,12 +123,14 @@ class MediaService: for ext in [".mp4", ".mkv", ".webm"]: vfile = item / f"recording{ext}" if vfile.exists(): + vfile = cls.ensure_browser_playable_video(vfile) url = cls.path_to_video_url(vfile) idx[item.name] = url prefix = item.name.split("_PASS_")[0].split("_FAIL_")[0] idx[prefix] = url break elif item.suffix in [".mp4", ".mkv", ".webm"]: + item = cls.ensure_browser_playable_video(item) url = cls.path_to_video_url(item) idx[item.stem] = url idx[item.name] = url @@ -85,12 +154,14 @@ class MediaService: v_url = None v_fp = row_dict.get("video_filepath") if v_fp and os.path.exists(v_fp): - v_url = cls.path_to_video_url(Path(v_fp)) + p = cls.ensure_browser_playable_video(Path(v_fp)) + v_url = cls.path_to_video_url(p) orig_rec = video_rec_map.get(s_id) if not v_url and orig_rec: p = Path(orig_rec) if p.exists(): + p = cls.ensure_browser_playable_video(p) v_url = cls.path_to_video_url(p) else: folder_name = p.parent.name @@ -105,12 +176,14 @@ class MediaService: for ext in [".mp4", ".webm", ".mkv"]: vfile = sess_dir / f"recording{ext}" if vfile.exists(): + vfile = cls.ensure_browser_playable_video(vfile) v_url = cls.path_to_video_url(vfile) break if not v_url: try: for item in sess_dir.iterdir(): if item.is_file() and item.suffix.lower() in [".mp4", ".webm", ".mkv"]: + item = cls.ensure_browser_playable_video(item) v_url = cls.path_to_video_url(item) break except Exception: diff --git a/apps/admin_console/services/task_queue_service.py b/apps/admin_console/services/task_queue_service.py index 99f35f8..0570201 100644 --- a/apps/admin_console/services/task_queue_service.py +++ b/apps/admin_console/services/task_queue_service.py @@ -32,6 +32,7 @@ try: ) from admin_console.core.state import state from admin_console.database.repositories.session_repository import session_repo + from admin_console.services.media_service import media_service from artemis.runtime import DeviceExecutionLock, process_supervisor except ImportError: from apps.admin_console.core.config import ( @@ -42,6 +43,7 @@ except ImportError: ) from apps.admin_console.core.state import state from apps.admin_console.database.repositories.session_repository import session_repo + from apps.admin_console.services.media_service import media_service from artemis.runtime import DeviceExecutionLock, process_supervisor @@ -199,6 +201,28 @@ class TaskQueueService: except ProcessLookupError: pass + @staticmethod + async def _wait_for_worker_process(proc: asyncio.subprocess.Process) -> int: + """Wait for worker process to exit, with watchdog fallback if PID was reaped externally.""" + while True: + try: + return await asyncio.wait_for(proc.wait(), timeout=1.0) + except asyncio.TimeoutError: + pid = getattr(proc, "pid", None) + if pid: + try: + import psutil + + p = psutil.Process(pid) + if not p.is_running() or p.status() == psutil.STATUS_ZOMBIE: + return proc.returncode if proc.returncode is not None else -15 + except (psutil.NoSuchProcess, ProcessLookupError): + return proc.returncode if proc.returncode is not None else -15 + except Exception: + pass + else: + return proc.returncode if proc.returncode is not None else -15 + @classmethod def ensure_worker_running(cls): """Guarantees that the background queue worker task is active and running.""" @@ -341,7 +365,7 @@ class TaskQueueService: await cls._terminate_worker_process(proc) # 3. Await subprocess completion - returncode = await proc.wait() + returncode = await cls._wait_for_worker_process(proc) print(f"[QueueWorker] Task [{sess_id}] exited with returncode {returncode}") # 4. Perform fallback database status update and notification @@ -360,12 +384,36 @@ class TaskQueueService: f"[QueueWorker] Preserved authoritative session {sess_id} " f"status '{new_status}'" ) - recording_error = "Task worker exited before recording finalization completed" - if session_repo.mark_recording_failed_if_pending(sess_id, recording_error): - cls._broadcast_event( - "recording_failed", - {"session_id": sess_id, "error": recording_error}, - ) + rec_info = session_repo.get_video_recording_for_session(sess_id) + rec_status = (rec_info or {}).get("status") + recovered_video_path = None + if rec_status != "ready": + try: + video_rec_map = session_repo.get_video_recordings_map() + video_idx = await asyncio.to_thread(media_service.build_video_index) + recovered_url = await asyncio.to_thread( + media_service.resolve_video_url, + {"session_id": sess_id}, + video_rec_map, + video_idx, + ) + if recovered_url: + session_repo.mark_recording_ready(sess_id, recovered_url) + cls._broadcast_event( + "recording_ready", + {"session_id": sess_id, "video_url": recovered_url}, + ) + recovered_video_path = recovered_url + except Exception as rec_err: + print(f"[QueueWorker] Error attempting recording recovery: {rec_err}") + + if not recovered_video_path and rec_status != "ready": + recording_error = "Task worker exited before recording finalization completed" + if session_repo.mark_recording_failed_if_pending(sess_id, recording_error): + cls._broadcast_event( + "recording_failed", + {"session_id": sess_id, "error": recording_error}, + ) state.active_connections.pop(sess_id, None) cls._broadcast_event( "session_ended", diff --git a/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.html b/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.html index dda92c9..17be5d7 100644 --- a/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.html +++ b/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.html @@ -263,28 +263,41 @@ [class.recording]="agentService.isCurrentSessionRunning()" [class.processing]="!agentService.isCurrentSessionRunning() && agentService.currentSessionRecordingStatus() === 'processing'" [class.has-video]="!agentService.isCurrentSessionRunning() && !!agentService.currentSessionVideoUrl()" + [class.has-steps]="!agentService.isCurrentSessionRunning() && !agentService.currentSessionVideoUrl() && agentService.hasCurrentSessionStepFrames()" + [class.failed]="!agentService.isCurrentSessionRunning() && !agentService.currentSessionVideoUrl() && !agentService.hasCurrentSessionStepFrames() && agentService.currentSessionRecordingStatus() === 'failed'" (click)="agentService.toggleVideoPlayer()" - [title]="agentService.isCurrentSessionRunning() ? 'Task is currently running (Recording screen)' : (agentService.currentSessionRecordingStatus() === 'processing' ? 'Preparing screen recording' : (agentService.currentSessionVideoUrl() ? 'Play Screen Recording' : 'Check Device Screen Recording'))" + [title]="getScreenRecordingButtonTitle()" > videocam - + progress_activity smart_display - + + slideshow + + + videocam_off + + videocam Recording... Preparing... Play Recording - Screen Recording + Step Replay + Recording Failed + Screen Recording + + {{ agentService.currentSessionStepFrames().length }} + diff --git a/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.scss b/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.scss index 8abd6f6..159858a 100644 --- a/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.scss +++ b/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.scss @@ -936,6 +936,56 @@ $stream-row-height: 25px; box-shadow: 0 1px 3px rgba(15, 23, 42, 0.08); } + &.has-steps { + background: rgba(240, 253, 250, 0.9); + border-color: rgba(94, 234, 212, 0.7); + box-shadow: 0 2px 6px -1px rgba(13, 148, 136, 0.12), inset 0 1px 1px rgba(255, 255, 255, 0.9); + + .steps-icon { + color: #0d9488; + } + + .btn-text { + color: #0f766e; + } + + &:hover:not(.active) { + background: rgba(204, 251, 241, 0.95); + border-color: rgba(45, 212, 191, 0.85); + + .steps-icon { + color: #0f766e; + } + } + } + + &.failed { + background: rgba(254, 242, 242, 0.8); + border-color: rgba(254, 202, 202, 0.9); + + .error-icon { + color: #ef4444; + } + + .btn-text { + color: #dc2626; + } + } + + .steps-count-badge { + display: inline-flex; + align-items: center; + justify-content: center; + padding: 1px 5px; + border-radius: 10px; + background: #0d9488; + color: #ffffff; + font-size: 10px; + font-weight: 700; + line-height: 1; + margin-left: 2px; + } + &.active { background: rgba(26, 115, 232, 0.95); border-color: rgba(26, 115, 232, 0.95); @@ -949,6 +999,11 @@ $stream-row-height: 25px; background: #ffffff; box-shadow: 0 0 6px rgba(255, 255, 255, 0.9); } + + .steps-count-badge { + background: #ffffff; + color: #1a73e8; + } } } } diff --git a/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.ts b/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.ts index 52c0afe..977a33a 100644 --- a/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.ts +++ b/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.ts @@ -936,6 +936,27 @@ export class AgentStreamComponent implements AfterViewInit { } } + public getScreenRecordingButtonTitle(): string { + if (this.agentService.isCurrentSessionRunning()) { + return 'Task is currently running (Recording screen)'; + } + if (this.agentService.currentSessionRecordingStatus() === 'processing') { + return 'Preparing screen recording...'; + } + if (this.agentService.currentSessionVideoUrl()) { + return 'Play Screen Recording Video'; + } + if (this.agentService.hasCurrentSessionStepFrames()) { + return this.agentService.currentSessionRecordingStatus() === 'failed' + ? 'Video recording failed — Click to replay step-by-step screenshots' + : 'Play step-by-step screenshots replay'; + } + if (this.agentService.currentSessionRecordingStatus() === 'failed') { + return 'Screen recording generation failed'; + } + return 'Screen Recording'; + } + public isDeviceActionTool(tool: any): boolean { return isDeviceActionTool(tool); } diff --git a/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.html b/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.html index cc240fd..37ecae2 100644 --- a/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.html +++ b/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.html @@ -33,16 +33,38 @@
- {{ agentService.recordingPlaybackStatus() === 'live' ? 'sensors' : (agentService.recordingPlaybackStatus() === 'processing' ? 'progress_activity' : 'smart_display') }} + {{ agentService.recordingPlaybackStatus() === 'live' ? 'sensors' : (agentService.playerMode() === 'steps' ? 'slideshow' : (agentService.recordingPlaybackStatus() === 'processing' ? 'progress_activity' : 'smart_display')) }} - {{ agentService.recordingPlaybackStatus() === 'live' ? 'Live Device Screen' : (agentService.recordingPlaybackStatus() === 'processing' ? 'Preparing Recording' : (agentService.activeVideoTitle() || 'Screen Recording')) }} + {{ agentService.recordingPlaybackStatus() === 'live' ? 'Live Device Screen' : (agentService.playerMode() === 'steps' ? ('Step Replay (' + (totalStepFrames() > 0 ? (activeStepIndex() + 1) + '/' + totalStepFrames() : '0') + ')') : (agentService.recordingPlaybackStatus() === 'processing' ? 'Preparing Recording' : (agentService.activeVideoTitle() || 'Screen Recording'))) }}
- -
+ +
+ + +
+ + +
+ + + + + + + + + + + + + Frame {{ activeStepIndex() + 1 }} / {{ totalStepFrames() }} + +
+ +
+ + + + + +
+
+
+ + + + +
- +
@@ -206,8 +319,8 @@
- -
+ +
videocam_off
diff --git a/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.scss b/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.scss index 0c54dd6..69908ce 100644 --- a/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.scss +++ b/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.scss @@ -176,6 +176,48 @@ gap: 4px; flex-shrink: 0; + .mode-toggle-group { + display: flex; + align-items: center; + background: rgba(241, 245, 249, 0.7); + backdrop-filter: blur(12px); + border: 1px solid rgba(226, 232, 240, 0.85); + border-radius: 8px; + padding: 2px; + margin-right: 4px; + gap: 2px; + + .mode-btn { + display: inline-flex; + align-items: center; + gap: 3px; + border: none; + background: transparent; + color: #475569; + font-size: 11px; + font-weight: 600; + padding: 2px 7px; + border-radius: 6px; + cursor: pointer; + transition: all 0.15s ease; + + .mode-icon { + font-size: 13px; + } + + &:hover:not(.active) { + color: #0f172a; + background: rgba(255, 255, 255, 0.8); + } + + &.active { + background: #0284c7; + color: #ffffff; + box-shadow: 0 1px 3px rgba(2, 132, 199, 0.3); + } + } + } + .speed-pill-group { display: flex; align-items: center; @@ -185,8 +227,6 @@ border-radius: 8px; padding: 2px; margin-right: 4px; - padding: 2px; - margin-right: 4px; .speed-btn { border: none; @@ -370,6 +410,98 @@ } } + .step-replay-container { + display: flex; + flex-direction: column; + background: #000000; + position: relative; + width: 100%; + overflow: hidden; + + .step-replay-notice { + display: flex; + align-items: center; + gap: 6px; + background: rgba(254, 243, 199, 0.95); + color: #92400e; + padding: 6px 12px; + font-size: 11.5px; + font-weight: 500; + border-bottom: 1px solid rgba(245, 158, 11, 0.3); + + .notice-icon { + font-size: 14px; + color: #d97706; + flex-shrink: 0; + } + } + + .step-image-viewport { + display: flex; + align-items: center; + justify-content: center; + position: relative; + background: #0a0a0c; + min-height: 240px; + max-height: 520px; + cursor: pointer; + overflow: hidden; + + .step-frame-img { + max-width: 100%; + max-height: 520px; + width: auto; + height: auto; + object-fit: contain; + display: block; + user-select: none; + transition: opacity 0.15s ease; + + &:hover { + opacity: 0.96; + } + } + + .step-frame-badge { + position: absolute; + bottom: 12px; + left: 12px; + right: 12px; + display: flex; + align-items: center; + gap: 6px; + background: rgba(15, 23, 42, 0.84); + backdrop-filter: blur(10px); + border: 1px solid rgba(255, 255, 255, 0.18); + border-radius: 8px; + padding: 6px 10px; + color: #ffffff; + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.45); + pointer-events: none; + animation: fadeIn 0.2s ease; + + .badge-step-no { + font-size: 11px; + font-weight: 700; + background: #0284c7; + padding: 1px 6px; + border-radius: 4px; + letter-spacing: 0.2px; + flex-shrink: 0; + } + + .badge-action-desc { + font-size: 11.5px; + font-weight: 500; + color: #f8fafc; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + } + } + } + .video-container { position: relative; width: 100%; diff --git a/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.ts b/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.ts index bbc688f..ddf3ea6 100644 --- a/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.ts +++ b/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.ts @@ -22,11 +22,13 @@ import { ElementRef, ViewChild, HostListener, - effect + effect, + OnDestroy } from '@angular/core'; import { CommonModule } from '@angular/common'; import { FormsModule } from '@angular/forms'; import { AgentService } from '../../services/agent.service'; +import { StepReplayFrame } from '../../core/models/stream.model'; @Component({ selector: 'app-floating-video-player', @@ -35,7 +37,7 @@ import { AgentService } from '../../services/agent.service'; templateUrl: './floating-video-player.component.html', styleUrl: './floating-video-player.component.scss' }) -export class FloatingVideoPlayerComponent { +export class FloatingVideoPlayerComponent implements OnDestroy { public agentService = inject(AgentService); @ViewChild('videoRef') videoRef?: ElementRef; @@ -60,6 +62,24 @@ export class FloatingVideoPlayerComponent { private pendingAutoplay = false; private pendingAbsoluteSeek: number | null = null; + // Step Replay state signals + public activeStepIndex = signal(0); + public isStepPlaying = signal(false); + private stepTimer: any = null; + + public stepFrames = computed(() => this.agentService.currentSessionStepFrames()); + public totalStepFrames = computed(() => this.stepFrames().length); + public currentStepFrame = computed(() => { + const frames = this.stepFrames(); + const idx = this.activeStepIndex(); + return frames[idx] || null; + }); + public stepProgressPercent = computed(() => { + const total = this.totalStepFrames(); + if (total <= 1) return 0; + return (this.activeStepIndex() / (total - 1)) * 100; + }); + // Available speed options public speedOptions = [0.5, 1.0, 1.5, 2.0, 4.0]; @@ -92,6 +112,16 @@ export class FloatingVideoPlayerComponent { { allowSignalWrites: true } ); + // Reset step playback state when session changes + effect( + () => { + const _ = this.stepFrames(); + this.activeStepIndex.set(0); + this.pauseStepPlay(); + }, + { allowSignalWrites: true } + ); + effect(() => { const request = this.agentService.videoSeekRequest(); if (!request) return; @@ -327,6 +357,9 @@ export class FloatingVideoPlayerComponent { if (v) { v.playbackRate = rate; } + if (this.isStepPlaying()) { + this.scheduleNextStepFrame(); + } } public toggleLoop(): void { @@ -361,4 +394,89 @@ export class FloatingVideoPlayerComponent { window.open(url, '_blank'); } } + + /** + * Step Replay Controls + */ + public toggleStepPlay(): void { + if (this.isStepPlaying()) { + this.pauseStepPlay(); + } else { + this.startStepPlay(); + } + } + + public startStepPlay(): void { + if (this.totalStepFrames() <= 0) return; + if (this.activeStepIndex() >= this.totalStepFrames() - 1) { + this.activeStepIndex.set(0); + } + this.isStepPlaying.set(true); + this.scheduleNextStepFrame(); + } + + public pauseStepPlay(): void { + this.isStepPlaying.set(false); + if (this.stepTimer) { + clearTimeout(this.stepTimer); + this.stepTimer = null; + } + } + + private scheduleNextStepFrame(): void { + if (this.stepTimer) clearTimeout(this.stepTimer); + const delay = Math.max(300, 1500 / this.playbackRate()); + this.stepTimer = setTimeout(() => { + if (!this.isStepPlaying()) return; + if (this.activeStepIndex() < this.totalStepFrames() - 1) { + this.activeStepIndex.update((i) => i + 1); + this.scheduleNextStepFrame(); + } else { + if (this.isLooping()) { + this.activeStepIndex.set(0); + this.scheduleNextStepFrame(); + } else { + this.pauseStepPlay(); + } + } + }, delay); + } + + public prevStepFrame(): void { + this.pauseStepPlay(); + this.activeStepIndex.update((i) => Math.max(0, i - 1)); + } + + public nextStepFrame(): void { + this.pauseStepPlay(); + this.activeStepIndex.update((i) => Math.min(this.totalStepFrames() - 1, i + 1)); + } + + public seekStepFrame(index: number): void { + this.activeStepIndex.set(Math.max(0, Math.min(this.totalStepFrames() - 1, index))); + } + + public onStepScrub(event: Event): void { + const input = event.target as HTMLInputElement; + this.seekStepFrame(parseInt(input.value, 10)); + } + + public restartStep(): void { + this.seekStepFrame(0); + this.startStepPlay(); + } + + public openImageInNewTab(): void { + const url = this.currentStepFrame()?.imageUrl; + if (url) { + window.open(url, '_blank'); + } + } + + public ngOnDestroy(): void { + if (this.stepTimer) { + clearTimeout(this.stepTimer); + this.stepTimer = null; + } + } } diff --git a/apps/showcase_ui/src/app/core/models/stream.model.ts b/apps/showcase_ui/src/app/core/models/stream.model.ts index 667aaca..b579aba 100644 --- a/apps/showcase_ui/src/app/core/models/stream.model.ts +++ b/apps/showcase_ui/src/app/core/models/stream.model.ts @@ -98,3 +98,14 @@ export interface CheckerResult { reason: string; } +export interface StepReplayFrame { + index: number; + stepNumber: number; + title: string; + imageUrl: string; + actionText?: string; + isPost?: boolean; + timestamp?: number; + phaseId?: string; +} + diff --git a/apps/showcase_ui/src/app/services/agent.service.spec.ts b/apps/showcase_ui/src/app/services/agent.service.spec.ts index 967aa80..c955e29 100644 --- a/apps/showcase_ui/src/app/services/agent.service.spec.ts +++ b/apps/showcase_ui/src/app/services/agent.service.spec.ts @@ -1,4 +1,4 @@ -import { signal } from '@angular/core'; +import { signal, computed } from '@angular/core'; import { of } from 'rxjs'; import { AgentService } from './agent.service'; @@ -198,6 +198,9 @@ describe('AgentService recording finalization lifecycle', () => { service.recordingPlaybackStatus = signal('idle'); service.recordingPlaybackMessage = signal(''); service.shouldAutoplayVideo = signal(true); + service.playerMode = signal<'video' | 'steps'>('video'); + (service as any).hasCurrentSessionStepFrames = computed(() => false); + (service as any).currentSessionStepFrames = computed(() => []); return service; } diff --git a/apps/showcase_ui/src/app/services/agent.service.ts b/apps/showcase_ui/src/app/services/agent.service.ts index a9bf6d0..d404e99 100644 --- a/apps/showcase_ui/src/app/services/agent.service.ts +++ b/apps/showcase_ui/src/app/services/agent.service.ts @@ -19,8 +19,9 @@ import { HttpClient } from '@angular/common/http'; import { Observable } from 'rxjs'; import { Session, ModelInfo, TaskQueueItem, AgentStatusResponse } from '../core/models/session.model'; -import { StepItemData } from '../core/models/stream.model'; -export type { Session, ModelInfo, TaskQueueItem, AgentStatusResponse, StepItemData }; +import { StepItemData, StepReplayFrame } from '../core/models/stream.model'; +import { extractStepReplayFrames } from '../utils/action-formatter.util'; +export type { Session, ModelInfo, TaskQueueItem, AgentStatusResponse, StepItemData, StepReplayFrame }; const SESSION_CACHE_KEY = 'artemis.sessions.v1'; @@ -158,12 +159,32 @@ export class AgentService { public recordingPlaybackMessage = signal(''); public shouldAutoplayVideo = signal(false); public videoSeekRequest = signal<{ seconds: number; requestId: number } | null>(null); + public playerMode = signal<'video' | 'steps'>('video'); private activeVideoSessionId: string | null = null; private videoSeekRequestId = 0; private videoRequestGeneration = 0; private videoRetryTimer: ReturnType | null = null; private videoWaitStartedAt = 0; + /** + * Computed step replay frames from current session logs + */ + public currentSessionStepFrames = computed(() => { + return extractStepReplayFrames(this.sessionLogs()); + }); + + public hasCurrentSessionStepFrames = computed(() => { + return this.currentSessionStepFrames().length > 0; + }); + + public setPlayerMode(mode: 'video' | 'steps'): void { + this.playerMode.set(mode); + } + + public togglePlayerMode(): void { + this.playerMode.update((mode) => (mode === 'video' ? 'steps' : 'video')); + } + /** * Clear user-pinned selection so subsequent runs automatically follow active runner */ @@ -1284,6 +1305,13 @@ export class AgentService { this.isVideoLoading.set(true); this.recordingPlaybackStatus.set('processing'); this.recordingPlaybackMessage.set('Loading screen recording...'); + if (targetUrl) { + this.playerMode?.set('video'); + } else if (this.hasCurrentSessionStepFrames?.()) { + this.playerMode?.set('steps'); + } else { + this.playerMode?.set('video'); + } this.requestSessionVideo(targetSessionId, this.videoRequestGeneration); } @@ -1341,6 +1369,7 @@ export class AgentService { this.recordingPlaybackMessage.set(''); this.activeVideoUrl.set(res.video_url); this.activeVideoSegments.set(res.video_segments || []); + this.playerMode?.set('video'); this.rawSessions.update((list) => list.map((s) => s.session_id === sessionId ? { ...s, video_url: res.video_url || undefined, recording_status: 'ready' } @@ -1368,6 +1397,9 @@ export class AgentService { ? 'Recording finalization failed.' : 'No screen recording is available for this task.') ); + if (this.hasCurrentSessionStepFrames?.()) { + this.playerMode?.set('steps'); + } }, error: () => { if (generation !== this.videoRequestGeneration || sessionId !== this.activeVideoSessionId) { @@ -1380,6 +1412,9 @@ export class AgentService { this.isVideoLoading.set(false); this.recordingPlaybackStatus.set('failed'); this.recordingPlaybackMessage.set('Unable to load the screen recording.'); + if (this.hasCurrentSessionStepFrames?.()) { + this.playerMode?.set('steps'); + } } }); } diff --git a/apps/showcase_ui/src/app/utils/action-formatter.util.spec.ts b/apps/showcase_ui/src/app/utils/action-formatter.util.spec.ts index 4464bb2..65ca813 100644 --- a/apps/showcase_ui/src/app/utils/action-formatter.util.spec.ts +++ b/apps/showcase_ui/src/app/utils/action-formatter.util.spec.ts @@ -1,6 +1,7 @@ import { getStepPreImageUrl, - getStepPostImageUrl + getStepPostImageUrl, + extractStepReplayFrames } from './action-formatter.util'; describe('action-formatter.util screenshot chaining', () => { @@ -109,4 +110,100 @@ describe('action-formatter.util screenshot chaining', () => { expect(preUrl).toBe('/images/pre123'); expect(postUrl).toBe('/images/post456'); }); + + it('should resolve post screenshot for primary action when generic_tools contains action traces', () => { + const stepData = { + step_id: 'step-02', + pre_image_name: 'pre_screen_hash', + post_image_name: 'post_screen_hash', + action_taken: { + action: 'click', + coordinates: [500, 928], + args: { target: [500, 928] }, + timestamp: 1000 + }, + generic_tools: [ + { + trace_id: 'trace-click-internal', + type: 'action', + name: 'click', + timestamp: 1001, + payload: { + args: { target: '[500, 928]', times: '1', delay_ms: '100' }, + result: { + outcome: 'Clicked successfully.', + post_image_name: 'post_screen_hash', + status: 'success' + } + } + } + ] + }; + + const preUrl = getStepPreImageUrl(stepData, stepData.action_taken); + const postUrl = getStepPostImageUrl(stepData, stepData.action_taken); + + expect(preUrl).toBe('/images/pre_screen_hash'); + expect(postUrl).toBe('/images/post_screen_hash'); + }); +}); + +describe('extractStepReplayFrames', () => { + it('should return an empty array for empty or invalid logs', () => { + expect(extractStepReplayFrames([])).toEqual([]); + expect(extractStepReplayFrames(null as any)).toEqual([]); + }); + + it('should extract ordered StepReplayFrames and deduplicate consecutive identical images', () => { + const logs = [ + { + type: 'step_updated', + data: { + step_id: 's-1', + step_number: 1, + pre_image_name: 'img_screen_1', + post_image_name: 'img_screen_2', + action_taken: { action: 'tap', coordinates: [500, 1000] }, + timestamp: 100 + } + }, + { + type: 'step_updated', + data: { + step_id: 's-2', + step_number: 2, + pre_image_name: 'img_screen_2', + post_image_name: 'img_screen_3', + action_taken: { action: 'input_text', text: 'hello' }, + timestamp: 200 + } + } + ]; + + const frames = extractStepReplayFrames(logs); + expect(frames.length).toBe(3); + expect(frames[0].title).toBe('Step 1 (Before Action)'); + expect(frames[0].imageUrl).toBe('/images/img_screen_1'); + expect(frames[1].title).toContain('Step 1'); + expect(frames[1].imageUrl).toBe('/images/img_screen_2'); + expect(frames[2].title).toContain('Step 2'); + expect(frames[2].imageUrl).toBe('/images/img_screen_3'); + }); + + it('should handle terminal step with only pre_image (e.g. cancelled / failed task)', () => { + const logs = [ + { + step_number: 1, + pre_image_name: 'img_start', + post_image_name: null, + action_taken: null, + timestamp: 100 + } + ]; + + const frames = extractStepReplayFrames(logs); + expect(frames.length).toBe(1); + expect(frames[0].imageUrl).toBe('/images/img_start'); + expect(frames[0].stepNumber).toBe(1); + }); }); diff --git a/apps/showcase_ui/src/app/utils/action-formatter.util.ts b/apps/showcase_ui/src/app/utils/action-formatter.util.ts index 95c5ca3..2606b75 100644 --- a/apps/showcase_ui/src/app/utils/action-formatter.util.ts +++ b/apps/showcase_ui/src/app/utils/action-formatter.util.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { ActionParam } from '../core/models/stream.model'; +import { ActionParam, StepReplayFrame } from '../core/models/stream.model'; import { extractNumbersFromCoordinateValue, isPureDirectionString, parseSequenceCoordinates } from './image-overlay.util'; import { cleanErrorMessage } from './tool-formatter.util'; @@ -698,7 +698,7 @@ export function resolveStepImageMap(stepData: any): Map if (explicitPost) { resolvedPost = explicitPost; hasExplicitPost = true; - } else if (e.isPrimary && !hadPrimaryFailure && !hasSubsequentActions) { + } else if (e.isPrimary && !hadPrimaryFailure) { resolvedPost = stepData.post_image_name || stepData.post_screenshot || null; } else if (e.item === lastActionItem && !hadPrimaryFailure) { resolvedPost = stepData.post_image_name || stepData.post_screenshot || null; @@ -765,3 +765,86 @@ export function getStepPostImageUrl(stepData: any, actionData?: any): string | n } return postUrl; } + +/** + * Extract an ordered array of visual StepReplayFrames from session logs or step blocks + */ +export function extractStepReplayFrames(logsOrSteps: any[]): StepReplayFrame[] { + if (!Array.isArray(logsOrSteps) || logsOrSteps.length === 0) { + return []; + } + + // Extract raw step data objects + const rawSteps: any[] = []; + for (const item of logsOrSteps) { + if (!item) continue; + if (item.type === 'step_updated' && item.data) { + rawSteps.push(item.data); + } else if (item.type === 'step' && item.data) { + rawSteps.push(item.data); + } else if (item.step_number !== undefined || item.step_id !== undefined) { + rawSteps.push(item); + } + } + + // Deduplicate by step_id or step_number, keeping latest + const stepMap = new Map(); + for (const step of rawSteps) { + const key = step.step_id || step.step_number; + if (key !== undefined) { + stepMap.set(key, step); + } + } + + const steps = Array.from(stepMap.values()).sort((a, b) => { + const numA = Number(a.step_number ?? 0); + const numB = Number(b.step_number ?? 0); + if (numA !== numB) return numA - numB; + return Number(a.timestamp ?? 0) - Number(b.timestamp ?? 0); + }); + + const frames: StepReplayFrame[] = []; + let lastImageUrl: string | null = null; + + for (const stepData of steps) { + const stepNum = Number(stepData.step_number ?? (frames.length + 1)); + const act = stepData.action_taken; + const title = act ? (getActionTitle(act) || 'Action') : 'Step'; + const coords = act ? getActionCoords(act) : ''; + const actionDesc = coords ? `${title} (${coords})` : title; + + const preUrl = getStepPreImageUrl(stepData, act); + const postUrl = getStepPostImageUrl(stepData, act); + + // If pre-image exists and is new, add it + if (preUrl && preUrl !== lastImageUrl) { + frames.push({ + index: frames.length, + stepNumber: stepNum, + title: `Step ${stepNum} (Before Action)`, + imageUrl: preUrl, + actionText: `Preparing: ${actionDesc}`, + isPost: false, + timestamp: stepData.timestamp + }); + lastImageUrl = preUrl; + } + + // If post-image exists and is different from current screen, add it + if (postUrl && postUrl !== lastImageUrl) { + frames.push({ + index: frames.length, + stepNumber: stepNum, + title: `Step ${stepNum}: ${actionDesc}`, + imageUrl: postUrl, + actionText: actionDesc, + isPost: true, + timestamp: stepData.timestamp + }); + lastImageUrl = postUrl; + } + } + + return frames; +} + diff --git a/artemis/agents/flash/flash_runner.md b/artemis/agents/flash/flash_runner.md index 0618206..51236ba 100644 --- a/artemis/agents/flash/flash_runner.md +++ b/artemis/agents/flash/flash_runner.md @@ -1,5 +1,5 @@ # ROLE & OBJECTIVE -You are **ARTEMIS Flash Runner**, an autonomous and highly efficient Android Device Execution Agent. Your goal is to accomplish the user's objective on the device. +You are an autonomous and highly efficient Android Device Execution Agent. Your goal is to accomplish the user's objective on the device. **Objective: {{ goal }}** diff --git a/artemis/controllers/unified_controller.py b/artemis/controllers/unified_controller.py index 11c75c4..f4b8ba4 100644 --- a/artemis/controllers/unified_controller.py +++ b/artemis/controllers/unified_controller.py @@ -697,7 +697,10 @@ class UnifiedMobileController: session.is_active = False if session.watchdog_task and not session.watchdog_task.done(): session.watchdog_task.cancel() - await session.watchdog_task + try: + await session.watchdog_task + except (asyncio.CancelledError, Exception): + pass try: process = session.process @@ -708,7 +711,14 @@ class UnifiedMobileController: logger.warning(f"Error terminating scrcpy process: {proc_e}") output_path = session.local_video_path - if not output_path or not output_path.exists(): + has_existing_recording = ( + (output_path and output_path.exists()) + or any( + Path(r.get("output_path", "")).exists() or Path(r.get("path", "")).exists() + for r in session.android_segment_records + ) + ) + if not has_existing_recording: message = "Recording file not found on disk" self._record_recording_failure(session, message) remove_active_session(device_id) @@ -726,31 +736,37 @@ class UnifiedMobileController: self._remux_segment_record(record) ) ) - conversion_results = [] if session.android_conversion_tasks: - conversion_results = await asyncio.gather( + await asyncio.gather( *session.android_conversion_tasks, return_exceptions=True ) mp4_paths = [ Path(record["output_path"]) for record in session.android_segment_records - if Path(record["output_path"]).exists() + if Path(record["output_path"]).exists() and Path(record["output_path"]).stat().st_size > 0 ] - manifest_path = await write_recording_manifest(output_path.parent, mp4_paths) - conversion_failed = ( - len(mp4_paths) != len(session.android_segment_records) - or any(result is not True for result in conversion_results) - or manifest_path is None - ) - if conversion_failed: + # Emergency fallback: if no segment MP4 was produced, attempt to remux local_video_path directly + if not mp4_paths and output_path and output_path.exists() and output_path.stat().st_size > 0: + fallback_mp4 = ( + output_path.parent / "recording.mp4" + if output_path.name != "recording.mp4" + else output_path.with_name("recording_converted.mp4") + ) + if await remux_recording_to_mp4(output_path, fallback_mp4): + mp4_paths.append(fallback_mp4) + + if not mp4_paths: message = "Recording finalization failed; no complete browser-safe video was produced" self._record_recording_failure(session, message) remove_active_session(device_id) return VideoRecordingResult(success=False, message=message) - remove_active_session(device_id) final_video_path = mp4_paths[0] + output_dir = final_video_path.parent + manifest_path = await write_recording_manifest(output_dir, mp4_paths) + + remove_active_session(device_id) # Persist update to local database if Data Engine is active if self.ctx and self.ctx.data_engine: @@ -764,12 +780,12 @@ class UnifiedMobileController: return VideoRecordingResult( success=True, - message=f"Recording stopped, saved {len(mp4_paths)} orientation-safe segments", - video_path=mp4_paths[0], + message=f"Recording stopped, saved {len(mp4_paths)} video segments", + video_path=final_video_path, video_id=session.video_id, generation=session.generation, sealed_until=session.sealed_until, - source_revision=f"{session.video_id}:{session.generation}:sealed", + source_revision=f"{session.video_id}:{session.generation}:ready", ) except Exception as e: diff --git a/artemis/interfaces/cli/commands/run.py b/artemis/interfaces/cli/commands/run.py index 2faf1d5..fae19a2 100644 --- a/artemis/interfaces/cli/commands/run.py +++ b/artemis/interfaces/cli/commands/run.py @@ -29,6 +29,7 @@ from artemis.sdk.types.task import AgentProfile from artemis.utils.cli_helpers import display_device_status from artemis.utils.logger import get_logger from artemis.utils.video import check_ffmpeg_available +import signal from rich.console import Console from rich.panel import Panel import typer @@ -292,6 +293,15 @@ def run_command( display_device_status(console, adb_client=adb_client) cancelled = False + original_sigterm = None + try: + if hasattr(signal, "SIGTERM"): + original_sigterm = signal.signal( + signal.SIGTERM, lambda s, f: (_ for _ in ()).throw(KeyboardInterrupt()) + ) + except Exception: + pass + try: asyncio.run( execute_task( @@ -335,5 +345,10 @@ def run_command( console.print(f"[bold red]Task execution failed:[/bold red] {e}") raise finally: + try: + if original_sigterm is not None and hasattr(signal, "SIGTERM"): + signal.signal(signal.SIGTERM, original_sigterm) + except Exception: + pass if cancelled: raise SystemExit(130) diff --git a/artemis/platform/darwin.py b/artemis/platform/darwin.py index 7bb866f..f4e870b 100644 --- a/artemis/platform/darwin.py +++ b/artemis/platform/darwin.py @@ -19,6 +19,7 @@ from pathlib import Path import shutil import signal import tempfile +import time from artemis.platform.base import IPlatform, IPlatformPaths, IPlatformProcess, OSType @@ -98,12 +99,47 @@ class DarwinPlatformProcess(IPlatformProcess): pass parent.send_signal(signal.SIGTERM) - gone, alive = psutil.wait_procs(children + [parent], timeout=timeout_seconds) - for p in alive: - try: - p.kill() - except (psutil.NoSuchProcess, psutil.AccessDenied): - pass + + # Avoid calling wait_procs on direct children of the current process. + # In POSIX, psutil.Process.wait() on a child of os.getpid() invokes + # os.waitpid(), which reaps the zombie and prevents asyncio / uvloop + # event loops from detecting that the child exited, causing proc.wait() + # to hang indefinitely. + current_pid = os.getpid() + external_procs = [ + p + for p in children + [parent] + if p.pid != current_pid and getattr(p, "ppid", lambda: None)() != current_pid + ] + direct_children = [ + p + for p in children + [parent] + if getattr(p, "ppid", lambda: None)() == current_pid + ] + + if external_procs: + gone, alive = psutil.wait_procs(external_procs, timeout=timeout_seconds) + for p in alive: + try: + p.kill() + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + + deadline = time.time() + timeout_seconds + for p in direct_children: + while time.time() < deadline: + try: + if not p.is_running() or p.status() == psutil.STATUS_ZOMBIE: + break + except (psutil.NoSuchProcess, psutil.AccessDenied): + break + time.sleep(0.05) + else: + try: + p.kill() + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + return True except Exception: try: diff --git a/artemis/platform/linux.py b/artemis/platform/linux.py index 1d47cfc..e29e7b3 100644 --- a/artemis/platform/linux.py +++ b/artemis/platform/linux.py @@ -19,6 +19,7 @@ from pathlib import Path import shutil import signal import tempfile +import time from artemis.platform.base import IPlatform, IPlatformPaths, IPlatformProcess, OSType @@ -113,12 +114,47 @@ class LinuxPlatformProcess(IPlatformProcess): pass parent.send_signal(signal.SIGTERM) - gone, alive = psutil.wait_procs(children + [parent], timeout=timeout_seconds) - for p in alive: - try: - p.kill() - except (psutil.NoSuchProcess, psutil.AccessDenied): - pass + + # Avoid calling wait_procs on direct children of the current process. + # In POSIX, psutil.Process.wait() on a child of os.getpid() invokes + # os.waitpid(), which reaps the zombie and prevents asyncio / uvloop + # event loops from detecting that the child exited, causing proc.wait() + # to hang indefinitely. + current_pid = os.getpid() + external_procs = [ + p + for p in children + [parent] + if p.pid != current_pid and getattr(p, "ppid", lambda: None)() != current_pid + ] + direct_children = [ + p + for p in children + [parent] + if getattr(p, "ppid", lambda: None)() == current_pid + ] + + if external_procs: + gone, alive = psutil.wait_procs(external_procs, timeout=timeout_seconds) + for p in alive: + try: + p.kill() + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + + deadline = time.time() + timeout_seconds + for p in direct_children: + while time.time() < deadline: + try: + if not p.is_running() or p.status() == psutil.STATUS_ZOMBIE: + break + except (psutil.NoSuchProcess, psutil.AccessDenied): + break + time.sleep(0.05) + else: + try: + p.kill() + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + return True except Exception: try: diff --git a/artemis/sdk/agent.py b/artemis/sdk/agent.py index 4bc0575..ab2a36c 100644 --- a/artemis/sdk/agent.py +++ b/artemis/sdk/agent.py @@ -645,23 +645,34 @@ class Agent: return output finally: if recording_started: - try: - logger.info(f"[{task_name}] Stopping automated screen recording...") + async def _safe_stop_recording(): + try: + logger.info(f"[{task_name}] Stopping automated screen recording...") + controller = get_controller(context) + return await controller.stop_video_recording() + except Exception as e: + logger.error(f"[{task_name}] Error stopping screen recording: {e}") + return None - controller = get_controller(context) - stop_result = await controller.stop_video_recording() - if stop_result.success and stop_result.video_path: - logger.info( - f"[{task_name}] Screen recording saved" - f" to: {stop_result.video_path}" - ) - else: - logger.warning( - f"[{task_name}] Failed to save screen" - f" recording: {stop_result.message}" - ) - except Exception as e: - logger.error(f"[{task_name}] Error stopping screen recording: {e}") + stop_task = asyncio.create_task(_safe_stop_recording()) + try: + stop_result = await asyncio.shield(stop_task) + except (asyncio.CancelledError, BaseException): + try: + stop_result = await asyncio.wait_for(stop_task, timeout=15.0) + except Exception: + stop_result = None + + if stop_result and stop_result.success and stop_result.video_path: + logger.info( + f"[{task_name}] Screen recording saved" + f" to: {stop_result.video_path}" + ) + elif stop_result: + logger.warning( + f"[{task_name}] Failed to save screen" + f" recording: {stop_result.message}" + ) except asyncio.CancelledError: err = f"[{task_name}] Task cancelled" logger.warning(err) diff --git a/artemis/utils/video.py b/artemis/utils/video.py index aae9b71..484284a 100644 --- a/artemis/utils/video.py +++ b/artemis/utils/video.py @@ -297,6 +297,36 @@ async def remux_recording_to_mp4(source_path: Path, output_path: Path) -> bool: if valid: temporary_path.replace(output_path) return True + + logger.warning( + f"Direct remux failed for {source_path} (code {process.returncode}), " + "attempting transcode fallback with ffmpeg..." + ) + # Fallback to ultrafast re-encoding in case MKV container was truncated or has timestamp irregularities + fallback_proc = await asyncio.create_subprocess_exec( + get_ffmpeg_path(), + "-y", + "-fflags", + "+genpts+discardcorrupt", + "-i", + str(source_path), + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + str(temporary_path), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + _fout, _ferr = await fallback_proc.communicate() + if fallback_proc.returncode == 0 and temporary_path.exists() and temporary_path.stat().st_size > 0: + temporary_path.replace(output_path) + return True + logger.error( f"Failed to remux recording segment (code {process.returncode}): " f"{stderr.decode(errors='replace')[-2000:]}" @@ -314,10 +344,8 @@ async def probe_video_segment(video_path: Path) -> dict[str, float | int]: get_ffprobe_path(), "-v", "error", - "-select_streams", - "v:0", "-show_entries", - "stream=width,height:format=duration", + "stream=width,height,duration,codec_type:format=duration", "-of", "json", str(video_path), @@ -329,11 +357,20 @@ async def probe_video_segment(video_path: Path) -> dict[str, float | int]: return {} try: payload = json.loads(stdout) - stream = (payload.get("streams") or [{}])[0] + streams = payload.get("streams") or [] + video_stream = next( + (s for s in streams if s.get("width") and s.get("height")), + next((s for s in streams if s.get("codec_type") == "video"), streams[0] if streams else {}) + ) + duration = float( + (payload.get("format") or {}).get("duration") + or video_stream.get("duration") + or 0 + ) return { - "duration": float((payload.get("format") or {}).get("duration") or 0), - "width": int(stream.get("width") or 0), - "height": int(stream.get("height") or 0), + "duration": duration, + "width": int(video_stream.get("width") or 0), + "height": int(video_stream.get("height") or 0), } except (TypeError, ValueError, json.JSONDecodeError): return {} @@ -380,8 +417,8 @@ async def write_recording_manifest(output_dir: Path, mp4_paths: list[Path]) -> P width = int(metadata.get("width", 0)) height = int(metadata.get("height", 0)) if duration <= 0 or width <= 0 or height <= 0: - logger.error(f"Recording segment failed validation: {path}") - return None + logger.warning(f"Recording segment skipped due to invalid metadata: {path}") + continue segments.append( { "file": path.name, diff --git a/tests/unit/admin_console/test_task_queue_service.py b/tests/unit/admin_console/test_task_queue_service.py index 0b3b145..f8233df 100644 --- a/tests/unit/admin_console/test_task_queue_service.py +++ b/tests/unit/admin_console/test_task_queue_service.py @@ -553,3 +553,54 @@ async def test_immediate_cancel_ignores_stale_ipc_and_runs_next_task(): await task except (asyncio.CancelledError, Exception): pass + + +@pytest.mark.asyncio +async def test_wait_for_worker_process_watchdog_handles_reaped_process(): + """Verify that _wait_for_worker_process never hangs even if the process was already reaped.""" + mock_proc = MagicMock() + mock_proc.pid = 999999 + mock_proc.returncode = None + + async def hanging_wait(): + await asyncio.sleep(10) + return 0 + + mock_proc.wait = AsyncMock(side_effect=hanging_wait) + + with patch("psutil.Process") as mock_psutil_proc: + mock_p = MagicMock() + mock_p.is_running.return_value = False + mock_psutil_proc.return_value = mock_p + + rc = await TaskQueueService._wait_for_worker_process(mock_proc) + assert rc == -15 + + +def test_darwin_terminate_process_tree_preserves_direct_child_for_asyncio(): + """Verify darwin terminate_process_tree does not pass direct children to psutil.wait_procs.""" + import os + from artemis.platform.darwin import DarwinPlatformProcess + + process = DarwinPlatformProcess() + current_pid = os.getpid() + + with ( + patch("psutil.Process") as mock_psutil_proc, + patch("psutil.wait_procs") as mock_wait_procs, + ): + parent = MagicMock() + parent.pid = 12345 + parent.ppid.return_value = current_pid + parent.children.return_value = [] + parent.is_running.return_value = False + + mock_psutil_proc.return_value = parent + mock_wait_procs.return_value = ([], []) + + success = process.terminate_process_tree(12345, timeout_seconds=0.1) + assert success is True + parent.send_signal.assert_called_once() + for call_args in mock_wait_procs.call_args_list: + procs_waited = call_args[0][0] + assert parent not in procs_waited