From 474e56ef8e9faa0c544ec61c5f0dd4bab4e887aa Mon Sep 17 00:00:00 2001 From: somew1nd Date: Sat, 22 Aug 2026 19:43:52 -0700 Subject: [PATCH] feat: support orientation-aware recording and device wakefulness Roll scrcpy recordings by device rotation and duration, and publish timeline manifests for playback and analysis. Add seamless segmented playback in the showcase UI and configurable stay-awake strategies for Android automation. --- .env.example | 3 + apps/admin_console/routers/media.py | 2 + apps/admin_console/services/media_service.py | 30 ++ .../floating-video-player.component.html | 2 +- .../floating-video-player.component.scss | 3 +- .../floating-video-player.component.ts | 73 +++- .../src/app/services/agent.service.ts | 21 +- artemis/clients/ui_automator_client.py | 170 ++++++++ artemis/controllers/android_controller.py | 60 +-- artemis/controllers/unified_controller.py | 395 ++++++++---------- artemis/drivers/android/adb_driver.py | 17 +- artemis/utils/video.py | 331 +++++++++++++++ tests/unit/test_ui_automator_client_awake.py | 165 ++++++++ tests/unit/test_unified_controller_video.py | 188 ++++++++- 14 files changed, 1150 insertions(+), 310 deletions(-) create mode 100644 tests/unit/test_ui_automator_client_awake.py diff --git a/.env.example b/.env.example index 27a9fc5..2c6b072 100644 --- a/.env.example +++ b/.env.example @@ -19,6 +19,9 @@ OCR_API_KEY= ADB_HOST=127.0.0.1 ADB_PORT=5037 # ADB_DEVICE_SERIAL= +# Keep the display awake, falling back to a wake lock or safe heartbeat when needed. +# Set to false if Artemis is used with a personal device whose screen should time out. +ARTEMIS_KEEP_DEVICE_AWAKE=true # Artemis Execution Defaults (flash or pro) ARTEMIS_DEFAULT_PROFILE=flash diff --git a/apps/admin_console/routers/media.py b/apps/admin_console/routers/media.py index ae40caf..4603ed6 100644 --- a/apps/admin_console/routers/media.py +++ b/apps/admin_console/routers/media.py @@ -92,10 +92,12 @@ async def get_session_video(session_id: str): row = session_repo.get_session_by_id(session_id) row_dict = dict(row) if row else {"session_id": session_id} v_url = media_service.resolve_video_url(row_dict, video_rec_map, video_idx) + video_segments = media_service.resolve_video_segments(v_url) return { "session_id": session_id, "has_video": bool(v_url), "video_url": v_url, + "video_segments": video_segments, } diff --git a/apps/admin_console/services/media_service.py b/apps/admin_console/services/media_service.py index 2e566de..fda93d5 100644 --- a/apps/admin_console/services/media_service.py +++ b/apps/admin_console/services/media_service.py @@ -127,6 +127,36 @@ class MediaService: return v_url + @classmethod + def resolve_video_segments(cls, video_url: str | None) -> list[dict[str, Any]]: + """Resolve a recording sidecar manifest into browser-safe segment URLs.""" + if not video_url or not video_url.startswith("/videos/"): + return [] + relative = urllib.parse.unquote(video_url.removeprefix("/videos/")) + video_path = (WORKSPACE_ROOT / relative).resolve() + manifest_path = video_path.parent / "recording.json" + if not manifest_path.is_file(): + return [] + try: + payload = json.loads(manifest_path.read_text(encoding="utf-8")) + segments = [] + for item in payload.get("segments", []): + segment_path = (manifest_path.parent / str(item["file"])).resolve() + if segment_path.parent != manifest_path.parent.resolve() or not segment_path.is_file(): + continue + segments.append( + { + "url": cls.path_to_video_url(segment_path), + "start": float(item.get("start", 0)), + "duration": float(item.get("duration", 0)), + "width": int(item.get("width", 0)), + "height": int(item.get("height", 0)), + } + ) + return segments + except (OSError, KeyError, TypeError, ValueError, json.JSONDecodeError): + return [] + @staticmethod def unwrap_payload(obj: Any) -> Any: """Unwraps trace payloads, storing base64 images as disk files and diff --git a/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.html b/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.html index 2e883fc..4cadb08 100644 --- a/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.html +++ b/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.html @@ -134,7 +134,7 @@