chenmingxuan/Artemis-Eapil
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 32 Wiki Activity

feat: support orientation-aware recording and device wakefulness

Browse Source 
Roll scrcpy recordings by device rotation and duration, and publish timeline manifests for playback and analysis.

Add seamless segmented playback in the showcase UI and configurable stay-awake strategies for Android automation.
This commit is contained in:
somew1nd committed 2026-08-22 19:45:02 -07:00
1 parent 160f43a74a
commit 474e56ef8e
14 files changed
+1150 -310
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

artemis/clients/ui_automator_client.py
+170
View File
@@ -21,8 +21,10 @@ Handles Maestro blocker detection and removal before connecting.
import base64
from io import BytesIO
import os
import re as _re
import subprocess
import threading
import time
from typing import TYPE_CHECKING
import xml.etree.ElementTree as ET
@@ -40,6 +42,129 @@ logger = get_logger(__name__)
MAESTRO_PACKAGE = "dev.mobile.maestro"
AWAKE_STRATEGY_STAY_ON = "stay_on"
AWAKE_STRATEGY_WAKE_LOCK = "wake_lock"
AWAKE_STRATEGY_HEARTBEAT = "heartbeat"
AWAKE_HEARTBEAT_INTERVAL_SECONDS = 5.0
def _run_awake_adb_command(
device_id: str, args: list[str], description: str
) -> subprocess.CompletedProcess[str] | None:
"""Run one non-fatal ADB command used by the awake strategy."""
try:
result = subprocess.run(
["adb", "-s", device_id, *args],
capture_output=True,
text=True,
timeout=10,
check=False,
)
if result.returncode != 0:
detail = (result.stderr or result.stdout).strip()
logger.warning(
f"Could not {description} on {device_id}: {detail or 'ADB command failed'}"
)
return result
except Exception as e:
logger.warning(f"Could not {description} on {device_id}: {e}")
return None
def _keep_device_awake(device_id: str) -> str | None:
"""Best-effort preparation of an Android device for UI automation.
``svc power stayon true`` asks Android to keep the display on while the
device has any external power source. It avoids changing the user's
normal battery-powered screen timeout. The wake and dismiss commands
also recover a device that went to sleep before Artemis connected;
``wm dismiss-keyguard`` cannot bypass a secure PIN/password keyguard.
The primary strategy is verified through PowerManager. If it is not
actually active, Artemis tries a shell-owned bright-screen wake lock. If
that command is unavailable, the caller starts a periodic KEYCODE_WAKEUP
heartbeat. Neither fallback changes the user's screen-timeout setting.
This must never make device connection fail. Some vendor Android builds
restrict one or more shell commands, so failures are logged only.
"""
enabled = os.environ.get("ARTEMIS_KEEP_DEVICE_AWAKE", "true").strip().lower()
if enabled in {"0", "false", "no", "off"}:
return None
commands = (
("enable stay-awake while powered", ["shell", "svc", "power", "stayon", "true"]),
("wake display", ["shell", "input", "keyevent", "KEYCODE_WAKEUP"]),
("dismiss non-secure keyguard", ["shell", "wm", "dismiss-keyguard"]),
)
for description, args in commands:
_run_awake_adb_command(device_id, args, description)
power_state = _run_awake_adb_command(
device_id, ["shell", "dumpsys", "power"], "verify stay-awake state"
)
if (
power_state is not None
and power_state.returncode == 0
and _re.search(r"\bmStayOn=true\b", power_state.stdout)
):
logger.info(f"Stay-awake primary strategy verified on {device_id}")
return AWAKE_STRATEGY_STAY_ON
logger.warning(
f"Stay-awake primary strategy is not active on {device_id}; "
"trying a screen-bright wake lock"
)
_run_awake_adb_command(
device_id,
[
"shell",
"cmd",
"power",
"set-wakelock",
"acquire",
"-d",
"0",
"SCREEN_BRIGHT_WAKE_LOCK",
],
"acquire a screen-bright wake lock",
)
wake_locks = _run_awake_adb_command(
device_id,
["shell", "cmd", "power", "set-wakelock", "list"],
"verify the screen-bright wake lock",
)
if (
wake_locks is not None
and wake_locks.returncode == 0
and _re.search(r"SCREEN_BRIGHT_WAKE_LOCK:.*\bheld=true\b", wake_locks.stdout)
):
logger.info(f"Screen-bright wake lock verified on {device_id}")
return AWAKE_STRATEGY_WAKE_LOCK
logger.warning(
f"Screen-bright wake lock is unavailable on {device_id}; "
f"using a {AWAKE_HEARTBEAT_INTERVAL_SECONDS:g}-second wakeup heartbeat"
)
return AWAKE_STRATEGY_HEARTBEAT
def _release_screen_wake_lock(device_id: str) -> None:
"""Release the shell-owned screen wake lock acquired for this session."""
_run_awake_adb_command(
device_id,
[
"shell",
"cmd",
"power",
"set-wakelock",
"release",
"-d",
"0",
"SCREEN_BRIGHT_WAKE_LOCK",
],
"release the screen-bright wake lock",
)
class _CyFunctionDetectorMeta(type):
@@ -257,6 +382,41 @@ class UIAutomatorClient:
"""
self._device_id = device_id
self._device: Device | None = None
self._awake_strategy: str | None = None
self._heartbeat_stop_event: threading.Event | None = None
self._heartbeat_thread: threading.Thread | None = None
def _start_awake_heartbeat(self) -> None:
"""Start a daemon heartbeat that safely refreshes Android user activity."""
if self._heartbeat_thread is not None and self._heartbeat_thread.is_alive():
return
stop_event = threading.Event()
self._heartbeat_stop_event = stop_event
def heartbeat() -> None:
while not stop_event.wait(AWAKE_HEARTBEAT_INTERVAL_SECONDS):
_run_awake_adb_command(
self._device_id,
["shell", "input", "keyevent", "KEYCODE_WAKEUP"],
"send the stay-awake heartbeat",
)
self._heartbeat_thread = threading.Thread(
target=heartbeat,
name=f"artemis-awake-{self._device_id}",
daemon=True,
)
self._heartbeat_thread.start()
def _stop_awake_heartbeat(self) -> None:
"""Stop the session heartbeat without delaying shutdown."""
if self._heartbeat_stop_event is not None:
self._heartbeat_stop_event.set()
if self._heartbeat_thread is not None:
self._heartbeat_thread.join(timeout=2.0)
self._heartbeat_stop_event = None
self._heartbeat_thread = None
def _ensure_connected(self) -> "Device":
"""Ensure connection to the device, handling Maestro blocker.
@@ -276,6 +436,12 @@ class UIAutomatorClient:
# Ensure Maestro is not blocking us
_ensure_maestro_not_installed(self._device_id)
# Configure this automation session before UIAutomator2 reads the screen.
if self._awake_strategy is None:
self._awake_strategy = _keep_device_awake(self._device_id)
if self._awake_strategy == AWAKE_STRATEGY_HEARTBEAT:
self._start_awake_heartbeat()
# Connect to device
logger.info(f"Connecting UIAutomator2 to device: {self._device_id}")
self._device = u2.connect(self._device_id)
@@ -400,6 +566,10 @@ class UIAutomatorClient:
def disconnect(self) -> None:
"""Disconnect from the device."""
self._stop_awake_heartbeat()
if self._awake_strategy == AWAKE_STRATEGY_WAKE_LOCK:
_release_screen_wake_lock(self._device_id)
self._awake_strategy = None
self._device = None
logger.info("UIAutomator2 client disconnected")
artemis/controllers/android_controller.py
+18 -42
View File
@@ -45,10 +45,12 @@ from artemis.utils.video import (
DEFAULT_MAX_DURATION_SECONDS,
RecordingSession,
VideoRecordingResult,
build_scrcpy_record_command,
cleanup_video_segments,
concatenate_videos,
get_active_session,
has_active_session,
normalize_recording_to_mp4,
remove_active_session,
set_active_session,
trim_video,
@@ -561,6 +563,8 @@ class AndroidDeviceController(MobileDeviceController):
start_time=time.time(),
data_engine_start_time=self.data_engine_start_time,
local_video_path=local_video_path,
capture_width=self.device_width,
capture_height=self.device_height,
)
# Persist to local database if Data Engine is active
@@ -578,18 +582,9 @@ class AndroidDeviceController(MobileDeviceController):
logger.error(f"Failed to persist video recording start to DB: {db_err}")
# Start scrcpy in background
cmd = [
"scrcpy",
"--serial",
self.device_id,
"--no-window",
"--record",
str(local_video_path),
"--record-format",
"mkv",
"--video-bit-rate",
"2M",
]
cmd = build_scrcpy_record_command(
"scrcpy", self.device_id, local_video_path, lock_capture_orientation=False
)
process = await asyncio.create_subprocess_exec(
*cmd,
@@ -760,18 +755,9 @@ class AndroidDeviceController(MobileDeviceController):
output_dir = mkv_path.parent
new_video_path = output_dir / f"recording_{session.android_segment_index}.mkv"
cmd = [
"scrcpy",
"--serial",
self.device_id,
"--no-window",
"--record",
str(new_video_path),
"--record-format",
"mkv",
"--video-bit-rate",
"2M",
]
cmd = build_scrcpy_record_command(
"scrcpy", self.device_id, new_video_path, lock_capture_orientation=False
)
process = await asyncio.create_subprocess_exec(
*cmd,
@@ -892,21 +878,11 @@ class AndroidDeviceController(MobileDeviceController):
)
async def _convert_mkv_to_mp4(self, mkv_path: Path, mp4_path: Path) -> bool:
"""Convert MKV to MP4 using ffmpeg."""
try:
process = await asyncio.create_subprocess_exec(
"ffmpeg",
"-y",
"-i",
str(mkv_path),
"-c",
"copy",
str(mp4_path),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
await process.wait()
return mp4_path.exists()
except Exception as e:
logger.error(f"Failed to convert MKV to MP4: {e}")
return False
"""Normalize MKV into a fixed-size, browser-safe MP4."""
session = get_active_session(self.device_id)
return await normalize_recording_to_mp4(
mkv_path,
mp4_path,
session.capture_width if session else self.device_width,
session.capture_height if session else self.device_height,
)
artemis/controllers/unified_controller.py
+176 -219
View File
@@ -15,6 +15,8 @@
import asyncio
import os
from pathlib import Path
import signal
import subprocess
import tempfile
import time
from typing import Any
@@ -33,17 +35,23 @@ from artemis.controllers.types import (
)
from artemis.utils.logger import get_logger
from artemis.utils.video import (
ANDROID_RECORDING_SEGMENT_SECONDS,
DEFAULT_MAX_DURATION_SECONDS,
RecordingSession,
VideoRecordingResult,
build_scrcpy_record_command,
cleanup_video_segments,
concatenate_videos,
get_android_display_state,
get_active_session,
get_ffmpeg_path,
has_active_session,
normalize_recording_to_mp4,
remux_recording_to_mp4,
render_timeline_clip,
remove_active_session,
set_active_session,
trim_video,
write_recording_manifest,
)
logger = get_logger(__name__)
@@ -63,6 +71,29 @@ class UnifiedMobileController:
def controller(self) -> Any:
return self._driver
@staticmethod
async def _spawn_scrcpy(command: list[str]) -> asyncio.subprocess.Process:
kwargs: dict[str, Any] = {
"stdout": asyncio.subprocess.PIPE,
"stderr": asyncio.subprocess.PIPE,
}
if os.name == "nt":
kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP
return await asyncio.create_subprocess_exec(*command, **kwargs)
@staticmethod
async def _stop_scrcpy(process: asyncio.subprocess.Process) -> None:
"""Ask scrcpy to flush its recorder before falling back to termination."""
if process.returncode is not None:
return
try:
process.send_signal(signal.CTRL_BREAK_EVENT if os.name == "nt" else signal.SIGINT)
await asyncio.wait_for(process.wait(), timeout=8.0)
except (ProcessLookupError, TimeoutError):
if process.returncode is None:
process.terminate()
await asyncio.wait_for(process.wait(), timeout=5.0)
async def tap_at(
self,
x: int,
@@ -301,68 +332,6 @@ class UnifiedMobileController:
message="Recording file not found",
)
# Check if scrcpy process terminated and restart if needed (synchronous fallback)
crashed = False
restart_message = ""
if session.process and session.process.returncode is not None:
logger.warning(f"scrcpy process for {device_id} has terminated unexpectedly")
crashed = True
if mkv_path.exists() and mkv_path not in session.android_video_segments:
session.android_video_segments.append(mkv_path)
try:
session.android_segment_index += 1
output_dir = mkv_path.parent
new_video_path = output_dir / f"recording_{session.android_segment_index}.mkv"
cmd = [
"scrcpy",
"--serial",
device_id,
"--no-window",
"--record",
str(new_video_path),
"--record-format",
"mkv",
"--video-bit-rate",
"2M",
]
process = await asyncio.create_subprocess_exec(
*cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
await asyncio.sleep(0.8)
if process.returncode is None:
session.process = process
session.local_video_path = new_video_path
mkv_path = new_video_path
restart_message = "Successfully restarted scrcpy."
else:
stderr = await process.stderr.read()
restart_message = f"Failed to restart scrcpy: {stderr.decode()}"
except Exception as e:
restart_message = f"Failed to restart scrcpy: {e}"
# Determine source file to trim from (support multi-segment seamless trimming)
all_segments = [p for p in session.android_video_segments if p.exists()]
if mkv_path.exists() and mkv_path not in all_segments:
all_segments.append(mkv_path)
if not all_segments:
return VideoRecordingResult(
success=False,
message="No recording segments found on disk",
)
temp_combined_file = None
if len(all_segments) > 1:
temp_combined_file = mkv_path.parent / "temp_combined_active.mkv"
concat_ok = await concatenate_videos(all_segments, temp_combined_file)
trim_source = (
temp_combined_file if (concat_ok and temp_combined_file.exists()) else mkv_path
)
else:
trim_source = all_segments[0]
current_time = time.time()
video_duration = current_time - session.start_time
@@ -415,8 +384,18 @@ class UnifiedMobileController:
success = False
for attempt in range(3):
success = await trim_video(
trim_source,
timeline_segments = [dict(record) for record in session.android_segment_records]
active_start = session.android_segment_started_at or session.start_time
if mkv_path.exists():
timeline_segments.append(
{
"path": mkv_path,
"start": max(0.0, active_start - session.start_time),
"end": max(0.0, current_time - session.start_time),
}
)
success = await render_timeline_clip(
timeline_segments,
video_start_relative_time,
video_end_relative_time,
trim_output_path,
@@ -426,13 +405,6 @@ class UnifiedMobileController:
logger.warning(f"ffmpeg trim attempt {attempt + 1} failed, retrying in 0.5s...")
await asyncio.sleep(0.5)
# Cleanup temp combined MKV if created
if temp_combined_file and temp_combined_file.exists():
try:
temp_combined_file.unlink()
except Exception:
pass
if not success or not trim_output_path.exists():
return VideoRecordingResult(
success=False,
@@ -445,11 +417,6 @@ class UnifiedMobileController:
duration = video_end_relative_time - video_start_relative_time
message = f"Video segment retrieved for range {actual_start:.1f}s to {actual_end:.1f}s"
if crashed:
message += (
f". Warning: scrcpy crashed. Data loss may have occurred. {restart_message}"
)
res = VideoRecordingResult(
success=True,
message=message,
@@ -469,61 +436,98 @@ class UnifiedMobileController:
message=f"Failed to get video segment: {e}",
)
async def _recording_watchdog(self, device_id: str) -> None:
"""Background watchdog to automatically detect crashes and recover scrcpy recording."""
while True:
session = get_active_session(device_id)
if not session or not session.is_active:
break
proc = session.process
if proc is None:
break
@staticmethod
def _segment_mp4_path(source_path: Path, index: int) -> Path:
return source_path.parent / ("recording.mp4" if index == 0 else f"recording_{index:03d}.mp4")
@staticmethod
async def _remux_segment_record(record: dict[str, Any]) -> bool:
source_path = Path(record["path"])
output_path = Path(record["output_path"])
success = await remux_recording_to_mp4(source_path, output_path)
if success:
record["path"] = output_path
try:
ret = await proc.wait()
source_path.unlink()
except OSError:
pass
return success
def _finalize_current_segment(self, session: RecordingSession, end_time: float) -> None:
source_path = session.local_video_path
if not source_path or not source_path.exists():
return
if any(Path(record["path"]) == source_path for record in session.android_segment_records):
return
start_time = session.android_segment_started_at or session.start_time
output_path = self._segment_mp4_path(source_path, session.android_segment_index)
session.android_video_segments.append(source_path)
session.android_segment_records.append(
{
"path": source_path,
"output_path": output_path,
"start": max(0.0, start_time - session.start_time),
"end": max(0.0, end_time - session.start_time),
"rotation": session.android_rotation,
"conversion_scheduled": True,
}
)
session.android_conversion_tasks.append(
asyncio.create_task(self._remux_segment_record(session.android_segment_records[-1]))
)
async def _start_next_recording_segment(
self, session: RecordingSession, display_state: tuple[int, int, int] | None
) -> bool:
session.android_segment_index += 1
output_dir = session.local_video_path.parent
new_video_path = output_dir / f"recording_{session.android_segment_index:03d}.mkv"
process = await self._spawn_scrcpy(
build_scrcpy_record_command("scrcpy", session.device_id, new_video_path)
)
await asyncio.sleep(0.8)
if process.returncode is not None:
stderr = await process.stderr.read()
logger.error(f"Failed to start scrcpy segment: {stderr.decode(errors='replace')}")
return False
session.process = process
session.local_video_path = new_video_path
session.android_segment_started_at = time.time()
if display_state:
session.android_rotation, session.capture_width, session.capture_height = display_state
return True
async def _recording_watchdog(self, device_id: str) -> None:
"""Roll fixed-orientation segments and recover scrcpy crashes."""
try:
while True:
await asyncio.sleep(0.5)
session = get_active_session(device_id)
if not session or not session.is_active:
break
logger.warning(
f"scrcpy recording process for {device_id} terminated unexpectedly (code {ret}). "
f"Auto-restarting recording segment {session.android_segment_index + 1}..."
if not session or not session.is_active or not session.process:
return
now = time.time()
display_state = await get_android_display_state(device_id)
rotated = bool(
display_state
and session.android_rotation is not None
and display_state[0] != session.android_rotation
)
if session.local_video_path and session.local_video_path.exists():
if session.local_video_path not in session.android_video_segments:
session.android_video_segments.append(session.local_video_path)
segment_age = now - (session.android_segment_started_at or session.start_time)
crashed = session.process.returncode is not None
if not crashed and not rotated and segment_age < ANDROID_RECORDING_SEGMENT_SECONDS:
continue
session.android_segment_index += 1
output_dir = session.local_video_path.parent
new_video_path = output_dir / f"recording_{session.android_segment_index}.mkv"
cmd = [
"scrcpy",
"--serial",
device_id,
"--no-window",
"--record",
str(new_video_path),
"--record-format",
"mkv",
"--video-bit-rate",
"2M",
]
new_proc = await asyncio.create_subprocess_exec(
*cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
session.process = new_proc
session.local_video_path = new_video_path
await asyncio.sleep(0.8)
if new_proc.returncode is not None:
stderr = await new_proc.stderr.read()
logger.error(f"Failed to auto-restart scrcpy recording: {stderr.decode()}")
break
except asyncio.CancelledError:
break
except Exception as e:
logger.error(f"Error in recording watchdog for {device_id}: {e}")
break
if not crashed:
await self._stop_scrcpy(session.process)
self._finalize_current_segment(session, time.time())
reason = "rotation" if rotated else "time limit" if not crashed else "recorder crash"
logger.info(f"Rolling screen recording segment after {reason}")
if not await self._start_next_recording_segment(session, display_state):
return
except asyncio.CancelledError:
return
except Exception as e:
logger.error(f"Error in recording supervisor for {device_id}: {e}")
async def start_video_recording(
self,
@@ -561,6 +565,7 @@ class UnifiedMobileController:
local_video_path = output_dir / "recording.mkv"
video_id = uuid4()
start_time = time.time()
display_state = await get_android_display_state(device_id)
data_engine_start_time = (
self.ctx.data_engine.session_start_time
if (self.ctx and self.ctx.data_engine)
@@ -573,8 +578,14 @@ class UnifiedMobileController:
start_time=start_time,
data_engine_start_time=data_engine_start_time,
local_video_path=local_video_path,
capture_width=getattr(getattr(self.ctx, "device", None), "device_width", None),
capture_height=getattr(getattr(self.ctx, "device", None), "device_height", None),
android_rotation=display_state[0] if display_state else None,
android_segment_started_at=start_time,
is_active=True,
)
if display_state:
session.capture_width, session.capture_height = display_state[1:]
# Persist to local database if Data Engine is active
if self.ctx and self.ctx.data_engine:
@@ -586,24 +597,9 @@ class UnifiedMobileController:
)
# Start scrcpy in background
cmd = [
"scrcpy",
"--serial",
device_id,
"--no-window",
"--record",
str(local_video_path),
"--record-format",
"mkv",
"--video-bit-rate",
"2M",
]
cmd = build_scrcpy_record_command("scrcpy", device_id, local_video_path)
process = await asyncio.create_subprocess_exec(
*cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
process = await self._spawn_scrcpy(cmd)
session.process = process
set_active_session(device_id, session)
@@ -667,13 +663,13 @@ class UnifiedMobileController:
session.is_active = False
if session.watchdog_task and not session.watchdog_task.done():
session.watchdog_task.cancel()
await session.watchdog_task
try:
process = session.process
if process is not None:
try:
process.terminate()
await asyncio.wait_for(process.wait(), timeout=5.0)
await self._stop_scrcpy(process)
except Exception as proc_e:
logger.warning(f"Error terminating scrcpy process: {proc_e}")
@@ -685,27 +681,26 @@ class UnifiedMobileController:
message="Recording file not found on disk",
)
# Concatenate previous segments if scrcpy crashed and restarted
if session.android_video_segments:
all_segments = session.android_video_segments
if output_path.exists() and output_path not in all_segments:
all_segments.append(output_path)
concatenated_mkv = output_path.parent / "full_recording.mkv"
if await concatenate_videos(all_segments, concatenated_mkv):
cleanup_video_segments(all_segments)
output_path = concatenated_mkv
# Convert MKV to web-streamable MP4
mp4_path = (
output_path.parent / "recording.mp4"
if output_path.name == "full_recording.mkv"
else output_path.with_suffix(".mp4")
)
success = await self._convert_mkv_to_mp4(output_path, mp4_path)
self._finalize_current_segment(session, time.time())
for record in session.android_segment_records:
if not record.get("conversion_scheduled"):
record["conversion_scheduled"] = True
session.android_conversion_tasks.append(
asyncio.create_task(
self._remux_segment_record(record)
)
)
if session.android_conversion_tasks:
await asyncio.gather(*session.android_conversion_tasks, return_exceptions=True)
mp4_paths = [
Path(record["output_path"])
for record in session.android_segment_records
if Path(record["output_path"]).exists()
]
await write_recording_manifest(output_path.parent, mp4_paths)
remove_active_session(device_id)
final_video_path = mp4_path if (success and mp4_path.exists()) else output_path
final_video_path = mp4_paths[0] if mp4_paths else output_path
# Persist update to local database if Data Engine is active
if self.ctx and self.ctx.data_engine:
@@ -717,16 +712,11 @@ class UnifiedMobileController:
end_time=time.time(),
)
if success and mp4_path.exists():
try:
if output_path != mp4_path and output_path.exists():
output_path.unlink()
except Exception:
pass
if mp4_paths:
return VideoRecordingResult(
success=True,
message=f"Recording stopped, saved to {mp4_path}",
video_path=mp4_path,
message=f"Recording stopped, saved {len(mp4_paths)} orientation-safe segments",
video_path=mp4_paths[0],
)
else:
return VideoRecordingResult(
@@ -744,52 +734,19 @@ class UnifiedMobileController:
)
async def _convert_mkv_to_mp4(self, mkv_path: Path, mp4_path: Path) -> bool:
"""Convert MKV to MP4 using ffmpeg with faststart for web streaming."""
if not mkv_path.exists():
return False
try:
# 1. Try fast stream copy with +faststart
process = await asyncio.create_subprocess_exec(
get_ffmpeg_path(),
"-y",
"-i",
str(mkv_path),
"-c",
"copy",
"-movflags",
"+faststart",
str(mp4_path),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
await process.wait()
if mp4_path.exists() and mp4_path.stat().st_size > 0:
return True
"""Normalize MKV into a fixed-size, browser-safe MP4.
# 2. Fallback to re-encoding if stream copy fails
logger.warning("ffmpeg fast copy failed, re-encoding MKV to MP4...")
process = await asyncio.create_subprocess_exec(
get_ffmpeg_path(),
"-y",
"-i",
str(mkv_path),
"-c:v",
"libx264",
"-preset",
"fast",
"-pix_fmt",
"yuv420p",
"-movflags",
"+faststart",
str(mp4_path),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
await process.wait()
return mp4_path.exists() and mp4_path.stat().st_size > 0
except Exception as e:
logger.error(f"Failed to convert MKV to MP4: {e}")
return False
Stream-copying a scrcpy H.264 track is unsafe because historical or
recovered recordings may contain resolution changes. Re-encoding onto
the initial capture canvas guarantees one coded size for the full MP4.
"""
session = get_active_session(self._get_device_id())
return await normalize_recording_to_mp4(
mkv_path,
mp4_path,
session.capture_width if session else None,
session.capture_height if session else None,
)
async def cleanup(self) -> None:
await self._driver.disconnect()
artemis/drivers/android/adb_driver.py
+7 -10
View File
@@ -28,6 +28,7 @@ from artemis.clients.ui_automator_client import (
from artemis.config.paths import get_temp_dir
from artemis.drivers.base import BaseDeviceDriver, KeyCode, ScreenData, SwipeDirection
from artemis.toolchain import find_ffmpeg, find_scrcpy
from artemis.utils.video import build_scrcpy_record_command
from artemis.utils.ui_filter import filter_ui_hierarchy
from artemis.utils.logger import get_logger
@@ -114,6 +115,8 @@ class AndroidAdbDriver(BaseDeviceDriver):
"""Clean up ongoing recordings and tunnels."""
if self._recording_process:
await self.stop_video_recording()
if self._ui_adb_client and hasattr(self._ui_adb_client, "disconnect"):
await asyncio.to_thread(self._ui_adb_client.disconnect)
async def get_screen_data(self, skip_settling: bool = False) -> ScreenData:
"""Captures screenshot and XML hierarchy concurrently."""
@@ -442,18 +445,12 @@ class AndroidAdbDriver(BaseDeviceDriver):
logger.info(f"Starting scrcpy video recording to {self._recording_mkv_path}...")
try:
scrcpy_bin = find_scrcpy()
cmd = [
cmd = build_scrcpy_record_command(
scrcpy_bin,
"--serial",
self.device_id,
"--no-window",
"--record",
str(self._recording_mkv_path),
"--record-format",
"mkv",
"--video-bit-rate",
"2M",
]
self._recording_mkv_path,
lock_capture_orientation=False,
)
self._scrcpy_process = await asyncio.create_subprocess_exec(
*cmd,
stdout=asyncio.subprocess.PIPE,
artemis/utils/video.py
+331
View File
@@ -19,8 +19,10 @@ Provides shared types and utilities for video recording across platforms.
import asyncio
import importlib.util
import json
from pathlib import Path
import platform
import re
import shutil
import subprocess
from typing import Any
@@ -34,6 +36,7 @@ from artemis.utils.logger import get_logger
logger = get_logger(__name__)
DEFAULT_MAX_DURATION_SECONDS = 900 # 15 minutes
ANDROID_RECORDING_SEGMENT_SECONDS = 1800
VIDEO_READY_DELAY_SECONDS = 1
ANDROID_DEVICE_VIDEO_PATH = "/sdcard/screen_recording.mp4"
ANDROID_MAX_RECORDING_DURATION_SECONDS = 180 # Android screenrecord limit
@@ -44,6 +47,37 @@ MAX_VIDEO_SIZE_MB = 500
MAX_VIDEO_SIZE_BYTES = MAX_VIDEO_SIZE_MB * 1024 * 1024
def build_scrcpy_record_command(
scrcpy_executable: str,
device_id: str,
output_path: Path,
video_bit_rate: str = "2M",
lock_capture_orientation: bool = True,
) -> list[str]:
"""Build the shared scrcpy command used by all recording paths.
Each recording segment locks the orientation present when scrcpy starts.
The recording supervisor starts a new segment when Android rotates, so a
segment never contains multiple coded sizes while still displaying the app
in its natural orientation.
"""
command = [
scrcpy_executable,
"--serial",
device_id,
"--no-window",
"--record",
str(output_path),
"--record-format",
"mkv",
"--video-bit-rate",
video_bit_rate,
]
if lock_capture_orientation:
command.append("--capture-orientation=@")
return command
class _CyFunctionDetectorMeta(type):
def __instancecheck__(self, instance):
name = type(instance).__name__
@@ -73,11 +107,17 @@ class RecordingSession(BaseModel):
process: Any = None
data_engine_start_time: float | None = None
local_video_path: Path | None = None
capture_width: int | None = None
capture_height: int | None = None
android_device_path: str = ANDROID_DEVICE_VIDEO_PATH
android_video_segments: list[Path] = []
android_segment_index: int = 0
android_restart_task: asyncio.Task | None = None
watchdog_task: asyncio.Task | None = None
android_rotation: int | None = None
android_segment_started_at: float | None = None
android_segment_records: list[dict[str, Any]] = []
android_conversion_tasks: list[asyncio.Task] = []
is_active: bool = True
errors: list[str] = []
@@ -138,6 +178,297 @@ def get_ffmpeg_path() -> str:
return "ffmpeg"
def get_ffprobe_path() -> str:
"""Get ffprobe for lightweight segment metadata inspection."""
return shutil.which("ffprobe") or "ffprobe"
async def normalize_recording_to_mp4(
source_path: Path,
output_path: Path,
capture_width: int | None,
capture_height: int | None,
) -> bool:
"""Re-encode a recording onto one fixed canvas for browser playback."""
if not source_path.exists():
return False
width = max(2, int(capture_width or 1080)) // 2 * 2
height = max(2, int(capture_height or 1920)) // 2 * 2
video_filter = (
f"scale={width}:{height}:force_original_aspect_ratio=decrease:"
f"force_divisible_by=2,pad={width}:{height}:"
"(ow-iw)/2:(oh-ih)/2:color=black,setsar=1"
)
try:
process = await asyncio.create_subprocess_exec(
get_ffmpeg_path(),
"-y",
"-fflags",
"+genpts",
"-i",
str(source_path),
"-map",
"0:v:0",
"-map",
"0:a?",
"-vf",
video_filter,
"-r",
"30",
"-fps_mode",
"cfr",
"-c:v",
"libx264",
"-preset",
"veryfast",
"-crf",
"23",
"-pix_fmt",
"yuv420p",
"-c:a",
"aac",
"-b:a",
"128k",
"-movflags",
"+faststart",
str(output_path),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
_stdout, stderr = await process.communicate()
if process.returncode == 0 and output_path.exists() and output_path.stat().st_size > 0:
return True
error_detail = stderr.decode(errors="replace")[-4000:]
logger.error(
f"Failed to normalize recording to MP4 (code {process.returncode}): {error_detail}"
)
if output_path.exists():
output_path.unlink()
return False
except Exception as e:
logger.error(f"Failed to normalize recording to MP4: {e}")
if output_path.exists():
output_path.unlink()
return False
async def remux_recording_to_mp4(source_path: Path, output_path: Path) -> bool:
"""Move a fixed-dimension recording segment into MP4 without re-encoding."""
if not source_path.exists() or source_path.stat().st_size == 0:
return False
try:
process = await asyncio.create_subprocess_exec(
get_ffmpeg_path(),
"-y",
"-fflags",
"+genpts",
"-i",
str(source_path),
"-map",
"0:v:0",
"-map",
"0:a?",
"-c",
"copy",
"-movflags",
"+faststart",
str(output_path),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
_stdout, stderr = await process.communicate()
valid = process.returncode == 0 and output_path.exists() and output_path.stat().st_size > 0
if valid:
return True
logger.error(
f"Failed to remux recording segment (code {process.returncode}): "
f"{stderr.decode(errors='replace')[-2000:]}"
)
except Exception as e:
logger.error(f"Failed to remux recording segment: {e}")
if output_path.exists():
output_path.unlink()
return False
async def probe_video_segment(video_path: Path) -> dict[str, float | int]:
"""Read duration and coded dimensions for a finalized segment."""
process = await asyncio.create_subprocess_exec(
get_ffprobe_path(),
"-v",
"error",
"-select_streams",
"v:0",
"-show_entries",
"stream=width,height:format=duration",
"-of",
"json",
str(video_path),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
stdout, _stderr = await process.communicate()
if process.returncode != 0:
return {}
try:
payload = json.loads(stdout)
stream = (payload.get("streams") or [{}])[0]
return {
"duration": float((payload.get("format") or {}).get("duration") or 0),
"width": int(stream.get("width") or 0),
"height": int(stream.get("height") or 0),
}
except (TypeError, ValueError, json.JSONDecodeError):
return {}
async def get_android_display_state(device_id: str) -> tuple[int, int, int] | None:
"""Return Android's current (rotation, width, height)."""
try:
process = await asyncio.create_subprocess_exec(
"adb",
"-s",
device_id,
"shell",
"dumpsys",
"window",
"displays",
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.DEVNULL,
)
stdout, _stderr = await asyncio.wait_for(process.communicate(), timeout=3.0)
text = stdout.decode(errors="replace")
rotation_match = re.search(r"\bmRotation=(\d+)", text)
size_match = re.search(r"\bcur=(\d+)x(\d+)", text)
if rotation_match and size_match:
return (
int(rotation_match.group(1)),
int(size_match.group(1)),
int(size_match.group(2)),
)
except (OSError, TimeoutError):
pass
return None
async def write_recording_manifest(output_dir: Path, mp4_paths: list[Path]) -> Path | None:
"""Write the browser playlist used for orientation-aware playback."""
segments = []
timeline = 0.0
for path in mp4_paths:
if not path.exists():
continue
metadata = await probe_video_segment(path)
duration = float(metadata.get("duration", 0))
segments.append(
{
"file": path.name,
"start": round(timeline, 3),
"duration": round(duration, 3),
"width": int(metadata.get("width", 0)),
"height": int(metadata.get("height", 0)),
}
)
timeline += duration
if not segments:
return None
manifest_path = output_dir / "recording.json"
manifest_path.write_text(
json.dumps(
{"version": 1, "duration": round(timeline, 3), "segments": segments},
indent=2,
),
encoding="utf-8",
)
return manifest_path
async def render_timeline_clip(
segments: list[dict[str, Any]],
start_time: float,
end_time: float,
output_path: Path,
canvas_width: int = 720,
canvas_height: int = 1280,
) -> bool:
"""Render one continuous analyzer clip from orientation-aware segments.
Only the requested time window is decoded. This keeps the video analyzer's
single-MP4 contract without ever re-encoding the full recording.
"""
overlaps = []
for segment in segments:
path = Path(segment["path"])
seg_start = float(segment["start"])
seg_end = float(segment["end"])
overlap_start = max(start_time, seg_start)
overlap_end = min(end_time, seg_end)
if path.exists() and overlap_end > overlap_start:
overlaps.append((path, overlap_start - seg_start, overlap_end - seg_start))
if not overlaps:
return False
command = [get_ffmpeg_path(), "-y", "-fflags", "+genpts"]
for path, _start, _end in overlaps:
command.extend(["-i", str(path)])
filter_parts = []
labels = []
for index, (_path, local_start, local_end) in enumerate(overlaps):
label = f"v{index}"
filter_parts.append(
f"[{index}:v:0]trim=start={local_start:.6f}:end={local_end:.6f},"
"setpts=PTS-STARTPTS,"
f"scale={canvas_width}:{canvas_height}:force_original_aspect_ratio=decrease:"
"force_divisible_by=2,"
f"pad={canvas_width}:{canvas_height}:(ow-iw)/2:(oh-ih)/2:color=black,"
f"setsar=1,fps=15,format=yuv420p[{label}]"
)
labels.append(f"[{label}]")
if len(labels) == 1:
filter_parts.append(f"{labels[0]}null[outv]")
else:
filter_parts.append(f"{''.join(labels)}concat=n={len(labels)}:v=1:a=0[outv]")
command.extend(
[
"-filter_complex",
";".join(filter_parts),
"-map",
"[outv]",
"-c:v",
"libx264",
"-preset",
"veryfast",
"-crf",
"23",
"-movflags",
"+faststart",
str(output_path),
]
)
try:
process = await asyncio.create_subprocess_exec(
*command,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
_stdout, stderr = await process.communicate()
if process.returncode == 0 and output_path.exists() and output_path.stat().st_size > 0:
return True
logger.error(
f"Failed to render analyzer timeline clip (code {process.returncode}): "
f"{stderr.decode(errors='replace')[-3000:]}"
)
except Exception as e:
logger.error(f"Failed to render analyzer timeline clip: {e}")
if output_path.exists():
output_path.unlink()
return False
def is_scrcpy_installed() -> bool:
"""Check if scrcpy is available in the system PATH."""