chenmingxuan/Artemis-Eapil
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 32 Wiki Activity

feat: support orientation-aware recording and device wakefulness

Browse Source 
Roll scrcpy recordings by device rotation and duration, and publish timeline manifests for playback and analysis.

Add seamless segmented playback in the showcase UI and configurable stay-awake strategies for Android automation.
This commit is contained in:
somew1nd committed 2026-08-22 19:45:02 -07:00
1 parent 160f43a74a
commit 474e56ef8e
14 files changed
+1150 -310
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

artemis/controllers/unified_controller.py
+176 -219
View File
@@ -15,6 +15,8 @@
import asyncio
import os
from pathlib import Path
import signal
import subprocess
import tempfile
import time
from typing import Any
@@ -33,17 +35,23 @@ from artemis.controllers.types import (
)
from artemis.utils.logger import get_logger
from artemis.utils.video import (
ANDROID_RECORDING_SEGMENT_SECONDS,
DEFAULT_MAX_DURATION_SECONDS,
RecordingSession,
VideoRecordingResult,
build_scrcpy_record_command,
cleanup_video_segments,
concatenate_videos,
get_android_display_state,
get_active_session,
get_ffmpeg_path,
has_active_session,
normalize_recording_to_mp4,
remux_recording_to_mp4,
render_timeline_clip,
remove_active_session,
set_active_session,
trim_video,
write_recording_manifest,
)
logger = get_logger(__name__)
@@ -63,6 +71,29 @@ class UnifiedMobileController:
def controller(self) -> Any:
return self._driver
@staticmethod
async def _spawn_scrcpy(command: list[str]) -> asyncio.subprocess.Process:
kwargs: dict[str, Any] = {
"stdout": asyncio.subprocess.PIPE,
"stderr": asyncio.subprocess.PIPE,
}
if os.name == "nt":
kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP
return await asyncio.create_subprocess_exec(*command, **kwargs)
@staticmethod
async def _stop_scrcpy(process: asyncio.subprocess.Process) -> None:
"""Ask scrcpy to flush its recorder before falling back to termination."""
if process.returncode is not None:
return
try:
process.send_signal(signal.CTRL_BREAK_EVENT if os.name == "nt" else signal.SIGINT)
await asyncio.wait_for(process.wait(), timeout=8.0)
except (ProcessLookupError, TimeoutError):
if process.returncode is None:
process.terminate()
await asyncio.wait_for(process.wait(), timeout=5.0)
async def tap_at(
self,
x: int,
@@ -301,68 +332,6 @@ class UnifiedMobileController:
message="Recording file not found",
)
# Check if scrcpy process terminated and restart if needed (synchronous fallback)
crashed = False
restart_message = ""
if session.process and session.process.returncode is not None:
logger.warning(f"scrcpy process for {device_id} has terminated unexpectedly")
crashed = True
if mkv_path.exists() and mkv_path not in session.android_video_segments:
session.android_video_segments.append(mkv_path)
try:
session.android_segment_index += 1
output_dir = mkv_path.parent
new_video_path = output_dir / f"recording_{session.android_segment_index}.mkv"
cmd = [
"scrcpy",
"--serial",
device_id,
"--no-window",
"--record",
str(new_video_path),
"--record-format",
"mkv",
"--video-bit-rate",
"2M",
]
process = await asyncio.create_subprocess_exec(
*cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
await asyncio.sleep(0.8)
if process.returncode is None:
session.process = process
session.local_video_path = new_video_path
mkv_path = new_video_path
restart_message = "Successfully restarted scrcpy."
else:
stderr = await process.stderr.read()
restart_message = f"Failed to restart scrcpy: {stderr.decode()}"
except Exception as e:
restart_message = f"Failed to restart scrcpy: {e}"
# Determine source file to trim from (support multi-segment seamless trimming)
all_segments = [p for p in session.android_video_segments if p.exists()]
if mkv_path.exists() and mkv_path not in all_segments:
all_segments.append(mkv_path)
if not all_segments:
return VideoRecordingResult(
success=False,
message="No recording segments found on disk",
)
temp_combined_file = None
if len(all_segments) > 1:
temp_combined_file = mkv_path.parent / "temp_combined_active.mkv"
concat_ok = await concatenate_videos(all_segments, temp_combined_file)
trim_source = (
temp_combined_file if (concat_ok and temp_combined_file.exists()) else mkv_path
)
else:
trim_source = all_segments[0]
current_time = time.time()
video_duration = current_time - session.start_time
@@ -415,8 +384,18 @@ class UnifiedMobileController:
success = False
for attempt in range(3):
success = await trim_video(
trim_source,
timeline_segments = [dict(record) for record in session.android_segment_records]
active_start = session.android_segment_started_at or session.start_time
if mkv_path.exists():
timeline_segments.append(
{
"path": mkv_path,
"start": max(0.0, active_start - session.start_time),
"end": max(0.0, current_time - session.start_time),
}
)
success = await render_timeline_clip(
timeline_segments,
video_start_relative_time,
video_end_relative_time,
trim_output_path,
@@ -426,13 +405,6 @@ class UnifiedMobileController:
logger.warning(f"ffmpeg trim attempt {attempt + 1} failed, retrying in 0.5s...")
await asyncio.sleep(0.5)
# Cleanup temp combined MKV if created
if temp_combined_file and temp_combined_file.exists():
try:
temp_combined_file.unlink()
except Exception:
pass
if not success or not trim_output_path.exists():
return VideoRecordingResult(
success=False,
@@ -445,11 +417,6 @@ class UnifiedMobileController:
duration = video_end_relative_time - video_start_relative_time
message = f"Video segment retrieved for range {actual_start:.1f}s to {actual_end:.1f}s"
if crashed:
message += (
f". Warning: scrcpy crashed. Data loss may have occurred. {restart_message}"
)
res = VideoRecordingResult(
success=True,
message=message,
@@ -469,61 +436,98 @@ class UnifiedMobileController:
message=f"Failed to get video segment: {e}",
)
async def _recording_watchdog(self, device_id: str) -> None:
"""Background watchdog to automatically detect crashes and recover scrcpy recording."""
while True:
session = get_active_session(device_id)
if not session or not session.is_active:
break
proc = session.process
if proc is None:
break
@staticmethod
def _segment_mp4_path(source_path: Path, index: int) -> Path:
return source_path.parent / ("recording.mp4" if index == 0 else f"recording_{index:03d}.mp4")
@staticmethod
async def _remux_segment_record(record: dict[str, Any]) -> bool:
source_path = Path(record["path"])
output_path = Path(record["output_path"])
success = await remux_recording_to_mp4(source_path, output_path)
if success:
record["path"] = output_path
try:
ret = await proc.wait()
source_path.unlink()
except OSError:
pass
return success
def _finalize_current_segment(self, session: RecordingSession, end_time: float) -> None:
source_path = session.local_video_path
if not source_path or not source_path.exists():
return
if any(Path(record["path"]) == source_path for record in session.android_segment_records):
return
start_time = session.android_segment_started_at or session.start_time
output_path = self._segment_mp4_path(source_path, session.android_segment_index)
session.android_video_segments.append(source_path)
session.android_segment_records.append(
{
"path": source_path,
"output_path": output_path,
"start": max(0.0, start_time - session.start_time),
"end": max(0.0, end_time - session.start_time),
"rotation": session.android_rotation,
"conversion_scheduled": True,
}
)
session.android_conversion_tasks.append(
asyncio.create_task(self._remux_segment_record(session.android_segment_records[-1]))
)
async def _start_next_recording_segment(
self, session: RecordingSession, display_state: tuple[int, int, int] | None
) -> bool:
session.android_segment_index += 1
output_dir = session.local_video_path.parent
new_video_path = output_dir / f"recording_{session.android_segment_index:03d}.mkv"
process = await self._spawn_scrcpy(
build_scrcpy_record_command("scrcpy", session.device_id, new_video_path)
)
await asyncio.sleep(0.8)
if process.returncode is not None:
stderr = await process.stderr.read()
logger.error(f"Failed to start scrcpy segment: {stderr.decode(errors='replace')}")
return False
session.process = process
session.local_video_path = new_video_path
session.android_segment_started_at = time.time()
if display_state:
session.android_rotation, session.capture_width, session.capture_height = display_state
return True
async def _recording_watchdog(self, device_id: str) -> None:
"""Roll fixed-orientation segments and recover scrcpy crashes."""
try:
while True:
await asyncio.sleep(0.5)
session = get_active_session(device_id)
if not session or not session.is_active:
break
logger.warning(
f"scrcpy recording process for {device_id} terminated unexpectedly (code {ret}). "
f"Auto-restarting recording segment {session.android_segment_index + 1}..."
if not session or not session.is_active or not session.process:
return
now = time.time()
display_state = await get_android_display_state(device_id)
rotated = bool(
display_state
and session.android_rotation is not None
and display_state[0] != session.android_rotation
)
if session.local_video_path and session.local_video_path.exists():
if session.local_video_path not in session.android_video_segments:
session.android_video_segments.append(session.local_video_path)
segment_age = now - (session.android_segment_started_at or session.start_time)
crashed = session.process.returncode is not None
if not crashed and not rotated and segment_age < ANDROID_RECORDING_SEGMENT_SECONDS:
continue
session.android_segment_index += 1
output_dir = session.local_video_path.parent
new_video_path = output_dir / f"recording_{session.android_segment_index}.mkv"
cmd = [
"scrcpy",
"--serial",
device_id,
"--no-window",
"--record",
str(new_video_path),
"--record-format",
"mkv",
"--video-bit-rate",
"2M",
]
new_proc = await asyncio.create_subprocess_exec(
*cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
session.process = new_proc
session.local_video_path = new_video_path
await asyncio.sleep(0.8)
if new_proc.returncode is not None:
stderr = await new_proc.stderr.read()
logger.error(f"Failed to auto-restart scrcpy recording: {stderr.decode()}")
break
except asyncio.CancelledError:
break
except Exception as e:
logger.error(f"Error in recording watchdog for {device_id}: {e}")
break
if not crashed:
await self._stop_scrcpy(session.process)
self._finalize_current_segment(session, time.time())
reason = "rotation" if rotated else "time limit" if not crashed else "recorder crash"
logger.info(f"Rolling screen recording segment after {reason}")
if not await self._start_next_recording_segment(session, display_state):
return
except asyncio.CancelledError:
return
except Exception as e:
logger.error(f"Error in recording supervisor for {device_id}: {e}")
async def start_video_recording(
self,
@@ -561,6 +565,7 @@ class UnifiedMobileController:
local_video_path = output_dir / "recording.mkv"
video_id = uuid4()
start_time = time.time()
display_state = await get_android_display_state(device_id)
data_engine_start_time = (
self.ctx.data_engine.session_start_time
if (self.ctx and self.ctx.data_engine)
@@ -573,8 +578,14 @@ class UnifiedMobileController:
start_time=start_time,
data_engine_start_time=data_engine_start_time,
local_video_path=local_video_path,
capture_width=getattr(getattr(self.ctx, "device", None), "device_width", None),
capture_height=getattr(getattr(self.ctx, "device", None), "device_height", None),
android_rotation=display_state[0] if display_state else None,
android_segment_started_at=start_time,
is_active=True,
)
if display_state:
session.capture_width, session.capture_height = display_state[1:]
# Persist to local database if Data Engine is active
if self.ctx and self.ctx.data_engine:
@@ -586,24 +597,9 @@ class UnifiedMobileController:
)
# Start scrcpy in background
cmd = [
"scrcpy",
"--serial",
device_id,
"--no-window",
"--record",
str(local_video_path),
"--record-format",
"mkv",
"--video-bit-rate",
"2M",
]
cmd = build_scrcpy_record_command("scrcpy", device_id, local_video_path)
process = await asyncio.create_subprocess_exec(
*cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
process = await self._spawn_scrcpy(cmd)
session.process = process
set_active_session(device_id, session)
@@ -667,13 +663,13 @@ class UnifiedMobileController:
session.is_active = False
if session.watchdog_task and not session.watchdog_task.done():
session.watchdog_task.cancel()
await session.watchdog_task
try:
process = session.process
if process is not None:
try:
process.terminate()
await asyncio.wait_for(process.wait(), timeout=5.0)
await self._stop_scrcpy(process)
except Exception as proc_e:
logger.warning(f"Error terminating scrcpy process: {proc_e}")
@@ -685,27 +681,26 @@ class UnifiedMobileController:
message="Recording file not found on disk",
)
# Concatenate previous segments if scrcpy crashed and restarted
if session.android_video_segments:
all_segments = session.android_video_segments
if output_path.exists() and output_path not in all_segments:
all_segments.append(output_path)
concatenated_mkv = output_path.parent / "full_recording.mkv"
if await concatenate_videos(all_segments, concatenated_mkv):
cleanup_video_segments(all_segments)
output_path = concatenated_mkv
# Convert MKV to web-streamable MP4
mp4_path = (
output_path.parent / "recording.mp4"
if output_path.name == "full_recording.mkv"
else output_path.with_suffix(".mp4")
)
success = await self._convert_mkv_to_mp4(output_path, mp4_path)
self._finalize_current_segment(session, time.time())
for record in session.android_segment_records:
if not record.get("conversion_scheduled"):
record["conversion_scheduled"] = True
session.android_conversion_tasks.append(
asyncio.create_task(
self._remux_segment_record(record)
)
)
if session.android_conversion_tasks:
await asyncio.gather(*session.android_conversion_tasks, return_exceptions=True)
mp4_paths = [
Path(record["output_path"])
for record in session.android_segment_records
if Path(record["output_path"]).exists()
]
await write_recording_manifest(output_path.parent, mp4_paths)
remove_active_session(device_id)
final_video_path = mp4_path if (success and mp4_path.exists()) else output_path
final_video_path = mp4_paths[0] if mp4_paths else output_path
# Persist update to local database if Data Engine is active
if self.ctx and self.ctx.data_engine:
@@ -717,16 +712,11 @@ class UnifiedMobileController:
end_time=time.time(),
)
if success and mp4_path.exists():
try:
if output_path != mp4_path and output_path.exists():
output_path.unlink()
except Exception:
pass
if mp4_paths:
return VideoRecordingResult(
success=True,
message=f"Recording stopped, saved to {mp4_path}",
video_path=mp4_path,
message=f"Recording stopped, saved {len(mp4_paths)} orientation-safe segments",
video_path=mp4_paths[0],
)
else:
return VideoRecordingResult(
@@ -744,52 +734,19 @@ class UnifiedMobileController:
)
async def _convert_mkv_to_mp4(self, mkv_path: Path, mp4_path: Path) -> bool:
"""Convert MKV to MP4 using ffmpeg with faststart for web streaming."""
if not mkv_path.exists():
return False
try:
# 1. Try fast stream copy with +faststart
process = await asyncio.create_subprocess_exec(
get_ffmpeg_path(),
"-y",
"-i",
str(mkv_path),
"-c",
"copy",
"-movflags",
"+faststart",
str(mp4_path),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
await process.wait()
if mp4_path.exists() and mp4_path.stat().st_size > 0:
return True
"""Normalize MKV into a fixed-size, browser-safe MP4.
# 2. Fallback to re-encoding if stream copy fails
logger.warning("ffmpeg fast copy failed, re-encoding MKV to MP4...")
process = await asyncio.create_subprocess_exec(
get_ffmpeg_path(),
"-y",
"-i",
str(mkv_path),
"-c:v",
"libx264",
"-preset",
"fast",
"-pix_fmt",
"yuv420p",
"-movflags",
"+faststart",
str(mp4_path),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
await process.wait()
return mp4_path.exists() and mp4_path.stat().st_size > 0
except Exception as e:
logger.error(f"Failed to convert MKV to MP4: {e}")
return False
Stream-copying a scrcpy H.264 track is unsafe because historical or
recovered recordings may contain resolution changes. Re-encoding onto
the initial capture canvas guarantees one coded size for the full MP4.
"""
session = get_active_session(self._get_device_id())
return await normalize_recording_to_mp4(
mkv_path,
mp4_path,
session.capture_width if session else None,
session.capture_height if session else None,
)
async def cleanup(self) -> None:
await self._driver.disconnect()