feat: support orientation-aware recording and device wakefulness
Roll scrcpy recordings by device rotation and duration, and publish timeline manifests for playback and analysis. Add seamless segmented playback in the showcase UI and configurable stay-awake strategies for Android automation.
This commit is contained in:
somew1nd committed
1 parent
160f43a74a
commit
474e56ef8e
14 files changed
+1150
-310
No files matched your search
@@ -0,0 +1,165 @@
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.clients.ui_automator_client import (
|
||||
AWAKE_STRATEGY_HEARTBEAT,
|
||||
AWAKE_STRATEGY_STAY_ON,
|
||||
AWAKE_STRATEGY_WAKE_LOCK,
|
||||
UIAutomatorClient,
|
||||
_keep_device_awake,
|
||||
)
|
||||
from artemis.drivers.android.adb_driver import AndroidAdbDriver
|
||||
|
||||
|
||||
def completed(stdout="", returncode=0, stderr=""):
|
||||
return MagicMock(returncode=returncode, stdout=stdout, stderr=stderr)
|
||||
|
||||
|
||||
@patch("artemis.clients.ui_automator_client.subprocess.run")
|
||||
def test_keep_device_awake_uses_verified_primary_strategy(mock_run):
|
||||
mock_run.side_effect = [
|
||||
completed(),
|
||||
completed(),
|
||||
completed(),
|
||||
completed(" mStayOn=true\n"),
|
||||
]
|
||||
|
||||
strategy = _keep_device_awake("device-123")
|
||||
|
||||
assert strategy == AWAKE_STRATEGY_STAY_ON
|
||||
assert mock_run.call_count == 4
|
||||
|
||||
|
||||
@patch("artemis.clients.ui_automator_client.subprocess.run")
|
||||
def test_keep_device_awake_falls_back_to_verified_screen_wake_lock(mock_run):
|
||||
mock_run.side_effect = [
|
||||
completed(),
|
||||
completed(),
|
||||
completed(),
|
||||
completed(" mStayOn=false\n"),
|
||||
completed("WakeLock{abc held=true, refCount=1}\n"),
|
||||
completed(
|
||||
"Display 0, wakelock type: SCREEN_BRIGHT_WAKE_LOCK: "
|
||||
"WakeLock{abc held=true, refCount=1}\n"
|
||||
),
|
||||
]
|
||||
|
||||
strategy = _keep_device_awake("device-123")
|
||||
|
||||
assert strategy == AWAKE_STRATEGY_WAKE_LOCK
|
||||
assert mock_run.call_args_list[-2].args[0] == [
|
||||
"adb",
|
||||
"-s",
|
||||
"device-123",
|
||||
"shell",
|
||||
"cmd",
|
||||
"power",
|
||||
"set-wakelock",
|
||||
"acquire",
|
||||
"-d",
|
||||
"0",
|
||||
"SCREEN_BRIGHT_WAKE_LOCK",
|
||||
]
|
||||
|
||||
|
||||
@patch("artemis.clients.ui_automator_client.subprocess.run")
|
||||
def test_keep_device_awake_uses_heartbeat_when_wake_lock_is_unavailable(mock_run):
|
||||
mock_run.side_effect = [
|
||||
completed(),
|
||||
completed(),
|
||||
completed(),
|
||||
completed(" mStayOn=false\n"),
|
||||
completed(returncode=1, stderr="unknown command"),
|
||||
completed("Wakelocks:\n"),
|
||||
]
|
||||
|
||||
strategy = _keep_device_awake("device-123")
|
||||
|
||||
assert strategy == AWAKE_STRATEGY_HEARTBEAT
|
||||
|
||||
|
||||
@patch.dict("os.environ", {"ARTEMIS_KEEP_DEVICE_AWAKE": "false"})
|
||||
@patch("artemis.clients.ui_automator_client.subprocess.run")
|
||||
def test_keep_device_awake_can_be_disabled(mock_run):
|
||||
assert _keep_device_awake("device-123") is None
|
||||
mock_run.assert_not_called()
|
||||
|
||||
|
||||
@patch("artemis.clients.ui_automator_client.u2.connect")
|
||||
@patch(
|
||||
"artemis.clients.ui_automator_client._keep_device_awake",
|
||||
return_value=AWAKE_STRATEGY_HEARTBEAT,
|
||||
)
|
||||
@patch("artemis.clients.ui_automator_client._ensure_maestro_not_installed")
|
||||
def test_new_ui_connection_starts_heartbeat_fallback(
|
||||
mock_remove_maestro, mock_keep_awake, mock_connect
|
||||
):
|
||||
client = UIAutomatorClient("device-123")
|
||||
client._start_awake_heartbeat = MagicMock()
|
||||
|
||||
client._ensure_connected()
|
||||
|
||||
mock_remove_maestro.assert_called_once_with("device-123")
|
||||
mock_keep_awake.assert_called_once_with("device-123")
|
||||
mock_connect.assert_called_once_with("device-123")
|
||||
client._start_awake_heartbeat.assert_called_once_with()
|
||||
assert client._awake_strategy == AWAKE_STRATEGY_HEARTBEAT
|
||||
|
||||
|
||||
@patch("artemis.clients.ui_automator_client._release_screen_wake_lock")
|
||||
def test_disconnect_releases_screen_wake_lock(mock_release):
|
||||
client = UIAutomatorClient("device-123")
|
||||
client._awake_strategy = AWAKE_STRATEGY_WAKE_LOCK
|
||||
client._stop_awake_heartbeat = MagicMock()
|
||||
|
||||
client.disconnect()
|
||||
|
||||
client._stop_awake_heartbeat.assert_called_once_with()
|
||||
mock_release.assert_called_once_with("device-123")
|
||||
assert client._awake_strategy is None
|
||||
|
||||
|
||||
@patch("artemis.clients.ui_automator_client.threading.Thread")
|
||||
@patch("artemis.clients.ui_automator_client.threading.Event")
|
||||
def test_heartbeat_thread_is_daemonized_and_stoppable(mock_event, mock_thread):
|
||||
stop_event = MagicMock()
|
||||
thread = MagicMock()
|
||||
mock_event.return_value = stop_event
|
||||
mock_thread.return_value = thread
|
||||
client = UIAutomatorClient("device-123")
|
||||
|
||||
client._start_awake_heartbeat()
|
||||
client._stop_awake_heartbeat()
|
||||
|
||||
assert mock_thread.call_args.kwargs["daemon"] is True
|
||||
thread.start.assert_called_once_with()
|
||||
stop_event.set.assert_called_once_with()
|
||||
thread.join.assert_called_once_with(timeout=2.0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_android_driver_disconnect_cleans_up_ui_client():
|
||||
ui_client = MagicMock()
|
||||
driver = AndroidAdbDriver(
|
||||
device_id="device-123",
|
||||
adb_client=MagicMock(),
|
||||
ui_adb_client=ui_client,
|
||||
)
|
||||
|
||||
await driver.disconnect()
|
||||
|
||||
ui_client.disconnect.assert_called_once_with()
|
||||
|
||||
|
||||
@patch("artemis.clients.ui_automator_client.subprocess.run")
|
||||
def test_keep_device_awake_continues_after_an_adb_failure(mock_run):
|
||||
mock_run.side_effect = [
|
||||
completed(returncode=1, stderr="not allowed"),
|
||||
OSError("adb temporarily unavailable"),
|
||||
completed(),
|
||||
completed("mStayOn=true\n"),
|
||||
]
|
||||
|
||||
assert _keep_device_awake("device-123") == AWAKE_STRATEGY_STAY_ON
|
||||
assert mock_run.call_count == 4
|
||||
@@ -14,23 +14,161 @@
|
||||
|
||||
"""Unit tests for UnifiedMobileController video recording and playback features."""
|
||||
|
||||
import subprocess
|
||||
import time
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
import cv2
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.controllers.unified_controller import UnifiedMobileController
|
||||
from artemis.drivers.mock.mock_driver import MockDeviceDriver
|
||||
from artemis.utils.video import (
|
||||
RecordingSession,
|
||||
build_scrcpy_record_command,
|
||||
get_ffmpeg_path,
|
||||
get_active_session,
|
||||
normalize_recording_to_mp4,
|
||||
render_timeline_clip,
|
||||
remove_active_session,
|
||||
set_active_session,
|
||||
)
|
||||
|
||||
|
||||
def test_scrcpy_recording_locks_each_segment_orientation(tmp_path):
|
||||
output_path = tmp_path / "recording.mkv"
|
||||
|
||||
command = build_scrcpy_record_command("scrcpy", "device-1", output_path)
|
||||
|
||||
assert "--capture-orientation=@" in command
|
||||
assert command[command.index("--record") + 1] == str(output_path)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_normalize_recording_handles_resolution_change(tmp_path):
|
||||
"""A portrait/landscape H.264 track must become one fixed-size MP4."""
|
||||
ffmpeg = get_ffmpeg_path()
|
||||
portrait = tmp_path / "portrait.mkv"
|
||||
landscape = tmp_path / "landscape.mkv"
|
||||
dynamic = tmp_path / "dynamic.mkv"
|
||||
output = tmp_path / "recording.mp4"
|
||||
|
||||
for size, path in (("108x242", portrait), ("242x108", landscape)):
|
||||
subprocess.run(
|
||||
[
|
||||
ffmpeg,
|
||||
"-y",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
f"testsrc=size={size}:rate=10:duration=0.5",
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
str(path),
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
|
||||
concat_file = tmp_path / "segments.txt"
|
||||
concat_file.write_text(
|
||||
f"file '{portrait.as_posix()}'\nfile '{landscape.as_posix()}'\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
subprocess.run(
|
||||
[
|
||||
ffmpeg,
|
||||
"-y",
|
||||
"-f",
|
||||
"concat",
|
||||
"-safe",
|
||||
"0",
|
||||
"-i",
|
||||
str(concat_file),
|
||||
"-c",
|
||||
"copy",
|
||||
str(dynamic),
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
|
||||
assert await normalize_recording_to_mp4(dynamic, output, 108, 242)
|
||||
|
||||
capture = cv2.VideoCapture(str(output))
|
||||
dimensions = set()
|
||||
frame_count = 0
|
||||
while True:
|
||||
ok, frame = capture.read()
|
||||
if not ok:
|
||||
break
|
||||
frame_count += 1
|
||||
dimensions.add((frame.shape[1], frame.shape[0]))
|
||||
capture.release()
|
||||
|
||||
assert frame_count >= 8
|
||||
assert dimensions == {(108, 242)}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_analyzer_clip_stays_continuous_across_orientation_segments(tmp_path):
|
||||
"""The analyzer must still receive one decodable MP4 across a rotation."""
|
||||
ffmpeg = get_ffmpeg_path()
|
||||
portrait = tmp_path / "recording.mkv"
|
||||
landscape = tmp_path / "recording_001.mkv"
|
||||
output = tmp_path / "agent_clip.mp4"
|
||||
for color, size, path in (
|
||||
("red", "108x242", portrait),
|
||||
("blue", "242x108", landscape),
|
||||
):
|
||||
subprocess.run(
|
||||
[
|
||||
ffmpeg,
|
||||
"-y",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
f"color={color}:size={size}:rate=15:duration=1",
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
str(path),
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
|
||||
assert await render_timeline_clip(
|
||||
[
|
||||
{"path": portrait, "start": 0.0, "end": 1.0},
|
||||
{"path": landscape, "start": 1.0, "end": 2.0},
|
||||
],
|
||||
0.25,
|
||||
1.75,
|
||||
output,
|
||||
canvas_width=180,
|
||||
canvas_height=320,
|
||||
)
|
||||
|
||||
capture = cv2.VideoCapture(str(output))
|
||||
dimensions = set()
|
||||
frames = 0
|
||||
while True:
|
||||
ok, frame = capture.read()
|
||||
if not ok:
|
||||
break
|
||||
dimensions.add((frame.shape[1], frame.shape[0]))
|
||||
frames += 1
|
||||
capture.release()
|
||||
assert dimensions == {(180, 320)}
|
||||
assert frames >= 20
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_ctx(tmp_path):
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
@@ -60,17 +198,21 @@ async def test_unified_controller_start_recording(mock_ctx, tmp_path):
|
||||
mock_proc.returncode = None
|
||||
|
||||
with patch("asyncio.create_subprocess_exec", AsyncMock(return_value=mock_proc)):
|
||||
with patch("asyncio.sleep", AsyncMock()):
|
||||
res = await controller.start_video_recording(output_dir=tmp_path)
|
||||
with patch(
|
||||
"artemis.controllers.unified_controller.get_android_display_state",
|
||||
AsyncMock(return_value=(0, 1080, 2424)),
|
||||
):
|
||||
with patch("asyncio.sleep", AsyncMock()):
|
||||
res = await controller.start_video_recording(output_dir=tmp_path)
|
||||
|
||||
assert res.success is True
|
||||
assert get_active_session("emulator-5554") is not None
|
||||
assert mock_ctx.data_engine.record_video_start.called
|
||||
assert res.success is True
|
||||
assert get_active_session("emulator-5554") is not None
|
||||
assert mock_ctx.data_engine.record_video_start.called
|
||||
|
||||
# Calling start again should report already in progress
|
||||
res2 = await controller.start_video_recording(output_dir=tmp_path)
|
||||
assert res2.success is False
|
||||
assert "already in progress" in res2.message
|
||||
# Calling start again should report already in progress
|
||||
res2 = await controller.start_video_recording(output_dir=tmp_path)
|
||||
assert res2.success is False
|
||||
assert "already in progress" in res2.message
|
||||
|
||||
remove_active_session("emulator-5554")
|
||||
|
||||
@@ -99,10 +241,11 @@ async def test_unified_controller_stop_recording(mock_ctx, tmp_path):
|
||||
)
|
||||
set_active_session("emulator-5554", session)
|
||||
|
||||
with patch.object(
|
||||
controller,
|
||||
"_convert_mkv_to_mp4",
|
||||
with patch(
|
||||
"artemis.controllers.unified_controller.remux_recording_to_mp4",
|
||||
AsyncMock(side_effect=lambda src, dst: dst.write_bytes(b"mp4 content") or True),
|
||||
), patch(
|
||||
"artemis.controllers.unified_controller.write_recording_manifest", AsyncMock()
|
||||
):
|
||||
res = await controller.stop_video_recording()
|
||||
|
||||
@@ -135,7 +278,7 @@ async def test_unified_controller_extract_segment_metadata(mock_ctx, tmp_path):
|
||||
set_active_session("emulator-5554", session)
|
||||
|
||||
with patch(
|
||||
"artemis.controllers.unified_controller.trim_video",
|
||||
"artemis.controllers.unified_controller.render_timeline_clip",
|
||||
AsyncMock(side_effect=lambda src, s, e, dst: dst.write_bytes(b"segment mp4") or True),
|
||||
):
|
||||
res = await controller.extract_segment_metadata(start_time=2.0, end_time=8.0)
|
||||
@@ -204,6 +347,14 @@ async def test_unified_controller_crash_recovery_and_multi_segment(mock_ctx, tmp
|
||||
data_engine_start_time=time.time() - 30.0,
|
||||
local_video_path=seg2,
|
||||
android_video_segments=[seg1],
|
||||
android_segment_records=[
|
||||
{
|
||||
"path": seg1,
|
||||
"output_path": tmp_path / "recording.mp4",
|
||||
"start": 0.0,
|
||||
"end": 15.0,
|
||||
}
|
||||
],
|
||||
android_segment_index=1,
|
||||
process=mock_proc,
|
||||
)
|
||||
@@ -211,14 +362,10 @@ async def test_unified_controller_crash_recovery_and_multi_segment(mock_ctx, tmp
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.controllers.unified_controller.concatenate_videos",
|
||||
AsyncMock(side_effect=lambda segs, out: out.write_bytes(b"combined mkv") or True),
|
||||
),
|
||||
patch.object(
|
||||
controller,
|
||||
"_convert_mkv_to_mp4",
|
||||
"artemis.controllers.unified_controller.remux_recording_to_mp4",
|
||||
AsyncMock(side_effect=lambda src, dst: dst.write_bytes(b"final mp4") or True),
|
||||
),
|
||||
patch("artemis.controllers.unified_controller.write_recording_manifest", AsyncMock()),
|
||||
):
|
||||
res = await controller.stop_video_recording()
|
||||
|
||||
@@ -261,7 +408,8 @@ async def test_unified_controller_timeline_alignment(mock_ctx, tmp_path):
|
||||
return True
|
||||
|
||||
with patch(
|
||||
"artemis.controllers.unified_controller.trim_video", AsyncMock(side_effect=mock_trim)
|
||||
"artemis.controllers.unified_controller.render_timeline_clip",
|
||||
AsyncMock(side_effect=mock_trim),
|
||||
):
|
||||
# Agent asks for system time range [5.0s, 10.0s]
|
||||
res = await controller.extract_segment_metadata(start_time=5.0, end_time=10.0)
|
||||
|
||||
Reference in new issue
Block a user