chenmingxuan
be6e8c2abd
Unify task scheduling and queue state, fix Android input and observation recovery, preserve screenshot paths, and match final review criteria accurately. Add collapsible copyable results and publish the complete 2.1.0 upgrade notes. Validation: 3189 backend tests, 326 frontend tests, 24 client tests, production build and quality gates.
804 lines
31 KiB
Python
804 lines
31 KiB
Python
# Copyright 2026 Google LLC
|
||
#
|
||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||
# you may not use this file except in compliance with the License.
|
||
# You may obtain a copy of the License at
|
||
#
|
||
# http://www.apache.org/licenses/LICENSE-2.0
|
||
#
|
||
# Unless required by applicable law or agreed to in writing, software
|
||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
# See the License for the specific language governing permissions and
|
||
# limitations under the License.
|
||
|
||
"""Unit tests for Artemis MCP Tools."""
|
||
|
||
import inspect
|
||
import json
|
||
import os
|
||
import shutil
|
||
import tempfile
|
||
from types import SimpleNamespace
|
||
import uuid
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import pytest
|
||
|
||
from mcp_server.tools import (
|
||
mobile_get_device_state,
|
||
mobile_inspect_trace,
|
||
mobile_manage_task,
|
||
mobile_run_task,
|
||
)
|
||
from artemis.runtime import trace_store
|
||
|
||
|
||
@pytest.fixture
|
||
def temp_trace_env(monkeypatch):
|
||
temp_dir = tempfile.mkdtemp()
|
||
monkeypatch.setattr(trace_store, "TRACES_DIR", temp_dir)
|
||
monkeypatch.setenv("ARTEMIS_STANDALONE", "1")
|
||
yield temp_dir
|
||
shutil.rmtree(temp_dir, ignore_errors=True)
|
||
|
||
|
||
def test_tool_signatures():
|
||
# mobile_run_task signature check
|
||
sig_run = inspect.signature(mobile_run_task)
|
||
assert "task_desc" in sig_run.parameters
|
||
assert "conversation_id" in sig_run.parameters
|
||
assert "model" in sig_run.parameters
|
||
assert "locked_app_package" in sig_run.parameters
|
||
assert "app_path" in sig_run.parameters
|
||
assert "expected_output_desc" in sig_run.parameters
|
||
assert "device_serial" in sig_run.parameters
|
||
assert "verification_level" in sig_run.parameters
|
||
assert "explorer_mode" in sig_run.parameters
|
||
assert sig_run.parameters["verification_level"].default is None
|
||
assert sig_run.parameters["explorer_mode"].default is None
|
||
# The tool description is the only schema an MCP caller sees.
|
||
assert "verification_level" in (mobile_run_task.__doc__ or "")
|
||
assert "explorer_mode" in (mobile_run_task.__doc__ or "")
|
||
|
||
# mobile_manage_task signature check
|
||
sig_manage = inspect.signature(mobile_manage_task)
|
||
assert "action" in sig_manage.parameters
|
||
assert "trace_id" in sig_manage.parameters
|
||
assert "instruction" in sig_manage.parameters
|
||
|
||
# mobile_get_device_state signature check
|
||
sig_device = inspect.signature(mobile_get_device_state)
|
||
assert "view_type" in sig_device.parameters
|
||
assert "device_serial" in sig_device.parameters
|
||
|
||
# mobile_inspect_trace signature check
|
||
sig_inspect = inspect.signature(mobile_inspect_trace)
|
||
assert "action" in sig_inspect.parameters
|
||
assert "trace_id" in sig_inspect.parameters
|
||
assert "step_number" in sig_inspect.parameters
|
||
assert "query" in sig_inspect.parameters
|
||
assert "step_range" in sig_inspect.parameters
|
||
assert sig_inspect.parameters["max_results"].default == 5
|
||
assert "'search'" in (mobile_inspect_trace.__doc__ or "")
|
||
|
||
|
||
def test_mobile_run_task_invalid_model():
|
||
with pytest.raises(ValueError, match="Invalid model"):
|
||
mobile_run_task(task_desc="test", conversation_id="conv-1", model="invalid_model")
|
||
|
||
|
||
def test_auto_plugins_do_not_silently_fall_back_to_a_different_executor(temp_trace_env):
|
||
with (
|
||
patch("artemis.plugins.store.PluginStore.selections", return_value=["desktop"]),
|
||
patch("mcp_server.tools.task_runner.subprocess.Popen") as popen,
|
||
):
|
||
result = mobile_run_task(task_desc="手机测试中操作电脑", model="flash")
|
||
assert result["status"] == "failed"
|
||
assert "统一任务服务" in result["error"]
|
||
popen.assert_not_called()
|
||
|
||
|
||
def test_mobile_run_task_reserves_and_passes_global_queue_ticket(temp_trace_env):
|
||
process = MagicMock(pid=43210)
|
||
with (
|
||
patch(
|
||
"mcp_server.tools.task_runner.DeviceExecutionLock.reserve",
|
||
return_value="queue-ticket-1",
|
||
) as reserve,
|
||
patch("mcp_server.tools.task_runner.DeviceExecutionLock.transfer_reservation") as transfer,
|
||
patch("mcp_server.tools.task_runner.subprocess.Popen", return_value=process) as popen,
|
||
):
|
||
result = mobile_run_task(
|
||
task_desc="Open Settings",
|
||
conversation_id="conv-1",
|
||
model="Flash",
|
||
)
|
||
|
||
reserve.assert_called_once()
|
||
transfer.assert_called_once_with(
|
||
"queue-ticket-1",
|
||
43210,
|
||
description="MCP task: Open Settings",
|
||
session_id=result["trace_id"],
|
||
ingress="mcp",
|
||
)
|
||
assert result["trace_id"]
|
||
assert result["device_serial"] == "auto-select"
|
||
assert popen.call_args.kwargs["env"]["ARTEMIS_DEVICE_QUEUE_TICKET"] == "queue-ticket-1"
|
||
assert popen.call_args.kwargs["env"]["ARTEMIS_TASK_INGRESS"] == "mcp"
|
||
status = trace_store.read_status(result["trace_id"])
|
||
assert status["queue_ticket"] == "queue-ticket-1"
|
||
assert status["device_serial"] is None
|
||
|
||
|
||
def test_mobile_run_task_with_device_serial(temp_trace_env):
|
||
process = MagicMock(pid=54321)
|
||
with (
|
||
patch(
|
||
"mcp_server.tools.task_runner.DeviceExecutionLock.reserve",
|
||
return_value="queue-ticket-dev",
|
||
) as reserve,
|
||
patch("mcp_server.tools.task_runner.DeviceExecutionLock.transfer_reservation") as transfer,
|
||
patch("mcp_server.tools.task_runner.subprocess.Popen", return_value=process) as popen,
|
||
# Hermetic device validation: the requested serial reports as ready
|
||
# regardless of what is attached to the host running the tests.
|
||
patch(
|
||
"artemis.runtime.device_pool.device_pool.try_list_devices",
|
||
return_value=[SimpleNamespace(serial="pixel-11-pro-001", state="device")],
|
||
),
|
||
):
|
||
result = mobile_run_task(
|
||
task_desc="Open Settings on target phone",
|
||
conversation_id="conv-2",
|
||
model="Flash",
|
||
device_serial="pixel-11-pro-001",
|
||
)
|
||
|
||
reserve.assert_called_once_with(
|
||
description="MCP task: Open Settings on target phone",
|
||
session_id=result["trace_id"],
|
||
ingress="mcp",
|
||
device_id="pixel-11-pro-001",
|
||
)
|
||
transfer.assert_called_once_with(
|
||
"queue-ticket-dev",
|
||
54321,
|
||
description="MCP task: Open Settings on target phone",
|
||
session_id=result["trace_id"],
|
||
ingress="mcp",
|
||
device_id="pixel-11-pro-001",
|
||
)
|
||
assert result["device_serial"] == "pixel-11-pro-001"
|
||
cmd = popen.call_args.args[0]
|
||
assert "--device-serial" in cmd
|
||
assert cmd[cmd.index("--device-serial") + 1] == "pixel-11-pro-001"
|
||
assert popen.call_args.kwargs["env"]["ADB_DEVICE_SERIAL"] == "pixel-11-pro-001"
|
||
assert popen.call_args.kwargs["env"]["ARTEMIS_DEVICE_ID"] == "pixel-11-pro-001"
|
||
|
||
status = trace_store.read_status(result["trace_id"])
|
||
assert status["device_serial"] == "pixel-11-pro-001"
|
||
|
||
|
||
@pytest.mark.parametrize("model", ["flash", "pro"])
|
||
def test_pc_run_task_uses_daemon_without_android_validation(temp_trace_env, monkeypatch, model):
|
||
monkeypatch.delenv("ARTEMIS_STANDALONE", raising=False)
|
||
with (
|
||
patch(
|
||
"mcp_server.tools.task_runner.ensure_daemon_running",
|
||
return_value=(True, "http://127.0.0.1:8000"),
|
||
),
|
||
patch(
|
||
"mcp_server.tools.task_runner._validate_device_serial",
|
||
side_effect=AssertionError("ADB must not be used"),
|
||
) as validate,
|
||
patch(
|
||
"mcp_server.tools.task_runner.submit_task_to_daemon",
|
||
side_effect=lambda **kwargs: {
|
||
"status": "started",
|
||
"tasks": [{"session_id": kwargs["session_id"]}],
|
||
},
|
||
) as submit,
|
||
patch("mcp_server.tools.task_runner.subprocess.Popen") as spawn,
|
||
):
|
||
result = mobile_run_task(
|
||
"Check desktop",
|
||
model=model,
|
||
device_serial="offline-phone",
|
||
runtime_overrides={"execution_target": "computer"},
|
||
)
|
||
assert result["status"] == "pending"
|
||
assert submit.call_args.kwargs["runtime_overrides"]["execution_target"] == "computer"
|
||
assert submit.call_args.kwargs["device_serial"] == "computer:local"
|
||
assert trace_store.read_status(result["trace_id"])["device_serial"] == "computer:local"
|
||
validate.assert_not_called()
|
||
spawn.assert_not_called()
|
||
|
||
|
||
def test_mobile_run_task_dispatched_to_daemon(temp_trace_env, monkeypatch):
|
||
monkeypatch.delenv("ARTEMIS_STANDALONE", raising=False)
|
||
with (
|
||
patch(
|
||
"mcp_server.tools.task_runner.ensure_daemon_running",
|
||
return_value=(True, "http://127.0.0.1:8000"),
|
||
),
|
||
patch(
|
||
"mcp_server.tools.task_runner.submit_task_to_daemon",
|
||
return_value={"status": "started", "tasks": [{"session_id": "daemon-sid-1"}]},
|
||
),
|
||
patch("mcp_server.tools.task_runner.subprocess.Popen") as popen,
|
||
):
|
||
result = mobile_run_task(
|
||
task_desc="Open Settings via Daemon",
|
||
conversation_id="conv-daemon",
|
||
model="Flash",
|
||
)
|
||
|
||
popen.assert_not_called()
|
||
assert result["trace_id"] == "daemon-sid-1"
|
||
assert result["status"] == "pending"
|
||
assert "enqueued via unified Artemis Daemon" in result["message"]
|
||
assert "stdout_log" in result
|
||
assert "stderr_log" in result
|
||
|
||
|
||
@pytest.mark.parametrize("lost_reply", [False, True])
|
||
@pytest.mark.parametrize("worker_status", ["pending", "running", "completed", "failed"])
|
||
def test_daemon_receipt_preserves_queue_and_worker_status(
|
||
temp_trace_env, monkeypatch, lost_reply, worker_status
|
||
):
|
||
monkeypatch.delenv("ARTEMIS_STANDALONE", raising=False)
|
||
queued = []
|
||
|
||
def submit(**kwargs):
|
||
sid = kwargs["session_id"]
|
||
assert trace_store.read_status(sid)["status"] == "pending"
|
||
queued.append({"session_id": sid, "status": "pending"})
|
||
if worker_status != "pending":
|
||
trace_store.update_trace_status(sid, worker_status)
|
||
return None if lost_reply else {"status": "started", "tasks": queued}
|
||
|
||
with (
|
||
patch(
|
||
"mcp_server.tools.task_runner.ensure_daemon_running",
|
||
return_value=(True, "http://127.0.0.1:8000"),
|
||
),
|
||
patch("mcp_server.tools.task_runner.submit_task_to_daemon", side_effect=submit),
|
||
patch(
|
||
"mcp_server.tools.task_runner.DeviceExecutionLock.get_queued_tasks",
|
||
side_effect=lambda: queued,
|
||
),
|
||
patch("mcp_server.tools.task_runner.subprocess.Popen") as spawn,
|
||
):
|
||
result = mobile_run_task("Queue state regression", model="Pro")
|
||
assert result["status"] == worker_status
|
||
assert trace_store.read_status(result["trace_id"])["status"] == worker_status
|
||
spawn.assert_not_called()
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
("kwargs", "match"),
|
||
[
|
||
({"verification_level": "paranoid"}, "Invalid verification_level"),
|
||
({"explorer_mode": "turbo"}, "Invalid explorer_mode"),
|
||
],
|
||
)
|
||
def test_mobile_run_task_rejects_unknown_pro_tuning_before_creating_a_trace(
|
||
temp_trace_env, kwargs, match
|
||
):
|
||
with pytest.raises(ValueError, match=match):
|
||
mobile_run_task(task_desc="Open Settings", model="Pro", **kwargs)
|
||
# Rejected before init_trace: nothing was written to the trace store.
|
||
assert os.listdir(temp_trace_env) == []
|
||
|
||
|
||
def test_mobile_run_task_forwards_pro_tuning_to_background_runner(temp_trace_env):
|
||
process = MagicMock(pid=777)
|
||
with (
|
||
patch("mcp_server.tools.task_runner.DeviceExecutionLock.reserve", return_value="t"),
|
||
patch("mcp_server.tools.task_runner.DeviceExecutionLock.transfer_reservation"),
|
||
patch("mcp_server.tools.task_runner.subprocess.Popen", return_value=process) as popen,
|
||
):
|
||
mobile_run_task(
|
||
task_desc="Audit the checkout flow",
|
||
model="Pro",
|
||
verification_level=" Strict ",
|
||
explorer_mode="ULTRA",
|
||
)
|
||
|
||
cmd = popen.call_args.args[0]
|
||
# Same flag spelling as `artemis run` / the admin-console worker, normalised values.
|
||
assert cmd[cmd.index("--verification-level") + 1] == "strict"
|
||
assert cmd[cmd.index("--explorer-pro-mode") + 1] == "ultra"
|
||
|
||
|
||
def test_mobile_run_task_omits_pro_tuning_flags_when_unset(temp_trace_env):
|
||
process = MagicMock(pid=778)
|
||
with (
|
||
patch("mcp_server.tools.task_runner.DeviceExecutionLock.reserve", return_value="t"),
|
||
patch("mcp_server.tools.task_runner.DeviceExecutionLock.transfer_reservation"),
|
||
patch("mcp_server.tools.task_runner.subprocess.Popen", return_value=process) as popen,
|
||
):
|
||
mobile_run_task(task_desc="Open Settings", model="Pro", verification_level=" ")
|
||
|
||
cmd = popen.call_args.args[0]
|
||
assert "--verification-level" not in cmd
|
||
assert "--explorer-pro-mode" not in cmd
|
||
|
||
|
||
def test_mobile_run_task_forwards_pro_tuning_to_daemon(temp_trace_env, monkeypatch):
|
||
monkeypatch.delenv("ARTEMIS_STANDALONE", raising=False)
|
||
with (
|
||
patch(
|
||
"mcp_server.tools.task_runner.ensure_daemon_running",
|
||
return_value=(True, "http://127.0.0.1:8000"),
|
||
),
|
||
patch(
|
||
"mcp_server.tools.task_runner.submit_task_to_daemon",
|
||
return_value={"status": "started", "tasks": [{"session_id": "daemon-sid-2"}]},
|
||
) as submit,
|
||
patch("mcp_server.tools.task_runner.subprocess.Popen") as popen,
|
||
):
|
||
result = mobile_run_task(
|
||
task_desc="Audit via Daemon",
|
||
model="Pro",
|
||
verification_level="Checkpoints",
|
||
explorer_mode="pro",
|
||
)
|
||
|
||
popen.assert_not_called()
|
||
assert result["trace_id"] == "daemon-sid-2"
|
||
# JSON field spelling of /api/run: verification_level / explorer_mode.
|
||
assert submit.call_args.kwargs["verification_level"] == "checkpoints"
|
||
assert submit.call_args.kwargs["explorer_mode"] == "pro"
|
||
|
||
|
||
def test_mobile_manage_task_unknown_trace(temp_trace_env):
|
||
res = mobile_manage_task(action="status", trace_id="non-existent-trace")
|
||
assert res["status"] == "unknown"
|
||
assert "not found" in res["message"]
|
||
|
||
|
||
def test_web_task_can_be_paused_and_guided_from_mcp(temp_trace_env, monkeypatch):
|
||
import sqlite3
|
||
from pathlib import Path
|
||
|
||
import artemis.runtime.task_control as control
|
||
|
||
monkeypatch.setattr(control, "TRACES_PATH", Path(temp_trace_env))
|
||
monkeypatch.setattr("mcp_server.tools.task_manager._pid_alive", lambda pid: True)
|
||
trace_id = str(uuid.uuid4())
|
||
db = Path(temp_trace_env) / "data_engine.db"
|
||
with sqlite3.connect(db) as conn:
|
||
conn.execute(
|
||
"CREATE TABLE sessions (session_id TEXT, status TEXT, pid INTEGER, "
|
||
"start_time REAL, end_time REAL, device_info TEXT)"
|
||
)
|
||
conn.execute(
|
||
"INSERT INTO sessions VALUES (?, 'running', 17, 1, NULL, ?)",
|
||
(trace_id, '{"device_id":"test-device"}'),
|
||
)
|
||
|
||
assert mobile_manage_task(action="status", trace_id=trace_id)["device_serial"] == "test-device"
|
||
assert mobile_manage_task(action="pause", trace_id=trace_id)["status"] == "pausing"
|
||
assert (Path(temp_trace_env) / trace_id / "pause.request").exists()
|
||
assert mobile_manage_task(action="status", trace_id=trace_id)["status"] == "pausing"
|
||
control._write_atomic(
|
||
control.control_dir(trace_id) / "task_control_state.json", {"status": "task_paused"}
|
||
)
|
||
guided = mobile_manage_task(
|
||
action="inject_instruction", trace_id=trace_id, instruction="先确认当前画面"
|
||
)
|
||
assert guided["status"] == "task_paused"
|
||
assert (Path(temp_trace_env) / trace_id / "resume.request").exists()
|
||
assert (
|
||
control.consume_instructions(control.control_dir(trace_id))["instruction"]
|
||
== "先确认当前画面"
|
||
)
|
||
guidance = mobile_manage_task(action="guidance_history", trace_id=trace_id)
|
||
assert guidance["instructions"][0]["status"] == "consumed"
|
||
assert guidance["instructions"][0]["instruction"] == "先确认当前画面"
|
||
|
||
|
||
def test_mobile_manage_task_status_and_stop(temp_trace_env):
|
||
trace_id = str(uuid.uuid4())
|
||
trace_store.init_trace(trace_id, "Test task", "Flash", "conv-1", device_serial="device-xyz")
|
||
|
||
# Status check
|
||
status_res = mobile_manage_task(action="status", trace_id=trace_id)
|
||
assert status_res["trace_id"] == trace_id
|
||
assert status_res["status"] == "running"
|
||
assert status_res["model"] == "Flash"
|
||
assert status_res["device_serial"] == "device-xyz"
|
||
|
||
# Stop without PID
|
||
stop_res = mobile_manage_task(action="stop", trace_id=trace_id)
|
||
assert stop_res["trace_id"] == trace_id
|
||
assert "PID" in stop_res["message"] or "Cannot stop" in stop_res["message"]
|
||
|
||
|
||
def test_mobile_manage_task_status_backfills_device_serial_from_db(temp_trace_env):
|
||
import sqlite3
|
||
import os
|
||
|
||
trace_id = str(uuid.uuid4())
|
||
trace_store.init_trace(trace_id, "Test task without serial", "Flash", "conv-1")
|
||
|
||
# Create dummy data_engine.db with session record
|
||
db_path = os.path.join(temp_trace_env, "data_engine.db")
|
||
conn = sqlite3.connect(db_path)
|
||
conn.execute(
|
||
"CREATE TABLE sessions (session_id TEXT PRIMARY KEY, device_info TEXT, pid INTEGER, start_time REAL)"
|
||
)
|
||
conn.execute(
|
||
"INSERT INTO sessions VALUES (?, ?, ?, ?)",
|
||
(trace_id, json.dumps({"device_id": "auto-chosen-serial-777"}), 12345, 1000.0),
|
||
)
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
status_res = mobile_manage_task(action="status", trace_id=trace_id)
|
||
assert status_res["device_serial"] == "auto-chosen-serial-777"
|
||
|
||
|
||
def test_mobile_manage_task_inject_instruction(temp_trace_env):
|
||
trace_id = str(uuid.uuid4())
|
||
trace_store.init_trace(trace_id, "Test task", "Pro", "conv-1")
|
||
|
||
# Missing instruction parameter
|
||
err_res = mobile_manage_task(action="inject_instruction", trace_id=trace_id)
|
||
assert "Missing required argument" in err_res["message"]
|
||
|
||
# Valid injection
|
||
ok_res = mobile_manage_task(
|
||
action="inject_instruction", trace_id=trace_id, instruction="Scroll down slowly"
|
||
)
|
||
assert "next safe checkpoint" in ok_res["message"]
|
||
assert "not guaranteed" in ok_res["message"]
|
||
assert ok_res["guidance"]["id"]
|
||
assert ok_res["guidance"]["status"] == "pending"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_mobile_get_device_state_hierarchy_without_ocr():
|
||
"""XML hierarchy remains available when OCR is not configured."""
|
||
controller = MagicMock()
|
||
controller.ctx.device.device_width = 1080
|
||
controller.ctx.device.device_height = 2400
|
||
controller.get_screen_data = AsyncMock(
|
||
return_value=SimpleNamespace(
|
||
base64="ZHVtbXk=",
|
||
elements=[
|
||
{
|
||
"class": "android.widget.Button",
|
||
"text": "Start",
|
||
"bounds": "[100,500][400,600]",
|
||
}
|
||
],
|
||
width=1080,
|
||
height=2400,
|
||
)
|
||
)
|
||
|
||
with (
|
||
patch("mcp_server.tools.device_state._get_controller", return_value=controller),
|
||
patch("mcp_server.tools.device_state.is_ocr_configured", return_value=False),
|
||
patch(
|
||
"mcp_server.tools.device_state.perform_ocr", new_callable=AsyncMock
|
||
) as perform_ocr_mock,
|
||
):
|
||
result = await mobile_get_device_state(view_type="hierarchy")
|
||
|
||
assert "Start" in result
|
||
perform_ocr_mock.assert_not_awaited()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_mobile_inspect_trace_invalid_action():
|
||
res = await mobile_inspect_trace(action="invalid_action", trace_id="trace-123")
|
||
assert "error" in res
|
||
assert "not supported" in res["message"]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_mobile_get_device_state_passes_device_serial():
|
||
"""Verify that device_serial is forwarded to _get_controller."""
|
||
controller = MagicMock()
|
||
controller.ctx.device.device_width = 1080
|
||
controller.ctx.device.device_height = 2400
|
||
controller.get_screen_data = AsyncMock(
|
||
return_value=SimpleNamespace(
|
||
base64="ZHVtbXk=",
|
||
elements=[],
|
||
width=1080,
|
||
height=2400,
|
||
)
|
||
)
|
||
|
||
with (
|
||
patch(
|
||
"mcp_server.tools.device_state._get_controller", return_value=controller
|
||
) as get_ctrl_mock,
|
||
patch("mcp_server.tools.device_state.is_ocr_configured", return_value=False),
|
||
):
|
||
await mobile_get_device_state(view_type="hierarchy", device_serial="device-serial-abc")
|
||
|
||
get_ctrl_mock.assert_called_once_with(device_serial="device-serial-abc")
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_mobile_inspect_trace_includes_device_serial(temp_trace_env):
|
||
import sqlite3
|
||
|
||
trace_id = str(uuid.uuid4())
|
||
trace_store.init_trace(
|
||
trace_id, "Inspect test", "Flash", "conv-1", device_serial="pixel-target-999"
|
||
)
|
||
|
||
# Create dummy database and table
|
||
db_path = os.path.join(temp_trace_env, "data_engine.db")
|
||
conn = sqlite3.connect(db_path)
|
||
conn.execute(
|
||
"CREATE TABLE sessions (session_id TEXT PRIMARY KEY, start_time REAL, device_info TEXT)"
|
||
)
|
||
conn.execute(
|
||
"CREATE TABLE steps (step_id TEXT PRIMARY KEY, session_id TEXT, step_number INTEGER, timestamp REAL, pre_image_name TEXT, post_image_name TEXT, summary TEXT, action_taken TEXT, operator_raw_thinking TEXT, last_execution_result TEXT, extra_metadata TEXT)"
|
||
)
|
||
conn.execute(
|
||
"CREATE TABLE traces (trace_id TEXT PRIMARY KEY, step_id TEXT, session_id TEXT, type TEXT, name TEXT, status TEXT, timestamp REAL, duration REAL, payload TEXT)"
|
||
)
|
||
conn.execute("INSERT INTO sessions VALUES (?, ?, ?)", (trace_id, 1000.0, None))
|
||
conn.executemany(
|
||
"INSERT INTO steps (step_id,session_id,step_number,timestamp) VALUES (?,?,?,?)",
|
||
[(f"actual-{n}", trace_id, n, 1000 + n) for n in range(1, 4)],
|
||
)
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
res_summary = await mobile_inspect_trace(action="view_summary", trace_id=trace_id, limit=2)
|
||
assert "**Device Serial:** `pixel-target-999`" in res_summary.content[0].text
|
||
assert res_summary.content[0].text.startswith("# ARTEMIS Flash Execution Summary")
|
||
assert res_summary.structuredContent["steps"] == {
|
||
"items": [
|
||
{"step_id": "actual-1", "step_number": 1},
|
||
{"step_id": "actual-2", "step_number": 2},
|
||
],
|
||
"total": 3,
|
||
"offset": 0,
|
||
"limit": 2,
|
||
"next_offset": 2,
|
||
}
|
||
last_page = await mobile_inspect_trace(
|
||
action="view_summary", trace_id=trace_id, offset=2, limit=2
|
||
)
|
||
assert last_page.structuredContent["steps"]["next_offset"] is None
|
||
assert last_page.structuredContent["steps"]["items"] == [
|
||
{"step_id": "actual-3", "step_number": 3}
|
||
]
|
||
|
||
|
||
def test_trace_store_device_serial(temp_trace_env):
|
||
trace_id = "test-store-serial"
|
||
data = trace_store.init_trace(trace_id, "desc", "Flash", "conv", device_serial="init-serial")
|
||
assert data["device_serial"] == "init-serial"
|
||
|
||
data_updated = trace_store.update_trace_device_serial(trace_id, "updated-serial")
|
||
assert data_updated["device_serial"] == "updated-serial"
|
||
|
||
read_back = trace_store.read_status(trace_id)
|
||
assert read_back["device_serial"] == "updated-serial"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_mcp_server_auto_registers_tools():
|
||
"""Verify that importing mcp_server.base auto-registers all mobile tools without manual tools import."""
|
||
from mcp_server.base import mcp
|
||
|
||
tools = await mcp.list_tools()
|
||
tool_names = {t.name for t in tools}
|
||
assert {
|
||
"mobile_run_task",
|
||
"mobile_manage_task",
|
||
"mobile_get_device_state",
|
||
"mobile_inspect_trace",
|
||
}.issubset(tool_names)
|
||
|
||
|
||
def test_mobile_manage_task_syncs_terminal_status_from_db(temp_trace_env):
|
||
import sqlite3
|
||
|
||
trace_id = str(uuid.uuid4())
|
||
trace_store.init_trace(trace_id, "Running task to sync", "Flash", "conv-sync")
|
||
|
||
db_path = os.path.join(temp_trace_env, "data_engine.db")
|
||
conn = sqlite3.connect(db_path)
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"""CREATE TABLE IF NOT EXISTS sessions (
|
||
session_id TEXT PRIMARY KEY,
|
||
status TEXT,
|
||
pid INTEGER,
|
||
start_time REAL,
|
||
end_time REAL,
|
||
device_info TEXT
|
||
)"""
|
||
)
|
||
cur.execute(
|
||
"INSERT INTO sessions (session_id, status, pid, start_time) VALUES (?, ?, ?, ?)",
|
||
(trace_id, "completed", 54321, 1000.0),
|
||
)
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
res = mobile_manage_task(action="status", trace_id=trace_id)
|
||
assert res["status"] == "completed"
|
||
# Ensure status.json was persisted
|
||
saved_st = trace_store.read_status(trace_id)
|
||
assert saved_st["status"] == "completed"
|
||
assert saved_st["pid"] == 54321
|
||
|
||
|
||
def test_mobile_manage_task_stop_via_daemon(temp_trace_env, monkeypatch):
|
||
monkeypatch.delenv("ARTEMIS_STANDALONE", raising=False)
|
||
trace_id = str(uuid.uuid4())
|
||
trace_store.init_trace(trace_id, "Daemon task to stop", "Flash", "conv-stop")
|
||
|
||
with (
|
||
patch("mcp_server.tools.task_manager.is_daemon_running", return_value=True),
|
||
patch("mcp_server.tools.task_manager.stop_task_on_daemon", return_value=True) as mock_stop,
|
||
):
|
||
res = mobile_manage_task(action="stop", trace_id=trace_id)
|
||
mock_stop.assert_called_once_with(trace_id)
|
||
assert res["status"] == "cancelled"
|
||
assert trace_store.read_status(trace_id)["status"] == "cancelled"
|
||
|
||
|
||
def _seed_trace(temp_dir: str, trace_id: str, *, with_image: bool = True) -> str:
|
||
"""Writes one Flash-style step (with a tool trace and a pre screenshot)
|
||
through the real storage layer, exactly as a live run would."""
|
||
import io
|
||
|
||
from PIL import Image
|
||
|
||
from artemis.data_engine.models import (
|
||
ImageRecord,
|
||
SessionMetadata,
|
||
StepRecord,
|
||
TraceRecord,
|
||
)
|
||
from artemis.data_engine.storage import StorageManager
|
||
|
||
storage = StorageManager(os.path.join(temp_dir, "data_engine.db"), temp_dir)
|
||
storage.create_session(
|
||
SessionMetadata(session_id=trace_id, initial_goal="Replay test", start_time=1000.0)
|
||
)
|
||
image_name = None
|
||
if with_image:
|
||
image_name = "a" * 64
|
||
images_dir = os.path.join(temp_dir, "images")
|
||
os.makedirs(images_dir, exist_ok=True)
|
||
buf = io.BytesIO()
|
||
Image.new("RGB", (1080, 2400), "white").save(buf, format="JPEG")
|
||
with open(os.path.join(images_dir, f"{image_name}.jpg"), "wb") as f:
|
||
f.write(buf.getvalue())
|
||
storage.create_image(
|
||
ImageRecord(
|
||
image_name=image_name,
|
||
ocr_result=[{"text": "Save ocrneedle", "bounds": [0, 0, 10, 10]}],
|
||
)
|
||
)
|
||
step_id = str(uuid.uuid4())
|
||
storage.create_step(
|
||
StepRecord(
|
||
step_id=step_id,
|
||
session_id=trace_id,
|
||
step_number=1,
|
||
timestamp=1012.0,
|
||
pre_image_name=image_name,
|
||
summary="Asked the explorer, then tapped Save.",
|
||
action_taken={"action": "click", "target": [540, 1200], "target_text": "Save"},
|
||
operator_raw_thinking="Confirm the toggle first.",
|
||
last_execution_result={"status": "success"},
|
||
extra_metadata={"width": 1080, "height": 2400},
|
||
)
|
||
)
|
||
storage.create_trace(
|
||
TraceRecord(
|
||
trace_id=str(uuid.uuid4()),
|
||
session_id=trace_id,
|
||
step_id=step_id,
|
||
type="tool",
|
||
name="ask_explorer",
|
||
status="success",
|
||
timestamp=1011.0,
|
||
duration=0.8,
|
||
payload={"args": {"question": "toggle?"}, "result": "The toggle is ON. " * 100},
|
||
)
|
||
)
|
||
return step_id
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_mobile_inspect_trace_step_details_replays_tool_calls_for_flash(temp_trace_env):
|
||
"""view_step_details works for Flash runs too and replays the recorded
|
||
tool calls through the same friendly-step path the live DataEngine uses."""
|
||
trace_id = str(uuid.uuid4())
|
||
trace_store.init_trace(trace_id, "Replay test", "Flash", "conv-1", device_serial="pixel-7")
|
||
_seed_trace(temp_trace_env, trace_id)
|
||
|
||
res = await mobile_inspect_trace(action="view_step_details", trace_id=trace_id, step_number=1)
|
||
assert "error" not in res
|
||
assert res["device_serial"] == "pixel-7"
|
||
details = res["details"]
|
||
# The header carries the session clock (session start 1000.0, step at 1012.0).
|
||
assert "- **Step 1 (T+00:12)**" in details
|
||
assert "[Screen]: Asked the explorer, then tapped Save." in details
|
||
assert "Confirm the toggle first." in details
|
||
assert "`ask_explorer(" in details
|
||
assert "The toggle is ON. " * 100 in details
|
||
# Coordinates are replayed normalized, not as stored pixels.
|
||
assert "[540, 1200]" not in details
|
||
assert "Tapped 'Save'" in details
|
||
|
||
missing = await mobile_inspect_trace(
|
||
action="view_step_details", trace_id=trace_id, step_number=9
|
||
)
|
||
assert missing["error"] == "Step not found"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_mobile_inspect_trace_step_screenshots_draw_the_overlay(temp_trace_env):
|
||
trace_id = str(uuid.uuid4())
|
||
trace_store.init_trace(trace_id, "Shot test", "Flash", "conv-1", device_serial="pixel-7")
|
||
_seed_trace(temp_trace_env, trace_id)
|
||
|
||
res = await mobile_inspect_trace(
|
||
action="view_step_screenshots", trace_id=trace_id, step_number=1
|
||
)
|
||
assert "error" not in res
|
||
assert res["before_screenshot"].startswith("file://")
|
||
assert res["before_screenshot"].endswith(f"{'a' * 64}.jpg")
|
||
assert res["after_screenshot"] is None
|
||
overlay = res["action_overlay_screenshot"]
|
||
assert overlay is not None
|
||
overlay_path = overlay[len("file://") :]
|
||
assert os.path.exists(overlay_path)
|
||
assert overlay_path.endswith("step_1_overlay.jpg")
|
||
# The overlay differs from the plain pre screenshot: the action was drawn.
|
||
with open(overlay_path, "rb") as f, open(res["before_screenshot"][7:], "rb") as g:
|
||
assert f.read() != g.read()
|
||
|
||
# Second call reuses the file it already wrote.
|
||
again = await mobile_inspect_trace(
|
||
action="view_step_screenshots", trace_id=trace_id, step_number=1
|
||
)
|
||
assert again["action_overlay_screenshot"] == overlay
|
||
|
||
no_step = await mobile_inspect_trace(
|
||
action="view_step_screenshots", trace_id=trace_id, step_number=None
|
||
)
|
||
assert no_step["error"] == "Missing parameter"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_mobile_inspect_trace_search_uses_the_shared_history_search(temp_trace_env):
|
||
trace_id = str(uuid.uuid4())
|
||
trace_store.init_trace(trace_id, "Search test", "Flash", "conv-1", device_serial="pixel-7")
|
||
_seed_trace(temp_trace_env, trace_id)
|
||
|
||
res = await mobile_inspect_trace(action="search", trace_id=trace_id, query="toggle")
|
||
assert "error" not in res
|
||
assert res["device_serial"] == "pixel-7"
|
||
assert "[Step 1 " in res["results"]
|
||
assert "Screen: Asked the explorer, then tapped Save." in res["results"]
|
||
|
||
# On-screen OCR text of the step's screenshot is part of the search surface.
|
||
res = await mobile_inspect_trace(action="search", trace_id=trace_id, query="ocrneedle")
|
||
assert "[Step 1 " in res["results"]
|
||
|
||
ledger = await mobile_inspect_trace(action="search", trace_id=trace_id, step_range=[1, 1])
|
||
assert "Action ledger for Steps 1–1" in ledger["results"]
|
||
|
||
empty = await mobile_inspect_trace(action="search", trace_id=trace_id)
|
||
assert empty["error"] == "Missing parameter"
|