feat(core): consolidate config hub, unify daemon scheduling across all entrypoints, and fix step replay

- Unified configuration and paths under artemis.config as single source of truth
- Consolidated execution across CLI, SDK, MCP, and Admin Console to default to Artemis Daemon
- Fixed frontend step replay to preserve failure steps with proper sequential numbering
- Implemented task deduplication, debouncing, and multi-device lock isolation
- Added comprehensive end-to-end multi-port integration tests for live phone and emulator
This commit is contained in:
somew1nd committed 2026-08-27 10:21:40 -07:00
1 parent e984d17635
commit 5f1ee18bc1
38 files changed
+1426 -203

No files matched your search

+227
View File
@@ -0,0 +1,227 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Comprehensive End-to-End Test for Artemis Multi-Port System.
Verifies end-to-end functionality across all entry points and ports:
1. Configuration & Default Model (gemini-3.5-flash-lite)
2. Daemon Server API & Multi-Device Discovery (/api/devices, /api/status)
3. Multi-Device Detection (Physical phone + Android Emulator)
4. MCP Server Tools (mobile_get_device_state, mobile_run_task, mobile_manage_task)
5. Python SDK (ArtemisClient with device targeting & concurrency modes)
6. CLI Entry Points (artemis run, batch, status)
7. Device Execution Lock Isolation (Multi-device concurrent execution)
"""
import asyncio
import json
import os
import subprocess
import sys
import time
import urllib.request
from pathlib import Path
import pytest
from artemis.config import (
DEFAULT_MODEL,
ROOT_DIR,
Settings,
settings,
get_config_path,
)
from artemis.runtime.daemon_client import (
ensure_daemon_running,
get_daemon_session,
get_daemon_status,
is_daemon_running,
submit_task_to_daemon,
stop_task_on_daemon,
)
from artemis.runtime.device_lock import DeviceExecutionLock
from artemis.runtime.device_pool import device_pool
from artemis.interfaces.sdk.client import ArtemisClient, ConcurrencyMode
from mcp_server.tools.device_state import mobile_get_device_state
from mcp_server.tools.task_runner import mobile_run_task
from mcp_server.tools.task_manager import mobile_manage_task
from mcp_server.utils import trace_store
@pytest.fixture(scope="module", autouse=True)
def ensure_daemon():
"""Ensure Artemis Daemon is running for E2E tests."""
assert ensure_daemon_running(wait_ready=True), "Artemis Daemon failed to start"
yield
def test_01_default_model_configured():
"""Verify default model configuration loads properly across config and router."""
assert DEFAULT_MODEL is not None
cfg_path = get_config_path("artemis.jsonc")
assert cfg_path.exists()
content = cfg_path.read_text(encoding="utf-8")
assert "default" in content
# Check Settings pydantic model
s = Settings()
assert s.ARTEMIS_DEFAULT_MODEL is not None
def test_02_daemon_device_discovery_sees_phone_and_emulator():
"""Verify Daemon API /api/devices detects both the physical phone and emulator."""
status = get_daemon_status()
assert status is not None
assert status.get("status") in ("idle", "running", "ok")
if "model_info" in status and status["model_info"]:
assert status["model_info"]["id"] is not None
req = urllib.request.Request("http://127.0.0.1:8000/api/devices")
with urllib.request.urlopen(req, timeout=5.0) as resp:
assert resp.status == 200
data = json.loads(resp.read().decode("utf-8"))
devices = data.get("devices", [])
serials = [d["serial"] for d in devices]
# Must detect both devices
assert any(s.startswith("emulator") for s in serials), f"Emulator not found in {serials}"
assert any(not s.startswith("emulator") for s in serials), f"Physical phone not found in {serials}"
# Check is_emulator flag classification
for d in devices:
if d["serial"].startswith("emulator"):
assert d["is_emulator"] is True
else:
assert d["is_emulator"] is False
@pytest.mark.asyncio
async def test_03_mcp_device_state_inspects_both_devices():
"""Verify MCP mobile_get_device_state reads element hierarchies from both devices."""
devices = device_pool.list_devices()
emulator_serial = next((d.serial for d in devices if d.is_emulator), None)
phone_serial = next((d.serial for d in devices if not d.is_emulator), None)
assert emulator_serial, f"No emulator connected in {devices}"
assert phone_serial, f"No physical phone connected in {devices}"
# Inspect emulator state
emu_state = await mobile_get_device_state(view_type="hierarchy", device_serial=emulator_serial)
assert isinstance(emu_state, str)
assert len(emu_state) > 0
assert "Bounds:" in emu_state or "Text:" in emu_state or len(emu_state.strip()) > 0
# Inspect physical phone state
phone_state = await mobile_get_device_state(view_type="hierarchy", device_serial=phone_serial)
assert isinstance(phone_state, str)
assert len(phone_state) > 0
assert "Bounds:" in phone_state or "Text:" in phone_state or len(phone_state.strip()) > 0
def test_04_mcp_run_and_manage_task_via_daemon():
"""Verify MCP mobile_run_task dispatches to Daemon and mobile_manage_task tracks it."""
devices = device_pool.list_devices()
emulator_serial = next((d.serial for d in devices if d.is_emulator), None)
assert emulator_serial, "No emulator connected"
res = mobile_run_task(
task_desc="E2E Test task from MCP port",
conversation_id="e2e-conv-mcp-1",
model="Flash",
device_serial=emulator_serial,
)
assert "trace_id" in res
trace_id = res["trace_id"]
assert res["status"] in ("started", "queued", "running")
# Verify tracking state via mobile_manage_task
status_res = mobile_manage_task(action="status", trace_id=trace_id)
assert status_res["trace_id"] == trace_id
assert "status" in status_res
assert status_res.get("device_serial") == emulator_serial
# Cleanup: stop the test task on Daemon and release locks
stop_task_on_daemon(session_id=trace_id)
time.sleep(1.5)
DeviceExecutionLock.cleanup_stale_locks()
def test_05_sdk_client_targeting_and_concurrency():
"""Verify Python SDK ArtemisClient correctly targets specific devices."""
devices = device_pool.list_devices()
emulator_serial = next((d.serial for d in devices if d.is_emulator), None)
phone_serial = next((d.serial for d in devices if not d.is_emulator), None)
assert emulator_serial, "Emulator serial required"
assert phone_serial, "Phone serial required"
client_emu = ArtemisClient(device_id=emulator_serial, concurrency_mode="per_device")
assert client_emu.device_serial == emulator_serial
assert client_emu.concurrency_mode == "per_device"
client_phone = ArtemisClient(device_id=phone_serial, concurrency_mode="per_device")
assert client_phone.device_serial == phone_serial
# Verify multi-device lock independence
DeviceExecutionLock.cleanup_stale_locks()
lock_emu = DeviceExecutionLock(device_id=emulator_serial)
lock_phone = DeviceExecutionLock(device_id=phone_serial)
# Both locks can be acquired simultaneously because device serials differ
acq1 = lock_emu.acquire(timeout=5.0)
assert acq1 is True, "Failed to acquire emulator lock"
try:
acq2 = lock_phone.acquire(timeout=5.0)
assert acq2 is True, "Failed to acquire phone lock concurrently (multi-device parallel mode)"
lock_phone.release()
finally:
lock_emu.release()
def test_06_cli_daemon_and_standalone_options():
"""Verify CLI commands properly expose and route through Daemon or standalone."""
# artemis status
res = subprocess.run([sys.executable, "-m", "artemis.main", "status"], capture_output=True, text=True)
assert res.returncode == 0
assert "online" in res.stdout.lower() or "running" in res.stdout.lower()
# artemis run --help exposes --standalone and --device
res_run = subprocess.run([sys.executable, "-m", "artemis.main", "run", "--help"], capture_output=True, text=True)
assert res_run.returncode == 0
assert "--standalone" in res_run.stdout
assert "--device" in res_run.stdout
# artemis batch --help exposes --standalone
res_batch = subprocess.run([sys.executable, "-m", "artemis.main", "batch", "--help"], capture_output=True, text=True)
assert res_batch.returncode == 0
assert "--standalone" in res_batch.stdout
def test_07_daemon_session_details_api():
"""Verify Daemon GET /api/sessions/{session_id} API endpoint."""
res = submit_task_to_daemon(
goal="E2E session query test",
profile="flash",
ingress="e2e_test",
)
assert "session_id" in res
session_id = res["session_id"]
# Poll session details
session_data = get_daemon_session(session_id)
assert session_data is not None
assert session_data["session_id"] == session_id
assert session_data["initial_goal"] == "E2E session query test"
# Clean up test task
stop_task_on_daemon(session_id=session_id)
@@ -52,9 +52,26 @@ def clean_state(tmp_path, monkeypatch):
state.queue_items.clear()
state.active_connections.clear()
state.cancelled_session_ids.clear()
state.active_session_id = None
state.current_process = None
state.current_goal = None
state.current_profile = None
state.was_stopped_manually = False
if hasattr(state, "recent_submissions"):
state.recent_submissions.clear()
if state.worker_task and not state.worker_task.done():
state.worker_task.cancel()
state.worker_task = None
from artemis.config.paths import get_temp_dir
from artemis.runtime.device_lock import DeviceExecutionLock
DeviceExecutionLock.cleanup_stale_locks()
queue_dir = get_temp_dir("device-locks") / "artemis-global-device.queue"
if queue_dir.exists():
for p in queue_dir.glob("*.wait"):
try:
p.unlink(missing_ok=True)
except OSError:
pass
def test_paused_error_reads_persisted_reason(tmp_path):
@@ -612,7 +629,10 @@ def test_darwin_terminate_process_tree_preserves_direct_child_for_asyncio():
@pytest.mark.asyncio
async def test_enqueue_tasks_unified_ingress():
"""Verify enqueue_tasks correctly propagates ingress, custom session_id, and conversation_id."""
with patch.object(TaskQueueService, "ensure_worker_running"):
with (
patch.object(TaskQueueService, "ensure_worker_running"),
patch("apps.admin_console.services.task_queue_service.DeviceExecutionLock.reserve", return_value="mock-ticket-unified"),
):
res = await task_queue_service.enqueue_tasks(
["Test unified goal"],
profile="flash",
@@ -801,4 +821,44 @@ def test_stop_tasks_queued_item_by_session_id():
assert update_status.call_args[0][1] == "cancelled"
@pytest.mark.asyncio
async def test_enqueue_tasks_deduplicates_by_session_id():
state.queue_items = []
state.active_session_id = None
with (
patch("apps.admin_console.services.task_queue_service.session_repo"),
patch("apps.admin_console.services.task_queue_service.DeviceExecutionLock.reserve", return_value="ticket-123"),
):
res1 = await task_queue_service.enqueue_tasks(["Task goal"], session_id="sid-dedup-1")
assert len(state.queue_items) == 1
assert res1["enqueued_count"] == 1
# Second submission with same session_id must not enqueue a duplicate
res2 = await task_queue_service.enqueue_tasks(["Task goal again"], session_id="sid-dedup-1")
assert len(state.queue_items) == 1
assert res2["enqueued_count"] == 0
assert res2["tasks"][0]["session_id"] == "sid-dedup-1"
@pytest.mark.asyncio
async def test_enqueue_tasks_debounces_rapid_identical_submissions():
state.queue_items = []
state.active_session_id = None
with (
patch("apps.admin_console.services.task_queue_service.session_repo"),
patch("apps.admin_console.services.task_queue_service.DeviceExecutionLock.reserve", return_value="ticket-456"),
):
res1 = await task_queue_service.enqueue_tasks(["Rapid duplicate goal"], device_serial="dev-1")
assert len(state.queue_items) == 1
assert res1["enqueued_count"] == 1
# Immediate second submission (within 1s) must debounce
res2 = await task_queue_service.enqueue_tasks(["Rapid duplicate goal"], device_serial="dev-1")
assert len(state.queue_items) == 1
assert res2["enqueued_count"] == 0
+33
View File
@@ -270,6 +270,8 @@ def test_update_step_execution_result_suppresses_identical_post_screenshot(tmp_p
pre_screenshot_bytes=image_bytes,
summary="Initial step",
)
for t in list(engine._pending_threads):
t.join()
step_record = engine.storage.get_steps(engine.current_session_id)[0]
assert step_record.post_image_name is None
@@ -293,3 +295,34 @@ def test_update_step_execution_result_suppresses_identical_post_screenshot(tmp_p
updated_step_distinct = engine.storage.get_steps(engine.current_session_id)[0]
assert updated_step_distinct.post_image_name == "distinct_new_hash_789"
def test_resumed_session_monotonic_step_numbering(tmp_path):
mock_ctx = MagicMock(spec=ArtemisContext)
mock_execution_setup = MagicMock()
mock_execution_setup.traces_path = str(tmp_path)
mock_ctx.execution_setup = mock_execution_setup
mock_ctx.device = None
# First runner records Step 1 and Step 2
engine1 = DataEngine(mock_ctx)
sid = engine1.start_session("Monotonic test")
engine1.record_step(summary="Step 1 action")
engine1.record_step(summary="Step 2 action")
for t in list(engine1._pending_threads):
t.join()
# Second runner starts or resumes the same session
engine2 = DataEngine(mock_ctx)
engine2.start_session("Resumed goal", session_id=sid)
assert engine2.current_step_number == 2
# Recording a new step must be Step 3, strictly monotonic
engine2.record_step(summary="Step 3 action")
for t in list(engine2._pending_threads):
t.join()
steps = engine2.storage.get_steps(sid)
assert len(steps) == 3
assert [s.step_number for s in steps] == [1, 2, 3]
+77
View File
@@ -38,6 +38,7 @@ from mcp_server.utils import trace_store
def temp_trace_env(monkeypatch):
temp_dir = tempfile.mkdtemp()
monkeypatch.setattr(trace_store, "TRACES_DIR", temp_dir)
monkeypatch.setenv("ARTEMIS_STANDALONE", "1")
yield temp_dir
shutil.rmtree(temp_dir, ignore_errors=True)
@@ -151,6 +152,33 @@ def test_mobile_run_task_with_device_serial(temp_trace_env):
assert status["device_serial"] == "pixel-11-pro-001"
def test_mobile_run_task_dispatched_to_daemon(temp_trace_env, monkeypatch):
monkeypatch.delenv("ARTEMIS_STANDALONE", raising=False)
with (
patch(
"artemis.runtime.ensure_daemon_running",
return_value=(True, "http://127.0.0.1:8000"),
),
patch(
"artemis.runtime.submit_task_to_daemon",
return_value={"status": "started", "tasks": [{"session_id": "daemon-sid-1"}]},
),
patch("mcp_server.tools.task_runner.subprocess.Popen") as popen,
):
result = mobile_run_task(
task_desc="Open Settings via Daemon",
conversation_id="conv-daemon",
model="Flash",
)
popen.assert_not_called()
assert result["trace_id"] == "daemon-sid-1"
assert result["status"] == "running"
assert "enqueued via unified Artemis Daemon" in result["message"]
assert "stdout_log" in result
assert "stderr_log" in result
def test_mobile_manage_task_unknown_trace(temp_trace_env):
res = mobile_manage_task(action="status", trace_id="non-existent-trace")
assert res["status"] == "unknown"
@@ -325,3 +353,52 @@ async def test_mcp_server_auto_registers_tools():
"mobile_get_device_state",
"mobile_inspect_trace",
}.issubset(tool_names)
def test_mobile_manage_task_syncs_terminal_status_from_db(temp_trace_env):
import sqlite3
trace_id = str(uuid.uuid4())
trace_store.init_trace(trace_id, "Running task to sync", "Flash", "conv-sync")
db_path = os.path.join(temp_trace_env, "data_engine.db")
conn = sqlite3.connect(db_path)
cur = conn.cursor()
cur.execute(
"""CREATE TABLE IF NOT EXISTS sessions (
session_id TEXT PRIMARY KEY,
status TEXT,
pid INTEGER,
start_time REAL,
end_time REAL,
device_info TEXT
)"""
)
cur.execute(
"INSERT INTO sessions (session_id, status, pid, start_time) VALUES (?, ?, ?, ?)",
(trace_id, "completed", 54321, 1000.0),
)
conn.commit()
conn.close()
res = mobile_manage_task(action="status", trace_id=trace_id)
assert res["status"] == "completed"
# Ensure status.json was persisted
saved_st = trace_store.read_status(trace_id)
assert saved_st["status"] == "completed"
assert saved_st["pid"] == 54321
def test_mobile_manage_task_stop_via_daemon(temp_trace_env, monkeypatch):
monkeypatch.delenv("ARTEMIS_STANDALONE", raising=False)
trace_id = str(uuid.uuid4())
trace_store.init_trace(trace_id, "Daemon task to stop", "Flash", "conv-stop")
with (
patch("artemis.runtime.is_daemon_running", return_value=True),
patch("artemis.runtime.stop_task_on_daemon", return_value=True) as mock_stop,
):
res = mobile_manage_task(action="stop", trace_id=trace_id)
mock_stop.assert_called_once_with(trace_id)
assert res["status"] == "cancelled"
assert trace_store.read_status(trace_id)["status"] == "cancelled"
+8
View File
@@ -75,3 +75,11 @@ def test_update_trace_status(temp_trace_env):
def test_read_nonexistent_status(temp_trace_env):
non_existent = str(uuid.uuid4())
assert trace_store.read_status(non_existent) is None
def test_trace_paths(temp_trace_env):
trace_id = "test-paths-id"
trace_dir = trace_store.get_trace_dir(trace_id)
assert trace_store.get_trace_notes_dir(trace_id) == f"{trace_dir}/notes"
assert trace_store.get_trace_stdout_log_path(trace_id) == f"{trace_dir}/stdout.log"
assert trace_store.get_trace_stderr_log_path(trace_id) == f"{trace_dir}/stderr.log"
+19
View File
@@ -116,3 +116,22 @@ def test_submit_task_to_daemon():
assert data["ingress"] == "mcp"
assert data["conversation_id"] == "conv-123"
def test_stop_task_on_daemon():
from artemis.runtime.daemon_client import stop_task_on_daemon
mock_resp = MagicMock()
mock_resp.status = 200
mock_resp.read.return_value = b'{"status": "stopped", "session_id": "test-sess-stop"}'
mock_resp.__enter__.return_value = mock_resp
with patch("urllib.request.urlopen", return_value=mock_resp) as mock_urlopen:
assert stop_task_on_daemon("test-sess-stop") is True
req = mock_urlopen.call_args[0][0]
data = json.loads(req.data.decode("utf-8"))
assert data["session_id"] == "test-sess-stop"
mock_resp.read.return_value = b'{"status": "no_running_task"}'
with patch("urllib.request.urlopen", return_value=mock_resp):
assert stop_task_on_daemon("non-existent-sess") is False
+57
View File
@@ -588,3 +588,60 @@ def test_agent_config_environment_variable_overrides(monkeypatch):
assert cfg.outputter.enabled is False
assert cfg.video_analyzer.enable_ledger is False
assert cfg.pro.video_analyzer.enable_ledger is False
def test_consolidated_workspace_and_admin_paths():
"""Verify consolidated workspace and admin paths exported from artemis.config."""
from artemis.config import (
DB_PATH,
IMAGES_DIR,
PAUSE_FILE,
REPLAY_BASE_DIR,
ROOT_DIR,
TEST_DATA_DIR,
TEST_OUTPUTS_DIR,
TRACES_PATH,
WORKSPACE_ROOT,
get_images_dir,
get_pause_file,
get_replay_dir,
get_test_data_dir,
get_test_outputs_dir,
)
assert WORKSPACE_ROOT == ROOT_DIR
assert PAUSE_FILE == get_pause_file()
assert PAUSE_FILE.name == ".artemis_paused"
assert REPLAY_BASE_DIR == get_replay_dir()
assert TEST_DATA_DIR == get_test_data_dir()
assert TEST_OUTPUTS_DIR == get_test_outputs_dir()
assert IMAGES_DIR == get_images_dir()
assert DB_PATH.name == "data_engine.db"
assert TRACES_PATH.name == "traces"
def test_admin_console_config_facade_backward_compatibility():
"""Verify apps.admin_console.core.config continues to re-export consolidated paths."""
from apps.admin_console.core.config import (
DB_PATH,
IMAGES_DIR,
PAUSE_FILE,
REPLAY_BASE_DIR,
TEST_DATA_DIR,
TEST_OUTPUTS_DIR,
TRACES_PATH,
WORKSPACE_ROOT,
init_ls_address,
)
import artemis.config as ac
assert WORKSPACE_ROOT == ac.WORKSPACE_ROOT
assert PAUSE_FILE == ac.PAUSE_FILE
assert REPLAY_BASE_DIR == ac.REPLAY_BASE_DIR
assert TEST_DATA_DIR == ac.TEST_DATA_DIR
assert TEST_OUTPUTS_DIR == ac.TEST_OUTPUTS_DIR
assert IMAGES_DIR == ac.IMAGES_DIR
assert DB_PATH == ac.DB_PATH
assert TRACES_PATH == ac.TRACES_PATH
assert callable(init_ls_address)