feat: add multi-device concurrency, server lifecycle CLI, and unified execution stream
- Implement DevicePool and DeviceExecutionLock concurrency modes (per-device FIFO vs global serialization) - Extend ArtemisClient SDK and CLI with explicit device targeting and concurrency strategy configuration - Add server lifecycle management (artemis restart, stop, status) with cross-platform PID discovery - Update MCP server tools to support multi-device dispatch, trace tagging, and per-device execution - Enhance Showcase UI with step-by-step visual replay, playback speed controls, and device serial badges - Add comprehensive unit test coverage and edge-case verification scripts
This commit is contained in:
1 parent
a36a66ce6d
commit
c2c90a6378
84 files changed
+5612
-645
No files matched your search
@@ -31,25 +31,37 @@ from artemis.utils.visualization import format_minimal_list_with_elements
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
async def mobile_get_device_state(view_type: str) -> str:
|
||||
async def mobile_get_device_state(view_type: str, device_serial: str | None = None) -> str:
|
||||
"""Real-time mobile device state observer (for debugging and validation).
|
||||
|
||||
Use this tool to inspect the latest mobile status or to assist in tracking
|
||||
the subagent's task progress.
|
||||
|
||||
Retrieves a real-time screenshot (returns local file URI) or a simplified
|
||||
element tree text list of the currently connected mobile device.
|
||||
element tree text list of the target mobile device.
|
||||
|
||||
### Device Selection Modes:
|
||||
1. **Direct Device Specification**: Provide `device_serial` to observe a specific
|
||||
phone or emulator (e.g. `device_serial="63191FDKX00062"`).
|
||||
2. **Automatic Device Selection**: Omit `device_serial` or leave it `None` to
|
||||
automatically inspect the default/connected device.
|
||||
|
||||
**Diagnostics & User Preference**:
|
||||
- When multiple devices are attached, prioritize letting the user specify or confirm the device.
|
||||
- Run `adb devices` to check connected devices and identify their serial numbers.
|
||||
|
||||
Args:
|
||||
view_type: The observation type. Supported values: - "screenshot":
|
||||
Captures the real-time mobile screen and saves it to the workspace,
|
||||
returning the local URI path of the image file (e.g.,
|
||||
"file:///path/to/screenshot.jpg"). - "hierarchy": Directly returns the
|
||||
simplified text-labeled element list, which is identical to what the
|
||||
mobile UI automation subagent sees when making decisions.
|
||||
view_type: The observation type. Supported values:
|
||||
- "screenshot": Captures the real-time mobile screen and saves it to the workspace,
|
||||
returning the local URI path of the image file (e.g., "file:///path/to/screenshot.jpg").
|
||||
- "hierarchy": Directly returns the simplified text-labeled element list, which is
|
||||
identical to what the mobile UI automation subagent sees when making decisions.
|
||||
device_serial: Optional Android device serial number (e.g., "63191FDKX00062" or
|
||||
"emulator-5554"). If specified, inspects state from that specific device. If omitted,
|
||||
inspects the default connected device.
|
||||
"""
|
||||
try:
|
||||
controller = _get_controller()
|
||||
controller = _get_controller(device_serial=device_serial)
|
||||
device_width = controller.ctx.device.device_width
|
||||
device_height = controller.ctx.device.device_height
|
||||
except Exception as e:
|
||||
|
||||
@@ -115,20 +115,21 @@ async def mobile_inspect_trace(action: str, trace_id: str, step_number: int | No
|
||||
|
||||
Use this tool DURING a running task or AFTER it has finished (or failed) to
|
||||
monitor progress, verify answers, diagnose agent errors, and debug.
|
||||
It provides comprehensive runtime information, raw agent outputs, and visual
|
||||
screenshots for both **Flash** and **Pro** tasks.
|
||||
It provides comprehensive runtime information (including the assigned
|
||||
`device_serial`), raw agent outputs, and visual screenshots for both
|
||||
**Flash** and **Pro** tasks.
|
||||
|
||||
### Available Actions:
|
||||
- **'view_summary'**: Retrieves a high-level execution summary across all
|
||||
steps.
|
||||
steps, including the target `device_serial`.
|
||||
* **Pro Model**: Displays a hierarchical task plan summary, showing the
|
||||
status and descriptions of all completed, active, and pending steps.
|
||||
status, device serial, and descriptions of all completed, active, and pending steps.
|
||||
* **Flash Model**: Displays an all-in-one execution chain containing the
|
||||
complete reasoning (thoughts/motivation) and action taken (`click`,
|
||||
complete reasoning (thoughts/motivation), device serial, and action taken (`click`,
|
||||
`input_text`, etc.) for every single step.
|
||||
|
||||
- **'view_step_screenshots'**: Retrieves local file paths of screenshots for
|
||||
a specific step (supports both Flash and Pro). Returns:
|
||||
a specific step (supports both Flash and Pro), along with `device_serial`. Returns:
|
||||
1. `before_screenshot`: The original device screenshot observed by the
|
||||
agent right before taking the action.
|
||||
2. `after_screenshot`: A post-action screenshot capturing the failure or
|
||||
@@ -139,9 +140,10 @@ async def mobile_inspect_trace(action: str, trace_id: str, step_number: int | No
|
||||
action visually marked (e.g., red circle for taps, red arrow for
|
||||
swipes). CRITICAL for verifying if the agent tapped the correct UI
|
||||
element.
|
||||
4. `device_serial`: The serial of the device on which this step was executed.
|
||||
|
||||
- **'view_step_details'**: Retrieves deep runtime logs, raw VLM thoughts,
|
||||
and exact actions for a specific step.
|
||||
and exact actions for a specific step, along with `device_serial`.
|
||||
* **Pro Model**: Fully supported. Returns detailed per-step diagnostic
|
||||
data.
|
||||
* **Flash Model**: DO NOT USE. (Flash reasoning and actions are already
|
||||
@@ -242,11 +244,34 @@ async def mobile_inspect_trace(action: str, trace_id: str, step_number: int | No
|
||||
|
||||
status_data = trace_store.read_status(trace_id)
|
||||
is_flash = status_data and status_data.get("model", "").lower() == "flash"
|
||||
device_serial = status_data.get("device_serial") if status_data else None
|
||||
if not device_serial and os.path.exists(db_path):
|
||||
try:
|
||||
conn_dev = sqlite3.connect(db_path)
|
||||
conn_dev.row_factory = sqlite3.Row
|
||||
cur_dev = conn_dev.cursor()
|
||||
cur_dev.execute(
|
||||
"SELECT device_info FROM sessions WHERE session_id = ? LIMIT 1",
|
||||
(trace_id,),
|
||||
)
|
||||
row_dev = cur_dev.fetchone()
|
||||
if row_dev and row_dev["device_info"]:
|
||||
try:
|
||||
d_info = json.loads(row_dev["device_info"])
|
||||
if isinstance(d_info, dict) and d_info.get("device_id"):
|
||||
device_serial = d_info["device_id"]
|
||||
except Exception:
|
||||
pass
|
||||
conn_dev.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
device_info_str = f" | **Device Serial:** `{device_serial}`" if device_serial else ""
|
||||
|
||||
if is_flash:
|
||||
summary_lines = [
|
||||
"# ARTEMIS Flash Execution Summary",
|
||||
f"**Session ID:** `{trace_id}` | **Model:** Flash\n",
|
||||
f"**Session ID:** `{trace_id}` | **Model:** Flash{device_info_str}\n",
|
||||
"---",
|
||||
"## Step-by-Step Execution Chain\n",
|
||||
]
|
||||
@@ -296,6 +321,8 @@ async def mobile_inspect_trace(action: str, trace_id: str, step_number: int | No
|
||||
last_n_detailed=0,
|
||||
all_detailed=False,
|
||||
)
|
||||
if device_serial and not summary_markdown.startswith("**Session ID:"):
|
||||
summary_markdown = f"**Session ID:** `{trace_id}` | **Model:** Pro{device_info_str}\n\n" + summary_markdown
|
||||
return summary_markdown
|
||||
except Exception as e:
|
||||
return {
|
||||
@@ -307,6 +334,7 @@ async def mobile_inspect_trace(action: str, trace_id: str, step_number: int | No
|
||||
elif action in ("view_step_screenshots", "view_step_details"):
|
||||
status_data = trace_store.read_status(trace_id)
|
||||
is_flash = status_data and status_data.get("model", "").lower() == "flash"
|
||||
device_serial = status_data.get("device_serial") if status_data else None
|
||||
|
||||
if action == "view_step_details" and is_flash:
|
||||
return {
|
||||
@@ -426,6 +454,7 @@ async def mobile_inspect_trace(action: str, trace_id: str, step_number: int | No
|
||||
|
||||
return {
|
||||
"trace_id": trace_id,
|
||||
"device_serial": device_serial,
|
||||
"step_number": step_dict["step_number"],
|
||||
"before_screenshot": pre_image,
|
||||
"after_screenshot": post_image,
|
||||
@@ -451,6 +480,7 @@ async def mobile_inspect_trace(action: str, trace_id: str, step_number: int | No
|
||||
|
||||
return {
|
||||
"trace_id": trace_id,
|
||||
"device_serial": device_serial,
|
||||
"step_number": step_dict["step_number"],
|
||||
"details": rendered_text,
|
||||
}
|
||||
|
||||
@@ -43,7 +43,16 @@ def mobile_manage_task(
|
||||
|
||||
### Available Actions:
|
||||
- **'status'**: Retrieves execution progress, elapsed time, and metadata.
|
||||
Returns a JSON object.
|
||||
Returns a JSON object containing:
|
||||
* `trace_id`: Unique identifier for the task.
|
||||
* `status`: Task lifecycle state ('running', 'completed', 'failed', 'cancelled').
|
||||
* `device_serial`: The Android device serial number assigned to/running this task
|
||||
(essential in multi-device environments to know which phone this task belongs to).
|
||||
* `task_desc`: The goal/description of the task.
|
||||
* `model`: 'Flash' or 'Pro'.
|
||||
* `elapsed_seconds`: Execution duration in seconds.
|
||||
* `progress`: Live progress data (current turn, latest thoughts and actions for Flash;
|
||||
active task plan for Pro).
|
||||
* **How to use the output**:
|
||||
- For **Pro Model**, check `progress.task_plan` to see if the
|
||||
agent's long-term plan aligns with your goal.
|
||||
@@ -113,9 +122,51 @@ def mobile_manage_task(
|
||||
end_time = status_data.get("end_time") or time.time()
|
||||
elapsed = round(end_time - start_time, 1) if start_time else 0
|
||||
|
||||
project_root = env_utils.get_project_root()
|
||||
db_path = os.path.join(trace_store.TRACES_DIR, "data_engine.db")
|
||||
if not os.path.exists(db_path):
|
||||
db_path = os.path.join(project_root, "traces", "data_engine.db")
|
||||
|
||||
device_serial = status_data.get("device_serial")
|
||||
if not device_serial and os.path.exists(db_path):
|
||||
try:
|
||||
conn_dev = sqlite3.connect(db_path)
|
||||
conn_dev.row_factory = sqlite3.Row
|
||||
cur_dev = conn_dev.cursor()
|
||||
cur_dev.execute(
|
||||
"SELECT device_info FROM sessions WHERE session_id = ? OR pid = ? ORDER BY start_time DESC LIMIT 1",
|
||||
(trace_id, pid),
|
||||
)
|
||||
row_dev = cur_dev.fetchone()
|
||||
if row_dev and row_dev["device_info"]:
|
||||
try:
|
||||
d_info = json.loads(row_dev["device_info"])
|
||||
if isinstance(d_info, dict) and d_info.get("device_id"):
|
||||
device_serial = d_info["device_id"]
|
||||
status_data["device_serial"] = device_serial
|
||||
trace_store.write_status(trace_id, status_data)
|
||||
except Exception:
|
||||
pass
|
||||
conn_dev.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not device_serial and pid:
|
||||
try:
|
||||
owners = DeviceExecutionLock.get_active_owners()
|
||||
for clean_id, owner in owners.items():
|
||||
if owner.pid == pid or owner.session_id == trace_id:
|
||||
device_serial = owner.device_id
|
||||
status_data["device_serial"] = device_serial
|
||||
trace_store.write_status(trace_id, status_data)
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
response: dict[str, Any] = {
|
||||
"trace_id": trace_id,
|
||||
"status": current_status,
|
||||
"device_serial": device_serial,
|
||||
"elapsed_seconds": elapsed,
|
||||
"task_desc": status_data.get("task_desc"),
|
||||
"model": status_data.get("model"),
|
||||
|
||||
@@ -32,6 +32,7 @@ def mobile_run_task(
|
||||
locked_app_package: str | None = None,
|
||||
app_path: str | None = None,
|
||||
expected_output_desc: str | None = None,
|
||||
device_serial: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Starts a specialized mobile UI automation subagent to autonomously plan and execute tasks on a connected Android device.
|
||||
|
||||
@@ -41,6 +42,23 @@ def mobile_run_task(
|
||||
straightforward UI scripts) and **Pro** (for complex, exploratory tasks
|
||||
requiring deep reasoning).
|
||||
|
||||
### Device Selection & Multi-Device Support (CRITICAL)
|
||||
ARTEMIS supports multi-device execution and provides two device selection modes:
|
||||
1. **Direct Device Specification**: Specify `device_serial` explicitly
|
||||
(e.g., `device_serial="63191FDKX00062"` or `device_serial="emulator-5554"`).
|
||||
Tasks targeting different devices run concurrently via per-device locking.
|
||||
2. **Automatic Device Selection**: If `device_serial` is omitted or `None`,
|
||||
ARTEMIS will automatically select the connected device or allocate an idle device
|
||||
from the device pool.
|
||||
|
||||
**Device Selection Policy & Diagnostics**:
|
||||
- **Prioritize User Choice**: When multiple devices or emulators are connected (or when
|
||||
user intent is not explicitly specified), **ALWAYS PRIORITIZE ASKING THE USER** to
|
||||
choose or confirm the target device before launching tasks.
|
||||
- **Device Diagnosis with `adb devices`**: Always use `adb devices` (or `adb devices -l`)
|
||||
to inspect connected hardware, verify authorization states (`device` vs `unauthorized`),
|
||||
and retrieve active device serials when diagnosing or before delegating tasks.
|
||||
|
||||
### Model Selection Guide (CRITICAL FOR ROUTING)
|
||||
You MUST choose the appropriate `model` based on the task complexity:
|
||||
- **Flash**: Best for simple, deterministic tasks that can be completed
|
||||
@@ -81,6 +99,7 @@ def mobile_run_task(
|
||||
containing:
|
||||
- `trace_id`: The unique session identifier of the task for status tracking
|
||||
and inspection.
|
||||
- `device_serial`: The assigned target device serial (or "auto-select").
|
||||
- `notes_dir`: (Pro Model Only) The directory where the subagent's execution
|
||||
files/notes are located.
|
||||
- `stdout_log`: The path to the execution framework log.
|
||||
@@ -120,6 +139,11 @@ def mobile_run_task(
|
||||
summarization agent will aggregate the interaction history
|
||||
(motivation, actions, videos) into a summary report saved as
|
||||
"output.md" in `notes_dir`. Ignored if `model="Flash"`.
|
||||
device_serial: Optional Android device serial number (e.g. "63191FDKX00062"
|
||||
or "emulator-5554"). If specified, binds execution strictly to that device,
|
||||
allowing multi-device concurrent automation. If omitted, ARTEMIS will
|
||||
automatically select an available device. Prioritize letting the user select
|
||||
the target device if multiple devices are attached.
|
||||
"""
|
||||
# 0. Validate and normalize model
|
||||
if model.lower() not in ("flash", "pro"):
|
||||
@@ -135,18 +159,31 @@ def mobile_run_task(
|
||||
task_desc=task_desc,
|
||||
model=canonical_model,
|
||||
conversation_id=conversation_id,
|
||||
device_serial=device_serial,
|
||||
)
|
||||
|
||||
# 3. Resolve project root, python executable, and background runner module
|
||||
# 3. Auto-spawn daemon if not already running (unless standalone forced)
|
||||
if os.environ.get("ARTEMIS_STANDALONE") != "1":
|
||||
try:
|
||||
from artemis.runtime import ensure_daemon_running
|
||||
|
||||
ensure_daemon_running(timeout=1.5)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 4. Resolve project root, python executable, and background runner module
|
||||
project_root = env_utils.get_project_root()
|
||||
python_exe = env_utils.resolve_python_executable(project_root)
|
||||
queue_ticket = DeviceExecutionLock.reserve(
|
||||
description=f"MCP task: {task_desc[:120]}",
|
||||
session_id=trace_id,
|
||||
ingress="mcp",
|
||||
)
|
||||
reserve_kwargs: dict[str, Any] = {
|
||||
"description": f"MCP task: {task_desc[:120]}",
|
||||
"session_id": trace_id,
|
||||
"ingress": "mcp",
|
||||
}
|
||||
if device_serial:
|
||||
reserve_kwargs["device_id"] = device_serial
|
||||
queue_ticket = DeviceExecutionLock.reserve(**reserve_kwargs)
|
||||
|
||||
# 4. Spawn the background task runner as an independent subprocess
|
||||
# 5. Spawn the background task runner as an independent subprocess
|
||||
try:
|
||||
cmd = [
|
||||
python_exe,
|
||||
@@ -167,10 +204,15 @@ def mobile_run_task(
|
||||
cmd.extend(["--app-path", app_path])
|
||||
if expected_output_desc and canonical_model != "Flash":
|
||||
cmd.extend(["--expected-output-desc", expected_output_desc])
|
||||
if device_serial:
|
||||
cmd.extend(["--device-serial", device_serial])
|
||||
|
||||
env = os.environ.copy()
|
||||
env["ARTEMIS_SESSION_ID"] = trace_id
|
||||
env["ARTEMIS_TASK_INGRESS"] = "mcp"
|
||||
if device_serial:
|
||||
env["ADB_DEVICE_SERIAL"] = device_serial
|
||||
env["ARTEMIS_DEVICE_ID"] = device_serial
|
||||
env[DeviceExecutionLock.QUEUE_TICKET_ENV] = queue_ticket
|
||||
|
||||
proc_kwargs = env_utils.get_detached_process_kwargs()
|
||||
@@ -180,12 +222,18 @@ def mobile_run_task(
|
||||
cwd=project_root,
|
||||
**proc_kwargs,
|
||||
)
|
||||
transfer_kwargs: dict[str, Any] = {
|
||||
"description": f"MCP task: {task_desc[:120]}",
|
||||
"session_id": trace_id,
|
||||
"ingress": "mcp",
|
||||
}
|
||||
if device_serial:
|
||||
transfer_kwargs["device_id"] = device_serial
|
||||
|
||||
DeviceExecutionLock.transfer_reservation(
|
||||
queue_ticket,
|
||||
proc.pid,
|
||||
description=f"MCP task: {task_desc[:120]}",
|
||||
session_id=trace_id,
|
||||
ingress="mcp",
|
||||
**transfer_kwargs,
|
||||
)
|
||||
|
||||
# 5. Record the PID of the spawned subprocess
|
||||
@@ -201,6 +249,7 @@ def mobile_run_task(
|
||||
|
||||
response_dict: dict[str, Any] = {
|
||||
"trace_id": trace_id,
|
||||
"device_serial": device_serial or "auto-select",
|
||||
"message": "Successfully started background task.",
|
||||
"stdout_log": stdout_log_path,
|
||||
"stderr_log": stderr_log_path,
|
||||
|
||||
Reference in new issue
Block a user