feat: add multi-device concurrency, server lifecycle CLI, and unified execution stream
- Implement DevicePool and DeviceExecutionLock concurrency modes (per-device FIFO vs global serialization) - Extend ArtemisClient SDK and CLI with explicit device targeting and concurrency strategy configuration - Add server lifecycle management (artemis restart, stop, status) with cross-platform PID discovery - Update MCP server tools to support multi-device dispatch, trace tagging, and per-device execution - Enhance Showcase UI with step-by-step visual replay, playback speed controls, and device serial badges - Add comprehensive unit test coverage and edge-case verification scripts
This commit is contained in:
1 parent
a36a66ce6d
commit
c2c90a6378
84 files changed
+5612
-645
No files matched your search
+11
-4
@@ -68,10 +68,17 @@ When you run `uv run artemis mcp --install all` (or target a specific IDE like `
|
||||
|
||||
## 🛠️ MCP Tools Overview
|
||||
|
||||
* **`mobile_run_task`**: Asynchronously launches an autonomous mobile automation task (`Flash` or `Pro` model).
|
||||
* **`mobile_manage_task`**: Manages task lifecycle (`status`, `stop`, `inject_instruction`).
|
||||
* **`mobile_get_device_state`**: Real-time observer (`screenshot` or OCR+XML `hierarchy`).
|
||||
* **`mobile_inspect_trace`**: Granular trace inspection, visual action overlays, and agent reasoning.
|
||||
* **`mobile_run_task`**: Asynchronously launches an autonomous mobile automation task (`Flash` or `Pro` model) with optional `device_serial` targeting.
|
||||
* **`mobile_manage_task`**: Manages task lifecycle (`status`, `stop`, `inject_instruction`), returning task state and assigned `device_serial`.
|
||||
* **`mobile_get_device_state`**: Real-time observer (`screenshot` or OCR+XML `hierarchy`) with optional `device_serial`.
|
||||
* **`mobile_inspect_trace`**: Granular trace inspection, visual action overlays, agent reasoning, and `device_serial` tracking.
|
||||
|
||||
### 📱 Device Selection & Multi-Device Execution
|
||||
ARTEMIS supports parallel execution across multiple connected Android devices and emulators:
|
||||
1. **Direct Device Specification**: Provide `device_serial` explicitly (e.g. `device_serial="63191FDKX00062"` or `device_serial="emulator-5554"`). Per-device execution locks allow distinct devices to run in parallel.
|
||||
2. **Automatic Device Selection**: Omit `device_serial` to let ARTEMIS auto-select an available ready device from the device pool.
|
||||
* **User Choice Priority**: When multiple devices/emulators are connected, AI agents should prioritize asking the user which device to run on.
|
||||
* **Device Diagnosis**: Inspect connected hardware, authorization status, and serials anytime via `adb devices` or `adb devices -l`.
|
||||
|
||||
## 🚀 How to Run & Configure
|
||||
|
||||
|
||||
@@ -114,6 +114,7 @@ async def run_task(
|
||||
locked_app_package: str | None = None,
|
||||
app_path: str | None = None,
|
||||
expected_output_desc: str | None = None,
|
||||
device_serial: str | None = None,
|
||||
):
|
||||
"""Executes the mobile automation agent task and logs all actions/results."""
|
||||
trace_dir = trace_store.get_trace_dir(trace_id)
|
||||
@@ -142,6 +143,7 @@ async def run_task(
|
||||
|
||||
agent = None
|
||||
adb_path = device_utils.resolve_adb_path()
|
||||
target_serial = device_serial
|
||||
|
||||
try:
|
||||
from artemis.sdk import Agent
|
||||
@@ -150,15 +152,36 @@ async def run_task(
|
||||
|
||||
connected_devices = device_utils.get_connected_devices(adb_path)
|
||||
|
||||
if connected_devices:
|
||||
print(
|
||||
f"✅ Detected active connected device(s): {connected_devices}. "
|
||||
f"Using the first available device: '{connected_devices[0]}'."
|
||||
)
|
||||
if device_serial:
|
||||
target_serial = device_serial
|
||||
if connected_devices and device_serial not in connected_devices:
|
||||
print(
|
||||
f"⚠️ Warning: Specified device serial '{device_serial}' was not detected in active ADB devices: {connected_devices}. "
|
||||
"Proceeding with target serial (will attempt direct ADB connection)..."
|
||||
)
|
||||
else:
|
||||
print(f"✅ Using specified target device: '{device_serial}'.")
|
||||
else:
|
||||
print("❌ No active connected devices detected. Booting emulator...")
|
||||
if not device_utils.ensure_emulator(adb_path=adb_path):
|
||||
raise RuntimeError("Failed to start or connect to the Android emulator.")
|
||||
if connected_devices:
|
||||
target_serial = settings.ADB_DEVICE_SERIAL or os.environ.get("ADB_DEVICE_SERIAL")
|
||||
if not target_serial:
|
||||
try:
|
||||
from artemis.runtime import device_pool
|
||||
target_serial = device_pool.select_device()
|
||||
except Exception:
|
||||
target_serial = connected_devices[0]
|
||||
print(
|
||||
f"✅ Detected active connected device(s): {connected_devices}. "
|
||||
f"Auto-selected device: '{target_serial}'."
|
||||
)
|
||||
else:
|
||||
print("❌ No active connected devices detected. Booting emulator...")
|
||||
if not device_utils.ensure_emulator(adb_path=adb_path):
|
||||
raise RuntimeError("Failed to start or connect to the Android emulator.")
|
||||
target_serial = "emulator-5554"
|
||||
|
||||
if target_serial:
|
||||
trace_store.update_trace_device_serial(trace_id, target_serial)
|
||||
|
||||
print("Initializing Artemis Agent...")
|
||||
from artemis.config import initialize_llm_config, settings
|
||||
@@ -173,6 +196,10 @@ async def run_task(
|
||||
if settings.ADB_HOST:
|
||||
config_builder.with_adb_server(host=settings.ADB_HOST, port=settings.ADB_PORT)
|
||||
|
||||
if target_serial:
|
||||
from artemis.context import DevicePlatform
|
||||
config_builder.for_device(DevicePlatform.ANDROID, target_serial)
|
||||
|
||||
config = config_builder.build()
|
||||
|
||||
agent = Agent(config=config)
|
||||
@@ -183,6 +210,12 @@ async def run_task(
|
||||
timeout_seconds=float(os.getenv("ARTEMIS_AGENT_INIT_TIMEOUT_SECONDS", 30)),
|
||||
)
|
||||
|
||||
actual_serial = getattr(getattr(agent, "_device_context", None), "device_id", None) or target_serial
|
||||
if actual_serial:
|
||||
target_serial = actual_serial
|
||||
trace_store.update_trace_device_serial(trace_id, actual_serial)
|
||||
print(f"📱 Bound to device serial: '{actual_serial}'")
|
||||
|
||||
print("Running task on agent...")
|
||||
task_builder = (
|
||||
agent.new_task(goal=task_desc)
|
||||
@@ -201,17 +234,20 @@ async def run_task(
|
||||
result = await agent.run_task(request=task_builder.build())
|
||||
print(f"Task completed. Result: {result}")
|
||||
|
||||
device_info_line = f"Device Serial: `{target_serial}`\n" if target_serial else ""
|
||||
|
||||
if isinstance(result, dict) and "status" in result and result.get("status") != "completed":
|
||||
error_explanation = result.get("explanation", "Task execution returned failed status.")
|
||||
print(f"Task finished with non-completed status: {error_explanation}", file=sys.stderr)
|
||||
trace_store.update_trace_status(
|
||||
trace_id, "failed", error=error_explanation, result=result
|
||||
trace_id, "failed", error=error_explanation, result=result, device_serial=target_serial
|
||||
)
|
||||
|
||||
formatted_result = json.dumps(result, indent=2, ensure_ascii=False)
|
||||
failure_msg = (
|
||||
f"Artemis background task finished with non-completed status.\n\n"
|
||||
f"Trace ID: {trace_id}\n"
|
||||
f"{device_info_line}"
|
||||
f"Goal: {task_desc}\n"
|
||||
f"Explanation: {error_explanation}\n\n"
|
||||
f"Result:\n```json\n{formatted_result}\n```\n\n"
|
||||
@@ -223,7 +259,7 @@ async def run_task(
|
||||
message=failure_msg,
|
||||
title="Artemis Task Incomplete",
|
||||
event_type="failed",
|
||||
payload={"trace_id": trace_id, "goal": task_desc, "result": result},
|
||||
payload={"trace_id": trace_id, "device_serial": target_serial, "goal": task_desc, "result": result},
|
||||
)
|
||||
return
|
||||
|
||||
@@ -236,7 +272,7 @@ async def run_task(
|
||||
"notes_dir to view more details."
|
||||
)
|
||||
|
||||
trace_store.update_trace_status(trace_id, "completed", result=result)
|
||||
trace_store.update_trace_status(trace_id, "completed", result=result, device_serial=target_serial)
|
||||
|
||||
if isinstance(result, dict):
|
||||
formatted_result = json.dumps(result, indent=2, ensure_ascii=False)
|
||||
@@ -248,6 +284,7 @@ async def run_task(
|
||||
success_msg = (
|
||||
"Artemis background task completed successfully.\n\n"
|
||||
f"Trace ID: {trace_id}\n"
|
||||
f"{device_info_line}"
|
||||
f"Goal: {task_desc}\n"
|
||||
f"{result_str}"
|
||||
)
|
||||
@@ -256,21 +293,22 @@ async def run_task(
|
||||
message=success_msg,
|
||||
title="Artemis Task Completed",
|
||||
event_type="completed",
|
||||
payload={"trace_id": trace_id, "goal": task_desc, "result": result},
|
||||
payload={"trace_id": trace_id, "device_serial": target_serial, "goal": task_desc, "result": result},
|
||||
)
|
||||
|
||||
except asyncio.CancelledError:
|
||||
print("Task was cancelled (asyncio.CancelledError)", file=sys.stderr)
|
||||
trace_store.update_trace_status(trace_id, "cancelled", error="Task was cancelled")
|
||||
trace_store.update_trace_status(trace_id, "cancelled", error="Task was cancelled", device_serial=target_serial)
|
||||
device_info_line = f"Device Serial: `{target_serial}`\n" if target_serial else ""
|
||||
cancel_msg = (
|
||||
f"Artemis background task was cancelled.\n\nTrace ID: {trace_id}\nGoal: {task_desc}\n"
|
||||
f"Artemis background task was cancelled.\n\nTrace ID: {trace_id}\n{device_info_line}Goal: {task_desc}\n"
|
||||
)
|
||||
notify(
|
||||
conversation_id=conversation_id,
|
||||
message=cancel_msg,
|
||||
title="Artemis Task Cancelled",
|
||||
event_type="cancelled",
|
||||
payload={"trace_id": trace_id, "goal": task_desc},
|
||||
payload={"trace_id": trace_id, "device_serial": target_serial, "goal": task_desc},
|
||||
)
|
||||
raise
|
||||
|
||||
@@ -289,11 +327,13 @@ async def run_task(
|
||||
frame_summary = f" (at {tb.tb_frame.f_code.co_name}:{tb.tb_lineno})" if tb else ""
|
||||
full_error_desc = f"{error_type}: {raw_error_msg}{frame_summary}"
|
||||
|
||||
trace_store.update_trace_status(trace_id, "failed", error=full_error_desc)
|
||||
trace_store.update_trace_status(trace_id, "failed", error=full_error_desc, device_serial=target_serial)
|
||||
|
||||
device_info_line = f"Device Serial: `{target_serial}`\n" if target_serial else ""
|
||||
failure_msg = (
|
||||
f"Artemis background task failed.\n\n"
|
||||
f"Trace ID: {trace_id}\n"
|
||||
f"{device_info_line}"
|
||||
f"Goal: {task_desc}\n"
|
||||
f"Error: {full_error_desc}\n\n"
|
||||
f'Check the logs using mobile_inspect_trace(action="view_summary", trace_id="{trace_id}") '
|
||||
@@ -304,7 +344,7 @@ async def run_task(
|
||||
message=failure_msg,
|
||||
title="Artemis Task Failed",
|
||||
event_type="failed",
|
||||
payload={"trace_id": trace_id, "goal": task_desc, "error": full_error_desc},
|
||||
payload={"trace_id": trace_id, "device_serial": target_serial, "goal": task_desc, "error": full_error_desc},
|
||||
)
|
||||
|
||||
finally:
|
||||
@@ -330,6 +370,7 @@ if __name__ == "__main__":
|
||||
parser.add_argument("--locked-app-package", help="Package name of app to lock execution to")
|
||||
parser.add_argument("--app-path", help="Path to local APK to install before running task")
|
||||
parser.add_argument("--expected-output-desc", help="Expected output description")
|
||||
parser.add_argument("--device-serial", help="Target specific device serial")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -342,5 +383,6 @@ if __name__ == "__main__":
|
||||
locked_app_package=args.locked_app_package,
|
||||
app_path=args.app_path,
|
||||
expected_output_desc=args.expected_output_desc,
|
||||
device_serial=args.device_serial,
|
||||
)
|
||||
)
|
||||
+11
-4
@@ -29,10 +29,17 @@ Deeply understand and select between ARTEMIS's dual execution models (**ARTEMIS
|
||||
- **Safety Net & Self-Healing (Failure Analyzer)**: Every action is vetted by an execution safety net. If an action fails or is blocked, the **Failure Analyzer** is triggered to run local step-by-step recovery sequences, achieving autonomous self-healing. This loop continues iteratively until the task terminates.
|
||||
- **Outputter (Optional)**: Synthesizes the entire execution trace into a human-readable report detailing every action step and visual result.
|
||||
|
||||
### 3. Device & Environment Constraints
|
||||
- **ADB & File Transfer**: While ARTEMIS excels at device automation, it operates within the device boundary. To extract diagnostic files, logs, or test artifacts from the mobile device to the host PC for further analysis, you must manually write and execute appropriate `adb` commands (e.g., `adb pull`).
|
||||
- **Hardware Prerequisites**: Running ARTEMIS requires a physically connected, fully authorized Android device (e.g., a Pixel phone) or an active emulator.
|
||||
- **Single-Device Policy**: ARTEMIS can only operate a single device at any given time. Ensure your workflows and code respect this concurrency limit.
|
||||
### 3. Device & Environment Constraints & Multi-Device Management
|
||||
- **Device Selection & Multi-Device Execution**: ARTEMIS supports multi-device execution and per-device concurrency. You can control device targeting via two modes:
|
||||
- **Direct Device Specification**: Explicitly provide the target phone's serial number via `device_serial` to `mobile_run_task` or `mobile_get_device_state`. Tasks targeting distinct devices run concurrently without blocking each other.
|
||||
- **Automatic Device Selection**: When `device_serial` is omitted or set to `None`, ARTEMIS automatically selects an available connected device or allocates an idle device from the device pool.
|
||||
- **Prioritize User Choice for Device Selection**:
|
||||
- When multiple connected devices or emulators are detected, or whenever device selection is ambiguous, **YOU MUST PRIORITIZE ASKING THE USER** to select or confirm their preferred device serial before launching a task.
|
||||
- **Device Diagnosis with `adb devices`**:
|
||||
- Use `adb devices` (or `adb devices -l`) via bash command execution to diagnose attached hardware, inspect connection status (`device`, `unauthorized`, `offline`), and retrieve device serials and models whenever preparing tasks or troubleshooting device issues.
|
||||
- **ADB & File Transfer**: While ARTEMIS excels at device automation, it operates within the device boundary. To extract diagnostic files, logs, or test artifacts from the mobile device to the host PC for further analysis, you must manually execute appropriate `adb` commands (e.g., `adb -s <serial> pull ...`).
|
||||
- **Hardware Prerequisites**: Running ARTEMIS requires at least one physically connected, fully authorized Android device (e.g., a Pixel phone) or an active emulator.
|
||||
- **Per-Device Mutual Exclusion**: ARTEMIS manages per-device execution mutexes (`DeviceExecutionLock`). A device can only execute a single task at a time (FIFO queue), while different devices can execute tasks in parallel.
|
||||
|
||||
### 4. Robust Test Code Design (The "Dynamic-First, Coordinate-Fallback" Philosophy)
|
||||
*If your task involves authoring test code, you must adhere to the following design principles for maximum reliability:*
|
||||
|
||||
@@ -31,25 +31,37 @@ from artemis.utils.visualization import format_minimal_list_with_elements
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
async def mobile_get_device_state(view_type: str) -> str:
|
||||
async def mobile_get_device_state(view_type: str, device_serial: str | None = None) -> str:
|
||||
"""Real-time mobile device state observer (for debugging and validation).
|
||||
|
||||
Use this tool to inspect the latest mobile status or to assist in tracking
|
||||
the subagent's task progress.
|
||||
|
||||
Retrieves a real-time screenshot (returns local file URI) or a simplified
|
||||
element tree text list of the currently connected mobile device.
|
||||
element tree text list of the target mobile device.
|
||||
|
||||
### Device Selection Modes:
|
||||
1. **Direct Device Specification**: Provide `device_serial` to observe a specific
|
||||
phone or emulator (e.g. `device_serial="63191FDKX00062"`).
|
||||
2. **Automatic Device Selection**: Omit `device_serial` or leave it `None` to
|
||||
automatically inspect the default/connected device.
|
||||
|
||||
**Diagnostics & User Preference**:
|
||||
- When multiple devices are attached, prioritize letting the user specify or confirm the device.
|
||||
- Run `adb devices` to check connected devices and identify their serial numbers.
|
||||
|
||||
Args:
|
||||
view_type: The observation type. Supported values: - "screenshot":
|
||||
Captures the real-time mobile screen and saves it to the workspace,
|
||||
returning the local URI path of the image file (e.g.,
|
||||
"file:///path/to/screenshot.jpg"). - "hierarchy": Directly returns the
|
||||
simplified text-labeled element list, which is identical to what the
|
||||
mobile UI automation subagent sees when making decisions.
|
||||
view_type: The observation type. Supported values:
|
||||
- "screenshot": Captures the real-time mobile screen and saves it to the workspace,
|
||||
returning the local URI path of the image file (e.g., "file:///path/to/screenshot.jpg").
|
||||
- "hierarchy": Directly returns the simplified text-labeled element list, which is
|
||||
identical to what the mobile UI automation subagent sees when making decisions.
|
||||
device_serial: Optional Android device serial number (e.g., "63191FDKX00062" or
|
||||
"emulator-5554"). If specified, inspects state from that specific device. If omitted,
|
||||
inspects the default connected device.
|
||||
"""
|
||||
try:
|
||||
controller = _get_controller()
|
||||
controller = _get_controller(device_serial=device_serial)
|
||||
device_width = controller.ctx.device.device_width
|
||||
device_height = controller.ctx.device.device_height
|
||||
except Exception as e:
|
||||
|
||||
@@ -115,20 +115,21 @@ async def mobile_inspect_trace(action: str, trace_id: str, step_number: int | No
|
||||
|
||||
Use this tool DURING a running task or AFTER it has finished (or failed) to
|
||||
monitor progress, verify answers, diagnose agent errors, and debug.
|
||||
It provides comprehensive runtime information, raw agent outputs, and visual
|
||||
screenshots for both **Flash** and **Pro** tasks.
|
||||
It provides comprehensive runtime information (including the assigned
|
||||
`device_serial`), raw agent outputs, and visual screenshots for both
|
||||
**Flash** and **Pro** tasks.
|
||||
|
||||
### Available Actions:
|
||||
- **'view_summary'**: Retrieves a high-level execution summary across all
|
||||
steps.
|
||||
steps, including the target `device_serial`.
|
||||
* **Pro Model**: Displays a hierarchical task plan summary, showing the
|
||||
status and descriptions of all completed, active, and pending steps.
|
||||
status, device serial, and descriptions of all completed, active, and pending steps.
|
||||
* **Flash Model**: Displays an all-in-one execution chain containing the
|
||||
complete reasoning (thoughts/motivation) and action taken (`click`,
|
||||
complete reasoning (thoughts/motivation), device serial, and action taken (`click`,
|
||||
`input_text`, etc.) for every single step.
|
||||
|
||||
- **'view_step_screenshots'**: Retrieves local file paths of screenshots for
|
||||
a specific step (supports both Flash and Pro). Returns:
|
||||
a specific step (supports both Flash and Pro), along with `device_serial`. Returns:
|
||||
1. `before_screenshot`: The original device screenshot observed by the
|
||||
agent right before taking the action.
|
||||
2. `after_screenshot`: A post-action screenshot capturing the failure or
|
||||
@@ -139,9 +140,10 @@ async def mobile_inspect_trace(action: str, trace_id: str, step_number: int | No
|
||||
action visually marked (e.g., red circle for taps, red arrow for
|
||||
swipes). CRITICAL for verifying if the agent tapped the correct UI
|
||||
element.
|
||||
4. `device_serial`: The serial of the device on which this step was executed.
|
||||
|
||||
- **'view_step_details'**: Retrieves deep runtime logs, raw VLM thoughts,
|
||||
and exact actions for a specific step.
|
||||
and exact actions for a specific step, along with `device_serial`.
|
||||
* **Pro Model**: Fully supported. Returns detailed per-step diagnostic
|
||||
data.
|
||||
* **Flash Model**: DO NOT USE. (Flash reasoning and actions are already
|
||||
@@ -242,11 +244,34 @@ async def mobile_inspect_trace(action: str, trace_id: str, step_number: int | No
|
||||
|
||||
status_data = trace_store.read_status(trace_id)
|
||||
is_flash = status_data and status_data.get("model", "").lower() == "flash"
|
||||
device_serial = status_data.get("device_serial") if status_data else None
|
||||
if not device_serial and os.path.exists(db_path):
|
||||
try:
|
||||
conn_dev = sqlite3.connect(db_path)
|
||||
conn_dev.row_factory = sqlite3.Row
|
||||
cur_dev = conn_dev.cursor()
|
||||
cur_dev.execute(
|
||||
"SELECT device_info FROM sessions WHERE session_id = ? LIMIT 1",
|
||||
(trace_id,),
|
||||
)
|
||||
row_dev = cur_dev.fetchone()
|
||||
if row_dev and row_dev["device_info"]:
|
||||
try:
|
||||
d_info = json.loads(row_dev["device_info"])
|
||||
if isinstance(d_info, dict) and d_info.get("device_id"):
|
||||
device_serial = d_info["device_id"]
|
||||
except Exception:
|
||||
pass
|
||||
conn_dev.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
device_info_str = f" | **Device Serial:** `{device_serial}`" if device_serial else ""
|
||||
|
||||
if is_flash:
|
||||
summary_lines = [
|
||||
"# ARTEMIS Flash Execution Summary",
|
||||
f"**Session ID:** `{trace_id}` | **Model:** Flash\n",
|
||||
f"**Session ID:** `{trace_id}` | **Model:** Flash{device_info_str}\n",
|
||||
"---",
|
||||
"## Step-by-Step Execution Chain\n",
|
||||
]
|
||||
@@ -296,6 +321,8 @@ async def mobile_inspect_trace(action: str, trace_id: str, step_number: int | No
|
||||
last_n_detailed=0,
|
||||
all_detailed=False,
|
||||
)
|
||||
if device_serial and not summary_markdown.startswith("**Session ID:"):
|
||||
summary_markdown = f"**Session ID:** `{trace_id}` | **Model:** Pro{device_info_str}\n\n" + summary_markdown
|
||||
return summary_markdown
|
||||
except Exception as e:
|
||||
return {
|
||||
@@ -307,6 +334,7 @@ async def mobile_inspect_trace(action: str, trace_id: str, step_number: int | No
|
||||
elif action in ("view_step_screenshots", "view_step_details"):
|
||||
status_data = trace_store.read_status(trace_id)
|
||||
is_flash = status_data and status_data.get("model", "").lower() == "flash"
|
||||
device_serial = status_data.get("device_serial") if status_data else None
|
||||
|
||||
if action == "view_step_details" and is_flash:
|
||||
return {
|
||||
@@ -426,6 +454,7 @@ async def mobile_inspect_trace(action: str, trace_id: str, step_number: int | No
|
||||
|
||||
return {
|
||||
"trace_id": trace_id,
|
||||
"device_serial": device_serial,
|
||||
"step_number": step_dict["step_number"],
|
||||
"before_screenshot": pre_image,
|
||||
"after_screenshot": post_image,
|
||||
@@ -451,6 +480,7 @@ async def mobile_inspect_trace(action: str, trace_id: str, step_number: int | No
|
||||
|
||||
return {
|
||||
"trace_id": trace_id,
|
||||
"device_serial": device_serial,
|
||||
"step_number": step_dict["step_number"],
|
||||
"details": rendered_text,
|
||||
}
|
||||
|
||||
@@ -43,7 +43,16 @@ def mobile_manage_task(
|
||||
|
||||
### Available Actions:
|
||||
- **'status'**: Retrieves execution progress, elapsed time, and metadata.
|
||||
Returns a JSON object.
|
||||
Returns a JSON object containing:
|
||||
* `trace_id`: Unique identifier for the task.
|
||||
* `status`: Task lifecycle state ('running', 'completed', 'failed', 'cancelled').
|
||||
* `device_serial`: The Android device serial number assigned to/running this task
|
||||
(essential in multi-device environments to know which phone this task belongs to).
|
||||
* `task_desc`: The goal/description of the task.
|
||||
* `model`: 'Flash' or 'Pro'.
|
||||
* `elapsed_seconds`: Execution duration in seconds.
|
||||
* `progress`: Live progress data (current turn, latest thoughts and actions for Flash;
|
||||
active task plan for Pro).
|
||||
* **How to use the output**:
|
||||
- For **Pro Model**, check `progress.task_plan` to see if the
|
||||
agent's long-term plan aligns with your goal.
|
||||
@@ -113,9 +122,51 @@ def mobile_manage_task(
|
||||
end_time = status_data.get("end_time") or time.time()
|
||||
elapsed = round(end_time - start_time, 1) if start_time else 0
|
||||
|
||||
project_root = env_utils.get_project_root()
|
||||
db_path = os.path.join(trace_store.TRACES_DIR, "data_engine.db")
|
||||
if not os.path.exists(db_path):
|
||||
db_path = os.path.join(project_root, "traces", "data_engine.db")
|
||||
|
||||
device_serial = status_data.get("device_serial")
|
||||
if not device_serial and os.path.exists(db_path):
|
||||
try:
|
||||
conn_dev = sqlite3.connect(db_path)
|
||||
conn_dev.row_factory = sqlite3.Row
|
||||
cur_dev = conn_dev.cursor()
|
||||
cur_dev.execute(
|
||||
"SELECT device_info FROM sessions WHERE session_id = ? OR pid = ? ORDER BY start_time DESC LIMIT 1",
|
||||
(trace_id, pid),
|
||||
)
|
||||
row_dev = cur_dev.fetchone()
|
||||
if row_dev and row_dev["device_info"]:
|
||||
try:
|
||||
d_info = json.loads(row_dev["device_info"])
|
||||
if isinstance(d_info, dict) and d_info.get("device_id"):
|
||||
device_serial = d_info["device_id"]
|
||||
status_data["device_serial"] = device_serial
|
||||
trace_store.write_status(trace_id, status_data)
|
||||
except Exception:
|
||||
pass
|
||||
conn_dev.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not device_serial and pid:
|
||||
try:
|
||||
owners = DeviceExecutionLock.get_active_owners()
|
||||
for clean_id, owner in owners.items():
|
||||
if owner.pid == pid or owner.session_id == trace_id:
|
||||
device_serial = owner.device_id
|
||||
status_data["device_serial"] = device_serial
|
||||
trace_store.write_status(trace_id, status_data)
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
response: dict[str, Any] = {
|
||||
"trace_id": trace_id,
|
||||
"status": current_status,
|
||||
"device_serial": device_serial,
|
||||
"elapsed_seconds": elapsed,
|
||||
"task_desc": status_data.get("task_desc"),
|
||||
"model": status_data.get("model"),
|
||||
|
||||
@@ -32,6 +32,7 @@ def mobile_run_task(
|
||||
locked_app_package: str | None = None,
|
||||
app_path: str | None = None,
|
||||
expected_output_desc: str | None = None,
|
||||
device_serial: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Starts a specialized mobile UI automation subagent to autonomously plan and execute tasks on a connected Android device.
|
||||
|
||||
@@ -41,6 +42,23 @@ def mobile_run_task(
|
||||
straightforward UI scripts) and **Pro** (for complex, exploratory tasks
|
||||
requiring deep reasoning).
|
||||
|
||||
### Device Selection & Multi-Device Support (CRITICAL)
|
||||
ARTEMIS supports multi-device execution and provides two device selection modes:
|
||||
1. **Direct Device Specification**: Specify `device_serial` explicitly
|
||||
(e.g., `device_serial="63191FDKX00062"` or `device_serial="emulator-5554"`).
|
||||
Tasks targeting different devices run concurrently via per-device locking.
|
||||
2. **Automatic Device Selection**: If `device_serial` is omitted or `None`,
|
||||
ARTEMIS will automatically select the connected device or allocate an idle device
|
||||
from the device pool.
|
||||
|
||||
**Device Selection Policy & Diagnostics**:
|
||||
- **Prioritize User Choice**: When multiple devices or emulators are connected (or when
|
||||
user intent is not explicitly specified), **ALWAYS PRIORITIZE ASKING THE USER** to
|
||||
choose or confirm the target device before launching tasks.
|
||||
- **Device Diagnosis with `adb devices`**: Always use `adb devices` (or `adb devices -l`)
|
||||
to inspect connected hardware, verify authorization states (`device` vs `unauthorized`),
|
||||
and retrieve active device serials when diagnosing or before delegating tasks.
|
||||
|
||||
### Model Selection Guide (CRITICAL FOR ROUTING)
|
||||
You MUST choose the appropriate `model` based on the task complexity:
|
||||
- **Flash**: Best for simple, deterministic tasks that can be completed
|
||||
@@ -81,6 +99,7 @@ def mobile_run_task(
|
||||
containing:
|
||||
- `trace_id`: The unique session identifier of the task for status tracking
|
||||
and inspection.
|
||||
- `device_serial`: The assigned target device serial (or "auto-select").
|
||||
- `notes_dir`: (Pro Model Only) The directory where the subagent's execution
|
||||
files/notes are located.
|
||||
- `stdout_log`: The path to the execution framework log.
|
||||
@@ -120,6 +139,11 @@ def mobile_run_task(
|
||||
summarization agent will aggregate the interaction history
|
||||
(motivation, actions, videos) into a summary report saved as
|
||||
"output.md" in `notes_dir`. Ignored if `model="Flash"`.
|
||||
device_serial: Optional Android device serial number (e.g. "63191FDKX00062"
|
||||
or "emulator-5554"). If specified, binds execution strictly to that device,
|
||||
allowing multi-device concurrent automation. If omitted, ARTEMIS will
|
||||
automatically select an available device. Prioritize letting the user select
|
||||
the target device if multiple devices are attached.
|
||||
"""
|
||||
# 0. Validate and normalize model
|
||||
if model.lower() not in ("flash", "pro"):
|
||||
@@ -135,18 +159,31 @@ def mobile_run_task(
|
||||
task_desc=task_desc,
|
||||
model=canonical_model,
|
||||
conversation_id=conversation_id,
|
||||
device_serial=device_serial,
|
||||
)
|
||||
|
||||
# 3. Resolve project root, python executable, and background runner module
|
||||
# 3. Auto-spawn daemon if not already running (unless standalone forced)
|
||||
if os.environ.get("ARTEMIS_STANDALONE") != "1":
|
||||
try:
|
||||
from artemis.runtime import ensure_daemon_running
|
||||
|
||||
ensure_daemon_running(timeout=1.5)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 4. Resolve project root, python executable, and background runner module
|
||||
project_root = env_utils.get_project_root()
|
||||
python_exe = env_utils.resolve_python_executable(project_root)
|
||||
queue_ticket = DeviceExecutionLock.reserve(
|
||||
description=f"MCP task: {task_desc[:120]}",
|
||||
session_id=trace_id,
|
||||
ingress="mcp",
|
||||
)
|
||||
reserve_kwargs: dict[str, Any] = {
|
||||
"description": f"MCP task: {task_desc[:120]}",
|
||||
"session_id": trace_id,
|
||||
"ingress": "mcp",
|
||||
}
|
||||
if device_serial:
|
||||
reserve_kwargs["device_id"] = device_serial
|
||||
queue_ticket = DeviceExecutionLock.reserve(**reserve_kwargs)
|
||||
|
||||
# 4. Spawn the background task runner as an independent subprocess
|
||||
# 5. Spawn the background task runner as an independent subprocess
|
||||
try:
|
||||
cmd = [
|
||||
python_exe,
|
||||
@@ -167,10 +204,15 @@ def mobile_run_task(
|
||||
cmd.extend(["--app-path", app_path])
|
||||
if expected_output_desc and canonical_model != "Flash":
|
||||
cmd.extend(["--expected-output-desc", expected_output_desc])
|
||||
if device_serial:
|
||||
cmd.extend(["--device-serial", device_serial])
|
||||
|
||||
env = os.environ.copy()
|
||||
env["ARTEMIS_SESSION_ID"] = trace_id
|
||||
env["ARTEMIS_TASK_INGRESS"] = "mcp"
|
||||
if device_serial:
|
||||
env["ADB_DEVICE_SERIAL"] = device_serial
|
||||
env["ARTEMIS_DEVICE_ID"] = device_serial
|
||||
env[DeviceExecutionLock.QUEUE_TICKET_ENV] = queue_ticket
|
||||
|
||||
proc_kwargs = env_utils.get_detached_process_kwargs()
|
||||
@@ -180,12 +222,18 @@ def mobile_run_task(
|
||||
cwd=project_root,
|
||||
**proc_kwargs,
|
||||
)
|
||||
transfer_kwargs: dict[str, Any] = {
|
||||
"description": f"MCP task: {task_desc[:120]}",
|
||||
"session_id": trace_id,
|
||||
"ingress": "mcp",
|
||||
}
|
||||
if device_serial:
|
||||
transfer_kwargs["device_id"] = device_serial
|
||||
|
||||
DeviceExecutionLock.transfer_reservation(
|
||||
queue_ticket,
|
||||
proc.pid,
|
||||
description=f"MCP task: {task_desc[:120]}",
|
||||
session_id=trace_id,
|
||||
ingress="mcp",
|
||||
**transfer_kwargs,
|
||||
)
|
||||
|
||||
# 5. Record the PID of the spawned subprocess
|
||||
@@ -201,6 +249,7 @@ def mobile_run_task(
|
||||
|
||||
response_dict: dict[str, Any] = {
|
||||
"trace_id": trace_id,
|
||||
"device_serial": device_serial or "auto-select",
|
||||
"message": "Successfully started background task.",
|
||||
"stdout_log": stdout_log_path,
|
||||
"stderr_log": stderr_log_path,
|
||||
|
||||
@@ -43,6 +43,7 @@ def init_trace(
|
||||
task_desc: str,
|
||||
model: str,
|
||||
conversation_id: str,
|
||||
device_serial: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Initializes the trace directory and creates the initial status.json file."""
|
||||
trace_dir = get_trace_dir(trace_id)
|
||||
@@ -54,6 +55,7 @@ def init_trace(
|
||||
"model": model,
|
||||
"conversation_id": conversation_id,
|
||||
"status": "running",
|
||||
"device_serial": device_serial,
|
||||
"start_time": time.time(),
|
||||
"end_time": None,
|
||||
"error": None,
|
||||
@@ -89,6 +91,7 @@ def update_trace_status(
|
||||
status: str,
|
||||
error: str | None = None,
|
||||
result: Any | None = None,
|
||||
device_serial: str | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Updates specific fields of the status.json for a given trace_id."""
|
||||
data = read_status(trace_id)
|
||||
@@ -103,6 +106,19 @@ def update_trace_status(
|
||||
data["error"] = error
|
||||
if result is not None:
|
||||
data["result"] = result
|
||||
if device_serial is not None:
|
||||
data["device_serial"] = device_serial
|
||||
|
||||
write_status(trace_id, data)
|
||||
return data
|
||||
|
||||
|
||||
def update_trace_device_serial(trace_id: str, device_serial: str) -> dict[str, Any] | None:
|
||||
"""Updates the device_serial field of the status.json for a given trace_id."""
|
||||
data = read_status(trace_id)
|
||||
if not data:
|
||||
return None
|
||||
|
||||
data["device_serial"] = device_serial
|
||||
write_status(trace_id, data)
|
||||
return data
|
||||
Reference in new issue
Block a user