feat: improve task execution, verification and packaging
Add execution incidents, action bursts and shared Flash history. Expose verification and Explorer settings with streamed check results. Support graceful cancellation, recording recovery and bundled UI resources. Add CI quality checks and clean up comments and documentation.
This commit is contained in:
1 parent
5ccf06d677
commit
67248c887e
289 files changed
+15883
-6919
No files matched your search
@@ -53,7 +53,7 @@ When connecting ARTEMIS to AI coding assistants like **Antigravity**, **Claude C
|
||||
The included [`rules.md`](./rules.md) file contains the **Mobile Testing Mindset (ARTEMIS Integration)** guideline. It teaches the AI agent how to properly collaborate with ARTEMIS:
|
||||
|
||||
1. **The Runnable Code Principle & Active Exploration**: Instructs the AI to never guess or hallucinate UI transitions. Instead, it must first explore the live app via ARTEMIS MCP tools (`mobile_run_task`, etc.) to discover and verify real-world interaction paths before writing test scripts.
|
||||
2. **Flash vs. Pro Routing Strategy**: Guides the AI to choose **Flash** for rapid, straightforward UI actions (< 30 steps) and **Pro** for complex multi-agent planning, polling loops, or video/log diagnosis.
|
||||
2. **Flash vs. Pro Routing Strategy**: Guides the AI to choose **Flash** for rapid, straightforward UI actions (no step cap by default) and **Pro** for tasks that need a persistent plan, verified checkpoints, polling loops, or ADB / video / log diagnosis.
|
||||
3. **Latency & Timing Compensation**: Clarifies the difference between AI exploratory latency (e.g., model decision intervals) and the deterministic timing requirements of final test code.
|
||||
4. **"Dynamic-First, Coordinate-Fallback" Locator Pattern**: Teaches the AI to prioritize dynamic UI locators (Resource IDs, OCR text, semantics) for layout resilience, while implementing absolute coordinate fallbacks for maximum execution reliability.
|
||||
|
||||
@@ -71,7 +71,7 @@ When you run `uv run artemis mcp --install all` (or target a specific IDE like `
|
||||
|
||||
## 🛠️ MCP Tools Overview
|
||||
|
||||
* **`mobile_run_task`**: Asynchronously launches an autonomous mobile automation task (`Flash` or `Pro` model) with optional `device_serial` targeting.
|
||||
* **`mobile_run_task`**: Asynchronously launches an autonomous mobile automation task (`Flash` or `Pro` model) with optional `device_serial` targeting. Pro runs can be tuned with `verification_level` (`off` | `final` | `checkpoints` | `strict` — how much the Checker audits) and `explorer_mode` (`flash` | `pro` | `ultra` — the Operator's perception depth).
|
||||
* **`mobile_manage_task`**: Manages task lifecycle (`status`, `stop`, `inject_instruction`), returning task state and assigned `device_serial`. Pass `release_loop=True` with `inject_instruction` to gracefully end a `[Loop:continuous]` monitoring task — this explicit signal (not "please stop" wording) is what unlocks the loop milestone's completion.
|
||||
* **`mobile_get_device_state`**: Real-time observer (`screenshot` or OCR+XML `hierarchy`) with optional `device_serial`.
|
||||
* **`mobile_inspect_trace`**: Granular trace inspection, visual action overlays, agent reasoning, and `device_serial` tracking.
|
||||
|
||||
@@ -119,8 +119,16 @@ async def run_task(
|
||||
app_path: str | None = None,
|
||||
expected_output_desc: str | None = None,
|
||||
device_serial: str | None = None,
|
||||
verification_level: str | None = None,
|
||||
explorer_pro_mode: str | None = None,
|
||||
):
|
||||
"""Executes the mobile automation agent task and logs all actions/results."""
|
||||
"""Executes the mobile automation agent task and logs all actions/results.
|
||||
|
||||
``verification_level`` ('off' | 'final' | 'checkpoints' | 'strict') and
|
||||
``explorer_pro_mode`` ('flash' | 'pro' | 'ultra') are Pro-profile tuning
|
||||
knobs mirroring ``artemis run --verification-level / --explorer-pro-mode``;
|
||||
the Flash profile ignores them.
|
||||
"""
|
||||
trace_dir = trace_store.get_trace_dir(trace_id)
|
||||
os.makedirs(trace_dir, exist_ok=True)
|
||||
stdout_log_path = os.path.join(trace_dir, "stdout.log")
|
||||
@@ -179,6 +187,7 @@ async def run_task(
|
||||
# Optional path: pool-based selection falls back to the
|
||||
# first connected device on any import or query failure.
|
||||
from artemis.runtime import device_pool
|
||||
|
||||
target_serial = device_pool.select_device()
|
||||
except Exception:
|
||||
target_serial = connected_devices[0]
|
||||
@@ -205,11 +214,16 @@ async def run_task(
|
||||
profile = AgentProfile(name="default", llm_config=initialize_llm_config())
|
||||
|
||||
config_builder = Builders.AgentConfig.with_default_profile(profile)
|
||||
if verification_level:
|
||||
config_builder.with_verification_level(verification_level)
|
||||
if explorer_pro_mode:
|
||||
config_builder.with_explorer(pro_mode=explorer_pro_mode)
|
||||
if settings.ADB_HOST:
|
||||
config_builder.with_adb_server(host=settings.ADB_HOST, port=settings.ADB_PORT)
|
||||
|
||||
if target_serial:
|
||||
from artemis.context import DevicePlatform
|
||||
|
||||
config_builder.for_device(DevicePlatform.ANDROID, target_serial)
|
||||
|
||||
config = config_builder.build()
|
||||
@@ -222,7 +236,9 @@ async def run_task(
|
||||
timeout_seconds=float(os.getenv("ARTEMIS_AGENT_INIT_TIMEOUT_SECONDS", 30)),
|
||||
)
|
||||
|
||||
actual_serial = getattr(getattr(agent, "_device_context", None), "device_id", None) or target_serial
|
||||
actual_serial = (
|
||||
getattr(getattr(agent, "_device_context", None), "device_id", None) or target_serial
|
||||
)
|
||||
if actual_serial:
|
||||
target_serial = actual_serial
|
||||
trace_store.update_trace_device_serial(trace_id, actual_serial)
|
||||
@@ -252,7 +268,11 @@ async def run_task(
|
||||
error_explanation = result.get("explanation", "Task execution returned failed status.")
|
||||
print(f"Task finished with non-completed status: {error_explanation}", file=sys.stderr)
|
||||
trace_store.update_trace_status(
|
||||
trace_id, "failed", error=error_explanation, result=result, device_serial=target_serial
|
||||
trace_id,
|
||||
"failed",
|
||||
error=error_explanation,
|
||||
result=result,
|
||||
device_serial=target_serial,
|
||||
)
|
||||
|
||||
formatted_result = json.dumps(result, indent=2, ensure_ascii=False)
|
||||
@@ -271,7 +291,12 @@ async def run_task(
|
||||
message=failure_msg,
|
||||
title="Artemis Task Incomplete",
|
||||
event_type="failed",
|
||||
payload={"trace_id": trace_id, "device_serial": target_serial, "goal": task_desc, "result": result},
|
||||
payload={
|
||||
"trace_id": trace_id,
|
||||
"device_serial": target_serial,
|
||||
"goal": task_desc,
|
||||
"result": result,
|
||||
},
|
||||
)
|
||||
return
|
||||
|
||||
@@ -284,7 +309,9 @@ async def run_task(
|
||||
"notes_dir to view more details."
|
||||
)
|
||||
|
||||
trace_store.update_trace_status(trace_id, "completed", result=result, device_serial=target_serial)
|
||||
trace_store.update_trace_status(
|
||||
trace_id, "completed", result=result, device_serial=target_serial
|
||||
)
|
||||
|
||||
if isinstance(result, dict):
|
||||
formatted_result = json.dumps(result, indent=2, ensure_ascii=False)
|
||||
@@ -305,16 +332,21 @@ async def run_task(
|
||||
message=success_msg,
|
||||
title="Artemis Task Completed",
|
||||
event_type="completed",
|
||||
payload={"trace_id": trace_id, "device_serial": target_serial, "goal": task_desc, "result": result},
|
||||
payload={
|
||||
"trace_id": trace_id,
|
||||
"device_serial": target_serial,
|
||||
"goal": task_desc,
|
||||
"result": result,
|
||||
},
|
||||
)
|
||||
|
||||
except asyncio.CancelledError:
|
||||
print("Task was cancelled (asyncio.CancelledError)", file=sys.stderr)
|
||||
trace_store.update_trace_status(trace_id, "cancelled", error="Task was cancelled", device_serial=target_serial)
|
||||
device_info_line = f"Device Serial: `{target_serial}`\n" if target_serial else ""
|
||||
cancel_msg = (
|
||||
f"Artemis background task was cancelled.\n\nTrace ID: {trace_id}\n{device_info_line}Goal: {task_desc}\n"
|
||||
trace_store.update_trace_status(
|
||||
trace_id, "cancelled", error="Task was cancelled", device_serial=target_serial
|
||||
)
|
||||
device_info_line = f"Device Serial: `{target_serial}`\n" if target_serial else ""
|
||||
cancel_msg = f"Artemis background task was cancelled.\n\nTrace ID: {trace_id}\n{device_info_line}Goal: {task_desc}\n"
|
||||
notify(
|
||||
conversation_id=conversation_id,
|
||||
message=cancel_msg,
|
||||
@@ -339,7 +371,9 @@ async def run_task(
|
||||
frame_summary = f" (at {tb.tb_frame.f_code.co_name}:{tb.tb_lineno})" if tb else ""
|
||||
full_error_desc = f"{error_type}: {raw_error_msg}{frame_summary}"
|
||||
|
||||
trace_store.update_trace_status(trace_id, "failed", error=full_error_desc, device_serial=target_serial)
|
||||
trace_store.update_trace_status(
|
||||
trace_id, "failed", error=full_error_desc, device_serial=target_serial
|
||||
)
|
||||
|
||||
device_info_line = f"Device Serial: `{target_serial}`\n" if target_serial else ""
|
||||
failure_msg = (
|
||||
@@ -356,7 +390,12 @@ async def run_task(
|
||||
message=failure_msg,
|
||||
title="Artemis Task Failed",
|
||||
event_type="failed",
|
||||
payload={"trace_id": trace_id, "device_serial": target_serial, "goal": task_desc, "error": full_error_desc},
|
||||
payload={
|
||||
"trace_id": trace_id,
|
||||
"device_serial": target_serial,
|
||||
"goal": task_desc,
|
||||
"error": full_error_desc,
|
||||
},
|
||||
)
|
||||
|
||||
finally:
|
||||
@@ -387,6 +426,14 @@ if __name__ == "__main__":
|
||||
parser.add_argument("--app-path", help="Path to local APK to install before running task")
|
||||
parser.add_argument("--expected-output-desc", help="Expected output description")
|
||||
parser.add_argument("--device-serial", help="Target specific device serial")
|
||||
parser.add_argument(
|
||||
"--verification-level",
|
||||
help="Pro-profile Checker preset: 'off', 'final', 'checkpoints' or 'strict'",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--explorer-pro-mode",
|
||||
help="Pro-profile Explorer perception version: 'flash', 'pro' or 'ultra'",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -400,5 +447,7 @@ if __name__ == "__main__":
|
||||
app_path=args.app_path,
|
||||
expected_output_desc=args.expected_output_desc,
|
||||
device_serial=args.device_serial,
|
||||
verification_level=args.verification_level,
|
||||
explorer_pro_mode=args.explorer_pro_mode,
|
||||
)
|
||||
)
|
||||
+6
-2
@@ -14,16 +14,20 @@
|
||||
|
||||
"""Shared FastMCP instance definition for the mobile automation MCP server."""
|
||||
|
||||
import logging
|
||||
|
||||
from mcp.server.fastmcp import FastMCP
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
try:
|
||||
from mcp.server.fastmcp.server import Settings as FastMCPSettings
|
||||
|
||||
FastMCPSettings.model_rebuild()
|
||||
except Exception:
|
||||
except Exception as exc:
|
||||
# Version-compat shim: older/newer FastMCP releases may not expose
|
||||
# Settings or need the rebuild; the server works without it.
|
||||
pass
|
||||
logger.debug("FastMCP Settings.model_rebuild skipped: %s", exc, exc_info=True)
|
||||
|
||||
# Define the shared FastMCP instance for external IDE and agent clients.
|
||||
mcp = FastMCP(
|
||||
|
||||
@@ -36,10 +36,10 @@ class AgentApiNotifier(BaseNotifier):
|
||||
token = os.environ.get("ANTIGRAVITY_CSRF_TOKEN")
|
||||
if addr and token:
|
||||
self._save_shared_env(addr, token)
|
||||
except Exception:
|
||||
except Exception as exc:
|
||||
# Environment recovery is opportunistic; notifier construction
|
||||
# must never fail because of it.
|
||||
pass
|
||||
logger.debug("[EnvRecovery] Skipped saving shared env: %s", exc, exc_info=True)
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
@@ -159,11 +159,7 @@ class AgentApiNotifier(BaseNotifier):
|
||||
for proc in psutil.process_iter(["create_time", "environ"]):
|
||||
try:
|
||||
env = proc.info.get("environ")
|
||||
if (
|
||||
env
|
||||
and "ANTIGRAVITY_LS_ADDRESS" in env
|
||||
and "ANTIGRAVITY_CSRF_TOKEN" in env
|
||||
):
|
||||
if env and "ANTIGRAVITY_LS_ADDRESS" in env and "ANTIGRAVITY_CSRF_TOKEN" in env:
|
||||
ctime = proc.info.get("create_time", 0.0) or 0.0
|
||||
proc_matches.append(
|
||||
(
|
||||
@@ -188,13 +184,9 @@ class AgentApiNotifier(BaseNotifier):
|
||||
local_token = None
|
||||
for var in env_vars:
|
||||
if var.startswith(b"ANTIGRAVITY_LS_ADDRESS="):
|
||||
local_addr = (
|
||||
var.split(b"=", 1)[1].decode("utf-8", errors="ignore")
|
||||
)
|
||||
local_addr = var.split(b"=", 1)[1].decode("utf-8", errors="ignore")
|
||||
elif var.startswith(b"ANTIGRAVITY_CSRF_TOKEN="):
|
||||
local_token = (
|
||||
var.split(b"=", 1)[1].decode("utf-8", errors="ignore")
|
||||
)
|
||||
local_token = var.split(b"=", 1)[1].decode("utf-8", errors="ignore")
|
||||
if local_addr and local_token:
|
||||
proc_matches.append((mtime, local_addr, local_token))
|
||||
except OSError:
|
||||
@@ -262,9 +254,7 @@ class AgentApiNotifier(BaseNotifier):
|
||||
logger.info(f"AgentAPI notification sent successfully via {addr}.")
|
||||
return True
|
||||
except Exception as err:
|
||||
logger.debug(
|
||||
f"Candidate session {addr} failed (attempt {attempt + 1}): {err}"
|
||||
)
|
||||
logger.debug(f"Candidate session {addr} failed (attempt {attempt + 1}): {err}")
|
||||
|
||||
# If all initial candidates failed, force a fresh process scan in case a new session started
|
||||
fresh_candidates = self._get_candidate_envs(force_proc_scan=True)
|
||||
|
||||
@@ -38,7 +38,11 @@ class DesktopNotifier(BaseNotifier):
|
||||
val = os.getenv("ARTEMIS_DESKTOP_NOTIFY", "").lower()
|
||||
if val in ("0", "false", "no", "off"):
|
||||
return False
|
||||
if os.getenv("CI", "").lower() in ("1", "true", "yes") and val not in ("1", "true", "yes"):
|
||||
if os.getenv("CI", "").lower() in ("1", "true", "yes") and val not in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
):
|
||||
return False
|
||||
if sys.platform == "linux" and not shutil.which("notify-send"):
|
||||
return False
|
||||
|
||||
+8
-7
@@ -18,15 +18,16 @@ As a senior test engineer, you must ensure that all test code you write is **ful
|
||||
Deeply understand and select between ARTEMIS's dual execution models (**ARTEMIS Flash** and **ARTEMIS Pro**) based on the task scenario, and master their corresponding workflows:
|
||||
|
||||
- **ARTEMIS Flash (Fast / Reactive Model)**:
|
||||
- **Applicable Scenarios**: Designed for simple, highly deterministic, short-to-medium range (typically `< 30-35` steps) lightweight UI operations or direct automation workflows.
|
||||
- **Execution Mechanism (Reactive Loop)**: Does not enter the complex LangGraph multi-node graph orchestration, and has no independent Planner, Validator, or Summarizer nodes. Instead, `FlashRunner` directly receives the user's goal, the latest structural layout (XML), and the real-time screenshot, rapidly performing "Observe-Think-Act" via a single LLM, calling action tools until finally reporting the task status.
|
||||
- **Applicable Scenarios**: Designed for simple, highly deterministic, lightweight UI operations or direct automation workflows. There is no step cap by default (history is compressed rather than truncated), so task length alone is not a reason to choose Pro; choose Pro when the task needs a persistent plan, verified checkpoints, notes or a written report, ADB / log diagnostics, or multi-branch exploration.
|
||||
- **Execution Mechanism (Reactive Loop)**: Does not enter the LangGraph multi-node orchestration: there is no Planner, no pre-execution safety net, and no Checker. `FlashRunner` receives the user's goal, the latest UI element list, and the live screenshot, performs "Observe-Think-Act" with a single LLM, and calls action tools until it reports the task status. It chains taps into one `click_sequence` to catch transient UI (auto-fading control bars, toasts) before they expire, and it can call `ask_explorer` for element grounding, `recall_history` to look up compressed earlier steps, and `video_analyzer` over the session recording. History lives in the same session transcript ledger as Pro (session-relative `T+mm:ss` clock, screenshots folded into visual summaries, older steps chunked into eras).
|
||||
|
||||
- **ARTEMIS Pro (Deep / Multi-Agent Graph-Driven Closed-Loop Model)**:
|
||||
- **Applicable Scenarios**: Designed for long-range, highly complex, dynamic multi-branch tasks, or tasks requiring deep system diagnostics, asynchronous validation, and self-healing exploratory retries.
|
||||
- **Execution Mechanism (Plan-Execute-Analyze-Summarize Closed-Loop)**:
|
||||
- **Planner**: Deconstructs complex, high-level testing goals into a structured, step-by-step execution plan.
|
||||
- **Operator**: Consumes the plan, analyzes the current screen state (via screenshots and XML layout trees), and executes precise device interactions.
|
||||
- **Safety Net & Self-Healing (Failure Analyzer)**: Every action is vetted by an execution safety net. If an action fails or is blocked, the **Failure Analyzer** is triggered to run local step-by-step recovery sequences, achieving autonomous self-healing. This loop continues iteratively until the task terminates.
|
||||
- **Applicable Scenarios**: Designed for long-range, highly complex, dynamic multi-branch tasks; continuous monitoring / polling loops; and tasks requiring deep system diagnostics (ADB, logs, video), verified checkpoints, or a detailed written report.
|
||||
- **Execution Mechanism (Plan-Execute-Verify-Summarize Closed-Loop)**:
|
||||
- **Planner**: Deconstructs complex, high-level testing goals into a living Markdown task plan with milestones and `verify` / `assert` check items. Later milestone edits get an advisory review (a hint plus reason back to the Operator; the plan is never rolled back).
|
||||
- **Operator**: Consumes the plan, analyzes the current screen state (screenshots plus the UI element list, with the Explorer for grounding at the `explorer_mode` perception depth: `flash` 1-shot, `pro` 3-turn, `ultra` deep zoom), keeps notes, and executes precise device interactions; it can also recall compressed history, analyze the session recording, and run ADB commands as a supplement to UI actions.
|
||||
- **Safety Net & Execution Incidents**: A single turn-ending action is vetted by a pre-execution safety net (XML-first, pixel fallback). A multi-action turn is a **fast-action burst** that fires back to back without the safety net, which is how the Operator defeats turn latency on transient UI. When an action is intercepted or fails, the system opens an **execution incident** that persists in the Operator's context (with the consecutive-failure count shown as a fact) until a later action executes successfully; recovery is the Operator's own decision and there is no separate repair agent.
|
||||
- **Checker (Verification)**: A read-only verifier with the same observation tools as the Operator audits plan-declared checkpoints and performs an exit final review against the original goal; `verification_level` selects the depth (`off`, `final` = exit review only and the default, `checkpoints` = every checkpoint plus exit review, `strict` = checkpoints with a larger repair budget where a failed assert halts the run).
|
||||
- **Outputter (Optional)**: Synthesizes the entire execution trace into a human-readable report detailing every action step and visual result.
|
||||
|
||||
### 3. Device & Environment Constraints & Multi-Device Management
|
||||
|
||||
@@ -74,7 +74,7 @@ try:
|
||||
self.log_file.flush()
|
||||
|
||||
sys.stderr = StderrTee(sys.stderr, os.path.join(_mcp_log_dir, "mcp_stderr.log"))
|
||||
except Exception:
|
||||
except OSError:
|
||||
# Debug logging and the stderr tee are optional diagnostics; they must
|
||||
# never prevent the MCP stdio server from starting.
|
||||
pass
|
||||
@@ -89,11 +89,10 @@ from artemis.runtime import shutdown_awake_service, start_awake_service
|
||||
|
||||
import threading
|
||||
|
||||
|
||||
def main(transport: str = "stdio", host: str = "127.0.0.1", port: int = 8001):
|
||||
"""Main entrypoint to run the Artemis Mobile Agent MCP server."""
|
||||
threading.Thread(
|
||||
target=start_awake_service, daemon=True, name="artemis-awake-init"
|
||||
).start()
|
||||
threading.Thread(target=start_awake_service, daemon=True, name="artemis-awake-init").start()
|
||||
try:
|
||||
if transport.lower() == "sse":
|
||||
mcp.run(transport="sse", host=host, port=port)
|
||||
|
||||
@@ -82,17 +82,13 @@ async def mobile_get_device_state(view_type: str, device_serial: str | None = No
|
||||
if is_ocr_configured():
|
||||
try:
|
||||
screen_height = device_data.height
|
||||
status_bar_height = _detect_status_bar_height(
|
||||
xml_hierarchy, screen_height
|
||||
)
|
||||
status_bar_height = _detect_status_bar_height(xml_hierarchy, screen_height)
|
||||
if status_bar_height > 0:
|
||||
cropped_b64, _, _ = _crop_image_remove_status_bar(
|
||||
latest_screenshot_b64, status_bar_height
|
||||
)
|
||||
raw_ocr_results = await perform_ocr(cropped_b64)
|
||||
ocr_results = _map_coordinates_back(
|
||||
raw_ocr_results, status_bar_height
|
||||
)
|
||||
ocr_results = _map_coordinates_back(raw_ocr_results, status_bar_height)
|
||||
else:
|
||||
ocr_results = await perform_ocr(latest_screenshot_b64)
|
||||
except Exception:
|
||||
|
||||
@@ -267,7 +267,10 @@ async def mobile_inspect_trace(action: str, trace_id: str, step_number: int | No
|
||||
all_detailed=False,
|
||||
)
|
||||
if device_serial and not summary_markdown.startswith("**Session ID:"):
|
||||
summary_markdown = f"**Session ID:** `{trace_id}` | **Model:** Pro{device_info_str}\n\n" + summary_markdown
|
||||
summary_markdown = (
|
||||
f"**Session ID:** `{trace_id}` | **Model:** Pro{device_info_str}\n\n"
|
||||
+ summary_markdown
|
||||
)
|
||||
return summary_markdown
|
||||
except Exception as e:
|
||||
return {
|
||||
@@ -381,7 +384,9 @@ async def mobile_inspect_trace(action: str, trace_id: str, step_number: int | No
|
||||
action_obj[0] if isinstance(action_obj, list) and action_obj else action_obj
|
||||
)
|
||||
|
||||
if isinstance(action_parsed, dict) and (action_parsed.get("action") or action_parsed.get("name")):
|
||||
if isinstance(action_parsed, dict) and (
|
||||
action_parsed.get("action") or action_parsed.get("name")
|
||||
):
|
||||
if not os.path.exists(overlay_abs_path):
|
||||
success_draw = _draw_action_overlay(
|
||||
path, action_parsed, overlay_abs_path
|
||||
@@ -422,7 +427,6 @@ async def mobile_inspect_trace(action: str, trace_id: str, step_number: int | No
|
||||
action=step_dict.get("action_taken"),
|
||||
result=step_dict.get("last_execution_result"),
|
||||
is_most_recent=False,
|
||||
for_failure_analyzer=False,
|
||||
)
|
||||
|
||||
return {
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
"""MCP Tool: mobile_manage_task."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import sqlite3
|
||||
@@ -34,6 +35,8 @@ from artemis.runtime import (
|
||||
)
|
||||
from artemis.runtime.process_probe import pid_is_alive
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_LIVENESS_FAILURE_ERROR = "Task runner process terminated unexpectedly."
|
||||
# Grace window covering the spawn race: the launcher pid may already have exited
|
||||
# while the real runner has not yet registered its own pid in the DataEngine DB.
|
||||
@@ -201,10 +204,15 @@ def _mark_liveness_failure(trace_id: str, status_data: dict[str, Any]) -> None:
|
||||
event_type="failed",
|
||||
payload={"trace_id": trace_id, "error": _LIVENESS_FAILURE_ERROR},
|
||||
)
|
||||
except Exception:
|
||||
except Exception as exc:
|
||||
# Notification dispatch is best-effort; the failure verdict is
|
||||
# already persisted in status.json.
|
||||
pass
|
||||
logger.debug(
|
||||
"Liveness-failure notification skipped for trace %s: %s",
|
||||
trace_id,
|
||||
exc,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
|
||||
@@ -20,12 +20,13 @@ import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from typing import Any
|
||||
from typing import Any, get_args
|
||||
import uuid
|
||||
|
||||
from mcp_server.base import mcp
|
||||
from mcp_server.notifiers import notify
|
||||
from mcp_server.utils import env_utils
|
||||
from artemis.config import ExplorerVersion, checker_overrides_for_level
|
||||
from artemis.config.runtime import read_ipc_port
|
||||
from artemis.runtime import (
|
||||
DeviceExecutionLock,
|
||||
@@ -116,10 +117,15 @@ def _watch_spawn(
|
||||
event_type="failed",
|
||||
payload={"trace_id": trace_id, "error": error_text},
|
||||
)
|
||||
except Exception:
|
||||
except Exception as exc:
|
||||
# Notification dispatch is best-effort; the failure verdict is
|
||||
# already persisted in status.json.
|
||||
pass
|
||||
logging.getLogger("mcp_server").debug(
|
||||
"Spawn-failure notification skipped for trace %s: %s",
|
||||
trace_id,
|
||||
exc,
|
||||
exc_info=True,
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
@@ -164,6 +170,37 @@ def _validate_device_serial(device_serial: str) -> dict[str, Any] | None:
|
||||
}
|
||||
|
||||
|
||||
_EXPLORER_MODES: tuple[str, ...] = get_args(ExplorerVersion)
|
||||
|
||||
|
||||
def _normalize_pro_tuning(
|
||||
verification_level: str | None, explorer_mode: str | None
|
||||
) -> tuple[str | None, str | None]:
|
||||
"""Validate and normalise the Pro-profile tuning knobs (strip + lower).
|
||||
|
||||
Raises:
|
||||
ValueError: with a caller-facing message when a value is not a known
|
||||
preset, so the tool rejects the call before any trace is created.
|
||||
"""
|
||||
level: str | None = None
|
||||
if verification_level is not None and str(verification_level).strip():
|
||||
level = str(verification_level).strip().lower()
|
||||
try:
|
||||
checker_overrides_for_level(level)
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"Invalid verification_level: {exc}") from exc
|
||||
|
||||
mode: str | None = None
|
||||
if explorer_mode is not None and str(explorer_mode).strip():
|
||||
mode = str(explorer_mode).strip().lower()
|
||||
if mode not in _EXPLORER_MODES:
|
||||
raise ValueError(
|
||||
f"Invalid explorer_mode {explorer_mode!r}. Must be one of: "
|
||||
+ ", ".join(_EXPLORER_MODES)
|
||||
)
|
||||
return level, mode
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def mobile_run_task(
|
||||
task_desc: str,
|
||||
@@ -173,6 +210,8 @@ def mobile_run_task(
|
||||
app_path: str | None = None,
|
||||
expected_output_desc: str | None = None,
|
||||
device_serial: str | None = None,
|
||||
verification_level: str | None = None,
|
||||
explorer_mode: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Starts an autonomous mobile UI automation subagent on a connected Android device.
|
||||
|
||||
@@ -183,15 +222,17 @@ def mobile_run_task(
|
||||
when `conversation_id` was provided; without it, rely on your fallback timer.
|
||||
|
||||
### Model selection (routing)
|
||||
- **Flash** (default, preferred): simple, deterministic tasks completable
|
||||
within ~30 UI steps. No log/video analysis, no ADB shell, no persistent
|
||||
plan or notes — unsuitable for exploration, complex error recovery,
|
||||
monitoring/polling, or tasks that must report back large amounts of
|
||||
detail.
|
||||
- **Flash** (default, preferred): simple, deterministic tasks with a clear
|
||||
UI path. No step cap by default (history is compressed, and the agent
|
||||
can recall earlier steps and query the session recording), but no ADB
|
||||
shell, no pre-execution safety net, no persistent plan or notes, and
|
||||
no verification or written report — unsuitable for exploration,
|
||||
multi-branch recovery, structured monitoring/polling, or tasks that
|
||||
must report back large amounts of detail.
|
||||
- **Pro**: complex, exploratory, or multi-branch tasks; continuous
|
||||
monitoring/polling; tasks needing ADB shell, system logs, video analysis,
|
||||
multi-step planning, or detailed written output (via notes and
|
||||
`expected_output_desc`).
|
||||
monitoring/polling; tasks needing ADB shell, system logs, verified
|
||||
checkpoints, multi-step planning, or detailed written output (via notes
|
||||
and `expected_output_desc`).
|
||||
|
||||
Timing is not precise: the agent's own inference adds ~5 s per step (Flash)
|
||||
or ~30 s per turn (Pro) on top of any requested waits — account for this
|
||||
@@ -226,11 +267,24 @@ def mobile_run_task(
|
||||
If omitted, an available device is selected automatically. When
|
||||
several devices are attached, confirm the target with the user first
|
||||
(`adb devices -l` lists serials and authorization states).
|
||||
verification_level: Optional, Pro only. Coarse Checker preset: `"off"`
|
||||
(no audit; the Operator self-reports), `"final"` (one exit review
|
||||
against the goal, the default), `"checkpoints"` (every plan
|
||||
checkpoint is verified + exit review), `"strict"` (checkpoints with
|
||||
a larger repair budget; a failed assert halts the run). Ignored for
|
||||
Flash.
|
||||
explorer_mode: Optional, Pro only. Explorer perception version used
|
||||
by the Operator: `"flash"` (1-shot detection, the default),
|
||||
`"pro"` (3-turn ReAct), `"ultra"` (deep pixel reasoning; slowest).
|
||||
Ignored for Flash.
|
||||
"""
|
||||
# 0. Validate and normalize model
|
||||
if model.lower() not in ("flash", "pro"):
|
||||
raise ValueError(f"Invalid model '{model}'. Must be either 'Flash' or 'Pro'.")
|
||||
canonical_model = "Flash" if model.lower() == "flash" else "Pro"
|
||||
# 0b. Validate the Pro tuning knobs before any trace exists so a typo is a
|
||||
# plain tool error rather than a failed trace on disk.
|
||||
verification_level, explorer_mode = _normalize_pro_tuning(verification_level, explorer_mode)
|
||||
|
||||
# 1. Generate a unique trace_id
|
||||
trace_id = str(uuid.uuid4())
|
||||
@@ -269,6 +323,8 @@ def mobile_run_task(
|
||||
session_id=trace_id,
|
||||
ingress="mcp",
|
||||
conversation_id=conversation_id,
|
||||
verification_level=verification_level,
|
||||
explorer_mode=explorer_mode,
|
||||
base_url=base_url,
|
||||
)
|
||||
if resp and resp.get("status") == "rejected":
|
||||
@@ -368,9 +424,7 @@ def mobile_run_task(
|
||||
"message": "Task rejected or timed out in Artemis Daemon. Standalone fallback blocked to prevent runner collision.",
|
||||
}
|
||||
except Exception as exc:
|
||||
logging.getLogger("mcp_server").warning(
|
||||
f"Daemon dispatch failed: {exc}"
|
||||
)
|
||||
logging.getLogger("mcp_server").warning(f"Daemon dispatch failed: {exc}")
|
||||
return {
|
||||
"trace_id": trace_id,
|
||||
"status": "failed",
|
||||
@@ -413,6 +467,10 @@ def mobile_run_task(
|
||||
cmd.extend(["--expected-output-desc", expected_output_desc])
|
||||
if device_serial:
|
||||
cmd.extend(["--device-serial", device_serial])
|
||||
if verification_level:
|
||||
cmd.extend(["--verification-level", verification_level])
|
||||
if explorer_mode:
|
||||
cmd.extend(["--explorer-pro-mode", explorer_mode])
|
||||
|
||||
env = os.environ.copy()
|
||||
env["ARTEMIS_SESSION_ID"] = trace_id
|
||||
|
||||
@@ -138,19 +138,24 @@ def ensure_emulator(
|
||||
if not emu_exe or not os.path.exists(emu_exe):
|
||||
return False
|
||||
|
||||
popen_kwargs = {"stdin": subprocess.DEVNULL, "stdout": subprocess.DEVNULL, "stderr": subprocess.DEVNULL}
|
||||
if sys.platform == "win32":
|
||||
popen_kwargs["creationflags"] = (
|
||||
subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.DETACHED_PROCESS # type: ignore[attr-defined]
|
||||
)
|
||||
else:
|
||||
popen_kwargs["start_new_session"] = True
|
||||
|
||||
try:
|
||||
subprocess.Popen(
|
||||
[emu_exe, "-avd", avd_name, "-no-snapshot", "-grpc", "8554"],
|
||||
**popen_kwargs,
|
||||
)
|
||||
command = [emu_exe, "-avd", avd_name, "-no-snapshot", "-grpc", "8554"]
|
||||
if sys.platform == "win32":
|
||||
subprocess.Popen(
|
||||
command,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
creationflags=(subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.DETACHED_PROCESS),
|
||||
)
|
||||
else:
|
||||
subprocess.Popen(
|
||||
command,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
start_new_session=True,
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
Reference in new issue
Block a user