chenmingxuan/Artemis-Eapil
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 32 Wiki Activity

feat: extract in-process MCP action layer, concurrent task dispatch, and spawn/reconcile hardening

Browse Source 
- Extract device action execution into artemis/mcp action layer:
  typed ActionResult, capability manifest with startup validation,
  AdbActuator as the single physical implementation, in-memory
  ActionSession (replaces stdio subprocess MCP in validator), and
  McpActionExecutor preserving the legacy executor contract.
  adb_server keeps its external stdio/SSE surface unchanged and now
  reuses the actuator implementation; import-time side effects moved
  into configure_stdio_mode().
- Capability-gated prompt assembly (prompt_assembly.py): tool enums and
  teaching segments render per available backend tools; full-set output
  is byte-identical to the previous static text (locked by tests).
- Admin console task queue: serial worker replaced by a per-device
  concurrent dispatcher (ARTEMIS_CONCURRENCY_MODE=global restores
  serial semantics), per-session cancellation, strict device binding on
  enqueue, unified stdio piping with tee to trace stdout.log, and
  shutdown cleanup of all active runs. Fix _pid_exists exception order
  so PermissionError is treated as alive.
- mcp_server reliability: spawn watchdog kills runners stuck before
  interpreter startup and releases device reservations; task state
  reconcile uses psutil liveness, prefers runner-reported pids, and
  adds a 45s startup grace to stop false failure reports on Windows;
  conversation_id is now optional end-to-end; explicit device_serial is
  validated at submission.
- CLI install now writes ~/.claude/rules/artemis.md for Claude Code and
  migrates/removes the legacy CLAUDE.md injection block; ~/.claude.json
  is never rewritten when it fails to parse.
- Tests: Windows-portable stdio lifecycle and path assertions; new
  suites for action layer, prompt assembly, spawn watchdog, and task
  reconcile (895 passed, 3 skipped).
This commit is contained in:
weilinliu55 committed 2026-08-27 18:25:27 -07:00
1 parent 60312abbb0
commit cbdca1497c
52 files changed
+5735 -1649
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

artemis/mcp/action_executor.py
+546
View File
@@ -0,0 +1,546 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""MCP-backed action executor for FlashRunner and FailureAnalyzer.
Same public contract as the legacy ``MobileActionExecutor`` --
``execute(name, args, tool_call_id, state) -> ToolExecutionResult`` -- but execution
routes through the in-process action MCP session, and dispatch is table-driven so a
backend's extension tools are callable without touching this file.
Kept agent-side (by design, not omission): argument normalization, element-index
resolution against LangGraph ``State``, smart-swipe resolution, act-then-observe
capture and ``State`` write-back, tracing (a ``ContextVar`` cannot cross the server's
task boundary -- see the plan's R2), and the non-device tools (``read_note``,
``list_notes``, ``ask_explorer``).
Status is carried by ``ActionResult.ok`` -- the historical substring sniffing on
``"Error"``/``"Failed"`` is gone, so typing the literal text "Failed" into a field no
longer marks the step as failed, and a successful tap with a failed screenshot is no
longer misreported as an error.
"""
import ast
import hashlib
import json
from pathlib import Path
from typing import Any
from artemis.config import resolve_explorer_version
from artemis.context import ArtemisContext
from artemis.controllers.unified_controller import UnifiedMobileController
from artemis.data_engine.trace import TraceSpan
from artemis.mcp.action_manifest import OPTIONAL_ACTIONS, REQUIRED_ACTIONS
from artemis.mcp.action_session import ActionSession, get_action_session
from artemis.mcp.action_types import ActionCode, ActionResult
from artemis.mcp.actuators.adb import AdbActuator
from artemis.agents.validator.tool_declarations import (
ToolExecutionResult,
normalize_coordinate_target,
)
from artemis.utils.coordinates import (
compute_smart_swipe_coordinates,
parse_swipe_parameters,
)
from artemis.utils.logger import get_logger
from artemis.utils.notes import (
format_list_notes_failure,
format_list_notes_success,
format_read_note_failure,
list_notes_info,
read_note_content,
)
logger = get_logger(__name__)
__all__ = ["McpActionExecutor"]
# Wording for wrapped exceptions; matches the historical executor `except` arms.
_EXCEPTION_PREFIX = {
"click": "Error during click",
"click_sequence": "Error executing click sequence",
"long_press": "Error during long press",
"input_text": "Error during input text",
"swipe": "Error during swipe",
"press_key": "Error during press_key",
"manage_app": "Error during manage_app",
"wait_for_delay": "Error during wait_for_delay",
"wait_for_text": "Error during wait_for_text",
}
class _ArgError(ValueError):
"""Argument-translation failure whose message is already fully formatted."""
class McpActionExecutor:
"""Routes agent tool calls through the unified action MCP session."""
def __init__(
self,
ctx: ArtemisContext,
controller: UnifiedMobileController | None = None,
actuator: AdbActuator | None = None,
):
self.ctx = ctx
self.actuator = (
actuator or getattr(ctx, "actuator", None) or AdbActuator(ctx, controller)
)
self.controller = self.actuator.controller
self._session: ActionSession | None = None
@property
def action_tool_names(self) -> frozenset[str]:
"""Device actions plus backend extension names -- the dynamic dispatch set."""
return (
(REQUIRED_ACTIONS | OPTIONAL_ACTIONS)
| {e.name for e in self.actuator.extensions()}
)
async def _session_or_start(self) -> ActionSession:
if self._session is None or not self._session.started:
self._session = await get_action_session(self.ctx, actuator=self.actuator)
return self._session
# --- Public entry ----------------------------------------------------------------
async def execute(
self,
name: str,
args: dict[str, Any],
tool_call_id: str,
state: Any,
) -> ToolExecutionResult:
"""Executes the named tool and wraps the outcome for the calling agent."""
raw_name = name.split(":")[-1] if ":" in name else name
if raw_name in ("read_note", "list_notes", "ask_explorer"):
return await self._execute_agent_tool(raw_name, name, args, tool_call_id, state)
if raw_name in self.action_tool_names:
return await self._execute_device_action(
raw_name, name, args, tool_call_id, state
)
return ToolExecutionResult(
tool_call_id=tool_call_id,
tool_name=name,
status="error",
text_summary=f"Error: Tool '{name}' not supported.",
)
# --- Device actions --------------------------------------------------------------
async def _execute_device_action(
self,
raw_name: str,
name: str,
args: dict[str, Any],
tool_call_id: str,
state: Any,
) -> ToolExecutionResult:
span = TraceSpan(name=raw_name, trace_type="action", ctx=self.ctx)
span.payload = {"args": args}
message = ""
res: ActionResult | None = None
img_bytes: bytes | None = None
shot_path: str | None = None
xml_list: str | None = None
with span:
try:
session = await self._session_or_start()
wire_name, wire_args, finalize = self._translate(raw_name, args, state)
extension = raw_name not in (REQUIRED_ACTIONS | OPTIONAL_ACTIONS)
if extension:
raw_result = await session.call_raw(wire_name, wire_args)
message = next(
(b.text for b in raw_result.content if getattr(b, "type", "") == "text"),
f"Extension tool '{raw_name}' completed.",
)
res = ActionResult.success(raw_name, message)
else:
res = await session.call(wire_name, wire_args)
message = finalize(res) if finalize else res.message
if res.ok or self._observe_despite_failure(raw_name, res):
obs = await session.observe(settle_ms=400)
if obs.ok:
shot_path = obs.screenshot_path
xml_list = obs.elements_text
if obs.hierarchy_ok:
state.indexed_points = [el["center"] for el in obs.elements]
state.indexed_elements = obs.elements
if shot_path:
try:
img_bytes = Path(shot_path).read_bytes()
except Exception as read_err:
logger.warning(
f"Failed to read observed screenshot: {read_err}"
)
except _ArgError as e:
res = ActionResult.failure(raw_name, str(e), code=ActionCode.INVALID_ARGS)
message = str(e)
except Exception as e:
prefix = _EXCEPTION_PREFIX.get(raw_name, f"Error during {raw_name}")
message = f"{prefix}: {e}"
res = ActionResult.failure(raw_name, message, detail=repr(e))
if shot_path:
state.latest_screenshot = shot_path
status = "success" if res.ok else "error"
post_image_name = None
if shot_path:
post_image_name = Path(shot_path).stem
elif img_bytes:
post_image_name = hashlib.sha256(img_bytes).hexdigest()
span.result = {
"outcome": message,
"post_image_name": post_image_name,
"has_xml": bool(xml_list),
"status": status,
}
if status == "error":
span.status = "failed"
span.error = message
return ToolExecutionResult(
tool_call_id=tool_call_id,
tool_name=name,
status=status,
text_summary=message,
screenshot_bytes=img_bytes,
screenshot_path=shot_path,
ui_elements_text=xml_list,
metadata={
"code": res.code.value,
"normalized_coordinates": res.normalized_coordinates,
},
)
@staticmethod
def _observe_despite_failure(raw_name: str, res: ActionResult) -> bool:
"""Failures that still observe the screen, mirroring the legacy executor.
A failed app launch and a wait_for_text timeout captured the screen; argument
and package errors returned without one.
"""
if raw_name == "manage_app":
return res.code not in (ActionCode.PACKAGE_NOT_FOUND, ActionCode.INVALID_ARGS)
if raw_name == "wait_for_text":
return res.code is ActionCode.TIMEOUT
return False
# --- Argument translation --------------------------------------------------------
def _translate(
self, raw_name: str, args: dict[str, Any], state: Any
) -> tuple[str, dict[str, Any], Any]:
"""Maps agent-facing args to wire args; returns (name, args, finalize).
``finalize(res)`` post-processes the ActionResult message where the historical
wording depended on client-side context (direction swipes).
"""
if raw_name == "click":
target = self._require_pair(
args.get("target") or args.get("coordinates") or [], "click"
)
return (
"click",
{
"target": list(target),
"times": args.get("times", 1),
"delay_ms": args.get("delay_ms", 100),
},
None,
)
if raw_name == "long_press":
target = self._require_pair(
args.get("target") or args.get("coordinates") or [], "long press"
)
return (
"long_press",
{
"target": list(target),
"duration_ms": args.get("duration_ms", args.get("duration", 1000)),
},
None,
)
if raw_name == "input_text":
raw_target = args.get("target") or args.get("coordinates")
target = None
if raw_target:
target = list(self._require_pair(raw_target, "input text"))
return (
"input_text",
{
"text": args.get("text", ""),
"target": target,
"clear_exist": args.get("clear_exist", True),
},
None,
)
if raw_name == "click_sequence":
return (
"click_sequence",
{
"sequence": self._resolve_sequence(args.get("sequence") or [], state),
"delay_ms": args.get("delay_ms", 50),
},
None,
)
if raw_name == "swipe":
return self._translate_swipe(args, state)
if raw_name == "press_key":
return "press_key", {"key": args.get("key", "BACK")}, None
if raw_name == "manage_app":
return (
"manage_app",
{
"action": args.get("action", "launch"),
"app_name": args.get("app_name", ""),
},
None,
)
if raw_name == "wait_for_delay":
return "wait_for_delay", {"time_in_ms": args.get("time_in_ms", 1000)}, None
if raw_name == "wait_for_text":
return (
"wait_for_text",
{
"text": args.get("text", ""),
"wait_state": args.get("wait_state") or "appear",
"timeout_ms": args.get("timeout_ms", 5000),
},
None,
)
# Backend extension: pass the arguments straight through.
return raw_name, dict(args), None
@staticmethod
def _require_pair(raw: Any, label: str) -> tuple[int, int]:
target = normalize_coordinate_target(raw)
if not isinstance(target, (list, tuple)) or len(target) != 2:
prefix = {
"click": "Error during click",
"long press": "Error during long press",
"input text": "Error during input text",
}[label]
raise _ArgError(f"{prefix}: Invalid target format: {raw}")
return int(target[0]), int(target[1])
def _resolve_sequence(self, sequence: Any, state: Any) -> list[list[int]]:
"""Resolves click_sequence entries (indices or pairs) to normalized pairs."""
if isinstance(sequence, str):
sequence_str = sequence.strip()
try:
sequence = json.loads(sequence_str)
except Exception:
try:
sequence = ast.literal_eval(sequence_str)
except Exception:
pass
if not isinstance(sequence, (list, tuple)):
raise _ArgError(
f"Error during click sequence: Invalid sequence format: {sequence}"
)
width = getattr(self.ctx.device, "device_width", 1080) if self.ctx.device else 1080
height = getattr(self.ctx.device, "device_height", 2400) if self.ctx.device else 2400
resolved: list[list[int]] = []
for raw_target in sequence:
target = normalize_coordinate_target(raw_target)
if isinstance(target, int):
idx = target
points = getattr(state, "indexed_points", []) or []
if not 1 <= idx <= len(points):
raise _ArgError(
"Error during click sequence: Index"
f" {idx} is out of range (available: {len(points)})"
)
px, py = int(points[idx - 1][0]), int(points[idx - 1][1])
nx = int(max(0, min(1000, round(px * 1000 / max(1, width)))))
ny = int(max(0, min(1000, round(py * 1000 / max(1, height)))))
elif isinstance(target, (list, tuple)) and len(target) == 2:
nx, ny = int(target[0]), int(target[1])
else:
raise _ArgError(
f"Error during click sequence: Invalid target format: {raw_target}"
)
resolved.append([nx, ny])
return resolved
def _translate_swipe(
self, args: dict[str, Any], state: Any
) -> tuple[str, dict[str, Any], Any]:
width = getattr(self.ctx.device, "device_width", 1080) if self.ctx.device else 1080
height = getattr(self.ctx.device, "device_height", 2400) if self.ctx.device else 2400
default_duration = args.get("duration", 400)
kind, target, final_duration = parse_swipe_parameters(
dict(args), default_duration=default_duration
)
if kind == "direction":
dir_name = target
x1, y1, x2, y2, smart_dur = compute_smart_swipe_coordinates(
direction=target,
target=args.get("target"),
indexed_elements=getattr(state, "indexed_elements", None) if state else None,
ui_hierarchy=getattr(state, "ui_tree", None) if state else None,
width=width,
height=height,
duration=final_duration,
)
def _norm(v: float, size: int) -> int:
return int(max(0, min(1000, round(v * 1000 / max(1, size)))))
def finalize(res: ActionResult) -> str:
if not res.ok:
return f"Error swiping {dir_name}: {res.detail}"
return f"Swipe completed successfully. Swiped {dir_name}."
return (
"swipe",
{
"start": [_norm(x1, width), _norm(y1, height)],
"end": [_norm(x2, width), _norm(y2, height)],
"duration_ms": smart_dur,
},
finalize,
)
if kind == "coords":
x1, y1, x2, y2 = target
def finalize(res: ActionResult) -> str:
if not res.ok:
return f"Error dragging: {res.detail}"
return (
"Swipe completed successfully. Swiped from"
f" [{x1}, {y1}] to [{x2}, {y2}]."
)
return (
"swipe",
{
"start": [int(x1), int(y1)],
"end": [int(x2), int(y2)],
"duration_ms": final_duration,
},
finalize,
)
raise _ArgError(f"Error during swipe: Invalid direction: {args}")
# --- Non-device tools (stay agent-side) ------------------------------------------
async def _execute_agent_tool(
self,
raw_name: str,
name: str,
args: dict[str, Any],
tool_call_id: str,
state: Any,
) -> ToolExecutionResult:
span = TraceSpan(name=raw_name, trace_type="tool", ctx=self.ctx)
span.payload = {"args": args}
with span:
if raw_name == "read_note":
text = await self._read_note(
args.get("key", ""), args.get("start_line"), args.get("end_line")
)
elif raw_name == "list_notes":
text = await self._list_notes()
else:
text = await self._ask_explorer(
args.get("query") or args.get("task_description") or "",
args.get("context_feedback"),
state,
)
span.result = {"outcome": text}
# These texts come from our own failure formatters, so the historical
# "Error" marker check is reliable here (unlike free-form device output).
return ToolExecutionResult(
tool_call_id=tool_call_id,
tool_name=name,
status="error" if "Error" in (text or "") else "success",
text_summary=text,
)
async def _read_note(
self, key: str, start_line: int | None, end_line: int | None
) -> str:
try:
base_dir = self.ctx.data_engine.base_dir if self.ctx.data_engine else None
if not base_dir:
return format_read_note_failure(key, "DataEngine not initialized.")
return read_note_content(base_dir, key, start_line, end_line)
except Exception as e:
return format_read_note_failure(key, str(e))
async def _list_notes(self) -> str:
try:
base_dir = self.ctx.data_engine.base_dir if self.ctx.data_engine else None
if not base_dir:
return format_list_notes_failure("DataEngine not initialized.")
return format_list_notes_success(list_notes_info(base_dir))
except Exception as e:
return format_list_notes_failure(str(e))
async def _ask_explorer(
self, query: str, context_feedback: str | None, state: Any
) -> str:
# Imported lazily: the explorer stack is heavy and recursive (it spawns an LLM
# sub-agent), which is also why it must never live behind the action server.
from artemis.tools.explorer_tool import _run_explorer_logic
try:
active_version = resolve_explorer_version(
self.ctx,
explicit_version=None,
agent_or_profile_name="validator",
)
result = await _run_explorer_logic(
self.ctx,
state,
query,
context_feedback or "",
version=active_version,
)
if isinstance(result, list):
texts = [
item["text"]
for item in result
if isinstance(item, dict) and item.get("type") == "text"
]
return "\n".join(texts) if texts else str(result)
return str(result)
except Exception as e:
return f"Error executing ask_explorer: {e}"