chenmingxuan/Artemis-Eapil
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 32 Wiki Activity

feat: extract in-process MCP action layer, concurrent task dispatch, and spawn/reconcile hardening

Browse Source 
- Extract device action execution into artemis/mcp action layer:
  typed ActionResult, capability manifest with startup validation,
  AdbActuator as the single physical implementation, in-memory
  ActionSession (replaces stdio subprocess MCP in validator), and
  McpActionExecutor preserving the legacy executor contract.
  adb_server keeps its external stdio/SSE surface unchanged and now
  reuses the actuator implementation; import-time side effects moved
  into configure_stdio_mode().
- Capability-gated prompt assembly (prompt_assembly.py): tool enums and
  teaching segments render per available backend tools; full-set output
  is byte-identical to the previous static text (locked by tests).
- Admin console task queue: serial worker replaced by a per-device
  concurrent dispatcher (ARTEMIS_CONCURRENCY_MODE=global restores
  serial semantics), per-session cancellation, strict device binding on
  enqueue, unified stdio piping with tee to trace stdout.log, and
  shutdown cleanup of all active runs. Fix _pid_exists exception order
  so PermissionError is treated as alive.
- mcp_server reliability: spawn watchdog kills runners stuck before
  interpreter startup and releases device reservations; task state
  reconcile uses psutil liveness, prefers runner-reported pids, and
  adds a 45s startup grace to stop false failure reports on Windows;
  conversation_id is now optional end-to-end; explicit device_serial is
  validated at submission.
- CLI install now writes ~/.claude/rules/artemis.md for Claude Code and
  migrates/removes the legacy CLAUDE.md injection block; ~/.claude.json
  is never rewritten when it fails to parse.
- Tests: Windows-portable stdio lifecycle and path assertions; new
  suites for action layer, prompt assembly, spawn watchdog, and task
  reconcile (895 passed, 3 skipped).
This commit is contained in:
weilinliu55 committed 2026-08-27 18:25:27 -07:00
1 parent 60312abbb0
commit cbdca1497c
52 files changed
+5735 -1649
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

artemis/mcp/action_server.py
+353
View File
@@ -0,0 +1,353 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""The unified device action MCP server.
``build_action_server`` turns an actuator into a ``FastMCP`` server that registers
**only** the tools the actuator implements, plus its extension tools -- so
``list_tools()`` is clean by construction and an absent optional tool is invisible to
every consumer.
Return protocol: every device action returns a ``CallToolResult`` whose
``structuredContent`` is an ``ActionResult`` -- ``ok``/``code`` carry the real status
(killing the historical ``"Error" in text`` sniffing) while ``content`` keeps the
historical human-readable message. ``isError`` is reserved for protocol-level
failures; a device refusing an action is ``isError=False, ok=False`` so it reaches the
model as an observation, not an exception.
Exceptions raised by the actuator are wrapped here with the historical
``"Error during X: {e}"`` wording the agents' transcripts already contain.
"""
import asyncio
from typing import Annotated, Any
from mcp.server.fastmcp import FastMCP
from mcp.types import CallToolResult, ImageContent, TextContent
from artemis.mcp.action_manifest import validate_actuator
from artemis.mcp.action_types import ActionCode, ActionResult, ObserveResult
from artemis.mcp.actuators.base import Actuator
from artemis.mcp.observation import observe as observe_impl
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
__all__ = ["build_action_server"]
# Exception wrapping per tool; wording matches the historical executor `except` arms.
_EXCEPTION_PREFIX = {
"click": "Error during click",
"click_sequence": "Error executing click sequence",
"long_press": "Error during long press",
"input_text": "Error during input text",
"swipe": "Error during swipe",
"press_key": "Error during press_key",
"manage_app": "Error during manage_app",
"wait_for_delay": "Error during wait_for_delay",
"wait_for_text": "Error during wait_for_text",
}
def _wrap(res: ActionResult) -> CallToolResult:
return CallToolResult(
content=[TextContent(type="text", text=res.message)],
structuredContent=res.model_dump(mode="json"),
isError=False,
)
def _wrap_exception(action: str, exc: Exception) -> CallToolResult:
prefix = _EXCEPTION_PREFIX.get(action, f"Error during {action}")
return _wrap(
ActionResult.failure(action, f"{prefix}: {exc}", detail=repr(exc))
)
def build_action_server(actuator: Actuator, name: str = "artemis_actions") -> FastMCP:
"""Builds a FastMCP server exposing exactly what ``actuator`` implements.
Raises ``ActuatorContractError`` when the actuator violates the manifest, so a
misconfigured backend fails before serving anything.
"""
validate_actuator(actuator)
caps = actuator.capabilities()
mcp = FastMCP(name)
# --- Device actions (registered only when implemented) ---------------------------
if "click" in caps:
@mcp.tool()
async def click(
target: list[int], times: int = 1, delay_ms: int = 100
) -> Annotated[CallToolResult, ActionResult]:
"""Tap at a 0-1000 normalized [x, y] coordinate."""
try:
return _wrap(
await actuator.click(
int(target[0]), int(target[1]), times=times, delay_ms=delay_ms
)
)
except Exception as e:
return _wrap_exception("click", e)
if "click_sequence" in caps:
@mcp.tool()
async def click_sequence(
sequence: list[list[int]], delay_ms: int = 50
) -> Annotated[CallToolResult, ActionResult]:
"""Tap a series of 0-1000 normalized [x, y] points in one atomic burst."""
try:
points = [(int(p[0]), int(p[1])) for p in sequence]
return _wrap(await actuator.click_sequence(points, delay_ms=delay_ms))
except Exception as e:
return _wrap_exception("click_sequence", e)
if "long_press" in caps:
@mcp.tool()
async def long_press(
target: list[int], duration_ms: int = 1000
) -> Annotated[CallToolResult, ActionResult]:
"""Long-press at a 0-1000 normalized [x, y] coordinate."""
try:
return _wrap(
await actuator.long_press(
int(target[0]), int(target[1]), duration_ms=duration_ms
)
)
except Exception as e:
return _wrap_exception("long_press", e)
if "input_text" in caps:
@mcp.tool()
async def input_text(
text: str,
target: list[int] | None = None,
clear_exist: bool = True,
) -> Annotated[CallToolResult, ActionResult]:
"""Type text, optionally focusing a 0-1000 normalized [x, y] target first."""
try:
norm = (int(target[0]), int(target[1])) if target else None
return _wrap(
await actuator.input_text(text, norm, clear_exist=clear_exist)
)
except Exception as e:
return _wrap_exception("input_text", e)
if "swipe" in caps:
@mcp.tool()
async def swipe(
start: list[int], end: list[int], duration_ms: int = 800
) -> Annotated[CallToolResult, ActionResult]:
"""Swipe between two 0-1000 normalized [x, y] points."""
try:
return _wrap(
await actuator.swipe(
(int(start[0]), int(start[1])),
(int(end[0]), int(end[1])),
duration_ms,
)
)
except Exception as e:
return _wrap_exception("swipe", e)
if "press_key" in caps:
@mcp.tool()
async def press_key(key: str) -> Annotated[CallToolResult, ActionResult]:
"""Press a device key (home, back, enter, delete, tab, search, menu, app_switch)."""
try:
return _wrap(await actuator.press_key(key))
except Exception as e:
return _wrap_exception("press_key", e)
if "manage_app" in caps:
@mcp.tool()
async def manage_app(
action: str, app_name: str
) -> Annotated[CallToolResult, ActionResult]:
"""Launch or stop an app by human-readable name or package."""
try:
return _wrap(await actuator.manage_app(action, app_name))
except Exception as e:
return _wrap_exception("manage_app", e)
if "wait_for_delay" in caps:
@mcp.tool()
async def wait_for_delay(
time_in_ms: int,
) -> Annotated[CallToolResult, ActionResult]:
"""Wait for a fixed number of milliseconds."""
try:
return _wrap(await actuator.wait_for_delay(time_in_ms))
except Exception as e:
return _wrap_exception("wait_for_delay", e)
if "wait_for_text" in caps:
@mcp.tool()
async def wait_for_text(
text: str, wait_state: str = "appear", timeout_ms: int = 5000
) -> Annotated[CallToolResult, ActionResult]:
"""Wait for text to appear on or disappear from the screen."""
try:
return _wrap(await actuator.wait_for_text(text, wait_state, timeout_ms))
except Exception as e:
return _wrap_exception("wait_for_text", e)
if "open_link" in caps:
@mcp.tool()
async def open_link(url: str) -> Annotated[CallToolResult, ActionResult]:
"""Open a URL on the device."""
try:
return _wrap(await actuator.open_link(url))
except Exception as e:
return _wrap_exception("open_link", e)
if "erase_one_char" in caps:
@mcp.tool()
async def erase_one_char() -> Annotated[CallToolResult, ActionResult]:
"""Erase a single character in the focused field."""
try:
return _wrap(await actuator.erase_one_char())
except Exception as e:
return _wrap_exception("erase_one_char", e)
if "focus_and_clear_text" in caps:
@mcp.tool()
async def focus_and_clear_text(
target: list[int],
) -> Annotated[CallToolResult, ActionResult]:
"""Focus the field at a 0-1000 normalized [x, y] and clear its text."""
try:
return _wrap(
await actuator.focus_and_clear_text(int(target[0]), int(target[1]))
)
except Exception as e:
return _wrap_exception("focus_and_clear_text", e)
# --- Internal observation tools (never declared to an LLM) -----------------------
if "observe_screen" in caps:
@mcp.tool()
async def observe_screen(
settle_ms: int = 400, include_image: bool = False
) -> Annotated[CallToolResult, ObserveResult]:
"""Capture and index the current screen (adapter-internal)."""
try:
obs, img_bytes = await observe_impl(
getattr(actuator, "ctx", None),
actuator.controller,
settle_ms=settle_ms,
)
content: list[Any] = [TextContent(type="text", text=obs.message)]
if include_image and img_bytes:
import base64 as _b64
content.append(
ImageContent(
type="image",
data=_b64.b64encode(img_bytes).decode("utf-8"),
mimeType="image/jpeg",
)
)
return CallToolResult(
content=content,
structuredContent=obs.model_dump(mode="json"),
isError=False,
)
except Exception as e:
obs = ObserveResult(
ok=False,
code=ActionCode.DEVICE_ERROR,
message=f"Error during observe_screen: {e}",
hierarchy_ok=False,
)
return CallToolResult(
content=[TextContent(type="text", text=obs.message)],
structuredContent=obs.model_dump(mode="json"),
isError=False,
)
# Timeouts on the two observation tools run INSIDE the tool, around the actuator
# call only. They must never be expressed as a transport-level read timeout
# (``read_timeout_seconds``) or a caller-side ``asyncio.wait_for``: both produce a
# late response to an abandoned request, which crashes the mcp-1.29 client
# receive loop (``_handle_response`` sends into the request's already-closed
# response stream) and bricks the in-memory session for every later caller.
# A server-side timeout yields a normal error payload and no late response.
if "take_screenshot" in caps:
@mcp.tool()
async def take_screenshot(timeout_ms: int | None = None) -> CallToolResult:
"""Return the current screen as base64 (adapter-internal, no persistence)."""
try:
coro = actuator.take_screenshot()
if timeout_ms:
coro = asyncio.wait_for(coro, timeout=timeout_ms / 1000.0)
b64 = await coro
payload: dict[str, Any] = {"ok": True, "image_b64": b64}
except (TimeoutError, asyncio.TimeoutError):
payload = {"ok": False, "error": f"screenshot timed out after {timeout_ms}ms"}
except Exception as e:
payload = {"ok": False, "error": str(e)}
return CallToolResult(
content=[TextContent(type="text", text="screenshot" if payload["ok"] else str(payload))],
structuredContent=payload,
isError=False,
)
if "get_ui_hierarchy" in caps:
@mcp.tool()
async def get_ui_hierarchy(timeout_ms: int | None = None) -> CallToolResult:
"""Return the raw UI element hierarchy (adapter-internal)."""
try:
coro = actuator.get_ui_elements()
if timeout_ms:
coro = asyncio.wait_for(coro, timeout=timeout_ms / 1000.0)
elements = await coro
payload: dict[str, Any] = {"ok": True, "elements": elements}
except (TimeoutError, asyncio.TimeoutError):
payload = {"ok": False, "error": f"ui hierarchy fetch timed out after {timeout_ms}ms"}
except Exception as e:
payload = {"ok": False, "error": str(e)}
return CallToolResult(
content=[TextContent(type="text", text="ui_hierarchy" if payload["ok"] else str(payload))],
structuredContent=payload,
isError=False,
)
# --- Backend extension tools -----------------------------------------------------
for ext in actuator.extensions():
mcp.add_tool(ext.handler, name=ext.name, description=ext.description)
logger.info(
f"Action server '{name}' built with {len(caps)} device actions and"
f" {len(actuator.extensions())} extension tools."
)
return mcp