chenmingxuan/Artemis-Eapil
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 32 Wiki Activity

feat: add agentic video analysis and clarify action results

Browse Source 
Report device actions as dispatched and preserve full failure details. Route
Flash through McpActionExecutor and use ToolFailure across helper tools and
agent loops instead of inferring failure from response text.

Require model-provided descriptions for coordinate targets. Keep observed
element metadata separate from those descriptions in safety checks, visual
summaries, history, and frontend labels.

Add Gemini Interactions API video analysis with configurable processing mode,
chunk size, and timeouts. Centralize Google provider capabilities and token
usage normalization, and update google-genai to 2.22.0.

Preserve system instructions on follow-up interactions, replan pending chunks
when falling back to static processing, and reject unanswered native video
and audio segments instead of caching placeholder results. Keep helper-tool
failure status intact through history tools and committee calls.

Validation: 1,762 Python unit tests passed (3 skipped), 40 frontend formatter
tests passed, and the production UI build completed. Ruff lint, formatting,
the quality ratchet, and git diff checks passed.
This commit is contained in:
somew1nd committed 2026-09-08 17:05:54 -07:00
1 parent 611d1a0947
commit a45daf23f8
120 files changed
+6301 -1710
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

artemis/mcp/action_executor.py
+91 -126
View File
@@ -14,22 +14,12 @@
"""MCP-backed action executor for the FlashRunner.
Same public contract as the legacy ``MobileActionExecutor`` --
``execute(name, args, tool_call_id, state) -> ToolExecutionResult`` -- but execution
routes through the in-process action MCP session, and dispatch is table-driven so a
backend's extension tools are callable without touching this file.
Device actions use the in-process MCP session. Argument normalization,
coordinate descriptions, smart swipes, post-action observations, state updates,
tracing, and helper tools are handled here on the agent side.
Kept agent-side (by design, not omission): argument normalization, element-index
resolution against LangGraph ``State``, smart-swipe resolution, act-then-observe
capture and ``State`` write-back, tracing (a ``ContextVar`` cannot cross the server's
task boundary -- see the plan's R2), and the non-device tools (``read_note``,
``list_notes``, ``ask_explorer``, ``video_analyzer`` and the shared history tools
``search_history`` / ``replay_steps`` / ``get_step_screenshot``).
Status is carried by ``ActionResult.ok`` -- the historical substring sniffing on
``"Error"``/``"Failed"`` is gone, so typing the literal text "Failed" into a field no
longer marks the step as failed, and a successful tap with a failed screenshot is no
longer misreported as an error.
Device action status comes from ``ActionResult.ok``; helper tools report failures
through ``ToolFailure`` or an explicit status.
"""
import ast
@@ -38,6 +28,7 @@ import json
from pathlib import Path
from typing import Any
from artemis.core.tool_failure import ToolFailure, is_tool_failure
from artemis.context import ArtemisContext
from artemis.controllers.unified_controller import UnifiedMobileController
from artemis.data_engine.trace import TraceSpan
@@ -56,7 +47,6 @@ from artemis.utils.coordinates import (
compute_smart_swipe_coordinates,
parse_swipe_parameters,
)
from artemis.utils.element_hit_test import hit_test_semantics
from artemis.utils.logger import get_logger
from artemis.utils.notes import (
format_list_notes_failure,
@@ -153,19 +143,13 @@ class McpActionExecutor:
img_bytes: bytes | None = None
shot_path: str | None = None
xml_list: str | None = None
target_semantics: dict[str, Any] | None = None
target_semantics: dict[str, Any] = {}
with span:
try:
session = await self._session_or_start()
wire_name, wire_args, finalize = self._translate(raw_name, args, state)
# Record-time enrichment: bare-coordinate targeted actions get
# best-effort element semantics from the pre-action frame (the
# post-action observe below overwrites state.indexed_elements,
# so this must happen before the device call).
target_semantics = self._target_semantics(raw_name, wire_args, state)
if target_semantics:
span.payload = {"args": {**args, **target_semantics}}
wire_name, wire_args, finalize, target_semantics = self._translate(
raw_name, args, state
)
extension = raw_name not in (REQUIRED_ACTIONS | OPTIONAL_ACTIONS)
if extension:
@@ -233,71 +217,11 @@ class McpActionExecutor:
},
)
def _target_semantics(
self, raw_name: str, wire_args: dict[str, Any], state: Any
) -> dict[str, Any] | None:
"""Best-effort element semantics for a coordinate-targeted action.
Hit tests the normalized (0-1000) target point(s) against the
pre-action frame's indexed elements. Single-target actions get the
flat ``target_*`` fields; multi-point actions (``click_sequence``,
``swipe``) hit test every point best-effort (M5), hoisting the FIRST
point's semantics to the main ``target_*`` label and keeping the
per-point results alongside. Returns None for actions without a
resolvable coordinate target; degrades to
``target_label_source: "none"`` when no element data covers the point.
"""
try:
elements = getattr(state, "indexed_elements", None) if state else None
if raw_name in ("click", "long_press", "input_text"):
target = wire_args.get("target")
if not (isinstance(target, (list, tuple)) and len(target) == 2):
return None
return self._hit_test_point(elements, target)
if raw_name == "click_sequence":
sequence = wire_args.get("sequence")
if not (isinstance(sequence, (list, tuple)) and sequence):
return None
points = [self._hit_test_point(elements, p) for p in sequence]
semantics: dict[str, Any] = dict(points[0])
semantics["points_semantics"] = points
return semantics
if raw_name == "swipe":
start = wire_args.get("start")
end = wire_args.get("end")
if not (isinstance(start, (list, tuple)) and len(start) == 2):
return None
semantics = dict(self._hit_test_point(elements, start))
semantics["start_semantics"] = self._hit_test_point(elements, start)
if isinstance(end, (list, tuple)) and len(end) == 2:
semantics["end_semantics"] = self._hit_test_point(elements, end)
return semantics
return None
except Exception as enrich_err:
logger.debug(f"Target semantics enrichment skipped for {raw_name}: {enrich_err}")
return {"target_label_source": "none"}
def _hit_test_point(self, elements: Any, target: Any) -> dict[str, Any]:
"""Hit tests one normalized (0-1000) point; never raises."""
try:
width = getattr(self.ctx.device, "device_width", 1080) if self.ctx.device else 1080
height = getattr(self.ctx.device, "device_height", 2400) if self.ctx.device else 2400
x_px = int(max(0, min(width - 1, float(target[0]) * width / 1000)))
y_px = int(max(0, min(height - 1, float(target[1]) * height / 1000)))
return hit_test_semantics(elements, x_px, y_px)
except Exception:
return {"target_label_source": "none"}
@staticmethod
def _observe_despite_failure(raw_name: str, res: ActionResult) -> bool:
"""Failures that still observe the screen, mirroring the legacy executor.
"""Capture the screen after launch failures and wait_for_text timeouts.
A failed app launch and a wait_for_text timeout captured the screen; argument
and package errors returned without one.
Argument and package lookup errors do not require an observation.
"""
if raw_name == "manage_app":
return res.code not in (ActionCode.PACKAGE_NOT_FOUND, ActionCode.INVALID_ARGS)
@@ -309,16 +233,19 @@ class McpActionExecutor:
def _translate(
self, raw_name: str, args: dict[str, Any], state: Any
) -> tuple[str, dict[str, Any], Any]:
"""Maps agent-facing args to wire args; returns (name, args, finalize).
) -> tuple[str, dict[str, Any], Any, dict[str, Any]]:
"""Maps agent-facing args to wire args; returns (name, args, finalize, recorded).
``finalize(res)`` post-processes the ActionResult message where the historical
wording depended on client-side context (direction swipes).
``finalize(res)`` adds client-side context to swipe result messages.
``recorded`` contains validated target descriptions in the Pro Operator's
record format. Focused input and directional swipes have no coordinate
target, so they do not record a description.
"""
if raw_name == "click":
target = self._require_pair(
args.get("target") or args.get("coordinates") or [], "click"
)
recorded = self._require_description(args, "click")
return (
"click",
{
@@ -327,12 +254,14 @@ class McpActionExecutor:
"delay_ms": args.get("delay_ms", 100),
},
None,
recorded,
)
if raw_name == "long_press":
target = self._require_pair(
args.get("target") or args.get("coordinates") or [], "long press"
)
recorded = self._require_description(args, "long press")
return (
"long_press",
{
@@ -340,13 +269,16 @@ class McpActionExecutor:
"duration_ms": args.get("duration_ms", args.get("duration", 1000)),
},
None,
recorded,
)
if raw_name == "input_text":
raw_target = args.get("target") or args.get("coordinates")
target = None
recorded = {}
if raw_target:
target = list(self._require_pair(raw_target, "input text"))
recorded = self._require_description(args, "input text")
return (
"input_text",
{
@@ -355,23 +287,27 @@ class McpActionExecutor:
"clear_exist": args.get("clear_exist", True),
},
None,
recorded,
)
if raw_name == "click_sequence":
sequence = self._resolve_sequence(args.get("sequence") or [])
recorded = self._require_descriptions(args, len(sequence))
return (
"click_sequence",
{
"sequence": self._resolve_sequence(args.get("sequence") or [], state),
"sequence": sequence,
"delay_ms": args.get("delay_ms", 50),
},
None,
recorded,
)
if raw_name == "swipe":
return self._translate_swipe(args, state)
if raw_name == "press_key":
return "press_key", {"key": args.get("key", "BACK")}, None
return "press_key", {"key": args.get("key", "BACK")}, None, {}
if raw_name == "manage_app":
return (
@@ -381,10 +317,11 @@ class McpActionExecutor:
"app_name": args.get("app_name", ""),
},
None,
{},
)
if raw_name == "wait_for_delay":
return "wait_for_delay", {"time_in_ms": args.get("time_in_ms", 1000)}, None
return "wait_for_delay", {"time_in_ms": args.get("time_in_ms", 1000)}, None, {}
if raw_name == "wait_for_text":
return (
@@ -395,10 +332,11 @@ class McpActionExecutor:
"timeout_ms": args.get("timeout_ms", 5000),
},
None,
{},
)
# Backend extension: pass the arguments straight through.
return raw_name, dict(args), None
return raw_name, dict(args), None, {}
@staticmethod
def _require_pair(raw: Any, label: str) -> tuple[int, int]:
@@ -412,8 +350,40 @@ class McpActionExecutor:
raise _ArgError(f"{prefix}: Invalid target format: {raw}")
return int(target[0]), int(target[1])
def _resolve_sequence(self, sequence: Any, state: Any) -> list[list[int]]:
"""Resolves click_sequence entries (indices or pairs) to normalized pairs."""
@staticmethod
def _require_description(args: dict[str, Any], label: str) -> dict[str, Any]:
"""Validate and trim the coordinate target description for the action record."""
description = args.get("target_description")
if not (isinstance(description, str) and description.strip()):
raise _ArgError(
f"Error during {label}: 'target_description' is required (what the"
" target is, in a few words)."
)
return {"target_description": description.strip()}
@staticmethod
def _require_descriptions(args: dict[str, Any], count: int) -> dict[str, Any]:
"""``click_sequence``: one description per sequence entry, in order.
Returns the cleaned statements as the recorded ``target_descriptions`` field.
"""
descriptions = args.get("target_descriptions")
if isinstance(descriptions, str):
descriptions = [descriptions]
if (
not isinstance(descriptions, (list, tuple))
or len(descriptions) != count
or not all(isinstance(d, str) and d.strip() for d in descriptions)
):
raise _ArgError(
"Error during click sequence: 'target_descriptions' is required, one"
f" non-empty entry per sequence entry ({count} expected)."
)
return {"target_descriptions": [d.strip() for d in descriptions]}
@staticmethod
def _resolve_sequence(sequence: Any) -> list[list[int]]:
"""Normalize click_sequence entries to coordinate pairs; reject element indices."""
if isinstance(sequence, str):
sequence_str = sequence.strip()
try:
@@ -427,31 +397,26 @@ class McpActionExecutor:
if not isinstance(sequence, (list, tuple)):
raise _ArgError(f"Error during click sequence: Invalid sequence format: {sequence}")
width = getattr(self.ctx.device, "device_width", 1080) if self.ctx.device else 1080
height = getattr(self.ctx.device, "device_height", 2400) if self.ctx.device else 2400
resolved: list[list[int]] = []
for raw_target in sequence:
for position, raw_target in enumerate(sequence, start=1):
target = normalize_coordinate_target(raw_target)
if isinstance(target, int):
idx = target
points = getattr(state, "indexed_points", []) or []
if not 1 <= idx <= len(points):
raise _ArgError(
"Error during click sequence: Index"
f" {idx} is out of range (available: {len(points)})"
)
px, py = int(points[idx - 1][0]), int(points[idx - 1][1])
nx = int(max(0, min(1000, round(px * 1000 / max(1, width)))))
ny = int(max(0, min(1000, round(py * 1000 / max(1, height)))))
elif isinstance(target, (list, tuple)) and len(target) == 2:
if isinstance(target, (int, float)) and not isinstance(target, bool):
raise _ArgError(
f"Error during click sequence: entry {position} ({raw_target!r}) is an"
" element index; click_sequence takes normalized [x, y] coordinate"
" pairs only. Use the element's coordinates from the element list,"
" or ask_explorer to locate it."
)
if isinstance(target, (list, tuple)) and len(target) == 2:
nx, ny = int(target[0]), int(target[1])
else:
raise _ArgError(f"Error during click sequence: Invalid target format: {raw_target}")
resolved.append([nx, ny])
return resolved
def _translate_swipe(self, args: dict[str, Any], state: Any) -> tuple[str, dict[str, Any], Any]:
def _translate_swipe(
self, args: dict[str, Any], state: Any
) -> tuple[str, dict[str, Any], Any, dict[str, Any]]:
width = getattr(self.ctx.device, "device_width", 1080) if self.ctx.device else 1080
height = getattr(self.ctx.device, "device_height", 2400) if self.ctx.device else 2400
default_duration = args.get("duration", 400)
@@ -477,7 +442,7 @@ class McpActionExecutor:
def finalize(res: ActionResult) -> str:
if not res.ok:
return f"Error swiping {dir_name}: {res.detail}"
return f"Swipe completed successfully. Swiped {dir_name}."
return f"Swiped {dir_name}."
return (
"swipe",
@@ -487,15 +452,17 @@ class McpActionExecutor:
"duration_ms": smart_dur,
},
finalize,
{},
)
if kind == "coords":
x1, y1, x2, y2 = target
recorded = self._require_description(args, "swipe")
def finalize(res: ActionResult) -> str:
if not res.ok:
return f"Error dragging: {res.detail}"
return f"Swipe completed successfully. Swiped from [{x1}, {y1}] to [{x2}, {y2}]."
return f"Swiped from [{x1}, {y1}] to [{x2}, {y2}] (normalized)."
return (
"swipe",
@@ -505,6 +472,7 @@ class McpActionExecutor:
"duration_ms": final_duration,
},
finalize,
recorded,
)
raise _ArgError(f"Error during swipe: Invalid direction: {args}")
@@ -537,7 +505,6 @@ class McpActionExecutor:
)
elif raw_name in HISTORY_TOOL_NAMES:
text, blocks = await self._history_tool(raw_name, args)
ok = True
else:
text, ok = await self._ask_explorer(
# ``task_description`` is the pre-contract argument name
@@ -551,13 +518,11 @@ class McpActionExecutor:
span.status = "failed"
span.error = text
# Note texts come from our own failure formatters, so the historical
# "Error" marker check is reliable there (unlike free-form device
# output); the Explorer and the video analyzer report their status
# explicitly and a history lookup's "no match" / "not found" answer
# is not an error.
# The Explorer and the video analyzer report their status explicitly;
# note and history texts carry it structurally (a ``ToolFailure``), so a
# history lookup's "no match" answer is an answer, not an error.
if ok is None:
ok = "Error" not in (text or "")
ok = not is_tool_failure(text)
return ToolExecutionResult(
tool_call_id=tool_call_id,
tool_name=name,
@@ -597,13 +562,13 @@ class McpActionExecutor:
"""
tool = history_tool_by_name(raw_name)
if tool is None:
return f"Error: Tool '{raw_name}' not supported.", None
return ToolFailure(f"Error: Tool '{raw_name}' not supported."), None
accepted = {k: v for k, v in args.items() if k in tool.args_schema.model_fields}
try:
result = await tool.execute(ctx=self.ctx, **accepted)
except Exception as e:
logger.error(f"Error running {raw_name}: {e}")
return f"{raw_name} failed: {e}", None
return ToolFailure(f"{raw_name} failed: {e}"), None
text, images = split_multimodal_result(result)
if images:
return text or "Screenshot attached.", result
artemis/mcp/action_manifest.py
+1 -1
View File
@@ -83,7 +83,7 @@ OPTIONAL_ACTIONS: frozenset[str] = frozenset(
"manage_app",
"wait_for_delay",
# The four below appear in no prompt at all. `wait_for_text` corroborates the
# classification: it is listed in the legacy ACTION_TOOL_NAMES but has never had
# classification: the former legacy executor dispatched it, but it has never had
# a ToolDeclaration, i.e. it has always been unreachable by any LLM.
"wait_for_text",
"open_link",
artemis/mcp/action_specs.py
+123 -14
View File
@@ -36,9 +36,19 @@ and declared**, not accidental:
be returned automatically").
* ``wire`` -- what the action MCP server serves, mirroring the actuator protocol
one-to-one: normalized coordinates, millisecond durations, no addressing sugar.
Client-side executors (``McpActionExecutor``, ``MobileActionExecutor``) lower the
The client-side executor (``McpActionExecutor``) lowers the
agent dialects onto it.
A coordinate target names nothing by itself, so both agent dialects pair it with
``target_description`` (``target_descriptions`` for a ``click_sequence``): the
model's own statement of what it is aiming at. It is required whenever the target
is a coordinate (always, in the declaration dialect), recorded verbatim on the
action, and never sent to the wire. An element index needs none -- its text,
bounds and id are read from the indexed list -- and the recorded fields stay
separate (``target_text``/``target_bounds``/... for observed elements,
``target_description`` for described coordinates) so no reader can mistake the
model's belief for an observation.
Each ``ActionSpec.param_bridge`` maps operator parameter names onto declaration
parameter names, so every rename (``duration`` -> ``duration_ms``, ``gesture`` ->
``action``) is written down exactly once. ``tests/unit/mcp/test_action_specs.py``
@@ -164,6 +174,22 @@ _COORD_PAIR_JSON = {
"items": {"type": "integer"},
}
#: Operator-dialect wording for the description that a coordinate target must carry.
#: An element index already names its element (its text, id and bounds are recorded
#: from the indexed list); a bare coordinate names nothing, so the model states what
#: it is aiming at and that statement is recorded as ``target_description``.
_OPERATOR_TARGET_DESCRIPTION = (
"What the target is, in a few words (e.g. 'play button', 'search input',"
" 'video body'). REQUIRED when target is a coordinate pair; ignored for an"
" element index."
)
#: Declaration-dialect wording (coordinates-only dialect: always required).
_DECLARATION_TARGET_DESCRIPTION = (
"What the target is, in a few words (e.g. 'play button', 'search input',"
" 'video body'). Required: a coordinate names nothing by itself."
)
# --- Wire bindings -------------------------------------------------------------------
# Each binding reproduces the exact conversion of the historical hand-written FastMCP
@@ -245,6 +271,13 @@ _SPECS: tuple[ActionSpec, ...] = (
"Click target. Can be an element index number (int, e.g. 3) OR"
" normalized coordinates (list of 2 integers, e.g. [500, 600]).",
),
ParamSpec(
"target_description",
str | None,
_OPERATOR_TARGET_DESCRIPTION,
required=False,
default=None,
),
ParamSpec(
"times",
int,
@@ -278,6 +311,10 @@ _SPECS: tuple[ActionSpec, ...] = (
"items": {"type": "integer"},
"description": "Normalized coordinates [x, y] in 0-1000 scale.",
},
"target_description": {
"type": "string",
"description": _DECLARATION_TARGET_DESCRIPTION,
},
"times": {
"type": "integer",
"description": (
@@ -289,7 +326,7 @@ _SPECS: tuple[ActionSpec, ...] = (
"description": "Delay between taps in milliseconds (default 100).",
},
},
"required": ["target"],
"required": ["target", "target_description"],
},
),
wire=WireDialect(
@@ -301,10 +338,17 @@ _SPECS: tuple[ActionSpec, ...] = (
),
bind=_wire_click,
),
param_bridge={"target": "target", "times": "times", "delay_ms": "delay_ms"},
param_bridge={
"target": "target",
"target_description": "target_description",
"times": "times",
"delay_ms": "delay_ms",
},
differences=(
"target: operator accepts an element index or a coordinate pair; the"
" declaration and wire dialects take coordinates only."
" target_description: recorded, never sent to the wire; the operator"
" requires it only for coordinate targets, the declaration always."
),
),
ActionSpec(
@@ -324,14 +368,22 @@ _SPECS: tuple[ActionSpec, ...] = (
"type": "array",
"items": {"type": "integer"},
"description": (
"Normalized coordinates [x, y] in 0-1000 scale (e.g., [500, 280]),"
" or a single integer element index."
"Normalized coordinates [x, y] in 0-1000 scale (e.g., [500, 280])."
),
},
"description": (
"List of targets to tap in sequence, e.g. [[500, 280], [885, 362]]."
),
},
"target_descriptions": {
"type": "array",
"items": {"type": "string"},
"description": (
"What each target is, in a few words, one entry per sequence"
" entry in the same order (e.g. ['video body', 'skip button'])."
" Required: a coordinate names nothing by itself."
),
},
"delay_ms": {
"type": "integer",
"description": (
@@ -339,7 +391,7 @@ _SPECS: tuple[ActionSpec, ...] = (
),
},
},
"required": ["sequence"],
"required": ["sequence", "target_descriptions"],
},
),
wire=WireDialect(
@@ -352,8 +404,9 @@ _SPECS: tuple[ActionSpec, ...] = (
),
differences=(
"Declared to Flash only (the Pro Operator chains actions as a fast-action"
" burst instead); sequence entries may be element indices at the declaration layer"
" (resolved client-side), while the wire takes coordinate pairs only."
" burst instead); sequence entries are normalized coordinate pairs in both"
" dialects (an element index is refused, never resolved client-side)."
" target_descriptions is recorded per entry and never sent to the wire."
),
),
ActionSpec(
@@ -371,6 +424,13 @@ _SPECS: tuple[ActionSpec, ...] = (
" 3) OR normalized coordinates (list of 2 integers, e.g. [500,"
" 600]).",
),
ParamSpec(
"target_description",
str | None,
_OPERATOR_TARGET_DESCRIPTION,
required=False,
default=None,
),
ParamSpec(
"duration",
int,
@@ -393,12 +453,16 @@ _SPECS: tuple[ActionSpec, ...] = (
"items": {"type": "integer"},
"description": "Normalized coordinates [x, y] in 0-1000 scale.",
},
"target_description": {
"type": "string",
"description": _DECLARATION_TARGET_DESCRIPTION,
},
"duration_ms": {
"type": "integer",
"description": "Duration of press in milliseconds (default 1000ms).",
},
},
"required": ["target"],
"required": ["target", "target_description"],
},
),
wire=WireDialect(
@@ -409,9 +473,15 @@ _SPECS: tuple[ActionSpec, ...] = (
),
bind=_wire_long_press,
),
param_bridge={"target": "target", "duration": "duration_ms"},
param_bridge={
"target": "target",
"target_description": "target_description",
"duration": "duration_ms",
},
differences=(
"target: operator accepts an element index or a coordinate pair."
" target_description: recorded, never sent to the wire; the operator"
" requires it only for coordinate targets, the declaration always."
" duration: the operator spells the duration parameter `duration` (its"
" structured decisions and recorded traces carry that key); the"
" declaration and wire dialects spell it `duration_ms`. Executors accept"
@@ -439,6 +509,13 @@ _SPECS: tuple[ActionSpec, ...] = (
" (int, e.g. 3) OR normalized coordinates (list of 2 integers,"
" e.g. [500, 600]).",
),
ParamSpec(
"target_description",
str | None,
_OPERATOR_TARGET_DESCRIPTION,
required=False,
default=None,
),
ParamSpec(
"clear_exist",
bool,
@@ -468,6 +545,10 @@ _SPECS: tuple[ActionSpec, ...] = (
"items": {"type": "integer"},
"description": "Coordinates [x, y] of the input box in 0-1000 scale.",
},
"target_description": {
"type": "string",
"description": _DECLARATION_TARGET_DESCRIPTION,
},
"clear_exist": {
"type": "boolean",
"description": (
@@ -476,7 +557,7 @@ _SPECS: tuple[ActionSpec, ...] = (
),
},
},
"required": ["text", "target"],
"required": ["text", "target", "target_description"],
},
),
wire=WireDialect(
@@ -488,11 +569,18 @@ _SPECS: tuple[ActionSpec, ...] = (
),
bind=_wire_input_text,
),
param_bridge={"target": "target", "text": "text", "clear_exist": "clear_exist"},
param_bridge={
"target": "target",
"target_description": "target_description",
"text": "text",
"clear_exist": "clear_exist",
},
differences=(
"target: operator accepts an element index or a coordinate pair, and"
" requires a target; the wire dialect allows omitting the target to type"
" into the already-focused field."
" into the already-focused field. target_description: recorded, never"
" sent to the wire; the operator requires it only for coordinate targets,"
" the declaration always."
),
),
ActionSpec(
@@ -510,6 +598,7 @@ _SPECS: tuple[ActionSpec, ...] = (
" end: End normalized coordinates [end_x, end_y] in 0-1000 scale.\n"
" target: Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within.\n"
" gesture: Backward-compatible parameter: direction string OR custom coordinates list [start_x, start_y, end_x, end_y] in 0-1000 scale.\n"
" target_description: What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling.\n"
" duration: Optional gesture duration in milliseconds (default 800)."
),
params=(
@@ -554,6 +643,15 @@ _SPECS: tuple[ActionSpec, ...] = (
required=False,
default=None,
),
ParamSpec(
"target_description",
str | None,
"What is being dragged, in a few words (e.g. 'brightness slider"
" knob'). REQUIRED for coordinate gestures ('start'/'end' or a"
" coordinates list); ignored for directional scrolling.",
required=False,
default=None,
),
ParamSpec(
"duration",
int | None,
@@ -609,6 +707,14 @@ _SPECS: tuple[ActionSpec, ...] = (
" OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale."
),
},
"target_description": {
"type": "string",
"description": (
"What is being dragged, in a few words (e.g. 'brightness slider"
" knob'). Required for coordinate gestures ('start'/'end' or a"
" coordinates list); ignored for directional scrolling."
),
},
"duration": {
"type": "integer",
"description": (
@@ -635,10 +741,13 @@ _SPECS: tuple[ActionSpec, ...] = (
"end": "end",
"target": "target",
"gesture": "action",
"target_description": "target_description",
"duration": "duration",
},
differences=(
"gesture/action: the legacy combined direction-or-coordinates parameter is"
"target_description: recorded, never sent to the wire; required by both"
" agent dialects for coordinate gestures only."
" gesture/action: the legacy combined direction-or-coordinates parameter is"
" spelled `gesture` in the operator dialect and `action` in the"
" declaration dialect (`action` collides with the operator's structured"
" decision verb field). Smart directional swipes exist only in the agent"
artemis/mcp/actuators/adb.py
+20 -17
View File
@@ -14,10 +14,9 @@
"""ADB-backed actuator: the reference implementation of the actuator contract.
The device-call bodies here were extracted from
``artemis.agents.validator.tool_declarations.MobileActionExecutor``; the outcome
message wording is preserved byte-for-byte because agent transcripts, trace payloads,
and existing tests assert on it.
Outcome messages describe the dispatched command, such as
"Tapped at [x, y] (normalized)." The agent verifies the effect from subsequent
observations.
Error convention: device-level refusals the controller *reports* (an ``error`` field,
a falsy success flag) come back as ``ActionResult(ok=False)`` with the historical
@@ -147,13 +146,13 @@ class AdbActuator:
f"Error executing click at step {i + 1}: {err}",
detail=str(err),
)
outcomes.append(f"Tapped at [{x}, {y}]")
outcomes.append(f"[{nx}, {ny}]")
if i < len(points) - 1:
await asyncio.sleep(max(0, delay_ms) / 1000.0)
return ActionResult.success(
"click_sequence",
f"Sequence clicked successfully: {'; '.join(outcomes)}",
f"Tapped in sequence at {'; '.join(outcomes)} (normalized).",
)
# --- Optional actions ------------------------------------------------------------
@@ -166,7 +165,7 @@ class AdbActuator:
return ActionResult.failure("click", f"Error executing click: {err}", detail=str(err))
return ActionResult.success(
"click",
f"Clicked at [{nx}, {ny}] (normalized) successfully.",
f"Tapped at [{nx}, {ny}] (normalized).",
normalized_coordinates=[int(nx), int(ny)],
)
@@ -182,7 +181,7 @@ class AdbActuator:
)
return ActionResult.success(
"long_press",
f"Long pressed at [{nx}, {ny}] (normalized) for {duration_ms}ms successfully.",
f"Long-pressed at [{nx}, {ny}] (normalized) for {duration_ms}ms.",
normalized_coordinates=[int(nx), int(ny)],
duration_ms=duration_ms,
)
@@ -227,7 +226,8 @@ class AdbActuator:
return ActionResult.success(
"input_text",
f"Executed typing '{text}'.",
f"Typed '{text}'"
+ (f" at [{coords[0]}, {coords[1]}] (normalized)." if coords else "."),
normalized_coordinates=coords,
)
@@ -246,7 +246,7 @@ class AdbActuator:
return ActionResult.failure("swipe", f"Error dragging: {err}", detail=str(err))
return ActionResult.success(
"swipe",
f"Swipe completed successfully. Swiped from [{nx1}, {ny1}] to [{nx2}, {ny2}].",
f"Swiped from [{nx1}, {ny1}] to [{nx2}, {ny2}] (normalized).",
normalized_coordinates=[int(nx1), int(ny1), int(nx2), int(ny2)],
duration_ms=duration_ms,
)
@@ -281,7 +281,7 @@ class AdbActuator:
if not res:
return ActionResult.failure("press_key", f"Error executing key press '{key}'.")
return ActionResult.success("press_key", f"Executed key press '{key}'.")
return ActionResult.success("press_key", f"Pressed key '{key}'.")
async def manage_app(self, action: str, app_name: str) -> ActionResult:
# Imported lazily: launch_app pulls in tool wrappers that are costly at import.
@@ -310,7 +310,7 @@ class AdbActuator:
if success:
return ActionResult.success(
"manage_app",
f"Launched app '{app_name}' ({target_pkg}) successfully.",
f"Launched app '{app_name}' ({target_pkg}); foreground confirmed.",
)
return ActionResult.failure(
"manage_app",
@@ -322,7 +322,9 @@ class AdbActuator:
res_term = self.controller.terminate_app(target_pkg)
if inspect.iscoroutine(res_term):
await res_term
return ActionResult.success("manage_app", f"Terminated app '{app_name}' successfully.")
return ActionResult.success(
"manage_app", f"Force-stopped app '{app_name}' ({target_pkg})."
)
return ActionResult.failure(
"manage_app",
f"Invalid manage_app action: {action}",
@@ -334,7 +336,7 @@ class AdbActuator:
await asyncio.sleep(delay_s)
return ActionResult.success(
"wait_for_delay",
f"Waited for {time_in_ms}ms successfully.",
f"Waited {time_in_ms}ms.",
duration_ms=int(time_in_ms),
)
@@ -360,7 +362,8 @@ class AdbActuator:
if found:
return ActionResult.success(
"wait_for_text",
f"Successfully waited for text '{text}' to {target_state}.",
f"Text '{text}' {'appeared' if target_state == 'appear' else 'disappeared'}"
f" in the UI tree after {int((time.time() - start) * 1000)}ms.",
)
return ActionResult.failure(
"wait_for_text",
@@ -371,7 +374,7 @@ class AdbActuator:
async def open_link(self, url: str) -> ActionResult:
success = await self.controller.open_url(url)
if success:
return ActionResult.success("open_link", f"Opened link '{url}' successfully.")
return ActionResult.success("open_link", f"Opened link '{url}'.")
return ActionResult.failure("open_link", f"Failed to open link '{url}'.")
async def erase_one_char(self) -> ActionResult:
@@ -394,7 +397,7 @@ class AdbActuator:
if success:
return ActionResult.success(
"focus_and_clear_text",
f"Cleared text at [{nx}, {ny}] (normalized) successfully.",
f"Cleared text at [{nx}, {ny}] (normalized).",
normalized_coordinates=[int(nx), int(ny)],
)
return ActionResult.failure("focus_and_clear_text", "Failed to erase text.")