chenmingxuan/Artemis-Eapil
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 32 Wiki Activity

feat: harden agent verification and runtime contracts

Browse Source 
Introduce two-gate checker settlement, explicit graph visibility and checkpoints, structured LLM reliability, centralized action specifications, and corresponding tests and design documentation.
This commit is contained in:
somew1nd committed 2026-08-31 20:42:24 -07:00
1 parent ddfa420945
commit cbe8f4f0c6
106 files changed
+11542 -4445
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

artemis/agents/validator/tool_declarations.py
+13 -224
View File
@@ -30,6 +30,7 @@ from artemis.context import ArtemisContext
from artemis.controllers.unified_controller import UnifiedMobileController
from artemis.core.tool_declaration import ToolDeclaration
from artemis.data_engine.trace import trace
from artemis.mcp.action_specs import tool_declaration
from artemis.mcp.action_types import ActionCode
from artemis.mcp.actuators.adb import AdbActuator
from artemis.mcp.observation import observe
@@ -187,9 +188,7 @@ async def capture_screenshot_and_parse_ui(
Thin adapter over :func:`artemis.mcp.observation.observe` that adds the LangGraph
``State`` write-back (indexed points/elements) the agents rely on.
"""
obs, screenshot_bytes = await observe(
ctx, controller, settle_ms=0 if skip_settling else 400
)
obs, screenshot_bytes = await observe(ctx, controller, settle_ms=0 if skip_settling else 400)
if not obs.ok:
return None, None, None
@@ -473,7 +472,7 @@ class MobileActionExecutor:
direction=target,
target=swipe_input.get("target"),
indexed_elements=getattr(state, "indexed_elements", None) if state else None,
ui_hierarchy=getattr(state, "ui_tree", None) if state else None,
ui_hierarchy=getattr(state, "latest_ui_hierarchy", None) if state else None,
width=width,
height=height,
duration=final_duration,
@@ -753,189 +752,19 @@ class MobileActionExecutor:
)
CLICK_TOOL = ToolDeclaration(
name="click",
description=(
"[ACTION] Tap/click on the target coordinate on the screen. The screen"
" after click will be returned automatically. For buttons, checkboxes,"
" tabs, icons, items in a list, tap ON the element. For text fields /"
" search bars / input boxes, tap INSIDE the input box to focus it."
),
parameters={
"type": "object",
"properties": {
"target": {
"type": "array",
"items": {"type": "integer"},
"description": "Normalized coordinates [x, y] in 0-1000 scale.",
},
"times": {
"type": "integer",
"description": "Number of taps to perform (default 1). Set to 2 for double click.",
},
"delay_ms": {
"type": "integer",
"description": "Delay between taps in milliseconds (default 100).",
},
},
"required": ["target"],
},
)
# Device-action declarations are generated from the canonical manifest
# (artemis/mcp/action_specs.py); the historical constant names remain the public API.
CLICK_TOOL = tool_declaration("click")
CLICK_SEQUENCE_TOOL = ToolDeclaration(
name="click_sequence",
description=(
"[ACTION] Executes a sequence of taps one by one in order on the"
" specified targets (e.g. [[500, 280], [885, 362]]). The screen will be returned ONLY after all clicks"
" in the sequence have completed."
),
parameters={
"type": "object",
"properties": {
"sequence": {
"type": "array",
"items": {
"type": "array",
"items": {"type": "integer"},
"description": (
"Normalized coordinates [x, y] in 0-1000 scale (e.g., [500, 280]),"
" or a single integer element index."
),
},
"description": "List of targets to tap in sequence, e.g. [[500, 280], [885, 362]].",
},
"delay_ms": {
"type": "integer",
"description": "Delay between consecutive taps in milliseconds (default 50ms).",
},
},
"required": ["sequence"],
},
)
CLICK_SEQUENCE_TOOL = tool_declaration("click_sequence")
LONG_PRESS_TOOL = ToolDeclaration(
name="long_press",
description=(
"[ACTION] Long press on a target coordinate on the screen. The screen"
" after long pressing will be returned automatically."
),
parameters={
"type": "object",
"properties": {
"target": {
"type": "array",
"items": {"type": "integer"},
"description": "Normalized coordinates [x, y] in 0-1000 scale.",
},
"duration_ms": {
"type": "integer",
"description": "Duration of press in milliseconds (default 1000ms).",
},
},
"required": ["target"],
},
)
LONG_PRESS_TOOL = tool_declaration("long_press")
INPUT_TEXT_TOOL = ToolDeclaration(
name="input_text",
description=(
"[ACTION] Type text into an input field on the screen. The screen after"
" typing will be returned automatically. Automatically taps inside the"
" input box at target [x, y] to focus before typing."
),
parameters={
"type": "object",
"properties": {
"text": {
"type": "string",
"description": "Text to type into the focused input field.",
},
"target": {
"type": "array",
"items": {"type": "integer"},
"description": "Coordinates [x, y] of the input box in 0-1000 scale.",
},
"clear_exist": {
"type": "boolean",
"description": (
"Whether to clear existing text in the input box before typing (default True)."
),
},
},
"required": ["text", "target"],
},
)
INPUT_TEXT_TOOL = tool_declaration("input_text")
SWIPE_TOOL = ToolDeclaration(
name="swipe",
description=(
"[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen. The screen and UI hierarchy after swipe will be returned automatically.\n\n"
"• Directional Scrolling ('direction' or 'action'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n"
"• Precise Coordinate Gestures ('start', 'end' or coordinates list): Best for local, fine-grained interactions such as adjusting sliders/SeekBars (e.g., volume, brightness, progress bars), drag-and-drop / list reordering, or as a reliable fallback when directional scrolling fails on specific containers. Always drag slightly PAST the target position to overcome touch slop and reliably trigger the update. When setting a slider to Maximum (100%) or Minimum (0%), swipe fully to the extreme boundary."
),
parameters={
"type": "object",
"properties": {
"direction": {
"type": "string",
"enum": ["up", "down", "left", "right"],
"description": (
"Direction for scrolling and swiping: 'up' (drags bottom-to-top, scrolling down to reveal content below),"
" 'down' (drags top-to-bottom, scrolling up to reveal content above),"
" 'left' (drags right-to-left, scrolling right),"
" 'right' (drags left-to-right, scrolling left)."
),
},
"start": {
"type": "array",
"items": {"type": "integer"},
"description": "Start normalized coordinates [start_x, start_y] in 0-1000 scale.",
},
"end": {
"type": "array",
"items": {"type": "integer"},
"description": "End normalized coordinates [end_x, end_y] in 0-1000 scale.",
},
"target": {
"description": "Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within.",
},
"action": {
"description": (
"Backward-compatible swipe gesture: smart direction string ('up', 'down', 'left', 'right')"
" OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale."
),
},
"duration": {
"type": "integer",
"description": (
"Optional swipe/drag duration in milliseconds (default 800). For drag-and-drop,"
" list reordering, or sliding/adjusting sliders (e.g., volume, brightness, SeekBars),"
" set duration >= 1000 (e.g. 1500). If omitted for directional swipe, duration is computed automatically."
),
},
},
},
)
SWIPE_TOOL = tool_declaration("swipe")
PRESS_KEY_TOOL = ToolDeclaration(
name="press_key",
description=(
"[ACTION] Press a physical or virtual system button. The screen after"
" pressing will be returned automatically."
),
parameters={
"type": "object",
"properties": {
"key": {
"type": "string",
"description": (
"Standard Android system button name (ENTER, BACK, HOME, APP_SWITCH)."
),
}
},
"required": ["key"],
},
)
PRESS_KEY_TOOL = tool_declaration("press_key")
READ_NOTE_TOOL = ToolDeclaration(
name="read_note",
@@ -966,49 +795,9 @@ LIST_NOTES_TOOL = ToolDeclaration(
parameters={"type": "object", "properties": {}},
)
MANAGE_APP_TOOL = ToolDeclaration(
name="manage_app",
description=(
"[ACTION] Launch or force stop a specified application. The screen"
" after launch/stop will be returned automatically."
),
parameters={
"type": "object",
"properties": {
"action": {
"type": "string",
"description": "The action type ('launch' or 'stop').",
},
"app_name": {
"type": "string",
"description": "Display name or package name of the application.",
},
},
"required": ["action", "app_name"],
},
)
MANAGE_APP_TOOL = tool_declaration("manage_app")
WAIT_FOR_DELAY_TOOL = ToolDeclaration(
name="wait_for_delay",
description=(
"[ACTION] Pause execution and wait for a specified duration in milliseconds."
" The screen after pause will be returned automatically."
),
parameters={
"type": "object",
"properties": {
"time_in_ms": {
"type": "integer",
"description": (
"The exact duration to pause in milliseconds (e.g., 2000 for 2s,"
" 60000 for 1 min, 180000 for 3 mins, 300000 for 5 mins)."
" Convert any required waiting duration into milliseconds."
),
}
},
"required": ["time_in_ms"],
},
)
WAIT_FOR_DELAY_TOOL = tool_declaration("wait_for_delay")
REPORT_FAILURE_ANALYSIS_TOOL = ToolDeclaration(
name="report_failure_analysis",
artemis/agents/validator/validator.py
+18 -32
View File
@@ -38,9 +38,10 @@ from artemis.context import ArtemisContext
from artemis.controllers.unified_controller import UnifiedMobileController
from artemis.data_engine.trace import CURRENT_TRACE_ID, trace
from artemis.graph.state import State
from artemis.graph.visibility import strict_state
from artemis.mcp.action_names import to_canonical_call
from artemis.mcp.action_session import ActionSession, get_action_session
from artemis.services.llm import get_llm
from artemis.services.llm import acomplete_structured, get_llm
from artemis.utils import image_diff, visualization
from artemis.utils.decorators import wrap_with_callbacks
from artemis.utils.logger import get_logger
@@ -154,11 +155,7 @@ class ValidatorNode:
async def _execute_validation_loop(self, state: State):
actions, error_msg = self._parse_decisions(state.structured_decisions)
if error_msg:
return await state.asanitize_update(
ctx=self.ctx,
update={},
agent="validator",
)
return {}
execution = []
failed_action = None
@@ -176,11 +173,7 @@ class ValidatorNode:
decision_screenshot_b64 = last_screenshot_b64
decision_screenshot_name = last_screenshot_name
except Exception:
return await state.asanitize_update(
ctx=self.ctx,
update={},
agent="validator",
)
return {}
step_id = None
if state.current_step_id:
@@ -355,7 +348,9 @@ class ValidatorNode:
if post_screenshot_b64:
decoded_bytes = base64.b64decode(post_screenshot_b64)
last_screenshot_b64 = post_screenshot_b64
logger.info(f"Successfully decoded post_screenshot_b64 ({len(decoded_bytes)} bytes)")
logger.info(
f"Successfully decoded post_screenshot_b64 ({len(decoded_bytes)} bytes)"
)
if self.ctx.data_engine:
post_image_name = self.ctx.data_engine.get_or_create_image(decoded_bytes)
last_screenshot_name = post_image_name
@@ -466,11 +461,7 @@ class ValidatorNode:
step_id, report, post_image_name=distinct_post_image_name
)
return await state.asanitize_update(
ctx=self.ctx,
update={"last_execution_result": report},
agent="validator",
)
return {"last_execution_result": report}
@wrap_with_callbacks(
before=lambda: logger.info("Starting Validator Agent..."),
@@ -479,6 +470,7 @@ class ValidatorNode:
)
@trace(type="agent", name="validator")
async def __call__(self, state: State):
state = strict_state(state, "validator")
step_id = None
if state.current_step_id:
@@ -1196,22 +1188,16 @@ class ValidatorNode:
HumanMessage(content=user_content),
]
# D. Invoke Universal VLM
response = await llm.ainvoke(messages)
output = response.content if isinstance(response.content, str) else ""
if isinstance(response.content, list):
output = "".join(
b.get("text", "")
for b in response.content
if isinstance(b, dict) and "text" in b
# D. Invoke Universal VLM. Parsing (fences, repair) and one
# corrective re-ask on unparseable JSON are handled by the
# structured-output layer; a StructuredOutputError lands in
# this attempt loop's except like any other attempt failure.
res_json = await acomplete_structured(llm, messages)
if not isinstance(res_json, dict):
raise ValueError(
"Pixel validation response parsed to"
f" {type(res_json).__name__}, expected a JSON object."
)
if output.startswith("```"):
code_match = re.search(r"```(?:json)?\s*([\s\S]*?)\s*```", output)
if code_match:
output = code_match.group(1).strip()
res_json = json.loads(output)
reasoning = res_json.get("reasoning", "")
is_present = res_json.get("is_present", True)
confidence = res_json.get("confidence", 1.0)