chenmingxuan/Artemis-Eapil
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 32 Wiki Activity

feat: harden agent verification and runtime contracts

Browse Source 
Introduce two-gate checker settlement, explicit graph visibility and checkpoints, structured LLM reliability, centralized action specifications, and corresponding tests and design documentation.
This commit is contained in:
somew1nd committed 2026-08-31 20:42:24 -07:00
1 parent ddfa420945
commit cbe8f4f0c6
106 files changed
+11542 -4445
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

artemis/tools/actions/__init__.py
-45
View File
@@ -1,45 +0,0 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Universal device action tools."""
from artemis.tools.actions.device_actions import (
ClickArgs,
InputTextArgs,
LongPressArgs,
PressKeyArgs,
SwipeArgs,
WaitForDelayArgs,
click,
input_text,
long_press,
press_key,
swipe,
wait_for_delay,
)
__all__ = [
"click",
"long_press",
"input_text",
"swipe",
"press_key",
"wait_for_delay",
"ClickArgs",
"LongPressArgs",
"InputTextArgs",
"SwipeArgs",
"PressKeyArgs",
"WaitForDelayArgs",
]
artemis/tools/actions/device_actions.py
-250
View File
@@ -1,250 +0,0 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Universal device action tools implemented on top of BaseDeviceDriver."""
import asyncio
from typing import Any, Literal
from artemis.drivers.base import BaseDeviceDriver
from artemis.tools.base import artemis_tool
from artemis.utils.coordinates import (
compute_smart_swipe_coordinates,
parse_swipe_parameters,
)
from artemis.utils.logger import get_logger
from pydantic import BaseModel, Field
logger = get_logger(__name__)
# Schemas
class ClickArgs(BaseModel):
target: list[int] = Field(
..., description="Click target normalized coordinates [x, y] in 0-1000 scale."
)
class LongPressArgs(BaseModel):
target: list[int] = Field(
..., description="Long press target normalized coordinates [x, y] in 0-1000 scale."
)
duration: int = Field(1000, description="Long press duration in milliseconds (default 1000).")
class InputTextArgs(BaseModel):
text: str = Field(..., description="The text content to input.")
target: list[int] = Field(
..., description="Input target field normalized coordinates [x, y] in 0-1000 scale."
)
clear_exist: bool = Field(
True, description="Whether to clear existing text in the input box before typing."
)
class SwipeArgs(BaseModel):
direction: Literal["up", "down", "left", "right"] | None = Field(
None,
description=(
"Direction for scrolling and swiping: 'up' (drags bottom-to-top, scrolling down to reveal content below),"
" 'down' (drags top-to-bottom, scrolling up to reveal content above),"
" 'left' (drags right-to-left, scrolling right),"
" 'right' (drags left-to-right, scrolling left)."
),
)
start: list[int] | None = Field(
None,
description=(
"Start normalized coordinates [start_x, start_y] in 0-1000 scale for precise,"
" local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop)."
),
)
end: list[int] | None = Field(
None,
description=(
"End normalized coordinates [end_x, end_y] in 0-1000 scale for precise,"
" local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop)."
),
)
action: Literal["up", "down", "left", "right"] | list[int] | None = Field(
None,
description=(
"Backward-compatible swipe gesture: smart direction string ('up', 'down', 'left', 'right')"
" OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale."
),
)
duration: int = Field(
800,
description=(
"Swipe/drag duration in milliseconds (default 800). For drag-and-drop, "
"list reordering, or sliding/adjusting sliders (e.g. volume, brightness, SeekBars), "
"set duration >= 1000 (e.g. 1500)."
),
)
class PressKeyArgs(BaseModel):
key: str = Field(
...,
description="Key name to press (e.g. 'home', 'back', 'enter', 'delete', 'power').",
)
class WaitForDelayArgs(BaseModel):
seconds: float = Field(
...,
description=(
"Duration in seconds to wait (e.g., 2 for 2s, 60 for 1 min, 180 for 3 mins)."
" Use whenever time needs to elapse for animations, loading, or scheduled delays."
),
)
# Universal Tool Definitions
@artemis_tool(
name="click",
description="[ACTION] Clicks on the specified screen coordinate [x, y] in normalized 0-1000 scale.",
args_schema=ClickArgs,
category="action",
)
async def click(target: list[int], driver: BaseDeviceDriver) -> str:
width, height = driver.screen_size
abs_x = int(target[0] * width / 1000.0)
abs_y = int(target[1] * height / 1000.0)
success = await driver.tap(abs_x, abs_y)
return (
f"Click executed at normalized {target} (pixels: {abs_x}, {abs_y})"
if success
else f"Click failed at {target}"
)
@artemis_tool(
name="long_press",
description="[ACTION] Long presses on the specified screen coordinate [x, y] in normalized 0-1000 scale.",
args_schema=LongPressArgs,
category="action",
)
async def long_press(
target: list[int], duration: int = 1000, driver: BaseDeviceDriver = None
) -> str:
width, height = driver.screen_size
abs_x = int(target[0] * width / 1000.0)
abs_y = int(target[1] * height / 1000.0)
success = await driver.long_press(abs_x, abs_y, duration_ms=duration)
return (
f"Long press executed at normalized {target} for {duration}ms"
if success
else f"Long press failed at {target}"
)
@artemis_tool(
name="input_text",
description="[ACTION] Clicks an input field target [x, y] and types text into it.",
args_schema=InputTextArgs,
category="action",
)
async def input_text(
text: str, target: list[int], clear_exist: bool = True, driver: BaseDeviceDriver = None
) -> str:
# First tap to focus
width, height = driver.screen_size
abs_x = int(target[0] * width / 1000.0)
abs_y = int(target[1] * height / 1000.0)
await driver.tap(abs_x, abs_y)
await asyncio.sleep(0.2)
# Then input text
success = await driver.input_text(text, clear_existing=clear_exist)
return (
f"Input text '{text}' into target {target}"
if success
else f"Failed to input text into {target}"
)
@artemis_tool(
name="swipe",
description=(
"[ACTION] Performs smart directional swipe for browsing, or precise coordinate drag/slider gestures."
),
args_schema=SwipeArgs,
category="action",
)
async def swipe(
action: Any = None,
direction: str | None = None,
start: list[int] | None = None,
end: list[int] | None = None,
duration: int = 800,
driver: BaseDeviceDriver = None,
**kwargs: Any,
) -> str:
params = dict(kwargs)
if action is not None:
params["action"] = action
if direction is not None:
params["direction"] = direction
if start is not None:
params["start"] = start
if end is not None:
params["end"] = end
params["duration"] = duration
kind, target, dur = parse_swipe_parameters(params, default_duration=duration)
if kind == "direction" and isinstance(target, str):
width, height = driver.screen_size
sx, sy, ex, ey, smart_dur = compute_smart_swipe_coordinates(
direction=target,
target=params.get("target"),
width=width,
height=height,
duration=dur,
)
success = await driver.swipe(sx, sy, ex, ey, duration_ms=smart_dur)
return f"Swiped {target}" if success else f"Swipe {target} failed"
elif kind == "coords" and isinstance(target, list) and len(target) == 4:
width, height = driver.screen_size
start_x = int(target[0] * width / 1000.0)
start_y = int(target[1] * height / 1000.0)
end_x = int(target[2] * width / 1000.0)
end_y = int(target[3] * height / 1000.0)
success = await driver.swipe(start_x, start_y, end_x, end_y, duration_ms=dur)
return (
f"Swiped from {target[:2]} to {target[2:]}" if success else "Swipe coordinates failed"
)
return f"Invalid swipe action parameter: {action or direction or (start, end)}"
@artemis_tool(
name="press_key",
description="[ACTION] Presses a hardware or system key on the device ('home', 'back', 'enter', 'delete', 'power').",
args_schema=PressKeyArgs,
category="action",
)
async def press_key(key: str, driver: BaseDeviceDriver) -> str:
success = await driver.press_key(key)
return f"Key '{key}' pressed" if success else f"Failed to press key '{key}'"
@artemis_tool(
name="wait_for_delay",
description="[ACTION] Pauses execution for a specified number of seconds to allow UI loading or transition.",
args_schema=WaitForDelayArgs,
category="action",
)
async def wait_for_delay(seconds: float, driver: BaseDeviceDriver = None) -> str:
await asyncio.sleep(seconds)
return f"Waited for {seconds} seconds."
artemis/tools/committee_tool.py
-7
View File
@@ -198,13 +198,6 @@ async def _execute_committee(
logger.error(f"Failed to build plan and history in committee: {e}")
history_summary += f"Error building plan and history: {e}"
# Append immediate failure reason if present in state
replan_reason = (
getattr(state, "operator_replan_reason", None) if state is not None else None
)
if replan_reason:
history_summary += f"\n\n### Current Failure Reason\n{replan_reason}\n"
# Read failed plans history to ensure the committee avoids
# repeating failed strategies
failed_plans_history = ""
artemis/tools/mobile/exec_tools.py
+11 -48
View File
@@ -17,10 +17,8 @@ from typing import Any, Literal
from langchain_core.messages import ToolMessage
from langchain_core.tools import BaseTool
from langgraph.types import Command
from pydantic import BaseModel, Field
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.controllers.unified_controller import UnifiedMobileController
from artemis.data_engine.trace import trace_langchain_tool
@@ -206,19 +204,12 @@ class ClickTool(ArtemisTool):
success = False
outcome = f"Error during click: {e}"
if st and callable(getattr(st, "asanitize_update", None)):
tool_message = ToolMessage(
if st is not None:
return ToolMessage(
tool_call_id=tcid or "",
content=outcome,
status="success" if success else "error",
)
return Command(
update=await st.asanitize_update(
ctx=ctx,
update={VALIDATOR_MESSAGES_KEY: [tool_message]},
agent="validator",
)
)
return outcome
@@ -306,19 +297,12 @@ class LongPressTool(ArtemisTool):
success = False
outcome = f"Error during long press: {e}"
if st and callable(getattr(st, "asanitize_update", None)):
tool_message = ToolMessage(
if st is not None:
return ToolMessage(
tool_call_id=tcid or "",
content=outcome,
status="success" if success else "error",
)
return Command(
update=await st.asanitize_update(
ctx=ctx,
update={VALIDATOR_MESSAGES_KEY: [tool_message]},
agent="validator",
)
)
return outcome
@@ -414,19 +398,12 @@ class InputTextTool(ArtemisTool):
success = False
outcome = f"Error during input text: {e}"
if st and callable(getattr(st, "asanitize_update", None)):
tool_message = ToolMessage(
if st is not None:
return ToolMessage(
tool_call_id=tcid or "",
content=outcome,
status="success" if success else "error",
)
return Command(
update=await st.asanitize_update(
ctx=ctx,
update={VALIDATOR_MESSAGES_KEY: [tool_message]},
agent="validator",
)
)
return outcome
@@ -545,7 +522,7 @@ class SwipeTool(ArtemisTool):
if kind == "direction" and isinstance(target, str):
indexed_elems = getattr(state, "indexed_elements", None) if state else None
ui_hier = getattr(state, "ui_tree", None) if state else None
ui_hier = getattr(state, "latest_ui_hierarchy", None) if state else None
start_x, start_y, end_x, end_y, smart_dur = compute_smart_swipe_coordinates(
direction=target,
target=params.get("target"),
@@ -575,19 +552,12 @@ class SwipeTool(ArtemisTool):
success = False
outcome = f"Error during swipe: {e}"
if st and callable(getattr(st, "asanitize_update", None)):
tool_message = ToolMessage(
if st is not None:
return ToolMessage(
tool_call_id=tcid or "",
content=outcome,
status="success" if success else "error",
)
return Command(
update=await st.asanitize_update(
ctx=ctx,
update={VALIDATOR_MESSAGES_KEY: [tool_message]},
agent="validator",
)
)
return outcome
@@ -664,19 +634,12 @@ class PressKeyTool(ArtemisTool):
success = False
outcome = f"Error during press key: {e}"
if st and callable(getattr(st, "asanitize_update", None)):
tool_message = ToolMessage(
if st is not None:
return ToolMessage(
tool_call_id=tcid or "",
content=outcome,
status="success" if success else "error",
)
return Command(
update=await st.asanitize_update(
ctx=ctx,
update={VALIDATOR_MESSAGES_KEY: [tool_message]},
agent="validator",
)
)
return outcome
artemis/tools/mobile/launch_app.py
+2 -13
View File
@@ -16,11 +16,9 @@ from typing import Any
from langchain_core.messages import ToolMessage
from langchain_core.tools import BaseTool
from langgraph.types import Command
from pydantic import BaseModel, Field
from artemis.agents.hopper.hopper import HopperOutput, hopper
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.controllers.platform_specific_commands_controller import (
list_packages_async,
@@ -176,23 +174,14 @@ class LaunchAppTool(ArtemisTool):
outcome = f"Error during launch app: {e}"
error_msg = str(e)
if st and callable(getattr(st, "asanitize_update", None)):
if st is not None:
additional_kwargs = {} if success else ({"error": error_msg} if error_msg else {})
tool_message = ToolMessage(
return ToolMessage(
tool_call_id=tcid or "",
content=outcome,
additional_kwargs=additional_kwargs,
status="success" if success else "error",
)
return Command(
update=await st.asanitize_update(
ctx=ctx,
update={
VALIDATOR_MESSAGES_KEY: [tool_message],
},
agent="validator",
)
)
return outcome
artemis/tools/mobile/ocr.py
+2 -13
View File
@@ -19,10 +19,8 @@ from typing import Any
from langchain_core.messages import ToolMessage
from langchain_core.tools import BaseTool
from langgraph.types import Command
from pydantic import BaseModel
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.data_engine.trace import trace_langchain_tool
from artemis.drivers.base import BaseDeviceDriver
@@ -169,8 +167,8 @@ class OcrRecognitionTool(ArtemisTool):
has_failed = True
error_message = f"Failed to perform OCR: {str(e)}"
if st and callable(getattr(st, "asanitize_update", None)):
tool_message = ToolMessage(
if st is not None:
return ToolMessage(
tool_call_id=tcid or "",
content=ocr_recognition_wrapper.on_failure_fn(error_message)
if has_failed
@@ -178,15 +176,6 @@ class OcrRecognitionTool(ArtemisTool):
additional_kwargs={"error": error_message} if has_failed else {},
status="error" if has_failed else "success",
)
return Command(
update=await st.asanitize_update(
ctx=ctx,
update={
VALIDATOR_MESSAGES_KEY: [tool_message],
},
agent="validator",
)
)
if has_failed:
return ocr_recognition_wrapper.on_failure_fn(error_message)
artemis/tools/object_detection_tool.py
+3 -3
View File
@@ -95,7 +95,7 @@ async def _run_object_detection_logic(
if not target_image:
error_msg = "No image path provided and no screenshot available in state."
if state and callable(getattr(state, "asanitize_update", None)):
if state is not None:
return await _create_error_command(ctx, state, tool_call_id, error_msg, active_wrapper)
return f"Object Detection failed: {error_msg}"
@@ -107,12 +107,12 @@ async def _run_object_detection_logic(
span.result = result
output = json.dumps(result) if not isinstance(result, str) else result
if state and callable(getattr(state, "asanitize_update", None)):
if state is not None:
return await _create_success_command(ctx, state, tool_call_id, output, active_wrapper)
return output
except Exception as e:
logger.error(f"Object detection execution failed: {e}")
if state and callable(getattr(state, "asanitize_update", None)):
if state is not None:
return await _create_error_command(ctx, state, tool_call_id, str(e), active_wrapper)
return f"Object Detection failed: {e}"
artemis/tools/scratchpad.py
+10 -82
View File
@@ -16,10 +16,8 @@ from typing import Any
from langchain_core.messages import ToolMessage
from langchain_core.tools.base import BaseTool
from langgraph.types import Command
from pydantic import BaseModel, Field
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.data_engine.trace import trace_langchain_tool
from artemis.drivers.base import BaseDeviceDriver
@@ -155,27 +153,13 @@ class SaveNoteTool(ArtemisTool):
agent_outcome = f"Failed to save note {k}.md: {e}"
status = "error"
if st and callable(getattr(st, "asanitize_update", None)):
tool_message = ToolMessage(
if st is not None:
return ToolMessage(
tool_call_id=tcid,
content=agent_outcome,
status=status,
)
current_agent = "operator"
if hasattr(st, "current_agent") and st.current_agent:
current_agent = st.current_agent
return Command(
update=await st.asanitize_update(
ctx=ctx,
update={
VALIDATOR_MESSAGES_KEY: [tool_message],
},
agent=current_agent,
),
)
return agent_outcome
@@ -249,27 +233,13 @@ class AppendNoteTool(ArtemisTool):
agent_outcome = f"Failed to append note {k}.md: {e}"
status = "error"
if st and callable(getattr(st, "asanitize_update", None)):
tool_message = ToolMessage(
if st is not None:
return ToolMessage(
tool_call_id=tcid,
content=agent_outcome,
status=status,
)
current_agent = "operator"
if hasattr(st, "current_agent") and st.current_agent:
current_agent = st.current_agent
return Command(
update=await st.asanitize_update(
ctx=ctx,
update={
VALIDATOR_MESSAGES_KEY: [tool_message],
},
agent=current_agent,
),
)
return agent_outcome
@@ -348,27 +318,13 @@ class UpdateNoteTool(ArtemisTool):
agent_outcome = f"Failed to update note '{k}': {e}"
status = "error"
if st and callable(getattr(st, "asanitize_update", None)):
tool_message = ToolMessage(
if st is not None:
return ToolMessage(
tool_call_id=tcid,
content=agent_outcome,
status=status,
)
current_agent = "operator"
if hasattr(st, "current_agent") and st.current_agent:
current_agent = st.current_agent
return Command(
update=await st.asanitize_update(
ctx=ctx,
update={
VALIDATOR_MESSAGES_KEY: [tool_message],
},
agent=current_agent,
),
)
return agent_outcome
@@ -447,27 +403,13 @@ class ReadNoteTool(ArtemisTool):
agent_outcome = f"Failed to read note '{k}': {e}"
status = "error"
if st and callable(getattr(st, "asanitize_update", None)):
tool_message = ToolMessage(
if st is not None:
return ToolMessage(
tool_call_id=tcid,
content=agent_outcome,
status=status,
)
current_agent = "operator"
if hasattr(st, "current_agent") and st.current_agent:
current_agent = st.current_agent
return Command(
update=await st.asanitize_update(
ctx=ctx,
update={
VALIDATOR_MESSAGES_KEY: [tool_message],
},
agent=current_agent,
),
)
return agent_outcome
@@ -523,27 +465,13 @@ class ListNotesTool(ArtemisTool):
agent_outcome = f"Failed to list notes: {e}"
status = "error"
if st and callable(getattr(st, "asanitize_update", None)):
tool_message = ToolMessage(
if st is not None:
return ToolMessage(
tool_call_id=tcid,
content=agent_outcome,
status=status,
)
current_agent = "operator"
if hasattr(st, "current_agent") and st.current_agent:
current_agent = st.current_agent
return Command(
update=await st.asanitize_update(
ctx=ctx,
update={
VALIDATOR_MESSAGES_KEY: [tool_message],
},
agent=current_agent,
),
)
return agent_outcome
artemis/tools/tool_wrapper.py
+4 -10
View File
@@ -19,13 +19,12 @@ from typing import Annotated, Any, get_args, get_origin
from unittest.mock import MagicMock, Mock
import uuid
from langchain_core.messages import ToolMessage
from langchain_core.tools import BaseTool
from langchain_core.tools.base import InjectedToolCallId
from langgraph.prebuilt import InjectedState
from langgraph.types import Command
from pydantic import BaseModel
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.data_engine import engine as engine_mod
from artemis.data_engine.trace import CURRENT_TRACE_ID, smart_serialize
@@ -169,12 +168,7 @@ async def invoke_tool_with_injection(
def get_tool_result_content(result: Any) -> Any:
"""Extracts the content from a tool result, handling Command objects."""
if isinstance(result, Command):
updates = result.update
if VALIDATOR_MESSAGES_KEY in updates:
msgs = updates[VALIDATOR_MESSAGES_KEY]
if msgs and hasattr(msgs[0], "content"):
return msgs[0].content
return result
"""Extracts the content from a tool result, handling ToolMessage returns."""
if isinstance(result, ToolMessage):
return result.content
return result
artemis/tools/video_tool.py
+2 -13
View File
@@ -23,11 +23,9 @@ from typing import Any, Literal
from langchain_core.messages import ToolMessage
from langchain_core.tools import BaseTool
from langgraph.types import Command
from pydantic import BaseModel, Field
from artemis.agents.video_analyzer.video_analyzer import VideoAnalyzer
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.data_engine.trace import trace_langchain_tool
from artemis.drivers.base import BaseDeviceDriver
@@ -148,21 +146,12 @@ class VideoAnalyzerTool(ArtemisTool):
agent_outcome = f"Error running video analyzer: {e}"
status = "error"
if st and callable(getattr(st, "asanitize_update", None)):
tool_message = ToolMessage(
if st is not None:
return ToolMessage(
tool_call_id=tcid or "",
content=agent_outcome,
status=status,
)
return Command(
update=await st.asanitize_update(
ctx=ctx,
update={
VALIDATOR_MESSAGES_KEY: [tool_message],
},
agent="validator",
),
)
if status == "failed":
return f"Video analysis failed: {agent_outcome}"