feat: initial release of ARTEMIS Android Agent & Automation Platform
ARTEMIS is an autonomous multimodal Android agent and mobile UI automation platform engineered for physical devices and emulators. It combines state-of-the-art multimodal Vision-Language Models (VLMs) with hierarchical closed-loop planning, resilient self-healing, and deep Android subsystem integration. Key Features & Architectural Highlights: - Dual-Engine Orchestration: * Pro Engine: Multi-agent closed-loop (Planner -> Operator -> Validator/Failure Analyzer -> Summarizer) powered by LangGraph. * Flash Engine: Ultra-fast reactive loop for deterministic, low-latency UI execution (<35 steps). - Multimodal Perception & Grounding: * Hybrid visual screenshot grounding, real-time OCR extraction, and XML accessibility hierarchy parsing. * Dynamic-First, Coordinate-Fallback locating strategy for maximum resilience against UI drift. - Autonomous Self-Healing & Diagnostics: * Failure analyzer and safety net for popup dismissal, crash recovery, and auto-retry. * Diagnostic sub-agents for log analysis, video recording inspection, and object detection. - Unified CLI & Python SDK: * Rich CLI with commands: init, doctor, run, trace, batch, bench, server, and mcp. * Fluent Python SDK with Pydantic structured output models and package locking. - Multi-Provider LLM Architecture: * Support for Google Gemini, OpenAI, Anthropic Claude, Vertex AI, OpenRouter, xAI Grok, and local endpoints (Ollama/vLLM). * Centralized configuration in config/artemis.jsonc. - Native MCP & Observability: * Model Context Protocol (MCP) server endpoints for AI IDEs. * Full trace recording with before/after screenshots, action overlays, and execution replays. Change-Id: I430b93ee2b76b54cc5a3c6eeccd15d2cf29830a1
This commit is contained in:
somew1nd committed
commit
206a9ce921
229 files changed
+49653
No files matched your search
@@ -0,0 +1,730 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import asyncio
|
||||
from typing import Any, Literal
|
||||
|
||||
from langchain_core.messages import ToolMessage
|
||||
from langchain_core.tools import BaseTool
|
||||
from langgraph.types import Command
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.controllers.unified_controller import UnifiedMobileController
|
||||
from artemis.data_engine.trace import trace_langchain_tool
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolCategory, ToolRegistry
|
||||
from artemis.tools.tool_wrapper import ToolWrapper
|
||||
from artemis.tools.types import CyFunctionDetector
|
||||
from artemis.utils.coordinates import parse_swipe_parameters
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class ClickArgs(BaseModel):
|
||||
"""Arguments schema for clicking on normalized coordinates."""
|
||||
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
target: list[int] = Field(
|
||||
...,
|
||||
description=("Click target normalized coordinates [x, y] in 0-1000 scale."),
|
||||
)
|
||||
|
||||
|
||||
class LongPressArgs(BaseModel):
|
||||
"""Arguments schema for long pressing on normalized coordinates."""
|
||||
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
target: list[int] = Field(
|
||||
...,
|
||||
description=("Long press target normalized coordinates [x, y] in 0-1000 scale."),
|
||||
)
|
||||
duration: int = Field(1000, description="Long press duration in milliseconds (default 1000).")
|
||||
|
||||
|
||||
class InputTextArgs(BaseModel):
|
||||
"""Arguments schema for inputting text into normalized coordinates."""
|
||||
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
text: str = Field(
|
||||
...,
|
||||
description="The text content to input. Supports multi-line strings with '\\n'.",
|
||||
)
|
||||
target: list[int] = Field(
|
||||
...,
|
||||
description=("Input target field normalized coordinates [x, y] in 0-1000 scale."),
|
||||
)
|
||||
clear_exist: bool = Field(
|
||||
True,
|
||||
description=(
|
||||
"Whether to clear existing text before typing (True to replace all"
|
||||
" text, False to append at the end of existing content)."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class SwipeArgs(BaseModel):
|
||||
"""Arguments schema for performing swipe gestures."""
|
||||
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
direction: Literal["up", "down", "left", "right"] | None = Field(
|
||||
None,
|
||||
description=(
|
||||
"Smart directional swipe ('up', 'down', 'left', 'right') for general page"
|
||||
" scrolling/browsing. Automatically applies safe vectors and a ~40% visual"
|
||||
" overlap anchor: 'up' (drags bottom-to-top, scrolling down to reveal content below),"
|
||||
" 'down' (drags top-to-bottom, scrolling up to reveal content above),"
|
||||
" 'left' (drags right-to-left, scrolling right),"
|
||||
" 'right' (drags left-to-right, scrolling left)."
|
||||
),
|
||||
)
|
||||
start: list[int] | None = Field(
|
||||
None,
|
||||
description=(
|
||||
"Start normalized coordinates [start_x, start_y] in 0-1000 scale for precise,"
|
||||
" local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop)."
|
||||
),
|
||||
)
|
||||
end: list[int] | None = Field(
|
||||
None,
|
||||
description=(
|
||||
"End normalized coordinates [end_x, end_y] in 0-1000 scale for precise,"
|
||||
" local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop)."
|
||||
),
|
||||
)
|
||||
action: Literal["up", "down", "left", "right"] | list[int] | None = Field(
|
||||
None,
|
||||
description=(
|
||||
"Backward-compatible swipe gesture: smart direction string ('up', 'down', 'left', 'right')"
|
||||
" OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale."
|
||||
),
|
||||
)
|
||||
duration: int = Field(
|
||||
400,
|
||||
description=(
|
||||
"Optional swipe/drag duration in milliseconds (default 400). For drag-and-drop,"
|
||||
" list reordering, or sliding/adjusting sliders (e.g., volume, brightness, SeekBars),"
|
||||
" set duration >= 1000 (e.g. 1500). If omitted for directional swipe, duration is computed automatically."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class PressKeyArgs(BaseModel):
|
||||
"""Arguments schema for pressing system keys."""
|
||||
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
key: Literal["ENTER", "BACK", "HOME", "APP_SWITCH"] = Field(
|
||||
...,
|
||||
description=("Standard Android system button name (ENTER, BACK, HOME, APP_SWITCH)."),
|
||||
)
|
||||
|
||||
|
||||
CLICK_DOCSTRING = "[ACTION] Click on the target normalized coordinates on the screen."
|
||||
|
||||
|
||||
class ClickTool(ArtemisTool):
|
||||
"""Universal tool for clicking on target normalized coordinates on the screen."""
|
||||
|
||||
def __init__(self, category: ToolCategory = "action"):
|
||||
super().__init__(
|
||||
name="click",
|
||||
description=CLICK_DOCSTRING,
|
||||
args_schema=ClickArgs,
|
||||
category=category,
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None,
|
||||
ctx: ArtemisContext | None = None,
|
||||
target: list[int] | None = None,
|
||||
tool_call_id: str | None = None,
|
||||
state: State | None = None,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
tgt = target if target is not None else (kwargs.get("target") or kwargs.get("Target") or [])
|
||||
tcid = tool_call_id if tool_call_id is not None else kwargs.get("tool_call_id")
|
||||
st = state if state is not None else kwargs.get("state")
|
||||
|
||||
width = 1080
|
||||
height = 2400
|
||||
if ctx and hasattr(ctx, "device") and ctx.device:
|
||||
width = getattr(ctx.device, "device_width", 1080)
|
||||
height = getattr(ctx.device, "device_height", 2400)
|
||||
elif driver and hasattr(driver, "screen_size"):
|
||||
width, height = driver.screen_size
|
||||
|
||||
try:
|
||||
if not tgt or len(tgt) < 2:
|
||||
raise ValueError(f"Invalid target coordinates: {tgt}")
|
||||
nx, ny = tgt[0], tgt[1]
|
||||
x = int(max(0, min(width - 1, nx * width / 1000)))
|
||||
y = int(max(0, min(height - 1, ny * height / 1000)))
|
||||
|
||||
if driver is not None and hasattr(driver, "tap"):
|
||||
success = await driver.tap(x, y)
|
||||
outcome = (
|
||||
f"Clicked at [{nx}, {ny}] (normalized) successfully."
|
||||
if success
|
||||
else f"Click failed at {tgt}"
|
||||
)
|
||||
elif ctx is not None:
|
||||
controller = UnifiedMobileController(ctx)
|
||||
result = await controller.tap_at(x, y)
|
||||
success = result.error is None
|
||||
outcome = (
|
||||
f"Clicked at [{nx}, {ny}] (normalized) successfully."
|
||||
if success
|
||||
else f"Failed to click: {result.error}"
|
||||
)
|
||||
else:
|
||||
success = False
|
||||
outcome = "Error during click: No driver or context provided."
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
success = False
|
||||
outcome = f"Error during click: {e}"
|
||||
|
||||
if st and callable(getattr(st, "asanitize_update", None)):
|
||||
tool_message = ToolMessage(
|
||||
tool_call_id=tcid or "",
|
||||
content=outcome,
|
||||
status="success" if success else "error",
|
||||
)
|
||||
return Command(
|
||||
update=await st.asanitize_update(
|
||||
ctx=ctx,
|
||||
update={VALIDATOR_MESSAGES_KEY: [tool_message]},
|
||||
agent="validator",
|
||||
)
|
||||
)
|
||||
|
||||
return outcome
|
||||
|
||||
|
||||
# Universal tool instance & aliases
|
||||
click = ClickTool()
|
||||
Click = ClickTool
|
||||
ToolRegistry.register(click)
|
||||
|
||||
|
||||
def get_click_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports click as a LangChain BaseTool."""
|
||||
return trace_langchain_tool(click.to_langchain_tool(ctx), ctx)
|
||||
|
||||
|
||||
LONG_PRESS_DOCSTRING = "[ACTION] Long press on the target normalized coordinates on the screen."
|
||||
|
||||
|
||||
class LongPressTool(ArtemisTool):
|
||||
"""Universal tool for long pressing on target normalized coordinates on the screen."""
|
||||
|
||||
def __init__(self, category: ToolCategory = "action"):
|
||||
super().__init__(
|
||||
name="long_press",
|
||||
description=LONG_PRESS_DOCSTRING,
|
||||
args_schema=LongPressArgs,
|
||||
category=category,
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None,
|
||||
ctx: ArtemisContext | None = None,
|
||||
target: list[int] | None = None,
|
||||
duration: int = 1000,
|
||||
tool_call_id: str | None = None,
|
||||
state: State | None = None,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
tgt = target if target is not None else (kwargs.get("target") or kwargs.get("Target") or [])
|
||||
dur = (
|
||||
duration
|
||||
if duration is not None
|
||||
else (kwargs.get("duration") or kwargs.get("Duration") or 1000)
|
||||
)
|
||||
tcid = tool_call_id if tool_call_id is not None else kwargs.get("tool_call_id")
|
||||
st = state if state is not None else kwargs.get("state")
|
||||
|
||||
width = 1080
|
||||
height = 2400
|
||||
if ctx and hasattr(ctx, "device") and ctx.device:
|
||||
width = getattr(ctx.device, "device_width", 1080)
|
||||
height = getattr(ctx.device, "device_height", 2400)
|
||||
elif driver and hasattr(driver, "screen_size"):
|
||||
width, height = driver.screen_size
|
||||
|
||||
try:
|
||||
if not tgt or len(tgt) < 2:
|
||||
raise ValueError(f"Invalid target coordinates: {tgt}")
|
||||
nx, ny = tgt[0], tgt[1]
|
||||
x = int(max(0, min(width - 1, nx * width / 1000)))
|
||||
y = int(max(0, min(height - 1, ny * height / 1000)))
|
||||
|
||||
if driver is not None and hasattr(driver, "long_press"):
|
||||
success = await driver.long_press(x, y, duration_ms=dur)
|
||||
outcome = (
|
||||
f"Long pressed at [{nx}, {ny}] (normalized) successfully."
|
||||
if success
|
||||
else f"Failed to long press: Long press failed at {tgt}"
|
||||
)
|
||||
elif ctx is not None:
|
||||
controller = UnifiedMobileController(ctx)
|
||||
result = await controller.tap_at(x, y, long_press=True, long_press_duration=dur)
|
||||
success = result.error is None
|
||||
outcome = (
|
||||
f"Long pressed at [{nx}, {ny}] (normalized) successfully."
|
||||
if success
|
||||
else f"Failed to long press: {result.error}"
|
||||
)
|
||||
else:
|
||||
success = False
|
||||
outcome = "Error during long press: No driver or context provided."
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
success = False
|
||||
outcome = f"Error during long press: {e}"
|
||||
|
||||
if st and callable(getattr(st, "asanitize_update", None)):
|
||||
tool_message = ToolMessage(
|
||||
tool_call_id=tcid or "",
|
||||
content=outcome,
|
||||
status="success" if success else "error",
|
||||
)
|
||||
return Command(
|
||||
update=await st.asanitize_update(
|
||||
ctx=ctx,
|
||||
update={VALIDATOR_MESSAGES_KEY: [tool_message]},
|
||||
agent="validator",
|
||||
)
|
||||
)
|
||||
|
||||
return outcome
|
||||
|
||||
|
||||
# Universal tool instance & aliases
|
||||
long_press = LongPressTool()
|
||||
LongPress = LongPressTool
|
||||
ToolRegistry.register(long_press)
|
||||
|
||||
|
||||
def get_long_press_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports long_press as a LangChain BaseTool."""
|
||||
return trace_langchain_tool(long_press.to_langchain_tool(ctx), ctx)
|
||||
|
||||
|
||||
INPUT_TEXT_DOCSTRING = "[ACTION] Type text into the target input field coordinates."
|
||||
|
||||
|
||||
class InputTextTool(ArtemisTool):
|
||||
"""Universal tool for typing text into target input field coordinates."""
|
||||
|
||||
def __init__(self, category: ToolCategory = "action"):
|
||||
super().__init__(
|
||||
name="input_text",
|
||||
description=INPUT_TEXT_DOCSTRING,
|
||||
args_schema=InputTextArgs,
|
||||
category=category,
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None,
|
||||
ctx: ArtemisContext | None = None,
|
||||
text: str | None = None,
|
||||
target: list[int] | None = None,
|
||||
clear_exist: bool = True,
|
||||
tool_call_id: str | None = None,
|
||||
state: State | None = None,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
txt = text if text is not None else (kwargs.get("text") or kwargs.get("Text") or "")
|
||||
tgt = target if target is not None else (kwargs.get("target") or kwargs.get("Target") or [])
|
||||
clr = (
|
||||
clear_exist
|
||||
if clear_exist is not None
|
||||
else kwargs.get("clear_exist", kwargs.get("ClearExist", True))
|
||||
)
|
||||
tcid = tool_call_id if tool_call_id is not None else kwargs.get("tool_call_id")
|
||||
st = state if state is not None else kwargs.get("state")
|
||||
|
||||
width = 1080
|
||||
height = 2400
|
||||
if ctx and hasattr(ctx, "device") and ctx.device:
|
||||
width = getattr(ctx.device, "device_width", 1080)
|
||||
height = getattr(ctx.device, "device_height", 2400)
|
||||
elif driver and hasattr(driver, "screen_size"):
|
||||
width, height = driver.screen_size
|
||||
|
||||
try:
|
||||
if not tgt or len(tgt) < 2:
|
||||
raise ValueError(f"Invalid target coordinates: {tgt}")
|
||||
nx, ny = tgt[0], tgt[1]
|
||||
x = int(max(0, min(width - 1, nx * width / 1000)))
|
||||
y = int(max(0, min(height - 1, ny * height / 1000)))
|
||||
|
||||
if driver is not None and hasattr(driver, "input_text"):
|
||||
await driver.tap(x, y)
|
||||
await asyncio.sleep(0.2)
|
||||
success = await driver.input_text(txt, clear_existing=clr)
|
||||
outcome = (
|
||||
f"Typed '{txt}' successfully."
|
||||
if success
|
||||
else f"Failed to input text into {tgt}"
|
||||
)
|
||||
elif ctx is not None:
|
||||
controller = UnifiedMobileController(ctx)
|
||||
tap_result = await controller.tap_at(x, y)
|
||||
if tap_result.error:
|
||||
raise RuntimeError(f"Failed to focus input field: {tap_result.error}")
|
||||
|
||||
if clr:
|
||||
await controller.erase_text()
|
||||
else:
|
||||
await controller.press_key("123") # Move to end
|
||||
|
||||
success = await controller.type_text(txt, clear_existing=False)
|
||||
outcome = f"Typed '{txt}' successfully." if success else "Failed to type text."
|
||||
else:
|
||||
success = False
|
||||
outcome = "Error during input text: No driver or context provided."
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
success = False
|
||||
outcome = f"Error during input text: {e}"
|
||||
|
||||
if st and callable(getattr(st, "asanitize_update", None)):
|
||||
tool_message = ToolMessage(
|
||||
tool_call_id=tcid or "",
|
||||
content=outcome,
|
||||
status="success" if success else "error",
|
||||
)
|
||||
return Command(
|
||||
update=await st.asanitize_update(
|
||||
ctx=ctx,
|
||||
update={VALIDATOR_MESSAGES_KEY: [tool_message]},
|
||||
agent="validator",
|
||||
)
|
||||
)
|
||||
|
||||
return outcome
|
||||
|
||||
|
||||
# Universal tool instance & aliases
|
||||
input_text = InputTextTool()
|
||||
InputText = InputTextTool
|
||||
ToolRegistry.register(input_text)
|
||||
|
||||
|
||||
def get_input_text_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports input_text as a LangChain BaseTool."""
|
||||
return trace_langchain_tool(input_text.to_langchain_tool(ctx), ctx)
|
||||
|
||||
|
||||
SWIPE_DOCSTRING = (
|
||||
"[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the"
|
||||
" screen.\n\n"
|
||||
"• Smart Directional Scrolling ('direction'): Best for general browsing"
|
||||
" and page scrolling. Automatically computes safe swipe vectors (locking"
|
||||
" axes to prevent edge/sidebar mis-clicks), retains a ~40% visual overlap"
|
||||
" anchor for zero-omission traversal, and prevents inertial flings.\n"
|
||||
"• Precise Coordinate Gestures ('start', 'end'): Best for local,"
|
||||
" fine-grained interactions such as adjusting sliders/SeekBars (e.g.,"
|
||||
" volume, brightness, progress bars) or drag-and-drop / list reordering."
|
||||
" Always drag slightly PAST the target position to overcome touch slop"
|
||||
" and reliably trigger the update. When setting a slider to Maximum (100%)"
|
||||
" or Minimum (0%), swipe fully to the extreme boundary."
|
||||
)
|
||||
|
||||
|
||||
class SwipeTool(ArtemisTool):
|
||||
"""Universal tool for performing swipe gestures on the screen."""
|
||||
|
||||
def __init__(self, category: ToolCategory = "action"):
|
||||
super().__init__(
|
||||
name="swipe",
|
||||
description=SWIPE_DOCSTRING,
|
||||
args_schema=SwipeArgs,
|
||||
category=category,
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-branches,too-many-locals,too-many-statements
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None,
|
||||
ctx: ArtemisContext | None = None,
|
||||
action: Literal["up", "down", "left", "right"] | list[int] | str | None = None,
|
||||
direction: Literal["up", "down", "left", "right"] | str | None = None,
|
||||
start: list[int] | None = None,
|
||||
end: list[int] | None = None,
|
||||
duration: int = 400,
|
||||
tool_call_id: str | None = None,
|
||||
state: State | None = None,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
params = dict(kwargs)
|
||||
if action is not None:
|
||||
params["action"] = action
|
||||
if direction is not None:
|
||||
params["direction"] = direction
|
||||
if start is not None:
|
||||
params["start"] = start
|
||||
if end is not None:
|
||||
params["end"] = end
|
||||
if duration is not None:
|
||||
params["duration"] = duration
|
||||
|
||||
kind, target, dur = parse_swipe_parameters(params, default_duration=duration or 400)
|
||||
|
||||
tcid = tool_call_id if tool_call_id is not None else kwargs.get("tool_call_id")
|
||||
st = state if state is not None else kwargs.get("state")
|
||||
|
||||
width = 1080
|
||||
height = 2400
|
||||
if ctx and hasattr(ctx, "device") and ctx.device:
|
||||
width = getattr(ctx.device, "device_width", 1080)
|
||||
height = getattr(ctx.device, "device_height", 2400)
|
||||
elif driver and hasattr(driver, "screen_size"):
|
||||
width, height = driver.screen_size
|
||||
|
||||
try:
|
||||
if kind is None or target is None:
|
||||
raise ValueError(
|
||||
f"Invalid swipe action parameter: {action or direction or (start, end) or kwargs}"
|
||||
)
|
||||
|
||||
if (
|
||||
driver is not None
|
||||
and hasattr(driver, "swipe_direction")
|
||||
and hasattr(driver, "swipe")
|
||||
):
|
||||
if kind == "direction" and isinstance(target, str):
|
||||
success = await driver.swipe_direction(target, duration_ms=dur)
|
||||
outcome = (
|
||||
"Swipe completed successfully." if success else f"Swipe {target} failed"
|
||||
)
|
||||
elif kind == "coords" and isinstance(target, list) and len(target) == 4:
|
||||
start_x = int(max(0, min(width - 1, target[0] * width / 1000)))
|
||||
start_y = int(max(0, min(height - 1, target[1] * height / 1000)))
|
||||
end_x = int(max(0, min(width - 1, target[2] * width / 1000)))
|
||||
end_y = int(max(0, min(height - 1, target[3] * height / 1000)))
|
||||
success = await driver.swipe(start_x, start_y, end_x, end_y, duration_ms=dur)
|
||||
outcome = (
|
||||
"Swipe completed successfully."
|
||||
if success
|
||||
else f"Swipe coordinates {target} failed"
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Invalid swipe action parameter: {target}")
|
||||
elif ctx is not None:
|
||||
controller = UnifiedMobileController(ctx)
|
||||
|
||||
if kind == "direction" and isinstance(target, str):
|
||||
if target == "up":
|
||||
coords = [
|
||||
width // 2,
|
||||
int(height * 0.8),
|
||||
width // 2,
|
||||
int(height * 0.2),
|
||||
]
|
||||
elif target == "down":
|
||||
coords = [
|
||||
width // 2,
|
||||
int(height * 0.2),
|
||||
width // 2,
|
||||
int(height * 0.8),
|
||||
]
|
||||
elif target == "left":
|
||||
coords = [
|
||||
int(width * 0.8),
|
||||
height // 2,
|
||||
int(width * 0.2),
|
||||
height // 2,
|
||||
]
|
||||
elif target == "right":
|
||||
coords = [
|
||||
int(width * 0.2),
|
||||
height // 2,
|
||||
int(width * 0.8),
|
||||
height // 2,
|
||||
]
|
||||
else:
|
||||
raise ValueError(f"Invalid direction: {target}")
|
||||
|
||||
start_x, start_y, end_x, end_y = coords
|
||||
elif kind == "coords" and isinstance(target, list) and len(target) == 4:
|
||||
nx1, ny1, nx2, ny2 = target
|
||||
start_x = int(max(0, min(width - 1, nx1 * width / 1000)))
|
||||
start_y = int(max(0, min(height - 1, ny1 * height / 1000)))
|
||||
end_x = int(max(0, min(width - 1, nx2 * width / 1000)))
|
||||
end_y = int(max(0, min(height - 1, ny2 * height / 1000)))
|
||||
else:
|
||||
raise ValueError(f"Invalid swipe action parameter: {target}")
|
||||
|
||||
error = await controller.swipe_coords(start_x, start_y, end_x, end_y, dur)
|
||||
success = error is None
|
||||
outcome = "Swipe completed successfully." if success else f"Swipe failed: {error}"
|
||||
else:
|
||||
success = False
|
||||
outcome = "Error during swipe: No driver or context provided."
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
success = False
|
||||
outcome = f"Error during swipe: {e}"
|
||||
|
||||
if st and callable(getattr(st, "asanitize_update", None)):
|
||||
tool_message = ToolMessage(
|
||||
tool_call_id=tcid or "",
|
||||
content=outcome,
|
||||
status="success" if success else "error",
|
||||
)
|
||||
return Command(
|
||||
update=await st.asanitize_update(
|
||||
ctx=ctx,
|
||||
update={VALIDATOR_MESSAGES_KEY: [tool_message]},
|
||||
agent="validator",
|
||||
)
|
||||
)
|
||||
|
||||
return outcome
|
||||
|
||||
|
||||
# Universal tool instance & aliases
|
||||
swipe = SwipeTool()
|
||||
Swipe = SwipeTool
|
||||
ToolRegistry.register(swipe)
|
||||
|
||||
|
||||
def get_swipe_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports swipe as a LangChain BaseTool."""
|
||||
return trace_langchain_tool(swipe.to_langchain_tool(ctx), ctx)
|
||||
|
||||
|
||||
PRESS_KEY_DOCSTRING = "[ACTION] Press a physical or virtual system button."
|
||||
|
||||
|
||||
class PressKeyTool(ArtemisTool):
|
||||
"""Universal tool for pressing a physical or virtual system button."""
|
||||
|
||||
def __init__(self, category: ToolCategory = "action"):
|
||||
super().__init__(
|
||||
name="press_key",
|
||||
description=PRESS_KEY_DOCSTRING,
|
||||
args_schema=PressKeyArgs,
|
||||
category=category,
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None,
|
||||
ctx: ArtemisContext | None = None,
|
||||
key: Literal["ENTER", "BACK", "HOME", "APP_SWITCH"] | str | None = None,
|
||||
tool_call_id: str | None = None,
|
||||
state: State | None = None,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
k = key if key is not None else (kwargs.get("key") or kwargs.get("Key") or "")
|
||||
tcid = tool_call_id if tool_call_id is not None else kwargs.get("tool_call_id")
|
||||
st = state if state is not None else kwargs.get("state")
|
||||
|
||||
try:
|
||||
if not k:
|
||||
raise ValueError("Key parameter is required.")
|
||||
|
||||
if driver is not None and hasattr(driver, "press_key"):
|
||||
success = await driver.press_key(k)
|
||||
outcome = (
|
||||
f"Pressed key '{k}' successfully." if success else f"Failed to press key '{k}'."
|
||||
)
|
||||
elif ctx is not None:
|
||||
controller = UnifiedMobileController(ctx)
|
||||
success = False
|
||||
if k == "ENTER":
|
||||
success = await controller.press_enter()
|
||||
elif k == "BACK":
|
||||
success = await controller.go_back()
|
||||
elif k == "HOME":
|
||||
success = await controller.go_home()
|
||||
elif k == "APP_SWITCH":
|
||||
success = await controller.press_key("KEYCODE_APP_SWITCH")
|
||||
else:
|
||||
success = await controller.press_key(k)
|
||||
|
||||
outcome = (
|
||||
f"Pressed key '{k}' successfully." if success else f"Failed to press key '{k}'."
|
||||
)
|
||||
else:
|
||||
success = False
|
||||
outcome = "Error during press key: No driver or context provided."
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
success = False
|
||||
outcome = f"Error during press key: {e}"
|
||||
|
||||
if st and callable(getattr(st, "asanitize_update", None)):
|
||||
tool_message = ToolMessage(
|
||||
tool_call_id=tcid or "",
|
||||
content=outcome,
|
||||
status="success" if success else "error",
|
||||
)
|
||||
return Command(
|
||||
update=await st.asanitize_update(
|
||||
ctx=ctx,
|
||||
update={VALIDATOR_MESSAGES_KEY: [tool_message]},
|
||||
agent="validator",
|
||||
)
|
||||
)
|
||||
|
||||
return outcome
|
||||
|
||||
|
||||
# Universal tool instance & aliases
|
||||
press_key = PressKeyTool()
|
||||
PressKey = PressKeyTool
|
||||
ToolRegistry.register(press_key)
|
||||
|
||||
|
||||
def get_press_key_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports press_key as a LangChain BaseTool."""
|
||||
return trace_langchain_tool(press_key.to_langchain_tool(ctx), ctx)
|
||||
|
||||
|
||||
# Wrappers to expose
|
||||
click_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_click_tool,
|
||||
on_success_fn=lambda *args, **kwargs: "Success",
|
||||
on_failure_fn=lambda *args, **kwargs: "Failure",
|
||||
)
|
||||
long_press_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_long_press_tool,
|
||||
on_success_fn=lambda *args, **kwargs: "Success",
|
||||
on_failure_fn=lambda *args, **kwargs: "Failure",
|
||||
)
|
||||
input_text_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_input_text_tool,
|
||||
on_success_fn=lambda *args, **kwargs: "Success",
|
||||
on_failure_fn=lambda *args, **kwargs: "Failure",
|
||||
)
|
||||
swipe_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_swipe_tool,
|
||||
on_success_fn=lambda *args, **kwargs: "Success",
|
||||
on_failure_fn=lambda *args, **kwargs: "Failure",
|
||||
)
|
||||
press_key_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_press_key_tool,
|
||||
on_success_fn=lambda *args, **kwargs: "Success",
|
||||
on_failure_fn=lambda *args, **kwargs: "Failure",
|
||||
)
|
||||
Reference in new issue
Block a user