Artemis-Eapil/artemis/tools/actions/device_actions.py

# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Universal device action tools implemented on top of BaseDeviceDriver."""

import asyncio
from typing import Any, Literal
from artemis.drivers.base import BaseDeviceDriver
from artemis.tools.base import artemis_tool
from artemis.utils.coordinates import (
    compute_smart_swipe_coordinates,
    parse_swipe_parameters,
)
from artemis.utils.logger import get_logger
from pydantic import BaseModel, Field

logger = get_logger(__name__)


# Schemas
class ClickArgs(BaseModel):
    target: list[int] = Field(
        ..., description="Click target normalized coordinates [x, y] in 0-1000 scale."
    )


class LongPressArgs(BaseModel):
    target: list[int] = Field(
        ..., description="Long press target normalized coordinates [x, y] in 0-1000 scale."
    )
    duration: int = Field(1000, description="Long press duration in milliseconds (default 1000).")


class InputTextArgs(BaseModel):
    text: str = Field(..., description="The text content to input.")
    target: list[int] = Field(
        ..., description="Input target field normalized coordinates [x, y] in 0-1000 scale."
    )
    clear_exist: bool = Field(
        True, description="Whether to clear existing text in the input box before typing."
    )


class SwipeArgs(BaseModel):
    direction: Literal["up", "down", "left", "right"] | None = Field(
        None,
        description=(
            "Direction for scrolling and swiping: 'up' (drags bottom-to-top, scrolling down to reveal content below),"
            " 'down' (drags top-to-bottom, scrolling up to reveal content above),"
            " 'left' (drags right-to-left, scrolling right),"
            " 'right' (drags left-to-right, scrolling left)."
        ),
    )
    start: list[int] | None = Field(
        None,
        description=(
            "Start normalized coordinates [start_x, start_y] in 0-1000 scale for precise,"
            " local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop)."
        ),
    )
    end: list[int] | None = Field(
        None,
        description=(
            "End normalized coordinates [end_x, end_y] in 0-1000 scale for precise,"
            " local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop)."
        ),
    )
    action: Literal["up", "down", "left", "right"] | list[int] | None = Field(
        None,
        description=(
            "Backward-compatible swipe gesture: smart direction string ('up', 'down', 'left', 'right')"
            " OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale."
        ),
    )
    duration: int = Field(
        800,
        description=(
            "Swipe/drag duration in milliseconds (default 800). For drag-and-drop, "
            "list reordering, or sliding/adjusting sliders (e.g. volume, brightness, SeekBars), "
            "set duration >= 1000 (e.g. 1500)."
        ),
    )


class PressKeyArgs(BaseModel):
    key: str = Field(
        ...,
        description="Key name to press (e.g. 'home', 'back', 'enter', 'delete', 'power').",
    )


class WaitForDelayArgs(BaseModel):
    seconds: float = Field(
        ...,
        description=(
            "Duration in seconds to wait (e.g., 2 for 2s, 60 for 1 min, 180 for 3 mins)."
            " Use whenever time needs to elapse for animations, loading, or scheduled delays."
        ),
    )


# Universal Tool Definitions
@artemis_tool(
    name="click",
    description="[ACTION] Clicks on the specified screen coordinate [x, y] in normalized 0-1000 scale.",
    args_schema=ClickArgs,
    category="action",
)
async def click(target: list[int], driver: BaseDeviceDriver) -> str:
    width, height = driver.screen_size
    abs_x = int(target[0] * width / 1000.0)
    abs_y = int(target[1] * height / 1000.0)
    success = await driver.tap(abs_x, abs_y)
    return (
        f"Click executed at normalized {target} (pixels: {abs_x}, {abs_y})"
        if success
        else f"Click failed at {target}"
    )


@artemis_tool(
    name="long_press",
    description="[ACTION] Long presses on the specified screen coordinate [x, y] in normalized 0-1000 scale.",
    args_schema=LongPressArgs,
    category="action",
)
async def long_press(
    target: list[int], duration: int = 1000, driver: BaseDeviceDriver = None
) -> str:
    width, height = driver.screen_size
    abs_x = int(target[0] * width / 1000.0)
    abs_y = int(target[1] * height / 1000.0)
    success = await driver.long_press(abs_x, abs_y, duration_ms=duration)
    return (
        f"Long press executed at normalized {target} for {duration}ms"
        if success
        else f"Long press failed at {target}"
    )


@artemis_tool(
    name="input_text",
    description="[ACTION] Clicks an input field target [x, y] and types text into it.",
    args_schema=InputTextArgs,
    category="action",
)
async def input_text(
    text: str, target: list[int], clear_exist: bool = True, driver: BaseDeviceDriver = None
) -> str:
    # First tap to focus
    width, height = driver.screen_size
    abs_x = int(target[0] * width / 1000.0)
    abs_y = int(target[1] * height / 1000.0)
    await driver.tap(abs_x, abs_y)
    await asyncio.sleep(0.2)

    # Then input text
    success = await driver.input_text(text, clear_existing=clear_exist)
    return (
        f"Input text '{text}' into target {target}"
        if success
        else f"Failed to input text into {target}"
    )


@artemis_tool(
    name="swipe",
    description=(
        "[ACTION] Performs smart directional swipe for browsing, or precise coordinate drag/slider gestures."
    ),
    args_schema=SwipeArgs,
    category="action",
)
async def swipe(
    action: Any = None,
    direction: str | None = None,
    start: list[int] | None = None,
    end: list[int] | None = None,
    duration: int = 800,
    driver: BaseDeviceDriver = None,
    **kwargs: Any,
) -> str:
    params = dict(kwargs)
    if action is not None:
        params["action"] = action
    if direction is not None:
        params["direction"] = direction
    if start is not None:
        params["start"] = start
    if end is not None:
        params["end"] = end
    params["duration"] = duration

    kind, target, dur = parse_swipe_parameters(params, default_duration=duration)

    if kind == "direction" and isinstance(target, str):
        width, height = driver.screen_size
        sx, sy, ex, ey, smart_dur = compute_smart_swipe_coordinates(
            direction=target,
            target=params.get("target"),
            width=width,
            height=height,
            duration=dur,
        )
        success = await driver.swipe(sx, sy, ex, ey, duration_ms=smart_dur)
        return f"Swiped {target}" if success else f"Swipe {target} failed"
    elif kind == "coords" and isinstance(target, list) and len(target) == 4:
        width, height = driver.screen_size
        start_x = int(target[0] * width / 1000.0)
        start_y = int(target[1] * height / 1000.0)
        end_x = int(target[2] * width / 1000.0)
        end_y = int(target[3] * height / 1000.0)
        success = await driver.swipe(start_x, start_y, end_x, end_y, duration_ms=dur)
        return (
            f"Swiped from {target[:2]} to {target[2:]}" if success else "Swipe coordinates failed"
        )
    return f"Invalid swipe action parameter: {action or direction or (start, end)}"


@artemis_tool(
    name="press_key",
    description="[ACTION] Presses a hardware or system key on the device ('home', 'back', 'enter', 'delete', 'power').",
    args_schema=PressKeyArgs,
    category="action",
)
async def press_key(key: str, driver: BaseDeviceDriver) -> str:
    success = await driver.press_key(key)
    return f"Key '{key}' pressed" if success else f"Failed to press key '{key}'"


@artemis_tool(
    name="wait_for_delay",
    description="[ACTION] Pauses execution for a specified number of seconds to allow UI loading or transition.",
    args_schema=WaitForDelayArgs,
    category="action",
)
async def wait_for_delay(seconds: float, driver: BaseDeviceDriver = None) -> str:
    await asyncio.sleep(seconds)
    return f"Waited for {seconds} seconds."