feat: add gesture sequences and calibrated pixel drags

artemis/mcp/action_specs.py

@@ -63,11 +63,13 @@ import types
 from typing import Annotated, Any, Literal, Union, get_args, get_origin
 
 from langchain_core.tools import StructuredTool
+from langchain_core.utils.function_calling import convert_to_openai_tool
 from mcp.types import CallToolResult
-from pydantic import Field, create_model
+from pydantic import BaseModel, Field, create_model
 
 from artemis.core.tool_declaration import ToolDeclaration
 from artemis.mcp.action_types import ActionResult
+from artemis.mcp.gestures import CoordinateSpace, DragCalibration, GestureStep
 
 __all__ = [
     "ACTION_SPECS",
@@ -213,6 +215,14 @@ async def _wire_swipe(actuator: Any, a: dict[str, Any]) -> ActionResult:
     )
 
 
+async def _wire_drag(actuator: Any, a: dict[str, Any]) -> ActionResult:
+    return await actuator.drag(**a)
+
+
+async def _wire_action_sequence(actuator: Any, a: dict[str, Any]) -> ActionResult:
+    return await actuator.action_sequence(a["steps"])
+
+
 async def _wire_press_key(actuator: Any, a: dict[str, Any]) -> ActionResult:
     return await actuator.press_key(a["key"])
 
@@ -244,6 +254,86 @@ async def _wire_focus_and_clear_text(actuator: Any, a: dict[str, Any]) -> Action
 # --- The manifest --------------------------------------------------------------------
 
 _SPECS: tuple[ActionSpec, ...] = (
+    ActionSpec(
+        name="action_sequence",
+        operator=OperatorDialect(
+            description=(
+                "[ACTION] Execute a short sequence of click, wait and drag steps in ONE call, "
+                "in order, without model turns or screenshots between members. Use for "
+                "temporary overlays: open the control, wait briefly, then operate it before "
+                "it disappears. Use observed coordinates, or positions learned from an "
+                "earlier observation of the same restored layout; element indices are not "
+                "accepted. Each click/drag needs target_description. All members are "
+                "validated before any input. The configured fast-action limit applies. "
+                "The first failure stops the remainder; receipts identify completed and "
+                "unexecuted members. Never replay the whole sequence after a partial result. "
+                "Verify the final screen before claiming the intended result."
+            ),
+            params=(
+                ParamSpec("steps", list[GestureStep], "Ordered click, wait and drag members."),
+            ),
+        ),
+        wire=WireDialect(
+            description="Execute validated short gesture steps consecutively; stop at the first failure.",
+            params=(ParamSpec("steps", list[GestureStep]),),
+            bind=_wire_action_sequence,
+        ),
+    ),
+    ActionSpec(
+        name="drag",
+        operator=OperatorDialect(
+            description=(
+                "[ACTION] Precisely drag a local slider or timeline. Use explicit start/end "
+                "or calibration from TWO observed numeric scale anchors. coordinate_space "
+                "is normalized (0..1000) or pixel (the current screenshot's actual pixels); "
+                "pixel avoids quantization on a zoomed scale. Default duration_ms=300: "
+                "slower is not always more precise and may trigger long-press recognition. "
+                "Do not overshoot the endpoint automatically. For calibration, axis=x/y; "
+                "mode=thumb moves the handle to target_value, mode=pan shifts the scale "
+                "under indicator_position. Use one numeric unit, e.g. seconds since midnight, "
+                "and two anchors from the CURRENT zoom/layout. Provide exactly one of end "
+                "or calibration. Returned geometry is a plan/dispatch receipt, not proof of "
+                "the selected value: compare the post-action scale/value and correct only "
+                "the observed residual; zoom further if the pixel resolution is insufficient."
+            ),
+            params=(
+                ParamSpec("start", list[int], "Observed handle or pan starting point [x, y]."),
+                ParamSpec(
+                    "target_description", str, "The slider, handle or timeline being dragged."
+                ),
+                ParamSpec(
+                    "end",
+                    list[int] | None,
+                    "Exact endpoint. Set null (or omit) when calibration is supplied; never compute both.",
+                    required=False,
+                ),
+                ParamSpec(
+                    "coordinate_space", CoordinateSpace, required=False, default="normalized"
+                ),
+                ParamSpec("duration_ms", int, required=False, default=300),
+                ParamSpec(
+                    "calibration",
+                    DragCalibration | None,
+                    "Observed numeric anchors that compute the endpoint. When supplied, end MUST be null; otherwise set calibration null.",
+                    required=False,
+                ),
+            ),
+        ),
+        wire=WireDialect(
+            description="Drag with exact coordinates or an observed two-anchor calibration.",
+            params=(
+                ParamSpec("start", list[int]),
+                ParamSpec("target_description", str),
+                ParamSpec("end", list[int] | None, required=False),
+                ParamSpec(
+                    "coordinate_space", CoordinateSpace, required=False, default="normalized"
+                ),
+                ParamSpec("duration_ms", int, required=False, default=300),
+                ParamSpec("calibration", DragCalibration | None, required=False),
+            ),
+            bind=_wire_drag,
+        ),
+    ),
     ActionSpec(
         name="click",
         operator=OperatorDialect(
@@ -470,7 +560,7 @@ _SPECS: tuple[ActionSpec, ...] = (
                 "[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen.\n"
                 "\n"
                 "• Directional Scrolling ('direction'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n"
-                "• Precise Coordinate Gestures ('start', 'end'): Best for local, fine-grained interactions such as adjusting sliders/SeekBars (e.g., volume, brightness, progress bars), drag-and-drop / list reordering, or as a reliable fallback when directional scrolling fails on specific containers. Always drag slightly PAST the target position to overcome touch slop and reliably trigger the update. When setting a slider to Maximum (100%) or Minimum (0%), swipe fully to the extreme boundary.\n"
+                "• Precise Coordinate Gestures ('start', 'end'): Use observed endpoints for local slider/timeline movement. Do not automatically overshoot: slow short gestures can trigger long-press recognition. Compare the actual post-action value/scale before adjusting again. When available, drag supports pixel coordinates and two-anchor calibration for fine positioning.\n"
                 "\n"
                 "Args:\n"
                 "    direction: Smart directional scrolling ('up', 'down', 'left', 'right'). Automatically computes safe swipe vectors, retaining 40% visual overlap: 'up' (reveal content below), 'down' (reveal content above), 'left', 'right'.\n"
@@ -684,6 +774,8 @@ ACTION_SPECS: dict[str, ActionSpec] = {spec.name: spec for spec in _SPECS}
 #: identical to the prompt's "Physical device actions" enumeration order).
 OPERATOR_SHELL_ORDER: tuple[str, ...] = (
     "click",
+    "action_sequence",
+    "drag",
     "input_text",
     "swipe",
     "press_key",
@@ -694,6 +786,8 @@ OPERATOR_SHELL_ORDER: tuple[str, ...] = (
 
 #: Exception wording per action; matches the historical executor `except` arms.
 EXCEPTION_PREFIXES: dict[str, str] = {
+    "action_sequence": "Error during action_sequence",
+    "drag": "Error during drag",
     "click": "Error during click",
     "click_sequence": "Error executing click sequence",
     "long_press": "Error during long press",
@@ -781,8 +875,18 @@ def _json_schema(annotation: Any) -> dict[str, Any]:
         raise TypeError(f"No JSON projection for parameter annotation {annotation!r}.") from None
 
 
-def _projected_parameters(dialect: OperatorDialect) -> dict[str, Any]:
+def _projected_parameters(dialect: OperatorDialect, name: str) -> dict[str, Any]:
     """The operator shell's parameters as a JSON-schema ``parameters`` object."""
+
+    def has_model(annotation: Any) -> bool:
+        return (isinstance(annotation, type) and issubclass(annotation, BaseModel)) or any(
+            has_model(arg) for arg in get_args(annotation)
+        )
+
+    if any(has_model(p.annotation) for p in dialect.params):
+        # Nullable alternatives matter for endpoint-versus-calibration requests:
+        # models must be able to send end=null instead of inventing an endpoint.
+        return convert_to_openai_tool(operator_shell_tool(name))["function"]["parameters"]
     properties: dict[str, Any] = {}
     required: list[str] = []
     for p in dialect.params:
@@ -814,7 +918,7 @@ def tool_declaration(name: str) -> ToolDeclaration:
         return ToolDeclaration(
             name=name,
             description=spec.operator.description,
-            parameters=_projected_parameters(spec.operator),
+            parameters=_projected_parameters(spec.operator, name),
         )
     raise ValueError(f"Action '{name}' has no agent dialect to declare.")