chenmingxuan/Artemis-Eapil
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 32 Wiki Activity

feat: add gesture sequences and calibrated pixel drags

Browse Source
This commit is contained in:
chenmingxuan committed 2026-11-02 11:53:27 +08:00
1 parent 2a3bbaa3fb
commit 479a185192
31 files changed
+2440 -44
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/fixtures/action_surfaces/operator_shells.json
Vendored
+371 -1
View File
@@ -50,6 +50,376 @@
}
}
},
"action_sequence": {
"type": "function",
"function": {
"name": "action_sequence",
"description": "[ACTION] Execute a short sequence of click, wait and drag steps in ONE call, in order, without model turns or screenshots between members. Use for temporary overlays: open the control, wait briefly, then operate it before it disappears. Use observed coordinates, or positions learned from an earlier observation of the same restored layout; element indices are not accepted. Each click/drag needs target_description. All members are validated before any input. The configured fast-action limit applies. The first failure stops the remainder; receipts identify completed and unexecuted members. Never replay the whole sequence after a partial result. Verify the final screen before claiming the intended result.",
"parameters": {
"properties": {
"steps": {
"description": "Ordered click, wait and drag members.",
"items": {
"anyOf": [
{
"additionalProperties": false,
"properties": {
"action": {
"const": "click",
"title": "Action",
"type": "string"
},
"target": {
"items": {
"type": "integer"
},
"maxItems": 2,
"minItems": 2,
"title": "Target",
"type": "array"
},
"target_description": {
"minLength": 1,
"title": "Target Description",
"type": "string"
},
"coordinate_space": {
"default": "normalized",
"enum": [
"normalized",
"pixel"
],
"title": "Coordinate Space",
"type": "string"
}
},
"required": [
"action",
"target",
"target_description"
],
"title": "ClickStep",
"type": "object"
},
{
"additionalProperties": false,
"properties": {
"start": {
"items": {
"type": "integer"
},
"maxItems": 2,
"minItems": 2,
"title": "Start",
"type": "array"
},
"target_description": {
"minLength": 1,
"title": "Target Description",
"type": "string"
},
"end": {
"anyOf": [
{
"items": {
"type": "integer"
},
"maxItems": 2,
"minItems": 2,
"type": "array"
},
{
"type": "null"
}
],
"default": null,
"description": "Exact endpoint; MUST be null when calibration is supplied.",
"title": "End"
},
"coordinate_space": {
"default": "normalized",
"enum": [
"normalized",
"pixel"
],
"title": "Coordinate Space",
"type": "string"
},
"duration_ms": {
"default": 300,
"maximum": 10000,
"minimum": 1,
"title": "Duration Ms",
"type": "integer"
},
"calibration": {
"anyOf": [
{
"additionalProperties": false,
"properties": {
"axis": {
"enum": [
"x",
"y"
],
"title": "Axis",
"type": "string"
},
"mode": {
"description": "thumb moves a handle to a value; pan moves a scale under a fixed indicator.",
"enum": [
"thumb",
"pan"
],
"title": "Mode",
"type": "string"
},
"anchors": {
"items": {
"additionalProperties": false,
"properties": {
"position": {
"description": "Observed axis coordinate in the selected coordinate space.",
"title": "Position",
"type": "number"
},
"value": {
"description": "Observed scale value, in one consistent numeric unit.",
"title": "Value",
"type": "number"
}
},
"required": [
"position",
"value"
],
"title": "ScaleAnchor",
"type": "object"
},
"maxItems": 2,
"minItems": 2,
"title": "Anchors",
"type": "array"
},
"target_value": {
"title": "Target Value",
"type": "number"
},
"indicator_position": {
"anyOf": [
{
"type": "number"
},
{
"type": "null"
}
],
"default": null,
"description": "Required for pan: the fixed indicator's observed axis position.",
"title": "Indicator Position"
}
},
"required": [
"axis",
"mode",
"anchors",
"target_value"
],
"title": "DragCalibration",
"type": "object"
},
{
"type": "null"
}
],
"default": null,
"description": "Computes the endpoint from observed anchors; then set end=null."
},
"action": {
"const": "drag",
"title": "Action",
"type": "string"
}
},
"required": [
"start",
"target_description",
"action"
],
"title": "DragStep",
"type": "object"
},
{
"additionalProperties": false,
"properties": {
"action": {
"const": "wait",
"title": "Action",
"type": "string"
},
"time_in_ms": {
"maximum": 10000,
"minimum": 0,
"title": "Time In Ms",
"type": "integer"
}
},
"required": [
"action",
"time_in_ms"
],
"title": "WaitStep",
"type": "object"
}
]
},
"type": "array"
}
},
"required": [
"steps"
],
"type": "object"
}
}
},
"drag": {
"type": "function",
"function": {
"name": "drag",
"description": "[ACTION] Precisely drag a local slider or timeline. Use explicit start/end or calibration from TWO observed numeric scale anchors. coordinate_space is normalized (0..1000) or pixel (the current screenshot's actual pixels); pixel avoids quantization on a zoomed scale. Default duration_ms=300: slower is not always more precise and may trigger long-press recognition. Do not overshoot the endpoint automatically. For calibration, axis=x/y; mode=thumb moves the handle to target_value, mode=pan shifts the scale under indicator_position. Use one numeric unit, e.g. seconds since midnight, and two anchors from the CURRENT zoom/layout. Provide exactly one of end or calibration. Returned geometry is a plan/dispatch receipt, not proof of the selected value: compare the post-action scale/value and correct only the observed residual; zoom further if the pixel resolution is insufficient.",
"parameters": {
"properties": {
"start": {
"description": "Observed handle or pan starting point [x, y].",
"items": {
"type": "integer"
},
"type": "array"
},
"target_description": {
"description": "The slider, handle or timeline being dragged.",
"type": "string"
},
"end": {
"anyOf": [
{
"items": {
"type": "integer"
},
"type": "array"
},
{
"type": "null"
}
],
"default": null,
"description": "Exact endpoint. Set null (or omit) when calibration is supplied; never compute both."
},
"coordinate_space": {
"default": "normalized",
"enum": [
"normalized",
"pixel"
],
"type": "string"
},
"duration_ms": {
"default": 300,
"type": "integer"
},
"calibration": {
"anyOf": [
{
"additionalProperties": false,
"properties": {
"axis": {
"enum": [
"x",
"y"
],
"title": "Axis",
"type": "string"
},
"mode": {
"description": "thumb moves a handle to a value; pan moves a scale under a fixed indicator.",
"enum": [
"thumb",
"pan"
],
"title": "Mode",
"type": "string"
},
"anchors": {
"items": {
"additionalProperties": false,
"properties": {
"position": {
"description": "Observed axis coordinate in the selected coordinate space.",
"title": "Position",
"type": "number"
},
"value": {
"description": "Observed scale value, in one consistent numeric unit.",
"title": "Value",
"type": "number"
}
},
"required": [
"position",
"value"
],
"title": "ScaleAnchor",
"type": "object"
},
"maxItems": 2,
"minItems": 2,
"title": "Anchors",
"type": "array"
},
"target_value": {
"title": "Target Value",
"type": "number"
},
"indicator_position": {
"anyOf": [
{
"type": "number"
},
{
"type": "null"
}
],
"default": null,
"description": "Required for pan: the fixed indicator's observed axis position.",
"title": "Indicator Position"
}
},
"required": [
"axis",
"mode",
"anchors",
"target_value"
],
"title": "DragCalibration",
"type": "object"
},
{
"type": "null"
}
],
"default": null,
"description": "Observed numeric anchors that compute the endpoint. When supplied, end MUST be null; otherwise set calibration null."
}
},
"required": [
"start",
"target_description"
],
"type": "object"
}
}
},
"input_text": {
"type": "function",
"function": {
@@ -105,7 +475,7 @@
"type": "function",
"function": {
"name": "swipe",
"description": "[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen.\n\n• Directional Scrolling ('direction'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n• Precise Coordinate Gestures ('start', 'end'): Best for local, fine-grained interactions such as adjusting sliders/SeekBars (e.g., volume, brightness, progress bars), drag-and-drop / list reordering, or as a reliable fallback when directional scrolling fails on specific containers. Always drag slightly PAST the target position to overcome touch slop and reliably trigger the update. When setting a slider to Maximum (100%) or Minimum (0%), swipe fully to the extreme boundary.\n\nArgs:\n direction: Smart directional scrolling ('up', 'down', 'left', 'right'). Automatically computes safe swipe vectors, retaining 40% visual overlap: 'up' (reveal content below), 'down' (reveal content above), 'left', 'right'.\n start: Start normalized coordinates [start_x, start_y] in 0-1000 scale.\n end: End normalized coordinates [end_x, end_y] in 0-1000 scale.\n target: Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within.\n gesture: Backward-compatible parameter: direction string OR custom coordinates list [start_x, start_y, end_x, end_y] in 0-1000 scale.\n target_description: What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling.\n duration: Optional gesture duration in milliseconds (default 800).",
"description": "[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen.\n\n• Directional Scrolling ('direction'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n• Precise Coordinate Gestures ('start', 'end'): Use observed endpoints for local slider/timeline movement. Do not automatically overshoot: slow short gestures can trigger long-press recognition. Compare the actual post-action value/scale before adjusting again. When available, drag supports pixel coordinates and two-anchor calibration for fine positioning.\n\nArgs:\n direction: Smart directional scrolling ('up', 'down', 'left', 'right'). Automatically computes safe swipe vectors, retaining 40% visual overlap: 'up' (reveal content below), 'down' (reveal content above), 'left', 'right'.\n start: Start normalized coordinates [start_x, start_y] in 0-1000 scale.\n end: End normalized coordinates [end_x, end_y] in 0-1000 scale.\n target: Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within.\n gesture: Backward-compatible parameter: direction string OR custom coordinates list [start_x, start_y, end_x, end_y] in 0-1000 scale.\n target_description: What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling.\n duration: Optional gesture duration in milliseconds (default 800).",
"parameters": {
"properties": {
"direction": {