feat: add gesture sequences and calibrated pixel drags
This commit is contained in:
chenmingxuan committed
1 parent
2a3bbaa3fb
commit
479a185192
31 files changed
+2440
-44
No files matched your search
+371
-1
@@ -161,7 +161,7 @@
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "swipe",
|
||||
"description": "[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen.\n\n• Directional Scrolling ('direction'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n• Precise Coordinate Gestures ('start', 'end'): Best for local, fine-grained interactions such as adjusting sliders/SeekBars (e.g., volume, brightness, progress bars), drag-and-drop / list reordering, or as a reliable fallback when directional scrolling fails on specific containers. Always drag slightly PAST the target position to overcome touch slop and reliably trigger the update. When setting a slider to Maximum (100%) or Minimum (0%), swipe fully to the extreme boundary.\n\nArgs:\n direction: Smart directional scrolling ('up', 'down', 'left', 'right'). Automatically computes safe swipe vectors, retaining 40% visual overlap: 'up' (reveal content below), 'down' (reveal content above), 'left', 'right'.\n start: Start normalized coordinates [start_x, start_y] in 0-1000 scale.\n end: End normalized coordinates [end_x, end_y] in 0-1000 scale.\n target: Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within.\n gesture: Backward-compatible parameter: direction string OR custom coordinates list [start_x, start_y, end_x, end_y] in 0-1000 scale.\n target_description: What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling.\n duration: Optional gesture duration in milliseconds (default 800).",
|
||||
"description": "[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen.\n\n• Directional Scrolling ('direction'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n• Precise Coordinate Gestures ('start', 'end'): Use observed endpoints for local slider/timeline movement. Do not automatically overshoot: slow short gestures can trigger long-press recognition. Compare the actual post-action value/scale before adjusting again. When available, drag supports pixel coordinates and two-anchor calibration for fine positioning.\n\nArgs:\n direction: Smart directional scrolling ('up', 'down', 'left', 'right'). Automatically computes safe swipe vectors, retaining 40% visual overlap: 'up' (reveal content below), 'down' (reveal content above), 'left', 'right'.\n start: Start normalized coordinates [start_x, start_y] in 0-1000 scale.\n end: End normalized coordinates [end_x, end_y] in 0-1000 scale.\n target: Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within.\n gesture: Backward-compatible parameter: direction string OR custom coordinates list [start_x, start_y, end_x, end_y] in 0-1000 scale.\n target_description: What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling.\n duration: Optional gesture duration in milliseconds (default 800).",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -310,5 +310,375 @@
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"DRAG_TOOL": {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "drag",
|
||||
"description": "[ACTION] Precisely drag a local slider or timeline. Use explicit start/end or calibration from TWO observed numeric scale anchors. coordinate_space is normalized (0..1000) or pixel (the current screenshot's actual pixels); pixel avoids quantization on a zoomed scale. Default duration_ms=300: slower is not always more precise and may trigger long-press recognition. Do not overshoot the endpoint automatically. For calibration, axis=x/y; mode=thumb moves the handle to target_value, mode=pan shifts the scale under indicator_position. Use one numeric unit, e.g. seconds since midnight, and two anchors from the CURRENT zoom/layout. Provide exactly one of end or calibration. Returned geometry is a plan/dispatch receipt, not proof of the selected value: compare the post-action scale/value and correct only the observed residual; zoom further if the pixel resolution is insufficient.",
|
||||
"parameters": {
|
||||
"properties": {
|
||||
"start": {
|
||||
"description": "Observed handle or pan starting point [x, y].",
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"target_description": {
|
||||
"description": "The slider, handle or timeline being dragged.",
|
||||
"type": "string"
|
||||
},
|
||||
"end": {
|
||||
"anyOf": [
|
||||
{
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Exact endpoint. Set null (or omit) when calibration is supplied; never compute both."
|
||||
},
|
||||
"coordinate_space": {
|
||||
"default": "normalized",
|
||||
"enum": [
|
||||
"normalized",
|
||||
"pixel"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"duration_ms": {
|
||||
"default": 300,
|
||||
"type": "integer"
|
||||
},
|
||||
"calibration": {
|
||||
"anyOf": [
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"axis": {
|
||||
"enum": [
|
||||
"x",
|
||||
"y"
|
||||
],
|
||||
"title": "Axis",
|
||||
"type": "string"
|
||||
},
|
||||
"mode": {
|
||||
"description": "thumb moves a handle to a value; pan moves a scale under a fixed indicator.",
|
||||
"enum": [
|
||||
"thumb",
|
||||
"pan"
|
||||
],
|
||||
"title": "Mode",
|
||||
"type": "string"
|
||||
},
|
||||
"anchors": {
|
||||
"items": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"position": {
|
||||
"description": "Observed axis coordinate in the selected coordinate space.",
|
||||
"title": "Position",
|
||||
"type": "number"
|
||||
},
|
||||
"value": {
|
||||
"description": "Observed scale value, in one consistent numeric unit.",
|
||||
"title": "Value",
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"position",
|
||||
"value"
|
||||
],
|
||||
"title": "ScaleAnchor",
|
||||
"type": "object"
|
||||
},
|
||||
"maxItems": 2,
|
||||
"minItems": 2,
|
||||
"title": "Anchors",
|
||||
"type": "array"
|
||||
},
|
||||
"target_value": {
|
||||
"title": "Target Value",
|
||||
"type": "number"
|
||||
},
|
||||
"indicator_position": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Required for pan: the fixed indicator's observed axis position.",
|
||||
"title": "Indicator Position"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"axis",
|
||||
"mode",
|
||||
"anchors",
|
||||
"target_value"
|
||||
],
|
||||
"title": "DragCalibration",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Observed numeric anchors that compute the endpoint. When supplied, end MUST be null; otherwise set calibration null."
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"start",
|
||||
"target_description"
|
||||
],
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
},
|
||||
"ACTION_SEQUENCE_TOOL": {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "action_sequence",
|
||||
"description": "[ACTION] Execute a short sequence of click, wait and drag steps in ONE call, in order, without model turns or screenshots between members. Use for temporary overlays: open the control, wait briefly, then operate it before it disappears. Use observed coordinates, or positions learned from an earlier observation of the same restored layout; element indices are not accepted. Each click/drag needs target_description. All members are validated before any input. The configured fast-action limit applies. The first failure stops the remainder; receipts identify completed and unexecuted members. Never replay the whole sequence after a partial result. Verify the final screen before claiming the intended result.",
|
||||
"parameters": {
|
||||
"properties": {
|
||||
"steps": {
|
||||
"description": "Ordered click, wait and drag members.",
|
||||
"items": {
|
||||
"anyOf": [
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"action": {
|
||||
"const": "click",
|
||||
"title": "Action",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"maxItems": 2,
|
||||
"minItems": 2,
|
||||
"title": "Target",
|
||||
"type": "array"
|
||||
},
|
||||
"target_description": {
|
||||
"minLength": 1,
|
||||
"title": "Target Description",
|
||||
"type": "string"
|
||||
},
|
||||
"coordinate_space": {
|
||||
"default": "normalized",
|
||||
"enum": [
|
||||
"normalized",
|
||||
"pixel"
|
||||
],
|
||||
"title": "Coordinate Space",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"action",
|
||||
"target",
|
||||
"target_description"
|
||||
],
|
||||
"title": "ClickStep",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"start": {
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"maxItems": 2,
|
||||
"minItems": 2,
|
||||
"title": "Start",
|
||||
"type": "array"
|
||||
},
|
||||
"target_description": {
|
||||
"minLength": 1,
|
||||
"title": "Target Description",
|
||||
"type": "string"
|
||||
},
|
||||
"end": {
|
||||
"anyOf": [
|
||||
{
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"maxItems": 2,
|
||||
"minItems": 2,
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Exact endpoint; MUST be null when calibration is supplied.",
|
||||
"title": "End"
|
||||
},
|
||||
"coordinate_space": {
|
||||
"default": "normalized",
|
||||
"enum": [
|
||||
"normalized",
|
||||
"pixel"
|
||||
],
|
||||
"title": "Coordinate Space",
|
||||
"type": "string"
|
||||
},
|
||||
"duration_ms": {
|
||||
"default": 300,
|
||||
"maximum": 10000,
|
||||
"minimum": 1,
|
||||
"title": "Duration Ms",
|
||||
"type": "integer"
|
||||
},
|
||||
"calibration": {
|
||||
"anyOf": [
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"axis": {
|
||||
"enum": [
|
||||
"x",
|
||||
"y"
|
||||
],
|
||||
"title": "Axis",
|
||||
"type": "string"
|
||||
},
|
||||
"mode": {
|
||||
"description": "thumb moves a handle to a value; pan moves a scale under a fixed indicator.",
|
||||
"enum": [
|
||||
"thumb",
|
||||
"pan"
|
||||
],
|
||||
"title": "Mode",
|
||||
"type": "string"
|
||||
},
|
||||
"anchors": {
|
||||
"items": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"position": {
|
||||
"description": "Observed axis coordinate in the selected coordinate space.",
|
||||
"title": "Position",
|
||||
"type": "number"
|
||||
},
|
||||
"value": {
|
||||
"description": "Observed scale value, in one consistent numeric unit.",
|
||||
"title": "Value",
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"position",
|
||||
"value"
|
||||
],
|
||||
"title": "ScaleAnchor",
|
||||
"type": "object"
|
||||
},
|
||||
"maxItems": 2,
|
||||
"minItems": 2,
|
||||
"title": "Anchors",
|
||||
"type": "array"
|
||||
},
|
||||
"target_value": {
|
||||
"title": "Target Value",
|
||||
"type": "number"
|
||||
},
|
||||
"indicator_position": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Required for pan: the fixed indicator's observed axis position.",
|
||||
"title": "Indicator Position"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"axis",
|
||||
"mode",
|
||||
"anchors",
|
||||
"target_value"
|
||||
],
|
||||
"title": "DragCalibration",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Computes the endpoint from observed anchors; then set end=null."
|
||||
},
|
||||
"action": {
|
||||
"const": "drag",
|
||||
"title": "Action",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"start",
|
||||
"target_description",
|
||||
"action"
|
||||
],
|
||||
"title": "DragStep",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"action": {
|
||||
"const": "wait",
|
||||
"title": "Action",
|
||||
"type": "string"
|
||||
},
|
||||
"time_in_ms": {
|
||||
"maximum": 10000,
|
||||
"minimum": 0,
|
||||
"title": "Time In Ms",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"action",
|
||||
"time_in_ms"
|
||||
],
|
||||
"title": "WaitStep",
|
||||
"type": "object"
|
||||
}
|
||||
]
|
||||
},
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"steps"
|
||||
],
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user