685 lines
27 KiB
JSON
685 lines
27 KiB
JSON
{
|
|
"CLICK_TOOL": {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "click",
|
|
"description": "[ACTION] Click on the target location on the screen (supports element index or absolute normalized coordinates).",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"target": {
|
|
"anyOf": [
|
|
{
|
|
"type": "integer"
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"type": "integer"
|
|
}
|
|
}
|
|
],
|
|
"description": "Click target. Can be an element index number (int, e.g. 3) OR normalized coordinates (list of 2 integers, e.g. [500, 600])."
|
|
},
|
|
"target_description": {
|
|
"type": "string",
|
|
"description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index."
|
|
},
|
|
"times": {
|
|
"type": "integer",
|
|
"description": "Number of consecutive clicks on this target. Use this for double-clicks or multi-clicks (e.g. 7 to enter developer mode). Default is 1."
|
|
},
|
|
"delay_ms": {
|
|
"type": "integer",
|
|
"description": "Delay in milliseconds between consecutive clicks. Default is 100."
|
|
}
|
|
},
|
|
"required": [
|
|
"target"
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"CLICK_SEQUENCE_TOOL": {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "click_sequence",
|
|
"description": "[ACTION] Executes a sequence of taps one by one in order on the specified targets (e.g. [[500, 280], [885, 362]]). The screen will be returned ONLY after all clicks in the sequence have completed.",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"sequence": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"description": "Normalized coordinates [x, y] in 0-1000 scale (e.g., [500, 280])."
|
|
},
|
|
"description": "List of targets to tap in sequence, e.g. [[500, 280], [885, 362]]."
|
|
},
|
|
"target_descriptions": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
},
|
|
"description": "What each target is, in a few words, one entry per sequence entry in the same order (e.g. ['video body', 'skip button']). Required: a coordinate names nothing by itself."
|
|
},
|
|
"delay_ms": {
|
|
"type": "integer",
|
|
"description": "Delay between consecutive taps in milliseconds (default 50ms)."
|
|
}
|
|
},
|
|
"required": [
|
|
"sequence",
|
|
"target_descriptions"
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"LONG_PRESS_TOOL": {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "long_press",
|
|
"description": "[ACTION] Long press on the target location on the screen (supports element index or absolute normalized coordinates).",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"target": {
|
|
"anyOf": [
|
|
{
|
|
"type": "integer"
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"type": "integer"
|
|
}
|
|
}
|
|
],
|
|
"description": "Long press target. Can be an element index number (int, e.g. 3) OR normalized coordinates (list of 2 integers, e.g. [500, 600])."
|
|
},
|
|
"target_description": {
|
|
"type": "string",
|
|
"description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index."
|
|
},
|
|
"duration": {
|
|
"type": "integer",
|
|
"description": "Long press duration in milliseconds (default 1000)."
|
|
}
|
|
},
|
|
"required": [
|
|
"target"
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"INPUT_TEXT_TOOL": {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "input_text",
|
|
"description": "[ACTION] Type text into the target input field (supports replacing whole text or appending to the end, and multi-line strings with '\\n').",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"text": {
|
|
"type": "string",
|
|
"description": "The text content to input. Supports multi-line content with '\\n'."
|
|
},
|
|
"target": {
|
|
"anyOf": [
|
|
{
|
|
"type": "integer"
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"type": "integer"
|
|
}
|
|
}
|
|
],
|
|
"description": "Input target field. Can be an input box element index number (int, e.g. 3) OR normalized coordinates (list of 2 integers, e.g. [500, 600])."
|
|
},
|
|
"target_description": {
|
|
"type": "string",
|
|
"description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index."
|
|
},
|
|
"clear_exist": {
|
|
"type": "boolean",
|
|
"description": "Whether to clear existing text before typing. True (default): clear/replace entire text. False: append at the end of existing content."
|
|
}
|
|
},
|
|
"required": [
|
|
"text",
|
|
"target"
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"SWIPE_TOOL": {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "swipe",
|
|
"description": "[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen.\n\n• Directional Scrolling ('direction'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n• Precise Coordinate Gestures ('start', 'end'): Use observed endpoints for local slider/timeline movement. Do not automatically overshoot: slow short gestures can trigger long-press recognition. Compare the actual post-action value/scale before adjusting again. When available, drag supports pixel coordinates and two-anchor calibration for fine positioning.\n\nArgs:\n direction: Smart directional scrolling ('up', 'down', 'left', 'right'). Automatically computes safe swipe vectors, retaining 40% visual overlap: 'up' (reveal content below), 'down' (reveal content above), 'left', 'right'.\n start: Start normalized coordinates [start_x, start_y] in 0-1000 scale.\n end: End normalized coordinates [end_x, end_y] in 0-1000 scale.\n target: Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within.\n gesture: Backward-compatible parameter: direction string OR custom coordinates list [start_x, start_y, end_x, end_y] in 0-1000 scale.\n target_description: What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling.\n duration: Optional gesture duration in milliseconds (default 800).",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"direction": {
|
|
"type": "string",
|
|
"enum": [
|
|
"up",
|
|
"down",
|
|
"left",
|
|
"right"
|
|
],
|
|
"description": "Direction for scrolling and swiping: 'up' (drags bottom-to-top, scrolling down to reveal content below), 'down' (drags top-to-bottom, scrolling up to reveal content above), 'left' (drags right-to-left, scrolling right), 'right' (drags left-to-right, scrolling left)."
|
|
},
|
|
"start": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"description": "Start normalized coordinates [start_x, start_y] in 0-1000 scale for precise, local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop)."
|
|
},
|
|
"end": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"description": "End normalized coordinates [end_x, end_y] in 0-1000 scale for precise, local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop)."
|
|
},
|
|
"target": {
|
|
"anyOf": [
|
|
{
|
|
"type": "integer"
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"type": "integer"
|
|
}
|
|
},
|
|
{
|
|
"type": "string"
|
|
}
|
|
],
|
|
"description": "Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within."
|
|
},
|
|
"gesture": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string",
|
|
"enum": [
|
|
"up",
|
|
"down",
|
|
"left",
|
|
"right"
|
|
]
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"type": "integer"
|
|
}
|
|
}
|
|
],
|
|
"description": "Backward-compatible swipe gesture: smart direction string ('up', 'down', 'left', 'right') OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale."
|
|
},
|
|
"target_description": {
|
|
"type": "string",
|
|
"description": "What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling."
|
|
},
|
|
"duration": {
|
|
"type": "integer",
|
|
"description": "Optional swipe/drag duration in milliseconds (default 800; computed automatically for directional swipes)."
|
|
}
|
|
},
|
|
"required": []
|
|
}
|
|
}
|
|
},
|
|
"PRESS_KEY_TOOL": {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "press_key",
|
|
"description": "[ACTION] Press a physical or virtual system button (e.g. ENTER, BACK, HOME, APP_SWITCH).",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"key": {
|
|
"type": "string",
|
|
"enum": [
|
|
"ENTER",
|
|
"BACK",
|
|
"HOME",
|
|
"APP_SWITCH"
|
|
],
|
|
"description": "Standard Android system button name (ENTER, BACK, HOME, APP_SWITCH)."
|
|
}
|
|
},
|
|
"required": [
|
|
"key"
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"MANAGE_APP_TOOL": {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "manage_app",
|
|
"description": "[ACTION] Launch or force stop a specified application.",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"action": {
|
|
"type": "string",
|
|
"enum": [
|
|
"launch",
|
|
"stop"
|
|
],
|
|
"description": "The action type."
|
|
},
|
|
"app_name": {
|
|
"type": "string",
|
|
"description": "Display name or package name of the application."
|
|
}
|
|
},
|
|
"required": [
|
|
"action",
|
|
"app_name"
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"WAIT_FOR_DELAY_TOOL": {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "wait_for_delay",
|
|
"description": "[ACTION] Pause execution and wait for a specified duration in milliseconds.\n\nUse this whenever you need time to elapse—whether for UI loading, animations,\nscreen transitions, or longer scheduled delays and intervals specified in the task.",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"time_in_ms": {
|
|
"type": "integer",
|
|
"description": "The exact duration to wait in milliseconds. Accurately convert the required time duration into milliseconds based on your objective or plan (e.g., 2000 for 2s, 5000 for 5s, 60000 for 1 minute, 180000 for 3 minutes, 300000 for 5 minutes)."
|
|
}
|
|
},
|
|
"required": [
|
|
"time_in_ms"
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"DRAG_TOOL": {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "drag",
|
|
"description": "[ACTION] Precisely drag a local slider or timeline. Use explicit start/end or calibration from TWO observed numeric scale anchors. coordinate_space is normalized (0..1000) or pixel (the current screenshot's actual pixels); pixel avoids quantization on a zoomed scale. Default duration_ms=300: slower is not always more precise and may trigger long-press recognition. Do not overshoot the endpoint automatically. For calibration, axis=x/y; mode=thumb moves the handle to target_value, mode=pan shifts the scale under indicator_position. Use one numeric unit, e.g. seconds since midnight, and two anchors from the CURRENT zoom/layout. Provide exactly one of end or calibration. Returned geometry is a plan/dispatch receipt, not proof of the selected value: compare the post-action scale/value and correct only the observed residual; zoom further if the pixel resolution is insufficient.",
|
|
"parameters": {
|
|
"properties": {
|
|
"start": {
|
|
"description": "Observed handle or pan starting point [x, y].",
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"type": "array"
|
|
},
|
|
"target_description": {
|
|
"description": "The slider, handle or timeline being dragged.",
|
|
"type": "string"
|
|
},
|
|
"end": {
|
|
"anyOf": [
|
|
{
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"type": "array"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"description": "Exact endpoint. Set null (or omit) when calibration is supplied; never compute both."
|
|
},
|
|
"coordinate_space": {
|
|
"default": "normalized",
|
|
"enum": [
|
|
"normalized",
|
|
"pixel"
|
|
],
|
|
"type": "string"
|
|
},
|
|
"duration_ms": {
|
|
"default": 300,
|
|
"type": "integer"
|
|
},
|
|
"calibration": {
|
|
"anyOf": [
|
|
{
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"axis": {
|
|
"enum": [
|
|
"x",
|
|
"y"
|
|
],
|
|
"title": "Axis",
|
|
"type": "string"
|
|
},
|
|
"mode": {
|
|
"description": "thumb moves a handle to a value; pan moves a scale under a fixed indicator.",
|
|
"enum": [
|
|
"thumb",
|
|
"pan"
|
|
],
|
|
"title": "Mode",
|
|
"type": "string"
|
|
},
|
|
"anchors": {
|
|
"items": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"position": {
|
|
"description": "Observed axis coordinate in the selected coordinate space.",
|
|
"title": "Position",
|
|
"type": "number"
|
|
},
|
|
"value": {
|
|
"description": "Observed scale value, in one consistent numeric unit.",
|
|
"title": "Value",
|
|
"type": "number"
|
|
}
|
|
},
|
|
"required": [
|
|
"position",
|
|
"value"
|
|
],
|
|
"title": "ScaleAnchor",
|
|
"type": "object"
|
|
},
|
|
"maxItems": 2,
|
|
"minItems": 2,
|
|
"title": "Anchors",
|
|
"type": "array"
|
|
},
|
|
"target_value": {
|
|
"title": "Target Value",
|
|
"type": "number"
|
|
},
|
|
"indicator_position": {
|
|
"anyOf": [
|
|
{
|
|
"type": "number"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"description": "Required for pan: the fixed indicator's observed axis position.",
|
|
"title": "Indicator Position"
|
|
}
|
|
},
|
|
"required": [
|
|
"axis",
|
|
"mode",
|
|
"anchors",
|
|
"target_value"
|
|
],
|
|
"title": "DragCalibration",
|
|
"type": "object"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"description": "Observed numeric anchors that compute the endpoint. When supplied, end MUST be null; otherwise set calibration null."
|
|
}
|
|
},
|
|
"required": [
|
|
"start",
|
|
"target_description"
|
|
],
|
|
"type": "object"
|
|
}
|
|
}
|
|
},
|
|
"ACTION_SEQUENCE_TOOL": {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "action_sequence",
|
|
"description": "[ACTION] Execute a short sequence of click, wait and drag steps in ONE call, in order, without model turns or screenshots between members. Use for temporary overlays: open the control, wait briefly, then operate it before it disappears. Use observed coordinates, or positions learned from an earlier observation of the same restored layout; element indices are not accepted. Each click/drag needs target_description. All members are validated before any input. The configured fast-action limit applies. The first failure stops the remainder; receipts identify completed and unexecuted members. Never replay the whole sequence after a partial result. Verify the final screen before claiming the intended result.",
|
|
"parameters": {
|
|
"properties": {
|
|
"steps": {
|
|
"description": "Ordered click, wait and drag members.",
|
|
"items": {
|
|
"anyOf": [
|
|
{
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"action": {
|
|
"const": "click",
|
|
"title": "Action",
|
|
"type": "string"
|
|
},
|
|
"target": {
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"maxItems": 2,
|
|
"minItems": 2,
|
|
"title": "Target",
|
|
"type": "array"
|
|
},
|
|
"target_description": {
|
|
"minLength": 1,
|
|
"title": "Target Description",
|
|
"type": "string"
|
|
},
|
|
"coordinate_space": {
|
|
"default": "normalized",
|
|
"enum": [
|
|
"normalized",
|
|
"pixel"
|
|
],
|
|
"title": "Coordinate Space",
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"action",
|
|
"target",
|
|
"target_description"
|
|
],
|
|
"title": "ClickStep",
|
|
"type": "object"
|
|
},
|
|
{
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"start": {
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"maxItems": 2,
|
|
"minItems": 2,
|
|
"title": "Start",
|
|
"type": "array"
|
|
},
|
|
"target_description": {
|
|
"minLength": 1,
|
|
"title": "Target Description",
|
|
"type": "string"
|
|
},
|
|
"end": {
|
|
"anyOf": [
|
|
{
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"maxItems": 2,
|
|
"minItems": 2,
|
|
"type": "array"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"description": "Exact endpoint; MUST be null when calibration is supplied.",
|
|
"title": "End"
|
|
},
|
|
"coordinate_space": {
|
|
"default": "normalized",
|
|
"enum": [
|
|
"normalized",
|
|
"pixel"
|
|
],
|
|
"title": "Coordinate Space",
|
|
"type": "string"
|
|
},
|
|
"duration_ms": {
|
|
"default": 300,
|
|
"maximum": 10000,
|
|
"minimum": 1,
|
|
"title": "Duration Ms",
|
|
"type": "integer"
|
|
},
|
|
"calibration": {
|
|
"anyOf": [
|
|
{
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"axis": {
|
|
"enum": [
|
|
"x",
|
|
"y"
|
|
],
|
|
"title": "Axis",
|
|
"type": "string"
|
|
},
|
|
"mode": {
|
|
"description": "thumb moves a handle to a value; pan moves a scale under a fixed indicator.",
|
|
"enum": [
|
|
"thumb",
|
|
"pan"
|
|
],
|
|
"title": "Mode",
|
|
"type": "string"
|
|
},
|
|
"anchors": {
|
|
"items": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"position": {
|
|
"description": "Observed axis coordinate in the selected coordinate space.",
|
|
"title": "Position",
|
|
"type": "number"
|
|
},
|
|
"value": {
|
|
"description": "Observed scale value, in one consistent numeric unit.",
|
|
"title": "Value",
|
|
"type": "number"
|
|
}
|
|
},
|
|
"required": [
|
|
"position",
|
|
"value"
|
|
],
|
|
"title": "ScaleAnchor",
|
|
"type": "object"
|
|
},
|
|
"maxItems": 2,
|
|
"minItems": 2,
|
|
"title": "Anchors",
|
|
"type": "array"
|
|
},
|
|
"target_value": {
|
|
"title": "Target Value",
|
|
"type": "number"
|
|
},
|
|
"indicator_position": {
|
|
"anyOf": [
|
|
{
|
|
"type": "number"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"description": "Required for pan: the fixed indicator's observed axis position.",
|
|
"title": "Indicator Position"
|
|
}
|
|
},
|
|
"required": [
|
|
"axis",
|
|
"mode",
|
|
"anchors",
|
|
"target_value"
|
|
],
|
|
"title": "DragCalibration",
|
|
"type": "object"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"description": "Computes the endpoint from observed anchors; then set end=null."
|
|
},
|
|
"action": {
|
|
"const": "drag",
|
|
"title": "Action",
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"start",
|
|
"target_description",
|
|
"action"
|
|
],
|
|
"title": "DragStep",
|
|
"type": "object"
|
|
},
|
|
{
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"action": {
|
|
"const": "wait",
|
|
"title": "Action",
|
|
"type": "string"
|
|
},
|
|
"time_in_ms": {
|
|
"maximum": 10000,
|
|
"minimum": 0,
|
|
"title": "Time In Ms",
|
|
"type": "integer"
|
|
}
|
|
},
|
|
"required": [
|
|
"action",
|
|
"time_in_ms"
|
|
],
|
|
"title": "WaitStep",
|
|
"type": "object"
|
|
}
|
|
]
|
|
},
|
|
"type": "array"
|
|
}
|
|
},
|
|
"required": [
|
|
"steps"
|
|
],
|
|
"type": "object"
|
|
}
|
|
}
|
|
}
|
|
}
|