2220 lines
79 KiB
JSON
2220 lines
79 KiB
JSON
{
|
|
"click": {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "click",
|
|
"description": "[ACTION] Click on the target location on the screen (supports element index or absolute normalized coordinates). Indexed resource targets are checked against live bounds before input; missing/moved targets refuse the tap. For floating or auto-hiding panels, use a guarded action_sequence with opener and target in the same call; an open panel in a screenshot may already have expired.",
|
|
"parameters": {
|
|
"properties": {
|
|
"target": {
|
|
"anyOf": [
|
|
{
|
|
"type": "integer"
|
|
},
|
|
{
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"type": "array"
|
|
}
|
|
],
|
|
"description": "Click target. Can be an element index number (int, e.g. 3) OR normalized coordinates (list of 2 integers, e.g. [500, 600])."
|
|
},
|
|
"target_description": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index."
|
|
},
|
|
"times": {
|
|
"default": 1,
|
|
"description": "Number of consecutive clicks on this target. Use this for double-clicks or multi-clicks (e.g. 7 to enter developer mode). Default is 1.",
|
|
"type": "integer"
|
|
},
|
|
"delay_ms": {
|
|
"default": 100,
|
|
"description": "Delay in milliseconds between consecutive clicks. Default is 100.",
|
|
"type": "integer"
|
|
}
|
|
},
|
|
"required": [
|
|
"target"
|
|
],
|
|
"type": "object"
|
|
}
|
|
}
|
|
},
|
|
"action_sequence": {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "action_sequence",
|
|
"description": "[ACTION] General Android composer: execute ensure_visible, click, wait and drag steps in ONE call, in order, without model turns between members; explicit visibility guards may read the live UI locally. Use for temporary overlays: open the control, wait briefly, then operate it before it disappears. Prefer ensure_visible(target=observed panel control ID, opener=observed UI opener ID) followed by click/drag. This checks live UI, skips opening an already-visible panel, or taps its opener ONCE and waits up to timeout_ms (default 1500, max 3000) for the target. Each preparation counts as one member. Include it again in EVERY panel-operation call, even when the previous screenshot showed the panel open; do not split preparation and operation into different model turns. It never retries a business action. Copy all resource IDs VERBATIM, including ':id/' separators, never ':id:'. If preparation fails, re-observe/correct the ID; never guess fallback coordinates. Legacy click.unless_visible is ONLY for an opener, NEVER for an operation such as a +/- button: visible means SKIP, not require-visible. For clicks prefer target=the exact observed resource_id string: live bounds are resolved locally, refusing absent or ambiguous targets. Otherwise use observed [x,y] in the declared coordinate_space; do not confuse original screenshot pixels with 0..1000 normalized coordinates. Element indices are not accepted. Each click/drag needs target_description. All members are validated before any input. The configured fast-action limit applies. The first failure stops the remainder; receipts identify completed and unexecuted members. Never replay the whole sequence after a partial result. Every wait counts as a member. click.times (1..10) performs consecutive taps within ONE member, like the ordinary click tool; prefer this over expanding repeated +/- taps into many members/model turns. Each resource-targeted tap is revalidated locally; missing targets stop remaining taps and receipts preserve completed_taps. For maximum/minimum prefer observed native +/- controls, confirming that further same-direction adjustment stops changing handle/ticks. An estimated drag endpoint can be imprecise. For every auto-hiding adjustment, including the reverse direction, keep opener/wait/operation in one call. A prior screenshot does not prove the panel is still open. Use click.unless_visible and drag.require_visible with an observed panel resource_id to avoid toggling an open panel closed or dragging its underlying content. Never invent resource IDs. Verify actual ticks/handle endpoints before claiming maximum or minimum.",
|
|
"parameters": {
|
|
"properties": {
|
|
"steps": {
|
|
"description": "Ordered UI preparation, click, wait, drag, pinch, touch_gesture, control_action and wait_until members. control_action members wrap their native selector/action arguments in request. Inline text is limited to 1024 characters; use a standalone control_action for long edits. Entire encoded sequence stays within the existing 16KiB argument safety bound.",
|
|
"items": {
|
|
"anyOf": [
|
|
{
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"action": {
|
|
"const": "click",
|
|
"title": "Action",
|
|
"type": "string"
|
|
},
|
|
"target": {
|
|
"anyOf": [
|
|
{
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"maxItems": 2,
|
|
"minItems": 2,
|
|
"type": "array"
|
|
},
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
}
|
|
],
|
|
"description": "Prefer an exact observed resource_id: resolve its live bounds before tapping, refusing absent/ambiguous targets. Otherwise use [x,y] in coordinate_space; never mix screenshot pixels with normalized coordinates.",
|
|
"title": "Target"
|
|
},
|
|
"target_description": {
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"title": "Target Description",
|
|
"type": "string"
|
|
},
|
|
"coordinate_space": {
|
|
"default": "normalized",
|
|
"enum": [
|
|
"normalized",
|
|
"pixel"
|
|
],
|
|
"title": "Coordinate Space",
|
|
"type": "string"
|
|
},
|
|
"times": {
|
|
"default": 1,
|
|
"description": "Consecutive taps within this ONE member, e.g. 10 for a scale +/- button; no model turns between taps.",
|
|
"maximum": 10,
|
|
"minimum": 1,
|
|
"title": "Times",
|
|
"type": "integer"
|
|
},
|
|
"delay_ms": {
|
|
"default": 100,
|
|
"maximum": 1000,
|
|
"minimum": 0,
|
|
"title": "Delay Ms",
|
|
"type": "integer"
|
|
},
|
|
"unless_visible": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"description": "Exact previously observed resource_id inside a toggled panel (not the opener). Skip this opener click when that control is already visible; otherwise click. Avoid closing a still-open panel.",
|
|
"title": "Unless Visible"
|
|
}
|
|
},
|
|
"required": [
|
|
"action",
|
|
"target",
|
|
"target_description"
|
|
],
|
|
"title": "ClickStep",
|
|
"type": "object"
|
|
},
|
|
{
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"start": {
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"maxItems": 2,
|
|
"minItems": 2,
|
|
"title": "Start",
|
|
"type": "array"
|
|
},
|
|
"target_description": {
|
|
"minLength": 1,
|
|
"title": "Target Description",
|
|
"type": "string"
|
|
},
|
|
"end": {
|
|
"anyOf": [
|
|
{
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"maxItems": 2,
|
|
"minItems": 2,
|
|
"type": "array"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"description": "Exact endpoint; MUST be null when calibration is supplied.",
|
|
"title": "End"
|
|
},
|
|
"coordinate_space": {
|
|
"default": "normalized",
|
|
"enum": [
|
|
"normalized",
|
|
"pixel"
|
|
],
|
|
"title": "Coordinate Space",
|
|
"type": "string"
|
|
},
|
|
"duration_ms": {
|
|
"default": 300,
|
|
"maximum": 10000,
|
|
"minimum": 1,
|
|
"title": "Duration Ms",
|
|
"type": "integer"
|
|
},
|
|
"calibration": {
|
|
"anyOf": [
|
|
{
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"axis": {
|
|
"enum": [
|
|
"x",
|
|
"y"
|
|
],
|
|
"title": "Axis",
|
|
"type": "string"
|
|
},
|
|
"mode": {
|
|
"description": "thumb moves a handle to a value; pan moves a scale under a fixed indicator.",
|
|
"enum": [
|
|
"thumb",
|
|
"pan"
|
|
],
|
|
"title": "Mode",
|
|
"type": "string"
|
|
},
|
|
"anchors": {
|
|
"items": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"position": {
|
|
"description": "Observed axis coordinate in the selected coordinate space.",
|
|
"title": "Position",
|
|
"type": "number"
|
|
},
|
|
"value": {
|
|
"description": "Observed scale value, in one consistent numeric unit.",
|
|
"title": "Value",
|
|
"type": "number"
|
|
}
|
|
},
|
|
"required": [
|
|
"position",
|
|
"value"
|
|
],
|
|
"title": "ScaleAnchor",
|
|
"type": "object"
|
|
},
|
|
"maxItems": 2,
|
|
"minItems": 2,
|
|
"title": "Anchors",
|
|
"type": "array"
|
|
},
|
|
"target_value": {
|
|
"title": "Target Value",
|
|
"type": "number"
|
|
},
|
|
"indicator_position": {
|
|
"anyOf": [
|
|
{
|
|
"type": "number"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"description": "Required for pan: the fixed indicator's observed axis position.",
|
|
"title": "Indicator Position"
|
|
}
|
|
},
|
|
"required": [
|
|
"axis",
|
|
"mode",
|
|
"anchors",
|
|
"target_value"
|
|
],
|
|
"title": "DragCalibration",
|
|
"type": "object"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"description": "Computes the endpoint from observed anchors; then set end=null."
|
|
},
|
|
"require_visible": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"description": "Exact previously observed resource_id of a panel control. Check live visibility before dragging; refuse input if absent. Use for auto-hiding controls.",
|
|
"title": "Require Visible"
|
|
},
|
|
"action": {
|
|
"const": "drag",
|
|
"title": "Action",
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"start",
|
|
"target_description",
|
|
"action"
|
|
],
|
|
"title": "DragStep",
|
|
"type": "object"
|
|
},
|
|
{
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"action": {
|
|
"const": "wait",
|
|
"title": "Action",
|
|
"type": "string"
|
|
},
|
|
"time_in_ms": {
|
|
"maximum": 10000,
|
|
"minimum": 0,
|
|
"title": "Time In Ms",
|
|
"type": "integer"
|
|
}
|
|
},
|
|
"required": [
|
|
"action",
|
|
"time_in_ms"
|
|
],
|
|
"title": "WaitStep",
|
|
"type": "object"
|
|
},
|
|
{
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"action": {
|
|
"const": "ensure_visible",
|
|
"title": "Action",
|
|
"type": "string"
|
|
},
|
|
"target": {
|
|
"description": "Panel control resource_id copied VERBATIM from observed elements, including the ':id/' separator. Never rewrite it as ':id:' or guess an ID.",
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"title": "Target",
|
|
"type": "string"
|
|
},
|
|
"opener": {
|
|
"description": "UI opener resource_id copied VERBATIM from observed elements, including ':id/' (not ':id:'). Tap ONCE only if target is absent; never use a business submission as an opener.",
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"title": "Opener",
|
|
"type": "string"
|
|
},
|
|
"target_description": {
|
|
"minLength": 1,
|
|
"title": "Target Description",
|
|
"type": "string"
|
|
},
|
|
"timeout_ms": {
|
|
"default": 1500,
|
|
"description": "Bounded live-UI wait AFTER opening, without extra model turns or repeated opener taps.",
|
|
"maximum": 3000,
|
|
"minimum": 100,
|
|
"title": "Timeout Ms",
|
|
"type": "integer"
|
|
}
|
|
},
|
|
"required": [
|
|
"action",
|
|
"target",
|
|
"opener",
|
|
"target_description"
|
|
],
|
|
"title": "EnsureVisibleStep",
|
|
"type": "object"
|
|
},
|
|
{
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"center": {
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"maxItems": 2,
|
|
"minItems": 2,
|
|
"title": "Center",
|
|
"type": "array"
|
|
},
|
|
"start_span": {
|
|
"exclusiveMinimum": 0,
|
|
"maximum": 20000,
|
|
"title": "Start Span",
|
|
"type": "integer"
|
|
},
|
|
"end_span": {
|
|
"exclusiveMinimum": 0,
|
|
"maximum": 20000,
|
|
"title": "End Span",
|
|
"type": "integer"
|
|
},
|
|
"angle_degrees": {
|
|
"default": 0,
|
|
"maximum": 360,
|
|
"minimum": -360,
|
|
"title": "Angle Degrees",
|
|
"type": "number"
|
|
},
|
|
"rotation_degrees": {
|
|
"default": 0,
|
|
"maximum": 360,
|
|
"minimum": -360,
|
|
"title": "Rotation Degrees",
|
|
"type": "number"
|
|
},
|
|
"duration_ms": {
|
|
"default": 800,
|
|
"maximum": 10000,
|
|
"minimum": 1,
|
|
"title": "Duration Ms",
|
|
"type": "integer"
|
|
},
|
|
"coordinate_space": {
|
|
"default": "normalized",
|
|
"enum": [
|
|
"normalized",
|
|
"pixel"
|
|
],
|
|
"title": "Coordinate Space",
|
|
"type": "string"
|
|
},
|
|
"target_description": {
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"title": "Target Description",
|
|
"type": "string"
|
|
},
|
|
"require_visible": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"title": "Require Visible"
|
|
},
|
|
"action": {
|
|
"const": "pinch",
|
|
"title": "Action",
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"center",
|
|
"start_span",
|
|
"end_span",
|
|
"target_description",
|
|
"action"
|
|
],
|
|
"title": "PinchStep",
|
|
"type": "object"
|
|
},
|
|
{
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"frames": {
|
|
"items": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"at_ms": {
|
|
"maximum": 10000,
|
|
"minimum": 0,
|
|
"title": "At Ms",
|
|
"type": "integer"
|
|
},
|
|
"points": {
|
|
"items": {
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"maxItems": 2,
|
|
"minItems": 2,
|
|
"type": "array"
|
|
},
|
|
"maxItems": 5,
|
|
"minItems": 1,
|
|
"title": "Points",
|
|
"type": "array"
|
|
}
|
|
},
|
|
"required": [
|
|
"at_ms",
|
|
"points"
|
|
],
|
|
"title": "TouchFrame",
|
|
"type": "object"
|
|
},
|
|
"maxItems": 32,
|
|
"minItems": 2,
|
|
"title": "Frames",
|
|
"type": "array"
|
|
},
|
|
"coordinate_space": {
|
|
"default": "normalized",
|
|
"enum": [
|
|
"normalized",
|
|
"pixel"
|
|
],
|
|
"title": "Coordinate Space",
|
|
"type": "string"
|
|
},
|
|
"target_description": {
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"title": "Target Description",
|
|
"type": "string"
|
|
},
|
|
"require_visible": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"title": "Require Visible"
|
|
},
|
|
"action": {
|
|
"const": "touch_gesture",
|
|
"title": "Action",
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"frames",
|
|
"target_description",
|
|
"action"
|
|
],
|
|
"title": "TouchGestureStep",
|
|
"type": "object"
|
|
},
|
|
{
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"action": {
|
|
"const": "control_action",
|
|
"title": "Action",
|
|
"type": "string"
|
|
},
|
|
"request": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"selector": {
|
|
"additionalProperties": false,
|
|
"description": "Every supplied attribute is an exact AND match against a fresh native node.",
|
|
"properties": {
|
|
"resource_id": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"title": "Resource Id"
|
|
},
|
|
"text": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 4096,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"title": "Text"
|
|
},
|
|
"content_desc": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"title": "Content Desc"
|
|
},
|
|
"class_name": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"title": "Class Name"
|
|
},
|
|
"package_name": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"title": "Package Name"
|
|
},
|
|
"bounds": {
|
|
"anyOf": [
|
|
{
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"maxItems": 4,
|
|
"minItems": 4,
|
|
"type": "array"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"title": "Bounds"
|
|
}
|
|
},
|
|
"title": "ControlSelector",
|
|
"type": "object"
|
|
},
|
|
"action": {
|
|
"enum": [
|
|
"click",
|
|
"long_click",
|
|
"focus",
|
|
"show_on_screen",
|
|
"set_progress",
|
|
"scroll_forward",
|
|
"scroll_backward",
|
|
"scroll_up",
|
|
"scroll_down",
|
|
"scroll_left",
|
|
"scroll_right",
|
|
"set_selection",
|
|
"replace_selection",
|
|
"set_text",
|
|
"ime_enter"
|
|
],
|
|
"title": "Action",
|
|
"type": "string"
|
|
},
|
|
"expected_token": {
|
|
"anyOf": [
|
|
{
|
|
"pattern": "^[a-f0-9]{64}$",
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"title": "Expected Token"
|
|
},
|
|
"value": {
|
|
"anyOf": [
|
|
{
|
|
"type": "number"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"title": "Value"
|
|
},
|
|
"text": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 1024,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"title": "Text"
|
|
},
|
|
"selection_start": {
|
|
"anyOf": [
|
|
{
|
|
"minimum": 0,
|
|
"type": "integer"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"title": "Selection Start"
|
|
},
|
|
"selection_end": {
|
|
"anyOf": [
|
|
{
|
|
"minimum": 0,
|
|
"type": "integer"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"title": "Selection End"
|
|
}
|
|
},
|
|
"required": [
|
|
"selector",
|
|
"action"
|
|
],
|
|
"title": "SequenceControlRequest",
|
|
"type": "object"
|
|
}
|
|
},
|
|
"required": [
|
|
"action",
|
|
"request"
|
|
],
|
|
"title": "ControlActionStep",
|
|
"type": "object"
|
|
},
|
|
{
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"selector": {
|
|
"additionalProperties": false,
|
|
"description": "Every supplied attribute is an exact AND match against a fresh native node.",
|
|
"properties": {
|
|
"resource_id": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"title": "Resource Id"
|
|
},
|
|
"text": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 4096,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"title": "Text"
|
|
},
|
|
"content_desc": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"title": "Content Desc"
|
|
},
|
|
"class_name": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"title": "Class Name"
|
|
},
|
|
"package_name": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"title": "Package Name"
|
|
},
|
|
"bounds": {
|
|
"anyOf": [
|
|
{
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"maxItems": 4,
|
|
"minItems": 4,
|
|
"type": "array"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"title": "Bounds"
|
|
}
|
|
},
|
|
"title": "ControlSelector",
|
|
"type": "object"
|
|
},
|
|
"state": {
|
|
"enum": [
|
|
"visible",
|
|
"absent",
|
|
"enabled",
|
|
"disabled",
|
|
"checked",
|
|
"unchecked",
|
|
"selected",
|
|
"unselected",
|
|
"text",
|
|
"value",
|
|
"stable"
|
|
],
|
|
"title": "State",
|
|
"type": "string"
|
|
},
|
|
"expected_text": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 4096,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"title": "Expected Text"
|
|
},
|
|
"expected_value": {
|
|
"anyOf": [
|
|
{
|
|
"type": "number"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"title": "Expected Value"
|
|
},
|
|
"tolerance": {
|
|
"default": 0,
|
|
"minimum": 0,
|
|
"title": "Tolerance",
|
|
"type": "number"
|
|
},
|
|
"stable_ms": {
|
|
"default": 300,
|
|
"maximum": 5000,
|
|
"minimum": 0,
|
|
"title": "Stable Ms",
|
|
"type": "integer"
|
|
},
|
|
"timeout_ms": {
|
|
"default": 1500,
|
|
"maximum": 3000,
|
|
"minimum": 1,
|
|
"title": "Timeout Ms",
|
|
"type": "integer"
|
|
},
|
|
"poll_interval_ms": {
|
|
"default": 200,
|
|
"maximum": 1000,
|
|
"minimum": 50,
|
|
"title": "Poll Interval Ms",
|
|
"type": "integer"
|
|
},
|
|
"action": {
|
|
"const": "wait_until",
|
|
"title": "Action",
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"selector",
|
|
"state",
|
|
"action"
|
|
],
|
|
"title": "WaitUntilStep",
|
|
"type": "object"
|
|
}
|
|
]
|
|
},
|
|
"maxItems": 10,
|
|
"minItems": 1,
|
|
"type": "array"
|
|
}
|
|
},
|
|
"required": [
|
|
"steps"
|
|
],
|
|
"type": "object"
|
|
}
|
|
}
|
|
},
|
|
"drag": {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "drag",
|
|
"description": "[ACTION] Precisely drag a local slider or timeline. Use explicit start/end or calibration from TWO observed numeric scale anchors. coordinate_space is normalized (0..1000) or pixel (the current screenshot's actual pixels); pixel avoids quantization on a zoomed scale. Default duration_ms=300. If a panned timeline coasts past the release position, try a slower pan (e.g. 1500ms), then wait for its ticks to stop before measuring the residual. A transient release-time bubble can disagree with the final fixed-pointer scale; do not treat it as final position proof. A permanently visible timeline can be panned independently of its zoom popup. Do not overshoot the endpoint automatically. For calibration, axis=x/y; mode=thumb moves the handle to target_value, mode=pan shifts the scale under indicator_position. Use one numeric unit, e.g. seconds since midnight, and two anchors from the CURRENT zoom/layout. Provide exactly one of end or calibration. Returned geometry is a plan/dispatch receipt, not proof of the selected value: compare the post-action scale/value and correct only the observed residual; zoom further if the pixel resolution is insufficient.",
|
|
"parameters": {
|
|
"properties": {
|
|
"start": {
|
|
"description": "Observed handle or pan starting point [x, y].",
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"type": "array"
|
|
},
|
|
"target_description": {
|
|
"description": "The slider, handle or timeline being dragged.",
|
|
"type": "string"
|
|
},
|
|
"end": {
|
|
"anyOf": [
|
|
{
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"type": "array"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"description": "Exact endpoint. Set null (or omit) when calibration is supplied; never compute both."
|
|
},
|
|
"coordinate_space": {
|
|
"default": "normalized",
|
|
"enum": [
|
|
"normalized",
|
|
"pixel"
|
|
],
|
|
"type": "string"
|
|
},
|
|
"duration_ms": {
|
|
"default": 300,
|
|
"type": "integer"
|
|
},
|
|
"calibration": {
|
|
"anyOf": [
|
|
{
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"axis": {
|
|
"enum": [
|
|
"x",
|
|
"y"
|
|
],
|
|
"title": "Axis",
|
|
"type": "string"
|
|
},
|
|
"mode": {
|
|
"description": "thumb moves a handle to a value; pan moves a scale under a fixed indicator.",
|
|
"enum": [
|
|
"thumb",
|
|
"pan"
|
|
],
|
|
"title": "Mode",
|
|
"type": "string"
|
|
},
|
|
"anchors": {
|
|
"items": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"position": {
|
|
"description": "Observed axis coordinate in the selected coordinate space.",
|
|
"title": "Position",
|
|
"type": "number"
|
|
},
|
|
"value": {
|
|
"description": "Observed scale value, in one consistent numeric unit.",
|
|
"title": "Value",
|
|
"type": "number"
|
|
}
|
|
},
|
|
"required": [
|
|
"position",
|
|
"value"
|
|
],
|
|
"title": "ScaleAnchor",
|
|
"type": "object"
|
|
},
|
|
"maxItems": 2,
|
|
"minItems": 2,
|
|
"title": "Anchors",
|
|
"type": "array"
|
|
},
|
|
"target_value": {
|
|
"title": "Target Value",
|
|
"type": "number"
|
|
},
|
|
"indicator_position": {
|
|
"anyOf": [
|
|
{
|
|
"type": "number"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"description": "Required for pan: the fixed indicator's observed axis position.",
|
|
"title": "Indicator Position"
|
|
}
|
|
},
|
|
"required": [
|
|
"axis",
|
|
"mode",
|
|
"anchors",
|
|
"target_value"
|
|
],
|
|
"title": "DragCalibration",
|
|
"type": "object"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"description": "Observed numeric anchors that compute the endpoint. When supplied, end MUST be null; otherwise set calibration null."
|
|
},
|
|
"require_visible": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"description": "Previously observed panel resource_id; refuse the drag if it is no longer visible."
|
|
}
|
|
},
|
|
"required": [
|
|
"start",
|
|
"target_description"
|
|
],
|
|
"type": "object"
|
|
}
|
|
}
|
|
},
|
|
"pinch": {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "pinch",
|
|
"description": "[ACTION] Synchronized two-finger pinch/rotation on an observed canvas. center and spans use coordinate_space: pixel is current screenshot pixels; normalized is 0..1000, with spans measured against the SHORT screen edge to preserve angles. angle_degrees starts from the horizontal axis, positive clockwise; rotation_degrees adds rotation. All paths are validated before input; off-screen paths fail without clamping. The span ratio is a gesture instruction, not proof of the app's resulting scale. Small spans may stay below the app's recognition threshold; choose a sufficiently large safe span inside the observed canvas and read the actual post-action ticks/value before recalibrating a precise drag.",
|
|
"parameters": {
|
|
"properties": {
|
|
"center": {
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"maxItems": 2,
|
|
"minItems": 2,
|
|
"type": "array"
|
|
},
|
|
"start_span": {
|
|
"exclusiveMinimum": 0,
|
|
"maximum": 20000,
|
|
"type": "integer"
|
|
},
|
|
"end_span": {
|
|
"exclusiveMinimum": 0,
|
|
"maximum": 20000,
|
|
"type": "integer"
|
|
},
|
|
"angle_degrees": {
|
|
"default": 0,
|
|
"maximum": 360,
|
|
"minimum": -360,
|
|
"type": "number"
|
|
},
|
|
"rotation_degrees": {
|
|
"default": 0,
|
|
"maximum": 360,
|
|
"minimum": -360,
|
|
"type": "number"
|
|
},
|
|
"duration_ms": {
|
|
"default": 800,
|
|
"maximum": 10000,
|
|
"minimum": 1,
|
|
"type": "integer"
|
|
},
|
|
"coordinate_space": {
|
|
"default": "normalized",
|
|
"enum": [
|
|
"normalized",
|
|
"pixel"
|
|
],
|
|
"type": "string"
|
|
},
|
|
"target_description": {
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
"require_visible": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
}
|
|
},
|
|
"required": [
|
|
"center",
|
|
"start_span",
|
|
"end_span",
|
|
"target_description"
|
|
],
|
|
"type": "object"
|
|
}
|
|
}
|
|
},
|
|
"touch_gesture": {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "touch_gesture",
|
|
"description": "[ACTION] Continuous synchronized native touch paths with 1..5 fingers. frames contain at_ms and points; times strictly increase from 0 to at most 10000ms, same finger count in every frame. Equal adjacent points hold contact; later points move WITHOUT lifting, enabling hold-then-drag and multi-segment gestures. Use 2..32 frames, observed geometry and explicit coordinate_space. All frames are validated before input. Completion reports Android gesture dispatch; verify that the app actually received the input and produced the intended effect.",
|
|
"parameters": {
|
|
"properties": {
|
|
"frames": {
|
|
"items": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"at_ms": {
|
|
"maximum": 10000,
|
|
"minimum": 0,
|
|
"type": "integer"
|
|
},
|
|
"points": {
|
|
"items": {
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"maxItems": 2,
|
|
"minItems": 2,
|
|
"type": "array"
|
|
},
|
|
"maxItems": 5,
|
|
"minItems": 1,
|
|
"type": "array"
|
|
}
|
|
},
|
|
"required": [
|
|
"at_ms",
|
|
"points"
|
|
],
|
|
"type": "object"
|
|
},
|
|
"maxItems": 32,
|
|
"minItems": 2,
|
|
"type": "array"
|
|
},
|
|
"coordinate_space": {
|
|
"default": "normalized",
|
|
"enum": [
|
|
"normalized",
|
|
"pixel"
|
|
],
|
|
"type": "string"
|
|
},
|
|
"target_description": {
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
"require_visible": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
}
|
|
},
|
|
"required": [
|
|
"frames",
|
|
"target_description"
|
|
],
|
|
"type": "object"
|
|
}
|
|
}
|
|
},
|
|
"read_control": {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "read_control",
|
|
"description": "[OBSERVE] Read ONE live native control by exact AND selector copied from the current UI. Use resource_id/text/content_desc/class_name and optional package_name/bounds to disambiguate. Returns current range/value, supported native actions, editable selection (UTF-16 offsets), identity token and visible subtree content_signature. Missing or ambiguous controls fail explicitly. Custom canvas controls may expose no range/actions; then use calibrated gestures and visual verification.",
|
|
"parameters": {
|
|
"properties": {
|
|
"selector": {
|
|
"additionalProperties": false,
|
|
"description": "Every supplied attribute is an exact AND match against a fresh native node.",
|
|
"properties": {
|
|
"resource_id": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
},
|
|
"text": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 4096,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
},
|
|
"content_desc": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
},
|
|
"class_name": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
},
|
|
"package_name": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
},
|
|
"bounds": {
|
|
"anyOf": [
|
|
{
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"maxItems": 4,
|
|
"minItems": 4,
|
|
"type": "array"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
}
|
|
},
|
|
"type": "object"
|
|
}
|
|
},
|
|
"required": [
|
|
"selector"
|
|
],
|
|
"type": "object"
|
|
}
|
|
}
|
|
},
|
|
"control_action": {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "control_action",
|
|
"description": "[ACTION] Perform ONE supported native action on a freshly resolved unique control. Read_control first; copy selector exactly and optionally expected_token to reject stale evidence. action=set_progress requires a finite in-range value; set_selection requires selection_start/end in UTF-16 code units (never split an emoji). set_text and replace_selection require text, including empty text; whitespace is preserved. Other actions take no text/value/selection. ime_enter invokes the advertised IME submit action. Unsupported actions fail without coordinate fallback or replay. Focus or selection changes may move the target; before subsequent edits, use wait_until state=stable when needed. STALE means no action started; observe again before rebuilding the request. Never resend an unknown write. An accepted action is not business verification; inspect the returned/current state.",
|
|
"parameters": {
|
|
"properties": {
|
|
"selector": {
|
|
"additionalProperties": false,
|
|
"description": "Every supplied attribute is an exact AND match against a fresh native node.",
|
|
"properties": {
|
|
"resource_id": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
},
|
|
"text": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 4096,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
},
|
|
"content_desc": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
},
|
|
"class_name": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
},
|
|
"package_name": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
},
|
|
"bounds": {
|
|
"anyOf": [
|
|
{
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"maxItems": 4,
|
|
"minItems": 4,
|
|
"type": "array"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
}
|
|
},
|
|
"type": "object"
|
|
},
|
|
"action": {
|
|
"enum": [
|
|
"click",
|
|
"long_click",
|
|
"focus",
|
|
"show_on_screen",
|
|
"set_progress",
|
|
"scroll_forward",
|
|
"scroll_backward",
|
|
"scroll_up",
|
|
"scroll_down",
|
|
"scroll_left",
|
|
"scroll_right",
|
|
"set_selection",
|
|
"replace_selection",
|
|
"set_text",
|
|
"ime_enter"
|
|
],
|
|
"type": "string"
|
|
},
|
|
"expected_token": {
|
|
"anyOf": [
|
|
{
|
|
"pattern": "^[a-f0-9]{64}$",
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
},
|
|
"value": {
|
|
"anyOf": [
|
|
{
|
|
"type": "number"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
},
|
|
"text": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 65536,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
},
|
|
"selection_start": {
|
|
"anyOf": [
|
|
{
|
|
"minimum": 0,
|
|
"type": "integer"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
},
|
|
"selection_end": {
|
|
"anyOf": [
|
|
{
|
|
"minimum": 0,
|
|
"type": "integer"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
}
|
|
},
|
|
"required": [
|
|
"selector",
|
|
"action"
|
|
],
|
|
"type": "object"
|
|
}
|
|
}
|
|
},
|
|
"wait_until": {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "wait_until",
|
|
"description": "[OBSERVE] Bounded polling of a unique live native control, replacing blind delays. state=text requires exact expected_text; value requires expected_value and optional tolerance. stable requires at least two unchanged native observations for stable_ms, not pixel stability. Other states inspect visible/enabled/checked/selected or absence. Ambiguity, unavailable page or transport failure never counts as absence. timeout_ms is at most 30000; no input is dispatched.",
|
|
"parameters": {
|
|
"properties": {
|
|
"selector": {
|
|
"additionalProperties": false,
|
|
"description": "Every supplied attribute is an exact AND match against a fresh native node.",
|
|
"properties": {
|
|
"resource_id": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
},
|
|
"text": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 4096,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
},
|
|
"content_desc": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
},
|
|
"class_name": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
},
|
|
"package_name": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
},
|
|
"bounds": {
|
|
"anyOf": [
|
|
{
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"maxItems": 4,
|
|
"minItems": 4,
|
|
"type": "array"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
}
|
|
},
|
|
"type": "object"
|
|
},
|
|
"state": {
|
|
"enum": [
|
|
"visible",
|
|
"absent",
|
|
"enabled",
|
|
"disabled",
|
|
"checked",
|
|
"unchecked",
|
|
"selected",
|
|
"unselected",
|
|
"text",
|
|
"value",
|
|
"stable"
|
|
],
|
|
"type": "string"
|
|
},
|
|
"expected_text": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 4096,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
},
|
|
"expected_value": {
|
|
"anyOf": [
|
|
{
|
|
"type": "number"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
},
|
|
"tolerance": {
|
|
"default": 0,
|
|
"minimum": 0,
|
|
"type": "number"
|
|
},
|
|
"stable_ms": {
|
|
"default": 300,
|
|
"maximum": 5000,
|
|
"minimum": 0,
|
|
"type": "integer"
|
|
},
|
|
"timeout_ms": {
|
|
"default": 5000,
|
|
"maximum": 30000,
|
|
"minimum": 1,
|
|
"type": "integer"
|
|
},
|
|
"poll_interval_ms": {
|
|
"default": 200,
|
|
"maximum": 1000,
|
|
"minimum": 50,
|
|
"type": "integer"
|
|
}
|
|
},
|
|
"required": [
|
|
"selector",
|
|
"state"
|
|
],
|
|
"type": "object"
|
|
}
|
|
}
|
|
},
|
|
"scroll_to": {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "scroll_to",
|
|
"description": "[ACTION] Bounded native scrolling INSIDE a uniquely observed container until its scoped target becomes visible. Copy exact container and target selectors; direction is forward/backward/up/down/left/right. After each scroll, container content and the scoped target's visibility/identity must remain unchanged for settle_ms before success or another scroll. Stops at max_scrolls, timeout, unsupported action or no content progress. Retains every scroll receipt and never repeats unacknowledged input. Finding a target does not click it or verify a business outcome.",
|
|
"parameters": {
|
|
"properties": {
|
|
"container": {
|
|
"additionalProperties": false,
|
|
"description": "Every supplied attribute is an exact AND match against a fresh native node.",
|
|
"properties": {
|
|
"resource_id": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
},
|
|
"text": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 4096,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
},
|
|
"content_desc": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
},
|
|
"class_name": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
},
|
|
"package_name": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
},
|
|
"bounds": {
|
|
"anyOf": [
|
|
{
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"maxItems": 4,
|
|
"minItems": 4,
|
|
"type": "array"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
}
|
|
},
|
|
"type": "object"
|
|
},
|
|
"target": {
|
|
"additionalProperties": false,
|
|
"description": "Every supplied attribute is an exact AND match against a fresh native node.",
|
|
"properties": {
|
|
"resource_id": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
},
|
|
"text": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 4096,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
},
|
|
"content_desc": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
},
|
|
"class_name": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
},
|
|
"package_name": {
|
|
"anyOf": [
|
|
{
|
|
"maxLength": 512,
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
},
|
|
"bounds": {
|
|
"anyOf": [
|
|
{
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"maxItems": 4,
|
|
"minItems": 4,
|
|
"type": "array"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null
|
|
}
|
|
},
|
|
"type": "object"
|
|
},
|
|
"direction": {
|
|
"default": "forward",
|
|
"enum": [
|
|
"forward",
|
|
"backward",
|
|
"up",
|
|
"down",
|
|
"left",
|
|
"right"
|
|
],
|
|
"type": "string"
|
|
},
|
|
"max_scrolls": {
|
|
"default": 10,
|
|
"maximum": 20,
|
|
"minimum": 1,
|
|
"type": "integer"
|
|
},
|
|
"timeout_ms": {
|
|
"default": 10000,
|
|
"maximum": 30000,
|
|
"minimum": 1,
|
|
"type": "integer"
|
|
},
|
|
"settle_ms": {
|
|
"default": 300,
|
|
"maximum": 3000,
|
|
"minimum": 50,
|
|
"type": "integer"
|
|
}
|
|
},
|
|
"required": [
|
|
"container",
|
|
"target"
|
|
],
|
|
"type": "object"
|
|
}
|
|
}
|
|
},
|
|
"input_text": {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "input_text",
|
|
"description": "[ACTION] Type text into the target input field (supports replacing whole text or appending to the end, and multi-line strings with '\\n').",
|
|
"parameters": {
|
|
"properties": {
|
|
"text": {
|
|
"description": "The text content to input. Supports multi-line content with '\\n'.",
|
|
"type": "string"
|
|
},
|
|
"target": {
|
|
"anyOf": [
|
|
{
|
|
"type": "integer"
|
|
},
|
|
{
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"type": "array"
|
|
}
|
|
],
|
|
"description": "Input target field. Can be an input box element index number (int, e.g. 3) OR normalized coordinates (list of 2 integers, e.g. [500, 600])."
|
|
},
|
|
"target_description": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index."
|
|
},
|
|
"clear_exist": {
|
|
"default": true,
|
|
"description": "Whether to clear existing text before typing. True (default): clear/replace entire text. False: append at the end of existing content.",
|
|
"type": "boolean"
|
|
}
|
|
},
|
|
"required": [
|
|
"text",
|
|
"target"
|
|
],
|
|
"type": "object"
|
|
}
|
|
}
|
|
},
|
|
"swipe": {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "swipe",
|
|
"description": "[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen.\n\n• Directional Scrolling ('direction'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n• Precise Coordinate Gestures ('start', 'end'): Use observed endpoints for local slider/timeline movement. Do not automatically overshoot: slow short gestures can trigger long-press recognition. Compare the actual post-action value/scale before adjusting again. When available, drag supports pixel coordinates and two-anchor calibration for fine positioning.\n\nArgs:\n direction: Smart directional scrolling ('up', 'down', 'left', 'right'). Automatically computes safe swipe vectors, retaining 40% visual overlap: 'up' (reveal content below), 'down' (reveal content above), 'left', 'right'.\n start: Start normalized coordinates [start_x, start_y] in 0-1000 scale.\n end: End normalized coordinates [end_x, end_y] in 0-1000 scale.\n target: Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within.\n gesture: Backward-compatible parameter: direction string OR custom coordinates list [start_x, start_y, end_x, end_y] in 0-1000 scale.\n target_description: What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling.\n duration: Optional gesture duration in milliseconds (default 800).",
|
|
"parameters": {
|
|
"properties": {
|
|
"direction": {
|
|
"anyOf": [
|
|
{
|
|
"enum": [
|
|
"up",
|
|
"down",
|
|
"left",
|
|
"right"
|
|
],
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"description": "Direction for scrolling and swiping: 'up' (drags bottom-to-top, scrolling down to reveal content below), 'down' (drags top-to-bottom, scrolling up to reveal content above), 'left' (drags right-to-left, scrolling right), 'right' (drags left-to-right, scrolling left)."
|
|
},
|
|
"start": {
|
|
"anyOf": [
|
|
{
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"type": "array"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"description": "Start normalized coordinates [start_x, start_y] in 0-1000 scale for precise, local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop)."
|
|
},
|
|
"end": {
|
|
"anyOf": [
|
|
{
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"type": "array"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"description": "End normalized coordinates [end_x, end_y] in 0-1000 scale for precise, local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop)."
|
|
},
|
|
"target": {
|
|
"anyOf": [
|
|
{
|
|
"type": "integer"
|
|
},
|
|
{
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"type": "array"
|
|
},
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"description": "Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within."
|
|
},
|
|
"gesture": {
|
|
"anyOf": [
|
|
{
|
|
"enum": [
|
|
"up",
|
|
"down",
|
|
"left",
|
|
"right"
|
|
],
|
|
"type": "string"
|
|
},
|
|
{
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"type": "array"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"description": "Backward-compatible swipe gesture: smart direction string ('up', 'down', 'left', 'right') OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale."
|
|
},
|
|
"target_description": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"description": "What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling."
|
|
},
|
|
"duration": {
|
|
"anyOf": [
|
|
{
|
|
"type": "integer"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"description": "Optional swipe/drag duration in milliseconds (default 800; computed automatically for directional swipes)."
|
|
}
|
|
},
|
|
"type": "object"
|
|
}
|
|
}
|
|
},
|
|
"press_key": {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "press_key",
|
|
"description": "[ACTION] Press a physical or virtual system button (e.g. ENTER, BACK, HOME, APP_SWITCH).",
|
|
"parameters": {
|
|
"properties": {
|
|
"key": {
|
|
"description": "Standard Android system button name (ENTER, BACK, HOME, APP_SWITCH).",
|
|
"enum": [
|
|
"ENTER",
|
|
"BACK",
|
|
"HOME",
|
|
"APP_SWITCH"
|
|
],
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"key"
|
|
],
|
|
"type": "object"
|
|
}
|
|
}
|
|
},
|
|
"manage_app": {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "manage_app",
|
|
"description": "[ACTION] Launch or force stop a specified application.",
|
|
"parameters": {
|
|
"properties": {
|
|
"action": {
|
|
"description": "The action type.",
|
|
"enum": [
|
|
"launch",
|
|
"stop"
|
|
],
|
|
"type": "string"
|
|
},
|
|
"app_name": {
|
|
"description": "Display name or package name of the application.",
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"action",
|
|
"app_name"
|
|
],
|
|
"type": "object"
|
|
}
|
|
}
|
|
},
|
|
"wait_for_delay": {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "wait_for_delay",
|
|
"description": "[ACTION] Pause execution and wait for a specified duration in milliseconds.\n\nUse this whenever you need time to elapse—whether for UI loading, animations,\nscreen transitions, or longer scheduled delays and intervals specified in the task.",
|
|
"parameters": {
|
|
"properties": {
|
|
"time_in_ms": {
|
|
"description": "The exact duration to wait in milliseconds. Accurately convert the required time duration into milliseconds based on your objective or plan (e.g., 2000 for 2s, 5000 for 5s, 60000 for 1 minute, 180000 for 3 minutes, 300000 for 5 minutes).",
|
|
"type": "integer"
|
|
}
|
|
},
|
|
"required": [
|
|
"time_in_ms"
|
|
],
|
|
"type": "object"
|
|
}
|
|
}
|
|
},
|
|
"long_press": {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "long_press",
|
|
"description": "[ACTION] Long press on the target location on the screen (supports element index or absolute normalized coordinates).",
|
|
"parameters": {
|
|
"properties": {
|
|
"target": {
|
|
"anyOf": [
|
|
{
|
|
"type": "integer"
|
|
},
|
|
{
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"type": "array"
|
|
}
|
|
],
|
|
"description": "Long press target. Can be an element index number (int, e.g. 3) OR normalized coordinates (list of 2 integers, e.g. [500, 600])."
|
|
},
|
|
"target_description": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
],
|
|
"default": null,
|
|
"description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index."
|
|
},
|
|
"duration": {
|
|
"default": 1000,
|
|
"description": "Long press duration in milliseconds (default 1000).",
|
|
"type": "integer"
|
|
}
|
|
},
|
|
"required": [
|
|
"target"
|
|
],
|
|
"type": "object"
|
|
}
|
|
}
|
|
}
|
|
}
|