{ "click": { "type": "function", "function": { "name": "click", "description": "[ACTION] Click on the target location on the screen (supports element index or absolute normalized coordinates). Indexed resource targets are checked against live bounds before input; missing/moved targets refuse the tap. For floating or auto-hiding panels, use a guarded action_sequence with opener and target in the same call; an open panel in a screenshot may already have expired.", "parameters": { "properties": { "target": { "anyOf": [ { "type": "integer" }, { "items": { "type": "integer" }, "type": "array" } ], "description": "Click target. Can be an element index number (int, e.g. 3) OR normalized coordinates (list of 2 integers, e.g. [500, 600])." }, "target_description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index." }, "times": { "default": 1, "description": "Number of consecutive clicks on this target. Use this for double-clicks or multi-clicks (e.g. 7 to enter developer mode). Default is 1.", "type": "integer" }, "delay_ms": { "default": 100, "description": "Delay in milliseconds between consecutive clicks. Default is 100.", "type": "integer" } }, "required": [ "target" ], "type": "object" } } }, "action_sequence": { "type": "function", "function": { "name": "action_sequence", "description": "[ACTION] Execute a short sequence of click, wait and drag steps in ONE call, in order, without model turns between members; explicit visibility guards may read the live UI locally. Use for temporary overlays: open the control, wait briefly, then operate it before it disappears. For clicks prefer target=the exact observed resource_id string: live bounds are resolved locally, refusing absent or ambiguous targets. Otherwise use observed [x,y] in the declared coordinate_space; do not confuse original screenshot pixels with 0..1000 normalized coordinates. Element indices are not accepted. Each click/drag needs target_description. All members are validated before any input. The configured fast-action limit applies. The first failure stops the remainder; receipts identify completed and unexecuted members. Never replay the whole sequence after a partial result. Every wait counts as a member. click.times (1..10) performs consecutive taps within ONE member, like the ordinary click tool; prefer this over expanding repeated +/- taps into many members/model turns. Each resource-targeted tap is revalidated locally; missing targets stop remaining taps and receipts preserve completed_taps. For maximum/minimum prefer observed native +/- controls, confirming that further same-direction adjustment stops changing handle/ticks. An estimated drag endpoint can be imprecise. For every auto-hiding adjustment, including the reverse direction, keep opener/wait/operation in one call. A prior screenshot does not prove the panel is still open. Use click.unless_visible and drag.require_visible with an observed panel resource_id to avoid toggling an open panel closed or dragging its underlying content. Never invent resource IDs. Verify actual ticks/handle endpoints before claiming maximum or minimum.", "parameters": { "properties": { "steps": { "description": "Ordered click, wait and drag members.", "items": { "anyOf": [ { "additionalProperties": false, "properties": { "action": { "const": "click", "title": "Action", "type": "string" }, "target": { "anyOf": [ { "items": { "type": "integer" }, "maxItems": 2, "minItems": 2, "type": "array" }, { "minLength": 1, "type": "string" } ], "description": "Prefer an exact observed resource_id: resolve its live bounds before tapping, refusing absent/ambiguous targets. Otherwise use [x,y] in coordinate_space; never mix screenshot pixels with normalized coordinates.", "title": "Target" }, "target_description": { "minLength": 1, "title": "Target Description", "type": "string" }, "coordinate_space": { "default": "normalized", "enum": [ "normalized", "pixel" ], "title": "Coordinate Space", "type": "string" }, "times": { "default": 1, "description": "Consecutive taps within this ONE member, e.g. 10 for a scale +/- button; no model turns between taps.", "maximum": 10, "minimum": 1, "title": "Times", "type": "integer" }, "delay_ms": { "default": 100, "maximum": 1000, "minimum": 0, "title": "Delay Ms", "type": "integer" }, "unless_visible": { "anyOf": [ { "minLength": 1, "type": "string" }, { "type": "null" } ], "default": null, "description": "Exact previously observed resource_id inside a toggled panel (not the opener). Skip this opener click when that control is already visible; otherwise click. Avoid closing a still-open panel.", "title": "Unless Visible" } }, "required": [ "action", "target", "target_description" ], "title": "ClickStep", "type": "object" }, { "additionalProperties": false, "properties": { "start": { "items": { "type": "integer" }, "maxItems": 2, "minItems": 2, "title": "Start", "type": "array" }, "target_description": { "minLength": 1, "title": "Target Description", "type": "string" }, "end": { "anyOf": [ { "items": { "type": "integer" }, "maxItems": 2, "minItems": 2, "type": "array" }, { "type": "null" } ], "default": null, "description": "Exact endpoint; MUST be null when calibration is supplied.", "title": "End" }, "coordinate_space": { "default": "normalized", "enum": [ "normalized", "pixel" ], "title": "Coordinate Space", "type": "string" }, "duration_ms": { "default": 300, "maximum": 10000, "minimum": 1, "title": "Duration Ms", "type": "integer" }, "calibration": { "anyOf": [ { "additionalProperties": false, "properties": { "axis": { "enum": [ "x", "y" ], "title": "Axis", "type": "string" }, "mode": { "description": "thumb moves a handle to a value; pan moves a scale under a fixed indicator.", "enum": [ "thumb", "pan" ], "title": "Mode", "type": "string" }, "anchors": { "items": { "additionalProperties": false, "properties": { "position": { "description": "Observed axis coordinate in the selected coordinate space.", "title": "Position", "type": "number" }, "value": { "description": "Observed scale value, in one consistent numeric unit.", "title": "Value", "type": "number" } }, "required": [ "position", "value" ], "title": "ScaleAnchor", "type": "object" }, "maxItems": 2, "minItems": 2, "title": "Anchors", "type": "array" }, "target_value": { "title": "Target Value", "type": "number" }, "indicator_position": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "default": null, "description": "Required for pan: the fixed indicator's observed axis position.", "title": "Indicator Position" } }, "required": [ "axis", "mode", "anchors", "target_value" ], "title": "DragCalibration", "type": "object" }, { "type": "null" } ], "default": null, "description": "Computes the endpoint from observed anchors; then set end=null." }, "require_visible": { "anyOf": [ { "minLength": 1, "type": "string" }, { "type": "null" } ], "default": null, "description": "Exact previously observed resource_id of a panel control. Check live visibility before dragging; refuse input if absent. Use for auto-hiding controls.", "title": "Require Visible" }, "action": { "const": "drag", "title": "Action", "type": "string" } }, "required": [ "start", "target_description", "action" ], "title": "DragStep", "type": "object" }, { "additionalProperties": false, "properties": { "action": { "const": "wait", "title": "Action", "type": "string" }, "time_in_ms": { "maximum": 10000, "minimum": 0, "title": "Time In Ms", "type": "integer" } }, "required": [ "action", "time_in_ms" ], "title": "WaitStep", "type": "object" } ] }, "maxItems": 10, "minItems": 1, "type": "array" } }, "required": [ "steps" ], "type": "object" } } }, "drag": { "type": "function", "function": { "name": "drag", "description": "[ACTION] Precisely drag a local slider or timeline. Use explicit start/end or calibration from TWO observed numeric scale anchors. coordinate_space is normalized (0..1000) or pixel (the current screenshot's actual pixels); pixel avoids quantization on a zoomed scale. Default duration_ms=300. If a panned timeline coasts past the release position, try a slower pan (e.g. 1500ms), then wait for its ticks to stop before measuring the residual. A transient release-time bubble can disagree with the final fixed-pointer scale; do not treat it as final position proof. A permanently visible timeline can be panned independently of its zoom popup. Do not overshoot the endpoint automatically. For calibration, axis=x/y; mode=thumb moves the handle to target_value, mode=pan shifts the scale under indicator_position. Use one numeric unit, e.g. seconds since midnight, and two anchors from the CURRENT zoom/layout. Provide exactly one of end or calibration. Returned geometry is a plan/dispatch receipt, not proof of the selected value: compare the post-action scale/value and correct only the observed residual; zoom further if the pixel resolution is insufficient.", "parameters": { "properties": { "start": { "description": "Observed handle or pan starting point [x, y].", "items": { "type": "integer" }, "type": "array" }, "target_description": { "description": "The slider, handle or timeline being dragged.", "type": "string" }, "end": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "Exact endpoint. Set null (or omit) when calibration is supplied; never compute both." }, "coordinate_space": { "default": "normalized", "enum": [ "normalized", "pixel" ], "type": "string" }, "duration_ms": { "default": 300, "type": "integer" }, "calibration": { "anyOf": [ { "additionalProperties": false, "properties": { "axis": { "enum": [ "x", "y" ], "title": "Axis", "type": "string" }, "mode": { "description": "thumb moves a handle to a value; pan moves a scale under a fixed indicator.", "enum": [ "thumb", "pan" ], "title": "Mode", "type": "string" }, "anchors": { "items": { "additionalProperties": false, "properties": { "position": { "description": "Observed axis coordinate in the selected coordinate space.", "title": "Position", "type": "number" }, "value": { "description": "Observed scale value, in one consistent numeric unit.", "title": "Value", "type": "number" } }, "required": [ "position", "value" ], "title": "ScaleAnchor", "type": "object" }, "maxItems": 2, "minItems": 2, "title": "Anchors", "type": "array" }, "target_value": { "title": "Target Value", "type": "number" }, "indicator_position": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "default": null, "description": "Required for pan: the fixed indicator's observed axis position.", "title": "Indicator Position" } }, "required": [ "axis", "mode", "anchors", "target_value" ], "title": "DragCalibration", "type": "object" }, { "type": "null" } ], "default": null, "description": "Observed numeric anchors that compute the endpoint. When supplied, end MUST be null; otherwise set calibration null." }, "require_visible": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "Previously observed panel resource_id; refuse the drag if it is no longer visible." } }, "required": [ "start", "target_description" ], "type": "object" } } }, "input_text": { "type": "function", "function": { "name": "input_text", "description": "[ACTION] Type text into the target input field (supports replacing whole text or appending to the end, and multi-line strings with '\\n').", "parameters": { "properties": { "text": { "description": "The text content to input. Supports multi-line content with '\\n'.", "type": "string" }, "target": { "anyOf": [ { "type": "integer" }, { "items": { "type": "integer" }, "type": "array" } ], "description": "Input target field. Can be an input box element index number (int, e.g. 3) OR normalized coordinates (list of 2 integers, e.g. [500, 600])." }, "target_description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index." }, "clear_exist": { "default": true, "description": "Whether to clear existing text before typing. True (default): clear/replace entire text. False: append at the end of existing content.", "type": "boolean" } }, "required": [ "text", "target" ], "type": "object" } } }, "swipe": { "type": "function", "function": { "name": "swipe", "description": "[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen.\n\n• Directional Scrolling ('direction'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n• Precise Coordinate Gestures ('start', 'end'): Use observed endpoints for local slider/timeline movement. Do not automatically overshoot: slow short gestures can trigger long-press recognition. Compare the actual post-action value/scale before adjusting again. When available, drag supports pixel coordinates and two-anchor calibration for fine positioning.\n\nArgs:\n direction: Smart directional scrolling ('up', 'down', 'left', 'right'). Automatically computes safe swipe vectors, retaining 40% visual overlap: 'up' (reveal content below), 'down' (reveal content above), 'left', 'right'.\n start: Start normalized coordinates [start_x, start_y] in 0-1000 scale.\n end: End normalized coordinates [end_x, end_y] in 0-1000 scale.\n target: Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within.\n gesture: Backward-compatible parameter: direction string OR custom coordinates list [start_x, start_y, end_x, end_y] in 0-1000 scale.\n target_description: What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling.\n duration: Optional gesture duration in milliseconds (default 800).", "parameters": { "properties": { "direction": { "anyOf": [ { "enum": [ "up", "down", "left", "right" ], "type": "string" }, { "type": "null" } ], "default": null, "description": "Direction for scrolling and swiping: 'up' (drags bottom-to-top, scrolling down to reveal content below), 'down' (drags top-to-bottom, scrolling up to reveal content above), 'left' (drags right-to-left, scrolling right), 'right' (drags left-to-right, scrolling left)." }, "start": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "Start normalized coordinates [start_x, start_y] in 0-1000 scale for precise, local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop)." }, "end": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "End normalized coordinates [end_x, end_y] in 0-1000 scale for precise, local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop)." }, "target": { "anyOf": [ { "type": "integer" }, { "items": { "type": "integer" }, "type": "array" }, { "type": "string" }, { "type": "null" } ], "default": null, "description": "Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within." }, "gesture": { "anyOf": [ { "enum": [ "up", "down", "left", "right" ], "type": "string" }, { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "Backward-compatible swipe gesture: smart direction string ('up', 'down', 'left', 'right') OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale." }, "target_description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling." }, "duration": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "Optional swipe/drag duration in milliseconds (default 800; computed automatically for directional swipes)." } }, "type": "object" } } }, "press_key": { "type": "function", "function": { "name": "press_key", "description": "[ACTION] Press a physical or virtual system button (e.g. ENTER, BACK, HOME, APP_SWITCH).", "parameters": { "properties": { "key": { "description": "Standard Android system button name (ENTER, BACK, HOME, APP_SWITCH).", "enum": [ "ENTER", "BACK", "HOME", "APP_SWITCH" ], "type": "string" } }, "required": [ "key" ], "type": "object" } } }, "manage_app": { "type": "function", "function": { "name": "manage_app", "description": "[ACTION] Launch or force stop a specified application.", "parameters": { "properties": { "action": { "description": "The action type.", "enum": [ "launch", "stop" ], "type": "string" }, "app_name": { "description": "Display name or package name of the application.", "type": "string" } }, "required": [ "action", "app_name" ], "type": "object" } } }, "wait_for_delay": { "type": "function", "function": { "name": "wait_for_delay", "description": "[ACTION] Pause execution and wait for a specified duration in milliseconds.\n\nUse this whenever you need time to elapse—whether for UI loading, animations,\nscreen transitions, or longer scheduled delays and intervals specified in the task.", "parameters": { "properties": { "time_in_ms": { "description": "The exact duration to wait in milliseconds. Accurately convert the required time duration into milliseconds based on your objective or plan (e.g., 2000 for 2s, 5000 for 5s, 60000 for 1 minute, 180000 for 3 minutes, 300000 for 5 minutes).", "type": "integer" } }, "required": [ "time_in_ms" ], "type": "object" } } }, "long_press": { "type": "function", "function": { "name": "long_press", "description": "[ACTION] Long press on the target location on the screen (supports element index or absolute normalized coordinates).", "parameters": { "properties": { "target": { "anyOf": [ { "type": "integer" }, { "items": { "type": "integer" }, "type": "array" } ], "description": "Long press target. Can be an element index number (int, e.g. 3) OR normalized coordinates (list of 2 integers, e.g. [500, 600])." }, "target_description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index." }, "duration": { "default": 1000, "description": "Long press duration in milliseconds (default 1000).", "type": "integer" } }, "required": [ "target" ], "type": "object" } } } }