feat(android): add state conditions and measured gesture feedback
Add optional ensure_state, gesture region and final value feedback shared by Flash and Pro. Preserve completed inputs when verification fails and stop repeated ineffective strategies. Regenerate pinned tool schemas for the new optional arguments.
This commit is contained in:
chenmingxuan committed
1 parent
6a69a595ef
commit
4ea7939d9c
20 files changed
+2779
-33
No files matched your search
+525
-1
@@ -103,6 +103,60 @@
|
||||
"title": "ControlActionStep",
|
||||
"type": "object"
|
||||
},
|
||||
"ControlCondition": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"state": {
|
||||
"enum": [
|
||||
"checked",
|
||||
"unchecked",
|
||||
"selected",
|
||||
"unselected",
|
||||
"text",
|
||||
"content_desc",
|
||||
"value"
|
||||
],
|
||||
"title": "State",
|
||||
"type": "string"
|
||||
},
|
||||
"expected_text": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 4096,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Expected Text"
|
||||
},
|
||||
"expected_value": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Expected Value"
|
||||
},
|
||||
"tolerance": {
|
||||
"default": 0,
|
||||
"minimum": 0,
|
||||
"title": "Tolerance",
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"state"
|
||||
],
|
||||
"title": "ControlCondition",
|
||||
"type": "object"
|
||||
},
|
||||
"ControlSelector": {
|
||||
"additionalProperties": false,
|
||||
"description": "Every supplied attribute is an exact AND match against a fresh native node.",
|
||||
@@ -242,6 +296,34 @@
|
||||
"default": null,
|
||||
"description": "Required for pan: the fixed indicator's observed axis position.",
|
||||
"title": "Indicator Position"
|
||||
},
|
||||
"tolerance": {
|
||||
"anyOf": [
|
||||
{
|
||||
"minimum": 0,
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Allowed value error. Reject a plan whose pixel rounding exceeds it; this is geometric error only, not proof of application precision.",
|
||||
"title": "Tolerance"
|
||||
},
|
||||
"min_movement_px": {
|
||||
"anyOf": [
|
||||
{
|
||||
"exclusiveMinimum": 0,
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Previously measured minimum effective drag distance in pixels. If this calibrated move is smaller, refuse input and zoom first. Never guess a device-wide threshold.",
|
||||
"title": "Min Movement Px"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -304,6 +386,24 @@
|
||||
"title": "Duration Ms",
|
||||
"type": "integer"
|
||||
},
|
||||
"region": {
|
||||
"anyOf": [
|
||||
{
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"maxItems": 4,
|
||||
"minItems": 4,
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Observed gesture surface rectangle in coordinate_space, excluding date/navigation rails and unrelated overlays.",
|
||||
"title": "Region"
|
||||
},
|
||||
"calibration": {
|
||||
"anyOf": [
|
||||
{
|
||||
@@ -464,6 +564,24 @@
|
||||
"default": null,
|
||||
"title": "Require Visible"
|
||||
},
|
||||
"region": {
|
||||
"anyOf": [
|
||||
{
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"maxItems": 4,
|
||||
"minItems": 4,
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Observed zoom surface bounds in coordinate_space; not the whole screen or date rail. Both fingers must remain inside.",
|
||||
"title": "Region"
|
||||
},
|
||||
"action": {
|
||||
"const": "pinch",
|
||||
"title": "Action",
|
||||
@@ -591,6 +709,25 @@
|
||||
],
|
||||
"default": null,
|
||||
"title": "Selection End"
|
||||
},
|
||||
"ensure_state": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/$defs/ControlCondition"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Desired observed state: skip an already satisfied action; otherwise write ONCE and verify by read-only polling. Never infer state from an icon or missing attributes."
|
||||
},
|
||||
"verify_timeout_ms": {
|
||||
"default": 1500,
|
||||
"maximum": 3000,
|
||||
"minimum": 1,
|
||||
"title": "Verify Timeout Ms",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -672,6 +809,24 @@
|
||||
"default": null,
|
||||
"title": "Require Visible"
|
||||
},
|
||||
"region": {
|
||||
"anyOf": [
|
||||
{
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"maxItems": 4,
|
||||
"minItems": 4,
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Observed gesture surface bounds in coordinate_space; every finger path must stay inside it.",
|
||||
"title": "Region"
|
||||
},
|
||||
"action": {
|
||||
"const": "touch_gesture",
|
||||
"title": "Action",
|
||||
@@ -686,6 +841,56 @@
|
||||
"title": "TouchGestureStep",
|
||||
"type": "object"
|
||||
},
|
||||
"ValueFeedback": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"selector": {
|
||||
"$ref": "#/$defs/ControlSelector",
|
||||
"description": "Unique, observed persistent value readout, not a transient drag tooltip. Copy its stable selector, excluding changing text."
|
||||
},
|
||||
"value_format": {
|
||||
"description": "range reads native range.current; number requires an entire numeric text; clock requires exactly one HH:MM:SS in readable text and returns seconds since midnight.",
|
||||
"enum": [
|
||||
"range",
|
||||
"number",
|
||||
"clock"
|
||||
],
|
||||
"title": "Value Format",
|
||||
"type": "string"
|
||||
},
|
||||
"target_value": {
|
||||
"title": "Target Value",
|
||||
"type": "number"
|
||||
},
|
||||
"tolerance": {
|
||||
"minimum": 0,
|
||||
"title": "Tolerance",
|
||||
"type": "number"
|
||||
},
|
||||
"stable_ms": {
|
||||
"default": 1000,
|
||||
"maximum": 5000,
|
||||
"minimum": 100,
|
||||
"title": "Stable Ms",
|
||||
"type": "integer"
|
||||
},
|
||||
"timeout_ms": {
|
||||
"default": 3000,
|
||||
"maximum": 10000,
|
||||
"minimum": 200,
|
||||
"title": "Timeout Ms",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"selector",
|
||||
"value_format",
|
||||
"target_value",
|
||||
"tolerance"
|
||||
],
|
||||
"title": "ValueFeedback",
|
||||
"type": "object"
|
||||
},
|
||||
"WaitStep": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
@@ -725,6 +930,7 @@
|
||||
"selected",
|
||||
"unselected",
|
||||
"text",
|
||||
"content_desc",
|
||||
"value",
|
||||
"stable"
|
||||
],
|
||||
@@ -830,6 +1036,17 @@
|
||||
},
|
||||
"title": "Steps",
|
||||
"type": "array"
|
||||
},
|
||||
"feedback": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/$defs/ValueFeedback"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -931,6 +1148,100 @@
|
||||
"description": "Drag with exact coordinates or an observed two-anchor calibration.",
|
||||
"inputSchema": {
|
||||
"$defs": {
|
||||
"ControlSelector": {
|
||||
"additionalProperties": false,
|
||||
"description": "Every supplied attribute is an exact AND match against a fresh native node.",
|
||||
"properties": {
|
||||
"resource_id": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 512,
|
||||
"minLength": 1,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Resource Id"
|
||||
},
|
||||
"text": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 4096,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Text"
|
||||
},
|
||||
"content_desc": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 512,
|
||||
"minLength": 1,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Content Desc"
|
||||
},
|
||||
"class_name": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 512,
|
||||
"minLength": 1,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Class Name"
|
||||
},
|
||||
"package_name": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 512,
|
||||
"minLength": 1,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Package Name"
|
||||
},
|
||||
"bounds": {
|
||||
"anyOf": [
|
||||
{
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"maxItems": 4,
|
||||
"minItems": 4,
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Bounds"
|
||||
}
|
||||
},
|
||||
"title": "ControlSelector",
|
||||
"type": "object"
|
||||
},
|
||||
"DragCalibration": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
@@ -976,6 +1287,34 @@
|
||||
"default": null,
|
||||
"description": "Required for pan: the fixed indicator's observed axis position.",
|
||||
"title": "Indicator Position"
|
||||
},
|
||||
"tolerance": {
|
||||
"anyOf": [
|
||||
{
|
||||
"minimum": 0,
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Allowed value error. Reject a plan whose pixel rounding exceeds it; this is geometric error only, not proof of application precision.",
|
||||
"title": "Tolerance"
|
||||
},
|
||||
"min_movement_px": {
|
||||
"anyOf": [
|
||||
{
|
||||
"exclusiveMinimum": 0,
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Previously measured minimum effective drag distance in pixels. If this calibrated move is smaller, refuse input and zoom first. Never guess a device-wide threshold.",
|
||||
"title": "Min Movement Px"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -1007,6 +1346,56 @@
|
||||
],
|
||||
"title": "ScaleAnchor",
|
||||
"type": "object"
|
||||
},
|
||||
"ValueFeedback": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"selector": {
|
||||
"$ref": "#/$defs/ControlSelector",
|
||||
"description": "Unique, observed persistent value readout, not a transient drag tooltip. Copy its stable selector, excluding changing text."
|
||||
},
|
||||
"value_format": {
|
||||
"description": "range reads native range.current; number requires an entire numeric text; clock requires exactly one HH:MM:SS in readable text and returns seconds since midnight.",
|
||||
"enum": [
|
||||
"range",
|
||||
"number",
|
||||
"clock"
|
||||
],
|
||||
"title": "Value Format",
|
||||
"type": "string"
|
||||
},
|
||||
"target_value": {
|
||||
"title": "Target Value",
|
||||
"type": "number"
|
||||
},
|
||||
"tolerance": {
|
||||
"minimum": 0,
|
||||
"title": "Tolerance",
|
||||
"type": "number"
|
||||
},
|
||||
"stable_ms": {
|
||||
"default": 1000,
|
||||
"maximum": 5000,
|
||||
"minimum": 100,
|
||||
"title": "Stable Ms",
|
||||
"type": "integer"
|
||||
},
|
||||
"timeout_ms": {
|
||||
"default": 3000,
|
||||
"maximum": 10000,
|
||||
"minimum": 200,
|
||||
"title": "Timeout Ms",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"selector",
|
||||
"value_format",
|
||||
"target_value",
|
||||
"tolerance"
|
||||
],
|
||||
"title": "ValueFeedback",
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"properties": {
|
||||
@@ -1050,6 +1439,34 @@
|
||||
"title": "Duration Ms",
|
||||
"type": "integer"
|
||||
},
|
||||
"region": {
|
||||
"anyOf": [
|
||||
{
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"maxItems": 4,
|
||||
"minItems": 4,
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Region"
|
||||
},
|
||||
"feedback": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/$defs/ValueFeedback"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null
|
||||
},
|
||||
"calibration": {
|
||||
"anyOf": [
|
||||
{
|
||||
@@ -2582,6 +2999,23 @@
|
||||
],
|
||||
"default": null,
|
||||
"title": "Require Visible"
|
||||
},
|
||||
"region": {
|
||||
"anyOf": [
|
||||
{
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"maxItems": 4,
|
||||
"minItems": 4,
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Region"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -2756,6 +3190,23 @@
|
||||
],
|
||||
"default": null,
|
||||
"title": "Require Visible"
|
||||
},
|
||||
"region": {
|
||||
"anyOf": [
|
||||
{
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"maxItems": 4,
|
||||
"minItems": 4,
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Region"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -3054,9 +3505,63 @@
|
||||
}
|
||||
},
|
||||
"control_action": {
|
||||
"description": "[ACTION] Perform ONE supported native action on a freshly resolved unique control. Read_control first; copy selector exactly and optionally expected_token to reject stale evidence. action=set_progress requires a finite in-range value; set_selection requires selection_start/end in UTF-16 code units (never split an emoji). set_text and replace_selection require text, including empty text; whitespace is preserved. Other actions take no text/value/selection. ime_enter invokes the advertised IME submit action. Unsupported actions fail without coordinate fallback or replay. Focus or selection changes may move the target; before subsequent edits, use wait_until state=stable when needed. STALE means no action started; observe again before rebuilding the request. Never resend an unknown write. An accepted action is not business verification; inspect the returned/current state.",
|
||||
"description": "[ACTION] Perform ONE supported native action on a freshly resolved unique control. Read_control first; copy selector exactly and optionally expected_token to reject stale evidence. For a desired state use ensure_state (checked/selected/text/content_desc/value), then verify_timeout_ms: skip an already satisfied state, otherwise write ONCE and poll read-only. Copy observed state semantics; do not assume selected means paused. Bind a stable selector excluding text/content_desc that the action changes. A failed postcondition does not undo the completed input and must not cause an automatic second toggle. action=set_progress requires a finite in-range value; set_selection requires selection_start/end in UTF-16 code units (never split an emoji). set_text and replace_selection require text, including empty text; whitespace is preserved. Other actions take no text/value/selection. ime_enter invokes the advertised IME submit action. Unsupported actions fail without coordinate fallback or replay. Focus or selection changes may move the target; before subsequent edits, use wait_until state=stable when needed. STALE means no action started; observe again before rebuilding the request. Never resend an unknown write. An accepted action is not business verification; inspect the returned/current state.",
|
||||
"inputSchema": {
|
||||
"$defs": {
|
||||
"ControlCondition": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"state": {
|
||||
"enum": [
|
||||
"checked",
|
||||
"unchecked",
|
||||
"selected",
|
||||
"unselected",
|
||||
"text",
|
||||
"content_desc",
|
||||
"value"
|
||||
],
|
||||
"title": "State",
|
||||
"type": "string"
|
||||
},
|
||||
"expected_text": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 4096,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Expected Text"
|
||||
},
|
||||
"expected_value": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Expected Value"
|
||||
},
|
||||
"tolerance": {
|
||||
"default": 0,
|
||||
"minimum": 0,
|
||||
"title": "Tolerance",
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"state"
|
||||
],
|
||||
"title": "ControlCondition",
|
||||
"type": "object"
|
||||
},
|
||||
"ControlSelector": {
|
||||
"additionalProperties": false,
|
||||
"description": "Every supplied attribute is an exact AND match against a fresh native node.",
|
||||
@@ -3240,6 +3745,24 @@
|
||||
],
|
||||
"default": null,
|
||||
"title": "Selection End"
|
||||
},
|
||||
"ensure_state": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/$defs/ControlCondition"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null
|
||||
},
|
||||
"verify_timeout_ms": {
|
||||
"default": 1500,
|
||||
"maximum": 3000,
|
||||
"minimum": 1,
|
||||
"title": "Verify Timeout Ms",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -3452,6 +3975,7 @@
|
||||
"selected",
|
||||
"unselected",
|
||||
"text",
|
||||
"content_desc",
|
||||
"value",
|
||||
"stable"
|
||||
],
|
||||
|
||||
+602
-2
@@ -54,7 +54,7 @@
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "action_sequence",
|
||||
"description": "[ACTION] General Android composer: execute ensure_visible, click, wait and drag steps in ONE call, in order, without model turns between members; explicit visibility guards may read the live UI locally. Use for temporary overlays: open the control, wait briefly, then operate it before it disappears. Prefer ensure_visible(target=observed panel control ID, opener=observed UI opener ID) followed by click/drag. This checks live UI, skips opening an already-visible panel, or taps its opener ONCE and waits up to timeout_ms (default 1500, max 3000) for the target. Each preparation counts as one member. Include it again in EVERY panel-operation call, even when the previous screenshot showed the panel open; do not split preparation and operation into different model turns. It never retries a business action. Copy all resource IDs VERBATIM, including ':id/' separators, never ':id:'. If preparation fails, re-observe/correct the ID; never guess fallback coordinates. Legacy click.unless_visible is ONLY for an opener, NEVER for an operation such as a +/- button: visible means SKIP, not require-visible. For clicks prefer target=the exact observed resource_id string: live bounds are resolved locally, refusing absent or ambiguous targets. Otherwise use observed [x,y] in the declared coordinate_space; do not confuse original screenshot pixels with 0..1000 normalized coordinates. Element indices are not accepted. Each click/drag needs target_description. All members are validated before any input. The configured fast-action limit applies. The first failure stops the remainder; receipts identify completed and unexecuted members. Never replay the whole sequence after a partial result. Every wait counts as a member. click.times (1..10) performs consecutive taps within ONE member, like the ordinary click tool; prefer this over expanding repeated +/- taps into many members/model turns. Each resource-targeted tap is revalidated locally; missing targets stop remaining taps and receipts preserve completed_taps. For maximum/minimum prefer observed native +/- controls, confirming that further same-direction adjustment stops changing handle/ticks. An estimated drag endpoint can be imprecise. For every auto-hiding adjustment, including the reverse direction, keep opener/wait/operation in one call. A prior screenshot does not prove the panel is still open. Use click.unless_visible and drag.require_visible with an observed panel resource_id to avoid toggling an open panel closed or dragging its underlying content. Never invent resource IDs. Verify actual ticks/handle endpoints before claiming maximum or minimum.",
|
||||
"description": "[ACTION] General Android composer: execute ensure_visible, click, wait and drag steps in ONE call, in order, without model turns between members; explicit visibility guards may read the live UI locally. Use for temporary overlays: open the control, wait briefly, then operate it before it disappears. Prefer ensure_visible(target=observed panel control ID, opener=observed UI opener ID) followed by click/drag. This checks live UI, skips opening an already-visible panel, or taps its opener ONCE and waits up to timeout_ms (default 1500, max 3000) for the target. Each preparation counts as one member. Include it again in EVERY panel-operation call, even when the previous screenshot showed the panel open; do not split preparation and operation into different model turns. It never retries a business action. Copy all resource IDs VERBATIM, including ':id/' separators, never ':id:'. If preparation fails, re-observe/correct the ID; never guess fallback coordinates. Legacy click.unless_visible is ONLY for an opener, NEVER for an operation such as a +/- button: visible means SKIP, not require-visible. For clicks prefer target=the exact observed resource_id string: live bounds are resolved locally, refusing absent or ambiguous targets. Otherwise use observed [x,y] in the declared coordinate_space; do not confuse original screenshot pixels with 0..1000 normalized coordinates. Element indices are not accepted. Each click/drag needs target_description. All members are validated before any input. The configured fast-action limit applies. The first failure stops the remainder; receipts identify completed and unexecuted members. Never replay the whole sequence after a partial result. Every wait counts as a member. click.times (1..10) performs consecutive taps within ONE member, like the ordinary click tool; prefer this over expanding repeated +/- taps into many members/model turns. Each resource-targeted tap is revalidated locally; missing targets stop remaining taps and receipts preserve completed_taps. For maximum/minimum prefer observed native +/- controls, confirming that further same-direction adjustment stops changing handle/ticks. An estimated drag endpoint can be imprecise. For every auto-hiding adjustment, including the reverse direction, keep opener/wait/operation in one call. A prior screenshot does not prove the panel is still open. Use click.unless_visible and drag.require_visible with an observed panel resource_id to avoid toggling an open panel closed or dragging its underlying content. Never invent resource IDs. Verify actual ticks/handle endpoints before claiming maximum or minimum. For a measurable target, bind feedback to its persistent native readout: final feedback is read AFTER ALL members, including any pause, finish. Never use a release-time tooltip. Use control_action.request.ensure_state for observed checked/selected/text/value state, so an already satisfied state is skipped rather than toggled. A playback icon alone does not prove a native boolean state: inspect it first. Feedback blocks the third consecutive ineffective strategy; zoom or increase the meaningful movement, then re-observe.",
|
||||
"parameters": {
|
||||
"properties": {
|
||||
"steps": {
|
||||
@@ -193,6 +193,24 @@
|
||||
"title": "Duration Ms",
|
||||
"type": "integer"
|
||||
},
|
||||
"region": {
|
||||
"anyOf": [
|
||||
{
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"maxItems": 4,
|
||||
"minItems": 4,
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Observed gesture surface rectangle in coordinate_space, excluding date/navigation rails and unrelated overlays.",
|
||||
"title": "Region"
|
||||
},
|
||||
"calibration": {
|
||||
"anyOf": [
|
||||
{
|
||||
@@ -258,6 +276,34 @@
|
||||
"default": null,
|
||||
"description": "Required for pan: the fixed indicator's observed axis position.",
|
||||
"title": "Indicator Position"
|
||||
},
|
||||
"tolerance": {
|
||||
"anyOf": [
|
||||
{
|
||||
"minimum": 0,
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Allowed value error. Reject a plan whose pixel rounding exceeds it; this is geometric error only, not proof of application precision.",
|
||||
"title": "Tolerance"
|
||||
},
|
||||
"min_movement_px": {
|
||||
"anyOf": [
|
||||
{
|
||||
"exclusiveMinimum": 0,
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Previously measured minimum effective drag distance in pixels. If this calibrated move is smaller, refuse input and zoom first. Never guess a device-wide threshold.",
|
||||
"title": "Min Movement Px"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -446,6 +492,24 @@
|
||||
"default": null,
|
||||
"title": "Require Visible"
|
||||
},
|
||||
"region": {
|
||||
"anyOf": [
|
||||
{
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"maxItems": 4,
|
||||
"minItems": 4,
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Observed zoom surface bounds in coordinate_space; not the whole screen or date rail. Both fingers must remain inside.",
|
||||
"title": "Region"
|
||||
},
|
||||
"action": {
|
||||
"const": "pinch",
|
||||
"title": "Action",
|
||||
@@ -531,6 +595,24 @@
|
||||
"default": null,
|
||||
"title": "Require Visible"
|
||||
},
|
||||
"region": {
|
||||
"anyOf": [
|
||||
{
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"maxItems": 4,
|
||||
"minItems": 4,
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Observed gesture surface bounds in coordinate_space; every finger path must stay inside it.",
|
||||
"title": "Region"
|
||||
},
|
||||
"action": {
|
||||
"const": "touch_gesture",
|
||||
"title": "Action",
|
||||
@@ -734,6 +816,76 @@
|
||||
],
|
||||
"default": null,
|
||||
"title": "Selection End"
|
||||
},
|
||||
"ensure_state": {
|
||||
"anyOf": [
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"state": {
|
||||
"enum": [
|
||||
"checked",
|
||||
"unchecked",
|
||||
"selected",
|
||||
"unselected",
|
||||
"text",
|
||||
"content_desc",
|
||||
"value"
|
||||
],
|
||||
"title": "State",
|
||||
"type": "string"
|
||||
},
|
||||
"expected_text": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 4096,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Expected Text"
|
||||
},
|
||||
"expected_value": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Expected Value"
|
||||
},
|
||||
"tolerance": {
|
||||
"default": 0,
|
||||
"minimum": 0,
|
||||
"title": "Tolerance",
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"state"
|
||||
],
|
||||
"title": "ControlCondition",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Desired observed state: skip an already satisfied action; otherwise write ONCE and verify by read-only polling. Never infer state from an icon or missing attributes."
|
||||
},
|
||||
"verify_timeout_ms": {
|
||||
"default": 1500,
|
||||
"maximum": 3000,
|
||||
"minimum": 1,
|
||||
"title": "Verify Timeout Ms",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -859,6 +1011,7 @@
|
||||
"selected",
|
||||
"unselected",
|
||||
"text",
|
||||
"content_desc",
|
||||
"value",
|
||||
"stable"
|
||||
],
|
||||
@@ -936,6 +1089,155 @@
|
||||
"maxItems": 10,
|
||||
"minItems": 1,
|
||||
"type": "array"
|
||||
},
|
||||
"feedback": {
|
||||
"anyOf": [
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"selector": {
|
||||
"additionalProperties": false,
|
||||
"description": "Unique, observed persistent value readout, not a transient drag tooltip. Copy its stable selector, excluding changing text.",
|
||||
"properties": {
|
||||
"resource_id": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 512,
|
||||
"minLength": 1,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Resource Id"
|
||||
},
|
||||
"text": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 4096,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Text"
|
||||
},
|
||||
"content_desc": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 512,
|
||||
"minLength": 1,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Content Desc"
|
||||
},
|
||||
"class_name": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 512,
|
||||
"minLength": 1,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Class Name"
|
||||
},
|
||||
"package_name": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 512,
|
||||
"minLength": 1,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Package Name"
|
||||
},
|
||||
"bounds": {
|
||||
"anyOf": [
|
||||
{
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"maxItems": 4,
|
||||
"minItems": 4,
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Bounds"
|
||||
}
|
||||
},
|
||||
"title": "ControlSelector",
|
||||
"type": "object"
|
||||
},
|
||||
"value_format": {
|
||||
"description": "range reads native range.current; number requires an entire numeric text; clock requires exactly one HH:MM:SS in readable text and returns seconds since midnight.",
|
||||
"enum": [
|
||||
"range",
|
||||
"number",
|
||||
"clock"
|
||||
],
|
||||
"title": "Value Format",
|
||||
"type": "string"
|
||||
},
|
||||
"target_value": {
|
||||
"title": "Target Value",
|
||||
"type": "number"
|
||||
},
|
||||
"tolerance": {
|
||||
"minimum": 0,
|
||||
"title": "Tolerance",
|
||||
"type": "number"
|
||||
},
|
||||
"stable_ms": {
|
||||
"default": 1000,
|
||||
"maximum": 5000,
|
||||
"minimum": 100,
|
||||
"title": "Stable Ms",
|
||||
"type": "integer"
|
||||
},
|
||||
"timeout_ms": {
|
||||
"default": 3000,
|
||||
"maximum": 10000,
|
||||
"minimum": 200,
|
||||
"title": "Timeout Ms",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"selector",
|
||||
"value_format",
|
||||
"target_value",
|
||||
"tolerance"
|
||||
],
|
||||
"title": "ValueFeedback",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Optional actual value/target comparison after the WHOLE sequence. clock uses seconds since midnight; exact bound and stable observation are checked locally, not inferred from dispatch."
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -990,6 +1292,172 @@
|
||||
"default": 300,
|
||||
"type": "integer"
|
||||
},
|
||||
"region": {
|
||||
"anyOf": [
|
||||
{
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"maxItems": 4,
|
||||
"minItems": 4,
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Observed gesture surface rectangle, in coordinate_space. Keep both endpoints inside."
|
||||
},
|
||||
"feedback": {
|
||||
"anyOf": [
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"selector": {
|
||||
"additionalProperties": false,
|
||||
"description": "Unique, observed persistent value readout, not a transient drag tooltip. Copy its stable selector, excluding changing text.",
|
||||
"properties": {
|
||||
"resource_id": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 512,
|
||||
"minLength": 1,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Resource Id"
|
||||
},
|
||||
"text": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 4096,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Text"
|
||||
},
|
||||
"content_desc": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 512,
|
||||
"minLength": 1,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Content Desc"
|
||||
},
|
||||
"class_name": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 512,
|
||||
"minLength": 1,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Class Name"
|
||||
},
|
||||
"package_name": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 512,
|
||||
"minLength": 1,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Package Name"
|
||||
},
|
||||
"bounds": {
|
||||
"anyOf": [
|
||||
{
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"maxItems": 4,
|
||||
"minItems": 4,
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Bounds"
|
||||
}
|
||||
},
|
||||
"title": "ControlSelector",
|
||||
"type": "object"
|
||||
},
|
||||
"value_format": {
|
||||
"description": "range reads native range.current; number requires an entire numeric text; clock requires exactly one HH:MM:SS in readable text and returns seconds since midnight.",
|
||||
"enum": [
|
||||
"range",
|
||||
"number",
|
||||
"clock"
|
||||
],
|
||||
"title": "Value Format",
|
||||
"type": "string"
|
||||
},
|
||||
"target_value": {
|
||||
"title": "Target Value",
|
||||
"type": "number"
|
||||
},
|
||||
"tolerance": {
|
||||
"minimum": 0,
|
||||
"title": "Tolerance",
|
||||
"type": "number"
|
||||
},
|
||||
"stable_ms": {
|
||||
"default": 1000,
|
||||
"maximum": 5000,
|
||||
"minimum": 100,
|
||||
"title": "Stable Ms",
|
||||
"type": "integer"
|
||||
},
|
||||
"timeout_ms": {
|
||||
"default": 3000,
|
||||
"maximum": 10000,
|
||||
"minimum": 200,
|
||||
"title": "Timeout Ms",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"selector",
|
||||
"value_format",
|
||||
"target_value",
|
||||
"tolerance"
|
||||
],
|
||||
"title": "ValueFeedback",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Read the persistent actual value after the drag. For drag then pause, put feedback on action_sequence instead."
|
||||
},
|
||||
"calibration": {
|
||||
"anyOf": [
|
||||
{
|
||||
@@ -1055,6 +1523,34 @@
|
||||
"default": null,
|
||||
"description": "Required for pan: the fixed indicator's observed axis position.",
|
||||
"title": "Indicator Position"
|
||||
},
|
||||
"tolerance": {
|
||||
"anyOf": [
|
||||
{
|
||||
"minimum": 0,
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Allowed value error. Reject a plan whose pixel rounding exceeds it; this is geometric error only, not proof of application precision.",
|
||||
"title": "Tolerance"
|
||||
},
|
||||
"min_movement_px": {
|
||||
"anyOf": [
|
||||
{
|
||||
"exclusiveMinimum": 0,
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Previously measured minimum effective drag distance in pixels. If this calibrated move is smaller, refuse input and zoom first. Never guess a device-wide threshold.",
|
||||
"title": "Min Movement Px"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -1162,6 +1658,23 @@
|
||||
}
|
||||
],
|
||||
"default": null
|
||||
},
|
||||
"region": {
|
||||
"anyOf": [
|
||||
{
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"maxItems": 4,
|
||||
"minItems": 4,
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Observed zoom surface bounds in coordinate_space; not the whole screen or date rail. Both fingers must remain inside."
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -1239,6 +1752,23 @@
|
||||
}
|
||||
],
|
||||
"default": null
|
||||
},
|
||||
"region": {
|
||||
"anyOf": [
|
||||
{
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"maxItems": 4,
|
||||
"minItems": 4,
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Observed gesture surface bounds in coordinate_space; every finger path must stay inside it."
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -1355,7 +1885,7 @@
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "control_action",
|
||||
"description": "[ACTION] Perform ONE supported native action on a freshly resolved unique control. Read_control first; copy selector exactly and optionally expected_token to reject stale evidence. action=set_progress requires a finite in-range value; set_selection requires selection_start/end in UTF-16 code units (never split an emoji). set_text and replace_selection require text, including empty text; whitespace is preserved. Other actions take no text/value/selection. ime_enter invokes the advertised IME submit action. Unsupported actions fail without coordinate fallback or replay. Focus or selection changes may move the target; before subsequent edits, use wait_until state=stable when needed. STALE means no action started; observe again before rebuilding the request. Never resend an unknown write. An accepted action is not business verification; inspect the returned/current state.",
|
||||
"description": "[ACTION] Perform ONE supported native action on a freshly resolved unique control. Read_control first; copy selector exactly and optionally expected_token to reject stale evidence. For a desired state use ensure_state (checked/selected/text/content_desc/value), then verify_timeout_ms: skip an already satisfied state, otherwise write ONCE and poll read-only. Copy observed state semantics; do not assume selected means paused. Bind a stable selector excluding text/content_desc that the action changes. A failed postcondition does not undo the completed input and must not cause an automatic second toggle. action=set_progress requires a finite in-range value; set_selection requires selection_start/end in UTF-16 code units (never split an emoji). set_text and replace_selection require text, including empty text; whitespace is preserved. Other actions take no text/value/selection. ime_enter invokes the advertised IME submit action. Unsupported actions fail without coordinate fallback or replay. Focus or selection changes may move the target; before subsequent edits, use wait_until state=stable when needed. STALE means no action started; observe again before rebuilding the request. Never resend an unknown write. An accepted action is not business verification; inspect the returned/current state.",
|
||||
"parameters": {
|
||||
"properties": {
|
||||
"selector": {
|
||||
@@ -1523,6 +2053,75 @@
|
||||
}
|
||||
],
|
||||
"default": null
|
||||
},
|
||||
"ensure_state": {
|
||||
"anyOf": [
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"state": {
|
||||
"enum": [
|
||||
"checked",
|
||||
"unchecked",
|
||||
"selected",
|
||||
"unselected",
|
||||
"text",
|
||||
"content_desc",
|
||||
"value"
|
||||
],
|
||||
"title": "State",
|
||||
"type": "string"
|
||||
},
|
||||
"expected_text": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 4096,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Expected Text"
|
||||
},
|
||||
"expected_value": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Expected Value"
|
||||
},
|
||||
"tolerance": {
|
||||
"default": 0,
|
||||
"minimum": 0,
|
||||
"title": "Tolerance",
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"state"
|
||||
],
|
||||
"title": "ControlCondition",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Desired observed state: skip an already satisfied action; otherwise write ONCE and verify by read-only polling. Never infer state from an icon or missing attributes."
|
||||
},
|
||||
"verify_timeout_ms": {
|
||||
"default": 1500,
|
||||
"maximum": 3000,
|
||||
"minimum": 1,
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -1638,6 +2237,7 @@
|
||||
"selected",
|
||||
"unselected",
|
||||
"text",
|
||||
"content_desc",
|
||||
"value",
|
||||
"stable"
|
||||
],
|
||||
|
||||
+602
-2
@@ -284,6 +284,172 @@
|
||||
"default": 300,
|
||||
"type": "integer"
|
||||
},
|
||||
"region": {
|
||||
"anyOf": [
|
||||
{
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"maxItems": 4,
|
||||
"minItems": 4,
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Observed gesture surface rectangle, in coordinate_space. Keep both endpoints inside."
|
||||
},
|
||||
"feedback": {
|
||||
"anyOf": [
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"selector": {
|
||||
"additionalProperties": false,
|
||||
"description": "Unique, observed persistent value readout, not a transient drag tooltip. Copy its stable selector, excluding changing text.",
|
||||
"properties": {
|
||||
"resource_id": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 512,
|
||||
"minLength": 1,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Resource Id"
|
||||
},
|
||||
"text": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 4096,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Text"
|
||||
},
|
||||
"content_desc": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 512,
|
||||
"minLength": 1,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Content Desc"
|
||||
},
|
||||
"class_name": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 512,
|
||||
"minLength": 1,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Class Name"
|
||||
},
|
||||
"package_name": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 512,
|
||||
"minLength": 1,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Package Name"
|
||||
},
|
||||
"bounds": {
|
||||
"anyOf": [
|
||||
{
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"maxItems": 4,
|
||||
"minItems": 4,
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Bounds"
|
||||
}
|
||||
},
|
||||
"title": "ControlSelector",
|
||||
"type": "object"
|
||||
},
|
||||
"value_format": {
|
||||
"description": "range reads native range.current; number requires an entire numeric text; clock requires exactly one HH:MM:SS in readable text and returns seconds since midnight.",
|
||||
"enum": [
|
||||
"range",
|
||||
"number",
|
||||
"clock"
|
||||
],
|
||||
"title": "Value Format",
|
||||
"type": "string"
|
||||
},
|
||||
"target_value": {
|
||||
"title": "Target Value",
|
||||
"type": "number"
|
||||
},
|
||||
"tolerance": {
|
||||
"minimum": 0,
|
||||
"title": "Tolerance",
|
||||
"type": "number"
|
||||
},
|
||||
"stable_ms": {
|
||||
"default": 1000,
|
||||
"maximum": 5000,
|
||||
"minimum": 100,
|
||||
"title": "Stable Ms",
|
||||
"type": "integer"
|
||||
},
|
||||
"timeout_ms": {
|
||||
"default": 3000,
|
||||
"maximum": 10000,
|
||||
"minimum": 200,
|
||||
"title": "Timeout Ms",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"selector",
|
||||
"value_format",
|
||||
"target_value",
|
||||
"tolerance"
|
||||
],
|
||||
"title": "ValueFeedback",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Read the persistent actual value after the drag. For drag then pause, put feedback on action_sequence instead."
|
||||
},
|
||||
"calibration": {
|
||||
"anyOf": [
|
||||
{
|
||||
@@ -349,6 +515,34 @@
|
||||
"default": null,
|
||||
"description": "Required for pan: the fixed indicator's observed axis position.",
|
||||
"title": "Indicator Position"
|
||||
},
|
||||
"tolerance": {
|
||||
"anyOf": [
|
||||
{
|
||||
"minimum": 0,
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Allowed value error. Reject a plan whose pixel rounding exceeds it; this is geometric error only, not proof of application precision.",
|
||||
"title": "Tolerance"
|
||||
},
|
||||
"min_movement_px": {
|
||||
"anyOf": [
|
||||
{
|
||||
"exclusiveMinimum": 0,
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Previously measured minimum effective drag distance in pixels. If this calibrated move is smaller, refuse input and zoom first. Never guess a device-wide threshold.",
|
||||
"title": "Min Movement Px"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -392,7 +586,7 @@
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "action_sequence",
|
||||
"description": "[ACTION] General Android composer: execute ensure_visible, click, wait and drag steps in ONE call, in order, without model turns between members; explicit visibility guards may read the live UI locally. Use for temporary overlays: open the control, wait briefly, then operate it before it disappears. Prefer ensure_visible(target=observed panel control ID, opener=observed UI opener ID) followed by click/drag. This checks live UI, skips opening an already-visible panel, or taps its opener ONCE and waits up to timeout_ms (default 1500, max 3000) for the target. Each preparation counts as one member. Include it again in EVERY panel-operation call, even when the previous screenshot showed the panel open; do not split preparation and operation into different model turns. It never retries a business action. Copy all resource IDs VERBATIM, including ':id/' separators, never ':id:'. If preparation fails, re-observe/correct the ID; never guess fallback coordinates. Legacy click.unless_visible is ONLY for an opener, NEVER for an operation such as a +/- button: visible means SKIP, not require-visible. For clicks prefer target=the exact observed resource_id string: live bounds are resolved locally, refusing absent or ambiguous targets. Otherwise use observed [x,y] in the declared coordinate_space; do not confuse original screenshot pixels with 0..1000 normalized coordinates. Element indices are not accepted. Each click/drag needs target_description. All members are validated before any input. The configured fast-action limit applies. The first failure stops the remainder; receipts identify completed and unexecuted members. Never replay the whole sequence after a partial result. Every wait counts as a member. click.times (1..10) performs consecutive taps within ONE member, like the ordinary click tool; prefer this over expanding repeated +/- taps into many members/model turns. Each resource-targeted tap is revalidated locally; missing targets stop remaining taps and receipts preserve completed_taps. For maximum/minimum prefer observed native +/- controls, confirming that further same-direction adjustment stops changing handle/ticks. An estimated drag endpoint can be imprecise. For every auto-hiding adjustment, including the reverse direction, keep opener/wait/operation in one call. A prior screenshot does not prove the panel is still open. Use click.unless_visible and drag.require_visible with an observed panel resource_id to avoid toggling an open panel closed or dragging its underlying content. Never invent resource IDs. Verify actual ticks/handle endpoints before claiming maximum or minimum.",
|
||||
"description": "[ACTION] General Android composer: execute ensure_visible, click, wait and drag steps in ONE call, in order, without model turns between members; explicit visibility guards may read the live UI locally. Use for temporary overlays: open the control, wait briefly, then operate it before it disappears. Prefer ensure_visible(target=observed panel control ID, opener=observed UI opener ID) followed by click/drag. This checks live UI, skips opening an already-visible panel, or taps its opener ONCE and waits up to timeout_ms (default 1500, max 3000) for the target. Each preparation counts as one member. Include it again in EVERY panel-operation call, even when the previous screenshot showed the panel open; do not split preparation and operation into different model turns. It never retries a business action. Copy all resource IDs VERBATIM, including ':id/' separators, never ':id:'. If preparation fails, re-observe/correct the ID; never guess fallback coordinates. Legacy click.unless_visible is ONLY for an opener, NEVER for an operation such as a +/- button: visible means SKIP, not require-visible. For clicks prefer target=the exact observed resource_id string: live bounds are resolved locally, refusing absent or ambiguous targets. Otherwise use observed [x,y] in the declared coordinate_space; do not confuse original screenshot pixels with 0..1000 normalized coordinates. Element indices are not accepted. Each click/drag needs target_description. All members are validated before any input. The configured fast-action limit applies. The first failure stops the remainder; receipts identify completed and unexecuted members. Never replay the whole sequence after a partial result. Every wait counts as a member. click.times (1..10) performs consecutive taps within ONE member, like the ordinary click tool; prefer this over expanding repeated +/- taps into many members/model turns. Each resource-targeted tap is revalidated locally; missing targets stop remaining taps and receipts preserve completed_taps. For maximum/minimum prefer observed native +/- controls, confirming that further same-direction adjustment stops changing handle/ticks. An estimated drag endpoint can be imprecise. For every auto-hiding adjustment, including the reverse direction, keep opener/wait/operation in one call. A prior screenshot does not prove the panel is still open. Use click.unless_visible and drag.require_visible with an observed panel resource_id to avoid toggling an open panel closed or dragging its underlying content. Never invent resource IDs. Verify actual ticks/handle endpoints before claiming maximum or minimum. For a measurable target, bind feedback to its persistent native readout: final feedback is read AFTER ALL members, including any pause, finish. Never use a release-time tooltip. Use control_action.request.ensure_state for observed checked/selected/text/value state, so an already satisfied state is skipped rather than toggled. A playback icon alone does not prove a native boolean state: inspect it first. Feedback blocks the third consecutive ineffective strategy; zoom or increase the meaningful movement, then re-observe.",
|
||||
"parameters": {
|
||||
"properties": {
|
||||
"steps": {
|
||||
@@ -531,6 +725,24 @@
|
||||
"title": "Duration Ms",
|
||||
"type": "integer"
|
||||
},
|
||||
"region": {
|
||||
"anyOf": [
|
||||
{
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"maxItems": 4,
|
||||
"minItems": 4,
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Observed gesture surface rectangle in coordinate_space, excluding date/navigation rails and unrelated overlays.",
|
||||
"title": "Region"
|
||||
},
|
||||
"calibration": {
|
||||
"anyOf": [
|
||||
{
|
||||
@@ -596,6 +808,34 @@
|
||||
"default": null,
|
||||
"description": "Required for pan: the fixed indicator's observed axis position.",
|
||||
"title": "Indicator Position"
|
||||
},
|
||||
"tolerance": {
|
||||
"anyOf": [
|
||||
{
|
||||
"minimum": 0,
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Allowed value error. Reject a plan whose pixel rounding exceeds it; this is geometric error only, not proof of application precision.",
|
||||
"title": "Tolerance"
|
||||
},
|
||||
"min_movement_px": {
|
||||
"anyOf": [
|
||||
{
|
||||
"exclusiveMinimum": 0,
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Previously measured minimum effective drag distance in pixels. If this calibrated move is smaller, refuse input and zoom first. Never guess a device-wide threshold.",
|
||||
"title": "Min Movement Px"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -784,6 +1024,24 @@
|
||||
"default": null,
|
||||
"title": "Require Visible"
|
||||
},
|
||||
"region": {
|
||||
"anyOf": [
|
||||
{
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"maxItems": 4,
|
||||
"minItems": 4,
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Observed zoom surface bounds in coordinate_space; not the whole screen or date rail. Both fingers must remain inside.",
|
||||
"title": "Region"
|
||||
},
|
||||
"action": {
|
||||
"const": "pinch",
|
||||
"title": "Action",
|
||||
@@ -869,6 +1127,24 @@
|
||||
"default": null,
|
||||
"title": "Require Visible"
|
||||
},
|
||||
"region": {
|
||||
"anyOf": [
|
||||
{
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"maxItems": 4,
|
||||
"minItems": 4,
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Observed gesture surface bounds in coordinate_space; every finger path must stay inside it.",
|
||||
"title": "Region"
|
||||
},
|
||||
"action": {
|
||||
"const": "touch_gesture",
|
||||
"title": "Action",
|
||||
@@ -1072,6 +1348,76 @@
|
||||
],
|
||||
"default": null,
|
||||
"title": "Selection End"
|
||||
},
|
||||
"ensure_state": {
|
||||
"anyOf": [
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"state": {
|
||||
"enum": [
|
||||
"checked",
|
||||
"unchecked",
|
||||
"selected",
|
||||
"unselected",
|
||||
"text",
|
||||
"content_desc",
|
||||
"value"
|
||||
],
|
||||
"title": "State",
|
||||
"type": "string"
|
||||
},
|
||||
"expected_text": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 4096,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Expected Text"
|
||||
},
|
||||
"expected_value": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Expected Value"
|
||||
},
|
||||
"tolerance": {
|
||||
"default": 0,
|
||||
"minimum": 0,
|
||||
"title": "Tolerance",
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"state"
|
||||
],
|
||||
"title": "ControlCondition",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Desired observed state: skip an already satisfied action; otherwise write ONCE and verify by read-only polling. Never infer state from an icon or missing attributes."
|
||||
},
|
||||
"verify_timeout_ms": {
|
||||
"default": 1500,
|
||||
"maximum": 3000,
|
||||
"minimum": 1,
|
||||
"title": "Verify Timeout Ms",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -1197,6 +1543,7 @@
|
||||
"selected",
|
||||
"unselected",
|
||||
"text",
|
||||
"content_desc",
|
||||
"value",
|
||||
"stable"
|
||||
],
|
||||
@@ -1274,6 +1621,155 @@
|
||||
"maxItems": 10,
|
||||
"minItems": 1,
|
||||
"type": "array"
|
||||
},
|
||||
"feedback": {
|
||||
"anyOf": [
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"selector": {
|
||||
"additionalProperties": false,
|
||||
"description": "Unique, observed persistent value readout, not a transient drag tooltip. Copy its stable selector, excluding changing text.",
|
||||
"properties": {
|
||||
"resource_id": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 512,
|
||||
"minLength": 1,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Resource Id"
|
||||
},
|
||||
"text": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 4096,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Text"
|
||||
},
|
||||
"content_desc": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 512,
|
||||
"minLength": 1,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Content Desc"
|
||||
},
|
||||
"class_name": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 512,
|
||||
"minLength": 1,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Class Name"
|
||||
},
|
||||
"package_name": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 512,
|
||||
"minLength": 1,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Package Name"
|
||||
},
|
||||
"bounds": {
|
||||
"anyOf": [
|
||||
{
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"maxItems": 4,
|
||||
"minItems": 4,
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Bounds"
|
||||
}
|
||||
},
|
||||
"title": "ControlSelector",
|
||||
"type": "object"
|
||||
},
|
||||
"value_format": {
|
||||
"description": "range reads native range.current; number requires an entire numeric text; clock requires exactly one HH:MM:SS in readable text and returns seconds since midnight.",
|
||||
"enum": [
|
||||
"range",
|
||||
"number",
|
||||
"clock"
|
||||
],
|
||||
"title": "Value Format",
|
||||
"type": "string"
|
||||
},
|
||||
"target_value": {
|
||||
"title": "Target Value",
|
||||
"type": "number"
|
||||
},
|
||||
"tolerance": {
|
||||
"minimum": 0,
|
||||
"title": "Tolerance",
|
||||
"type": "number"
|
||||
},
|
||||
"stable_ms": {
|
||||
"default": 1000,
|
||||
"maximum": 5000,
|
||||
"minimum": 100,
|
||||
"title": "Stable Ms",
|
||||
"type": "integer"
|
||||
},
|
||||
"timeout_ms": {
|
||||
"default": 3000,
|
||||
"maximum": 10000,
|
||||
"minimum": 200,
|
||||
"title": "Timeout Ms",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"selector",
|
||||
"value_format",
|
||||
"target_value",
|
||||
"tolerance"
|
||||
],
|
||||
"title": "ValueFeedback",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Optional actual value/target comparison after the WHOLE sequence. clock uses seconds since midnight; exact bound and stable observation are checked locally, not inferred from dispatch."
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -1423,6 +1919,23 @@
|
||||
}
|
||||
],
|
||||
"default": null
|
||||
},
|
||||
"region": {
|
||||
"anyOf": [
|
||||
{
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"maxItems": 4,
|
||||
"minItems": 4,
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Observed zoom surface bounds in coordinate_space; not the whole screen or date rail. Both fingers must remain inside."
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -1500,6 +2013,23 @@
|
||||
}
|
||||
],
|
||||
"default": null
|
||||
},
|
||||
"region": {
|
||||
"anyOf": [
|
||||
{
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"maxItems": 4,
|
||||
"minItems": 4,
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Observed gesture surface bounds in coordinate_space; every finger path must stay inside it."
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -1616,7 +2146,7 @@
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "control_action",
|
||||
"description": "[ACTION] Perform ONE supported native action on a freshly resolved unique control. Read_control first; copy selector exactly and optionally expected_token to reject stale evidence. action=set_progress requires a finite in-range value; set_selection requires selection_start/end in UTF-16 code units (never split an emoji). set_text and replace_selection require text, including empty text; whitespace is preserved. Other actions take no text/value/selection. ime_enter invokes the advertised IME submit action. Unsupported actions fail without coordinate fallback or replay. Focus or selection changes may move the target; before subsequent edits, use wait_until state=stable when needed. STALE means no action started; observe again before rebuilding the request. Never resend an unknown write. An accepted action is not business verification; inspect the returned/current state.",
|
||||
"description": "[ACTION] Perform ONE supported native action on a freshly resolved unique control. Read_control first; copy selector exactly and optionally expected_token to reject stale evidence. For a desired state use ensure_state (checked/selected/text/content_desc/value), then verify_timeout_ms: skip an already satisfied state, otherwise write ONCE and poll read-only. Copy observed state semantics; do not assume selected means paused. Bind a stable selector excluding text/content_desc that the action changes. A failed postcondition does not undo the completed input and must not cause an automatic second toggle. action=set_progress requires a finite in-range value; set_selection requires selection_start/end in UTF-16 code units (never split an emoji). set_text and replace_selection require text, including empty text; whitespace is preserved. Other actions take no text/value/selection. ime_enter invokes the advertised IME submit action. Unsupported actions fail without coordinate fallback or replay. Focus or selection changes may move the target; before subsequent edits, use wait_until state=stable when needed. STALE means no action started; observe again before rebuilding the request. Never resend an unknown write. An accepted action is not business verification; inspect the returned/current state.",
|
||||
"parameters": {
|
||||
"properties": {
|
||||
"selector": {
|
||||
@@ -1784,6 +2314,75 @@
|
||||
}
|
||||
],
|
||||
"default": null
|
||||
},
|
||||
"ensure_state": {
|
||||
"anyOf": [
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"state": {
|
||||
"enum": [
|
||||
"checked",
|
||||
"unchecked",
|
||||
"selected",
|
||||
"unselected",
|
||||
"text",
|
||||
"content_desc",
|
||||
"value"
|
||||
],
|
||||
"title": "State",
|
||||
"type": "string"
|
||||
},
|
||||
"expected_text": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 4096,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Expected Text"
|
||||
},
|
||||
"expected_value": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Expected Value"
|
||||
},
|
||||
"tolerance": {
|
||||
"default": 0,
|
||||
"minimum": 0,
|
||||
"title": "Tolerance",
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"state"
|
||||
],
|
||||
"title": "ControlCondition",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"description": "Desired observed state: skip an already satisfied action; otherwise write ONCE and verify by read-only polling. Never infer state from an icon or missing attributes."
|
||||
},
|
||||
"verify_timeout_ms": {
|
||||
"default": 1500,
|
||||
"maximum": 3000,
|
||||
"minimum": 1,
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -1899,6 +2498,7 @@
|
||||
"selected",
|
||||
"unselected",
|
||||
"text",
|
||||
"content_desc",
|
||||
"value",
|
||||
"stable"
|
||||
],
|
||||
|
||||
@@ -71,6 +71,81 @@ async def run(name, args, native):
|
||||
return await execute_native(name, args, 400, 800, native)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"condition,fields",
|
||||
[
|
||||
({"state": "unchecked"}, {"checked": False}),
|
||||
({"state": "text", "expected_text": " exact "}, {"text": " exact "}),
|
||||
({"state": "content_desc", "expected_text": "播放"}, {"content_desc": "播放"}),
|
||||
({"state": "value", "expected_value": 3.0}, {}),
|
||||
],
|
||||
)
|
||||
async def test_ensure_state_already_satisfied_never_toggles(condition, fields):
|
||||
native = AsyncMock(return_value=observed(**fields))
|
||||
result = await run(
|
||||
"control_action",
|
||||
{"selector": SELECTOR, "action": "click", "ensure_state": condition},
|
||||
native,
|
||||
)
|
||||
assert result["success"] and result["skipped"]
|
||||
assert result["execution"] == "observed"
|
||||
assert [call.args[0] for call in native.await_args_list] == ["inspect_control"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ensure_state_missing_native_boolean_refuses_input():
|
||||
native = AsyncMock(return_value=observed(checked=None))
|
||||
result = await run(
|
||||
"control_action",
|
||||
{"selector": SELECTOR, "action": "click", "ensure_state": {"state": "unchecked"}},
|
||||
native,
|
||||
)
|
||||
assert not result["success"] and result["execution"] == "not_started"
|
||||
native.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ensure_state_writes_once_then_reads_actual_state():
|
||||
native = AsyncMock(side_effect=[observed(checked=True), completed(), observed(checked=False)])
|
||||
result = await run(
|
||||
"control_action",
|
||||
{"selector": SELECTOR, "action": "click", "ensure_state": {"state": "unchecked"}},
|
||||
native,
|
||||
)
|
||||
assert result["success"] and result["verification"] == "condition_satisfied"
|
||||
assert [call.args[0] for call in native.await_args_list] == [
|
||||
"inspect_control",
|
||||
"control_action",
|
||||
"inspect_control",
|
||||
]
|
||||
args = native.await_args_list[1].args[1]
|
||||
assert "ensure_state" not in args and "verify_timeout_ms" not in args
|
||||
assert args["expected_token"] == TOKEN
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_postcondition_preserves_completed_input_without_retoggling(clock):
|
||||
async def native(command, args, **kwargs):
|
||||
return completed() if command == "control_action" else observed(checked=True)
|
||||
|
||||
native = AsyncMock(side_effect=native)
|
||||
result = await run(
|
||||
"control_action",
|
||||
{
|
||||
"selector": SELECTOR,
|
||||
"action": "click",
|
||||
"ensure_state": {"state": "unchecked"},
|
||||
"verify_timeout_ms": 400,
|
||||
},
|
||||
native,
|
||||
)
|
||||
assert not result["success"] and result["code"] == "TIMEOUT"
|
||||
assert result["execution"] == "completed"
|
||||
assert result["native_receipt"]["success"]
|
||||
assert sum(c.args[0] == "control_action" for c in native.await_args_list) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("name", ["pinch", "touch_gesture"])
|
||||
async def test_all_gesture_geometry_is_validated_before_a_single_native_dispatch(name):
|
||||
|
||||
@@ -0,0 +1,346 @@
|
||||
"""Regress real actuator dispatch, state verification and value convergence without devices."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.mcp import value_feedback
|
||||
from artemis.mcp.action_manifest import DEVICE_ACTIONS
|
||||
from artemis.mcp.actuators import MockActuator
|
||||
from artemis.mcp.gestures import DragRequest, plan_drag
|
||||
from artemis.mcp.native_actions import PinchRequest, plan_pinch
|
||||
from artemis.mcp.value_feedback import read_value
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def deterministic_clock(monkeypatch):
|
||||
now = [0.0]
|
||||
|
||||
async def sleep(seconds):
|
||||
now[0] += seconds
|
||||
|
||||
monkeypatch.setattr(value_feedback, "monotonic", lambda: now[0])
|
||||
monkeypatch.setattr(value_feedback, "sleep", sleep)
|
||||
monkeypatch.setattr(
|
||||
"artemis.config.agent.load_agent_config",
|
||||
lambda: SimpleNamespace(
|
||||
pro=SimpleNamespace(execution=SimpleNamespace(max_burst_actions=10))
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def setup_actuator(*, current="100", after="100", readable=True):
|
||||
actuator = MockActuator(width=720, height=1600, capabilities=DEVICE_ACTIONS)
|
||||
state = {"text": current, "playing": False}
|
||||
events = []
|
||||
|
||||
async def native(command, args, **kwargs):
|
||||
events.append(command)
|
||||
if command == "control_action":
|
||||
state["playing"] = not state["playing"]
|
||||
return {"success": True, "execution": "completed", "code": "OK"}
|
||||
is_player = args["selector"]["resource_id"] == "player"
|
||||
return {
|
||||
"success": True,
|
||||
"execution": "observed",
|
||||
"code": "OK",
|
||||
"node": {
|
||||
"token": "a" * 64,
|
||||
"content_signature": "b" * 64,
|
||||
"visible": True,
|
||||
"enabled": True,
|
||||
"password": False,
|
||||
"actions": ["click"],
|
||||
"text": state["text"] if readable else None,
|
||||
"content_desc": ("暂停" if state["playing"] else "播放")
|
||||
if is_player
|
||||
else "readout",
|
||||
},
|
||||
}
|
||||
|
||||
async def swipe(*args):
|
||||
events.append("swipe")
|
||||
state.update(text=after, playing=True)
|
||||
return None
|
||||
|
||||
actuator.controller.native_request = AsyncMock(side_effect=native)
|
||||
actuator.controller.swipe_coords = AsyncMock(side_effect=swipe)
|
||||
actuator.controller.tap_at = AsyncMock(return_value=SimpleNamespace(error=None))
|
||||
return actuator, state, events
|
||||
|
||||
|
||||
def feedback(**overrides):
|
||||
return {
|
||||
"selector": {"resource_id": "readout"},
|
||||
"value_format": "number",
|
||||
"target_value": 90.0,
|
||||
"tolerance": 1.0,
|
||||
"stable_ms": 100,
|
||||
"timeout_ms": 500,
|
||||
**overrides,
|
||||
}
|
||||
|
||||
|
||||
def drag(**overrides):
|
||||
return {
|
||||
"start": [300, 1050],
|
||||
"end": [325, 1050],
|
||||
"coordinate_space": "pixel",
|
||||
"target_description": "observed timeline",
|
||||
**overrides,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_two_ineffective_attempts_block_third_but_allow_meaningful_strategy_change():
|
||||
actuator, _, _ = setup_actuator()
|
||||
for _ in range(2):
|
||||
result = await actuator.drag(**drag(feedback=feedback()))
|
||||
assert not result.ok and result.metadata["execution"] == "completed"
|
||||
assert result.metadata["value_feedback"]["after_value"] == 100
|
||||
result = await actuator.drag(**drag(feedback=feedback()))
|
||||
assert not result.ok and "NO_PROGRESS" in result.message
|
||||
assert result.metadata["execution"] == "not_started"
|
||||
assert actuator.controller.swipe_coords.await_count == 2
|
||||
# Tiny endpoint jitter is the same strategy, not an escape from the guard.
|
||||
await actuator.drag(**drag(end=[328, 1050], feedback=feedback()))
|
||||
assert actuator.controller.swipe_coords.await_count == 2
|
||||
await actuator.drag(**drag(end=[424, 1050], feedback=feedback()))
|
||||
assert actuator.controller.swipe_coords.await_count == 3
|
||||
# History belongs to this actuator/task, not a global cache.
|
||||
other, _, _ = setup_actuator()
|
||||
await other.drag(**drag(feedback=feedback()))
|
||||
assert other.controller.swipe_coords.await_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_changed_actual_value_resets_stalled_history():
|
||||
actuator, state, _ = setup_actuator()
|
||||
for _ in range(2):
|
||||
await actuator.drag(**drag(feedback=feedback()))
|
||||
state["text"] = "95"
|
||||
await actuator.drag(**drag(feedback=feedback()))
|
||||
assert actuator.controller.swipe_coords.await_count == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_one_pixel_change_across_power_of_two_is_still_the_same_strategy():
|
||||
actuator, _, _ = setup_actuator()
|
||||
for endpoint in (331, 332, 333):
|
||||
await actuator.drag(**drag(end=[endpoint, 1050], feedback=feedback()))
|
||||
assert actuator.controller.swipe_coords.await_count == 2
|
||||
# A substantially slower gesture is a meaningful new approach.
|
||||
await actuator.drag(**drag(end=[333, 1050], duration_ms=1500, feedback=feedback()))
|
||||
assert actuator.controller.swipe_coords.await_count == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sequence_reads_final_value_only_after_pause_and_skips_redundant_toggle():
|
||||
actuator, state, events = setup_actuator(
|
||||
current="11-02 周一 17:43:12", after="11-02 周一 17:42:17"
|
||||
)
|
||||
pause = {
|
||||
"action": "control_action",
|
||||
"request": {
|
||||
"action": "click",
|
||||
"selector": {"resource_id": "player"},
|
||||
"ensure_state": {"state": "content_desc", "expected_text": "播放"},
|
||||
},
|
||||
}
|
||||
result = await actuator.action_sequence(
|
||||
[pause, {"action": "drag", **drag()}, pause],
|
||||
feedback=feedback(
|
||||
value_format="clock", target_value=17.0 * 3600 + 42 * 60 + 17, tolerance=5.0
|
||||
),
|
||||
)
|
||||
assert result.ok and not state["playing"]
|
||||
assert result.metadata["skipped_steps"] == 1
|
||||
assert result.metadata["value_feedback"]["residual"] == 0
|
||||
observation = result.metadata["value_feedback"]["after_observation"]
|
||||
assert observation["node"]["text"] == "11-02 周一 17:42:17"
|
||||
assert observation["stable_for_ms"] >= 100 and observation["samples"] >= 2
|
||||
assert result.metadata["verification"] == "value_matched"
|
||||
assert events.count("control_action") == 1
|
||||
assert events.index("swipe") < events.index("control_action")
|
||||
assert events[-1] == "inspect_control"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_value_refuses_input_and_postwrite_read_failure_preserves_receipt():
|
||||
actuator, _, _ = setup_actuator(readable=False)
|
||||
result = await actuator.drag(**drag(feedback=feedback()))
|
||||
assert not result.ok and result.metadata["execution"] == "not_started"
|
||||
actuator.controller.swipe_coords.assert_not_awaited()
|
||||
actuator, _, _ = setup_actuator(after="unreadable")
|
||||
result = await actuator.drag(**drag(feedback=feedback()))
|
||||
assert not result.ok and result.metadata["execution"] == "completed"
|
||||
assert result.metadata["coordinates"] == [300, 1050, 325, 1050]
|
||||
assert result.metadata["verification"] == "unavailable"
|
||||
actuator.controller.swipe_coords.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_feedback_unavailable_has_no_coordinate_fallback():
|
||||
actuator = MockActuator(capabilities=DEVICE_ACTIONS - {"read_control", "wait_until"})
|
||||
actuator.controller.swipe_coords = AsyncMock()
|
||||
result = await actuator.drag(**drag(feedback=feedback()))
|
||||
assert not result.ok and result.code == "UNSUPPORTED"
|
||||
actuator.controller.swipe_coords.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_whole_sequence_rejects_bad_region_before_any_click():
|
||||
actuator, _, _ = setup_actuator()
|
||||
result = await actuator.action_sequence(
|
||||
[
|
||||
{"action": "click", "target": [50, 50], "target_description": "opener"},
|
||||
{"action": "drag", **drag(region=[400, 970, 650, 1118])},
|
||||
]
|
||||
)
|
||||
assert not result.ok
|
||||
actuator.controller.tap_at.assert_not_awaited()
|
||||
actuator.controller.swipe_coords.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_state_verification_keeps_completed_members_and_stops_next_input():
|
||||
actuator, _, _ = setup_actuator()
|
||||
|
||||
async def unchanged(command, args, **kwargs):
|
||||
if command == "control_action":
|
||||
return {"success": True, "execution": "completed", "code": "OK"}
|
||||
return {
|
||||
"success": True,
|
||||
"execution": "observed",
|
||||
"code": "OK",
|
||||
"node": {
|
||||
"token": "a" * 64,
|
||||
"visible": True,
|
||||
"enabled": True,
|
||||
"text": "100",
|
||||
"content_desc": "暂停",
|
||||
"actions": ["click"],
|
||||
},
|
||||
}
|
||||
|
||||
actuator.controller.native_request = AsyncMock(side_effect=unchanged)
|
||||
result = await actuator.action_sequence(
|
||||
[
|
||||
{"action": "drag", **drag()},
|
||||
{
|
||||
"action": "control_action",
|
||||
"request": {
|
||||
"action": "click",
|
||||
"selector": {"resource_id": "player"},
|
||||
"ensure_state": {"state": "content_desc", "expected_text": "播放"},
|
||||
"verify_timeout_ms": 1,
|
||||
},
|
||||
},
|
||||
{"action": "click", "target": [50, 50], "target_description": "must not execute"},
|
||||
],
|
||||
feedback=feedback(),
|
||||
)
|
||||
assert not result.ok and result.metadata["execution"] == "partial"
|
||||
assert [r["execution"] for r in result.metadata["sequence_results"]] == [
|
||||
"completed",
|
||||
"completed",
|
||||
"not_started",
|
||||
]
|
||||
assert result.metadata["sequence_results"][1]["verification"] == "condition_not_met"
|
||||
actuator.controller.tap_at.assert_not_awaited()
|
||||
actuator.controller.swipe_coords.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transient_post_action_read_failure_can_settle_without_replaying_input():
|
||||
actuator, _, events = setup_actuator(after="90")
|
||||
original = actuator.controller.native_request.side_effect
|
||||
failed = False
|
||||
|
||||
async def transient(command, args, **kwargs):
|
||||
nonlocal failed
|
||||
if "swipe" in events and not failed:
|
||||
failed = True
|
||||
return {"success": False, "execution": "not_started", "code": "READ_FAILED"}
|
||||
return await original(command, args, **kwargs)
|
||||
|
||||
actuator.controller.native_request = AsyncMock(side_effect=transient)
|
||||
result = await actuator.drag(**drag(feedback=feedback()))
|
||||
assert result.ok and result.metadata["value_feedback"]["after_value"] == 90
|
||||
actuator.controller.swipe_coords.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_feedback_timeout_never_repeats_a_completed_input(monkeypatch):
|
||||
actuator, state, _ = setup_actuator()
|
||||
original = value_feedback.sleep
|
||||
|
||||
async def moving(seconds):
|
||||
state["text"] = str(float(state["text"]) + 1)
|
||||
await original(seconds)
|
||||
|
||||
monkeypatch.setattr(value_feedback, "sleep", moving)
|
||||
result = await actuator.drag(**drag(feedback=feedback()))
|
||||
assert not result.ok and result.code == "TIMEOUT"
|
||||
assert result.metadata["execution"] == "completed"
|
||||
assert result.metadata["verification"] == "unavailable"
|
||||
actuator.controller.swipe_coords.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text", ["17:42:17 / 17:43:12", "25:42:17", "117:42:17", "17:42:170", "17:42"]
|
||||
)
|
||||
def test_clock_readout_rejects_ambiguous_or_invalid_times(text):
|
||||
with pytest.raises(ValueError):
|
||||
read_value({"visible": True, "text": text}, "clock")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"value_format,node",
|
||||
[
|
||||
("number", {"text": "0"}),
|
||||
("clock", {"text": "00:00:00"}),
|
||||
("range", {"range": {"min": 0, "current": 0, "max": 10}}),
|
||||
],
|
||||
)
|
||||
def test_zero_is_an_actual_value(value_format, node):
|
||||
assert read_value({"visible": True, **node}, value_format) == 0
|
||||
|
||||
|
||||
def test_precision_guard_uses_observed_minimum_and_reports_geometric_budget():
|
||||
calibration = {
|
||||
"axis": "x",
|
||||
"mode": "thumb",
|
||||
"anchors": [{"position": 100.0, "value": 0.0}, {"position": 200.0, "value": 1000.0}],
|
||||
"target_value": 125.0,
|
||||
"tolerance": 1.0,
|
||||
}
|
||||
request = {
|
||||
"start": [100, 1050],
|
||||
"coordinate_space": "pixel",
|
||||
"target_description": "observed scale",
|
||||
"calibration": calibration,
|
||||
}
|
||||
with pytest.raises(ValueError, match="rounding"):
|
||||
plan_drag(DragRequest.model_validate(request), 720, 1600)
|
||||
calibration.update(tolerance=5.0, min_movement_px=20.0)
|
||||
with pytest.raises(ValueError, match="minimum effective"):
|
||||
plan_drag(DragRequest.model_validate(request), 720, 1600)
|
||||
calibration.update(min_movement_px=10.0)
|
||||
plan = plan_drag(DragRequest.model_validate(request), 720, 1600)
|
||||
assert plan["calibration"]["pixel_error_budget"] == 0.5
|
||||
assert plan["calibration"]["rounding_error_value"] == 5
|
||||
|
||||
|
||||
def test_both_pinch_fingers_must_stay_in_observed_surface():
|
||||
request = PinchRequest(
|
||||
center=[360, 656],
|
||||
start_span=40,
|
||||
end_span=80,
|
||||
coordinate_space="pixel",
|
||||
target_description="observed timeline",
|
||||
region=[200, 970, 650, 1118],
|
||||
)
|
||||
with pytest.raises(ValueError, match="region"):
|
||||
plan_pinch(request, 720, 1600)
|
||||
Reference in new issue
Block a user