655 lines
24 KiB
JSON
655 lines
24 KiB
JSON
[
|
|
{
|
|
"name": "list_windows",
|
|
"description": "Find the authorized application window. If its PID is unknown, call with app only, e.g. {\"app\":\"Brave\",\"on_screen_only\":true}; omit pid or use null, never 0. Use the returned matching pid/window_id for later actions. Empty filtered results are not proof that the app is unavailable; follow lookup.suggested_arguments when present.",
|
|
"inputSchema": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"app": {
|
|
"description": "Application name substring (case-insensitive). Start with this when the target PID is unknown. app and a non-null pid are combined filters; do not copy the PID of an unrelated foreground app.",
|
|
"type": [
|
|
"string",
|
|
"null"
|
|
]
|
|
},
|
|
"on_screen_only": {
|
|
"description": "When true, drop windows that aren't currently on-screen. Default false.",
|
|
"type": "boolean"
|
|
},
|
|
"pid": {
|
|
"description": "Known positive Windows process ID from a matching list_windows result. Unknown: omit or null; never 0. With app, both filters must match the same window.",
|
|
"type": [
|
|
"integer",
|
|
"null"
|
|
],
|
|
"minimum": 1,
|
|
"maximum": 4294967295
|
|
}
|
|
},
|
|
"type": "object",
|
|
"required": []
|
|
},
|
|
"annotations": {
|
|
"readOnlyHint": true,
|
|
"idempotentHint": true
|
|
}
|
|
},
|
|
{
|
|
"name": "get_window_state",
|
|
"description": "Read fresh window screenshot and compact UIA tree. Every [N] uses element_token=<snapshot_id>:N; a new snapshot invalidates old tokens. Pixel x/y are screenshot-local; use capture_id. since=latest returns changes. Keep the new snapshot_id.",
|
|
"inputSchema": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"max_elements": {
|
|
"description": "Cap on the total number of UIA nodes walked. Truncates depth-first; markdown and structured elements truncate together. Default 250 (5 000 with full_output:true); the response states when the tree was cut. Raise it to read further into a la",
|
|
"minimum": 1,
|
|
"type": "integer"
|
|
},
|
|
"max_image_dimension": {
|
|
"description": "Per-call long-edge limit for the returned screenshot, in pixels (aspect ratio preserved). Explicit values override configured behavior; 0 returns native resolution. Omit to preserve the configured default.",
|
|
"minimum": 0,
|
|
"type": "integer"
|
|
},
|
|
"pid": {
|
|
"description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.",
|
|
"type": "integer"
|
|
},
|
|
"query": {
|
|
"description": "Optional case-insensitive substring. Projects both tree_markdown and structured elements to matches plus ancestors while preserving original indices. Compare total_element_count with returned_element_count.",
|
|
"type": "string"
|
|
},
|
|
"since": {
|
|
"description": "A `snapshot_id` from an earlier get_window_state of the same window, or \"latest\" for this window's most recent read with the same query/max_elements/max_depth. Use it on every re-read after acting. Returns only what changed against that sna",
|
|
"type": "string"
|
|
},
|
|
"window_id": {
|
|
"description": "HWND of the target window. Must belong to `pid`. Enumerate via `list_windows` or read from `launch_app`'s `windows` array.",
|
|
"type": "integer"
|
|
}
|
|
},
|
|
"required": [
|
|
"pid",
|
|
"window_id"
|
|
],
|
|
"type": "object"
|
|
},
|
|
"annotations": {
|
|
"readOnlyHint": true,
|
|
"idempotentHint": true
|
|
}
|
|
},
|
|
{
|
|
"name": "verify_state",
|
|
"description": "Read-only assertion for exact window. satisfied/unsatisfied/unknown describes the observation, not the whole task verdict.",
|
|
"inputSchema": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"expect": {
|
|
"description": "One to eight predicates, combined with logical AND.",
|
|
"items": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"element": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"enabled": {
|
|
"type": "boolean"
|
|
},
|
|
"exists": {
|
|
"description": "Assert that at least one trusted element matches the selector.\n\nElement walks are not yet exhaustive on every platform, so absence\ncannot be proven. `false` is rejected instead of returning an\nindefinitely-unknown predicate.",
|
|
"type": "boolean"
|
|
},
|
|
"selected": {
|
|
"type": "boolean"
|
|
},
|
|
"selector": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"label_contains": {
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
"role": {
|
|
"minLength": 1,
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [],
|
|
"type": "object"
|
|
},
|
|
"value_equals": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"selector"
|
|
],
|
|
"type": "object"
|
|
},
|
|
"window": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"bounds": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"height": {
|
|
"type": "number"
|
|
},
|
|
"tolerance_px": {
|
|
"maximum": 100,
|
|
"minimum": 0,
|
|
"type": "number"
|
|
},
|
|
"width": {
|
|
"type": "number"
|
|
},
|
|
"x": {
|
|
"type": "number"
|
|
},
|
|
"y": {
|
|
"type": "number"
|
|
}
|
|
},
|
|
"required": [
|
|
"x",
|
|
"y",
|
|
"width",
|
|
"height"
|
|
],
|
|
"type": "object"
|
|
},
|
|
"exists": {
|
|
"type": "boolean"
|
|
}
|
|
},
|
|
"required": [],
|
|
"type": "object"
|
|
}
|
|
},
|
|
"required": [],
|
|
"type": "object"
|
|
},
|
|
"maxItems": 8,
|
|
"minItems": 1,
|
|
"type": "array"
|
|
},
|
|
"include_screenshot": {
|
|
"description": "Return the final window screenshot as image content for a multimodal\ncaller. The driver does not interpret that image.",
|
|
"type": "boolean"
|
|
},
|
|
"pid": {
|
|
"description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.",
|
|
"minimum": 1,
|
|
"type": "integer"
|
|
},
|
|
"stable_samples": {
|
|
"default": 2,
|
|
"description": "Consecutive satisfied samples required before returning success.",
|
|
"maximum": 5,
|
|
"minimum": 1,
|
|
"type": "integer"
|
|
},
|
|
"timeout_ms": {
|
|
"default": 5000,
|
|
"description": "Bounded wait. Zero performs one sample.",
|
|
"maximum": 10000,
|
|
"minimum": 0,
|
|
"type": "integer"
|
|
},
|
|
"window_id": {
|
|
"description": "Exact native window identifier.",
|
|
"type": "integer"
|
|
}
|
|
},
|
|
"required": [
|
|
"pid",
|
|
"window_id",
|
|
"expect"
|
|
],
|
|
"type": "object"
|
|
},
|
|
"annotations": {
|
|
"readOnlyHint": true,
|
|
"idempotentHint": true
|
|
}
|
|
},
|
|
{
|
|
"name": "click",
|
|
"description": "Click the latest UIA element_token. This tool has no pixel coordinates. For visual targets use click_at. Returns effect and fresh observation; old tokens become stale.",
|
|
"inputSchema": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"button": {
|
|
"description": "Mouse button. Default \"left\".",
|
|
"enum": [
|
|
"left",
|
|
"right",
|
|
"middle"
|
|
],
|
|
"type": "string"
|
|
},
|
|
"count": {
|
|
"description": "Click count — 1 (single), 2 (double), 3 (triple). Default 1.",
|
|
"maximum": 3,
|
|
"minimum": 1,
|
|
"type": "integer"
|
|
},
|
|
"delivery_mode": {
|
|
"description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button",
|
|
"enum": [
|
|
"background",
|
|
"foreground"
|
|
],
|
|
"type": "string"
|
|
},
|
|
"element_token": {
|
|
"description": "Element handle `<snapshot_id>:<row>` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current sna",
|
|
"pattern": "^s[0-9a-f]{8}:[0-9]+$",
|
|
"type": "string"
|
|
},
|
|
"pid": {
|
|
"description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.",
|
|
"type": "integer"
|
|
},
|
|
"window_id": {
|
|
"description": "HWND of the target window. Omit when element_token is supplied (the token carries it).",
|
|
"type": "integer"
|
|
}
|
|
},
|
|
"type": "object",
|
|
"required": [
|
|
"pid",
|
|
"window_id",
|
|
"element_token"
|
|
]
|
|
},
|
|
"annotations": {
|
|
"readOnlyHint": false,
|
|
"idempotentHint": false
|
|
}
|
|
},
|
|
{
|
|
"name": "drag",
|
|
"description": "Drag between screenshot-local coordinates. Returns actual delivery effect and fresh post-action observation.",
|
|
"inputSchema": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"delivery_mode": {
|
|
"description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button",
|
|
"enum": [
|
|
"background",
|
|
"foreground"
|
|
],
|
|
"type": "string"
|
|
},
|
|
"duration_ms": {
|
|
"description": "Wall-clock duration of drag path. Default: 500.",
|
|
"maximum": 10000,
|
|
"minimum": 0,
|
|
"type": "integer"
|
|
},
|
|
"from_x": {
|
|
"description": "Drag-start X in window-local screenshot pixels.",
|
|
"type": "number"
|
|
},
|
|
"from_y": {
|
|
"description": "Drag-start Y in window-local screenshot pixels.",
|
|
"type": "number"
|
|
},
|
|
"pid": {
|
|
"description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.",
|
|
"type": "integer"
|
|
},
|
|
"to_x": {
|
|
"description": "Drag-end X in window-local screenshot pixels.",
|
|
"type": "number"
|
|
},
|
|
"to_y": {
|
|
"description": "Drag-end Y in window-local screenshot pixels.",
|
|
"type": "number"
|
|
},
|
|
"window_id": {
|
|
"description": "Target window handle (HWND). Optional — driver picks frontmost window of pid when omitted.",
|
|
"type": "integer"
|
|
}
|
|
},
|
|
"required": [
|
|
"pid",
|
|
"window_id",
|
|
"from_x",
|
|
"from_y",
|
|
"to_x",
|
|
"to_y"
|
|
],
|
|
"type": "object"
|
|
},
|
|
"annotations": {
|
|
"readOnlyHint": false,
|
|
"idempotentHint": false
|
|
}
|
|
},
|
|
{
|
|
"name": "type_text",
|
|
"description": "Type text into the focused target window or specified latest element. Fresh observation follows. If background is unsupported, explicitly choose foreground within task scope.",
|
|
"inputSchema": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"delivery_mode": {
|
|
"description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button",
|
|
"enum": [
|
|
"background",
|
|
"foreground"
|
|
],
|
|
"type": "string"
|
|
},
|
|
"element_token": {
|
|
"description": "Opaque element handle from get_window_state. When supplied, type_text writes through UIA ValuePattern and reads that exact element back by handle. The shared ActionResult is confirmed only when the complete expected value is synchronously v",
|
|
"pattern": "^s[0-9a-f]{8}:[0-9]+$",
|
|
"type": "string"
|
|
},
|
|
"pid": {
|
|
"description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.",
|
|
"type": "integer"
|
|
},
|
|
"text": {
|
|
"description": "Text to insert at the focused element's cursor.",
|
|
"type": "string"
|
|
},
|
|
"window_id": {
|
|
"description": "HWND of the target window. Omit when element_token is supplied (the token carries it).",
|
|
"type": "integer"
|
|
},
|
|
"x": {
|
|
"description": "Window-local screenshot-pixel X of the field to type into — the element px action form. Pass x,y (no element_token) and the tool pixel-clicks there to establish real renderer focus, then types. Use for Chromium/Electron inputs the UIA/WM_CH",
|
|
"type": "number"
|
|
},
|
|
"y": {
|
|
"description": "Window-local screenshot-pixel Y of the field (see x).",
|
|
"type": "number"
|
|
}
|
|
},
|
|
"required": [
|
|
"pid",
|
|
"window_id",
|
|
"text"
|
|
],
|
|
"type": "object"
|
|
},
|
|
"annotations": {
|
|
"readOnlyHint": false,
|
|
"idempotentHint": false
|
|
}
|
|
},
|
|
{
|
|
"name": "press_key",
|
|
"description": "Send key and modifiers to the exact window, optionally focus element_token or x/y first. Fresh observation follows.",
|
|
"inputSchema": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"delivery_mode": {
|
|
"description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button",
|
|
"enum": [
|
|
"background",
|
|
"foreground"
|
|
],
|
|
"type": "string"
|
|
},
|
|
"element_token": {
|
|
"description": "Element handle `<snapshot_id>:<row>` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current sna",
|
|
"pattern": "^s[0-9a-f]{8}:[0-9]+$",
|
|
"type": "string"
|
|
},
|
|
"key": {
|
|
"description": "Key name (return, tab, escape, up, down, left, right, space, delete, home, end, pageup, pagedown, f1-f12, letter, digit).",
|
|
"type": "string"
|
|
},
|
|
"modifiers": {
|
|
"description": "Optional modifier names held while the key is pressed (ctrl/shift/alt/win).",
|
|
"items": {
|
|
"type": "string"
|
|
},
|
|
"type": "array"
|
|
},
|
|
"pid": {
|
|
"description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.",
|
|
"type": "integer"
|
|
},
|
|
"window_id": {
|
|
"description": "HWND for the target window. Omit when element_token is supplied; otherwise auto-resolves the pid's first visible window.",
|
|
"type": "integer"
|
|
},
|
|
"x": {
|
|
"description": "Window-local screenshot-pixel X — the element px action form: pixel-click there to focus, then send the key. Use when the key must go to a Chromium/Electron surface the UIA path can't focus. Pass with y, no element_token.",
|
|
"type": "number"
|
|
},
|
|
"y": {
|
|
"description": "Window-local screenshot-pixel Y (see x).",
|
|
"type": "number"
|
|
}
|
|
},
|
|
"required": [
|
|
"pid",
|
|
"window_id",
|
|
"key"
|
|
],
|
|
"type": "object"
|
|
},
|
|
"annotations": {
|
|
"readOnlyHint": false,
|
|
"idempotentHint": false
|
|
}
|
|
},
|
|
{
|
|
"name": "set_value",
|
|
"description": "Set the latest UIA element_token value. Returns delivery effect/readback and fresh screenshot/tree.",
|
|
"inputSchema": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"delivery_mode": {
|
|
"description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button",
|
|
"enum": [
|
|
"background",
|
|
"foreground"
|
|
],
|
|
"type": "string"
|
|
},
|
|
"element_token": {
|
|
"description": "Element handle `<snapshot_id>:<row>` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current sna",
|
|
"pattern": "^s[0-9a-f]{8}:[0-9]+$",
|
|
"type": "string"
|
|
},
|
|
"pid": {
|
|
"description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.",
|
|
"type": "integer"
|
|
},
|
|
"value": {
|
|
"description": "New value. UIA will coerce to the element's native type.",
|
|
"type": "string"
|
|
},
|
|
"window_id": {
|
|
"description": "HWND of the window. Omit when element_token is supplied (the token carries it).",
|
|
"type": "integer"
|
|
}
|
|
},
|
|
"required": [
|
|
"pid",
|
|
"window_id",
|
|
"value"
|
|
],
|
|
"type": "object"
|
|
},
|
|
"annotations": {
|
|
"readOnlyHint": false,
|
|
"idempotentHint": false
|
|
}
|
|
},
|
|
{
|
|
"name": "scroll",
|
|
"description": "Scroll the target window/element at x,y. amount=1..50, by=line/page. Verify content changed in fresh returned observation.",
|
|
"inputSchema": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"amount": {
|
|
"description": "Number of scroll ticks. Default 3.",
|
|
"maximum": 50,
|
|
"minimum": 1,
|
|
"type": "integer"
|
|
},
|
|
"by": {
|
|
"description": "Scroll granularity. Default: line.",
|
|
"enum": [
|
|
"line",
|
|
"page"
|
|
],
|
|
"type": "string"
|
|
},
|
|
"delivery_mode": {
|
|
"description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button",
|
|
"enum": [
|
|
"background",
|
|
"foreground"
|
|
],
|
|
"type": "string"
|
|
},
|
|
"direction": {
|
|
"description": "Scroll direction.",
|
|
"enum": [
|
|
"up",
|
|
"down",
|
|
"left",
|
|
"right"
|
|
],
|
|
"type": "string"
|
|
},
|
|
"element_token": {
|
|
"description": "Element handle `<snapshot_id>:<row>` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current sna",
|
|
"pattern": "^s[0-9a-f]{8}:[0-9]+$",
|
|
"type": "string"
|
|
},
|
|
"pid": {
|
|
"description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.",
|
|
"type": "integer"
|
|
},
|
|
"window_id": {
|
|
"description": "HWND of the target window. Omit when element_token is supplied; otherwise auto-resolves the pid's first visible window.",
|
|
"type": "integer"
|
|
},
|
|
"x": {
|
|
"description": "With pid/window_id: window-local screenshot X used to target a nested scroll surface in foreground mode. Without pid/window_id: screen-absolute X for desktop scope. Must be paired with y.",
|
|
"type": "number"
|
|
},
|
|
"y": {
|
|
"description": "With pid/window_id: window-local screenshot Y used to target a nested scroll surface in foreground mode. Without pid/window_id: screen-absolute Y for desktop scope. Must be paired with x.",
|
|
"type": "number"
|
|
}
|
|
},
|
|
"required": [
|
|
"pid",
|
|
"window_id",
|
|
"direction"
|
|
],
|
|
"type": "object"
|
|
},
|
|
"annotations": {
|
|
"readOnlyHint": false,
|
|
"idempotentHint": false
|
|
}
|
|
},
|
|
{
|
|
"name": "save_screenshot",
|
|
"description": "Save fresh window screenshot to this task trace. Plain .png filename only; no overwrite. Returns verified attachment hash, screenshot and latest tree.",
|
|
"inputSchema": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"pid": {
|
|
"type": "integer",
|
|
"minimum": 1,
|
|
"description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID."
|
|
},
|
|
"window_id": {
|
|
"type": "integer",
|
|
"minimum": 1
|
|
},
|
|
"filename": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"pid",
|
|
"window_id",
|
|
"filename"
|
|
]
|
|
},
|
|
"annotations": {
|
|
"readOnlyHint": false,
|
|
"idempotentHint": false
|
|
}
|
|
},
|
|
{
|
|
"name": "click_at",
|
|
"description": "Click a visual target at x,y in the latest window screenshot, with its exact capture_id. Never pass an element token. Returns effect and fresh observation.",
|
|
"inputSchema": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"button": {
|
|
"description": "Mouse button. Default \"left\".",
|
|
"enum": [
|
|
"left",
|
|
"right",
|
|
"middle"
|
|
],
|
|
"type": "string"
|
|
},
|
|
"capture_id": {
|
|
"description": "Optional ID from get_window_state or get_desktop_state. When present, x and y are admitted against that exact capture and the ID is consumed before native dispatch.",
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
"count": {
|
|
"description": "Click count — 1 (single), 2 (double), 3 (triple). Default 1.",
|
|
"maximum": 3,
|
|
"minimum": 1,
|
|
"type": "integer"
|
|
},
|
|
"delivery_mode": {
|
|
"description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button",
|
|
"enum": [
|
|
"background",
|
|
"foreground"
|
|
],
|
|
"type": "string"
|
|
},
|
|
"pid": {
|
|
"description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.",
|
|
"type": "integer"
|
|
},
|
|
"window_id": {
|
|
"description": "HWND of the target window. Omit when element_token is supplied (the token carries it).",
|
|
"type": "integer"
|
|
},
|
|
"x": {
|
|
"description": "X in window-local screenshot pixels — same space as the PNG get_window_state returns. Must be provided together with y.",
|
|
"type": "number"
|
|
},
|
|
"y": {
|
|
"description": "Y in window-local screenshot pixels. Must be provided together with x.",
|
|
"type": "number"
|
|
}
|
|
},
|
|
"type": "object",
|
|
"required": [
|
|
"pid",
|
|
"window_id",
|
|
"x",
|
|
"y",
|
|
"capture_id"
|
|
]
|
|
},
|
|
"annotations": {
|
|
"readOnlyHint": false,
|
|
"idempotentHint": false
|
|
}
|
|
}
|
|
]
|