669 lines
25 KiB
JSON
669 lines
25 KiB
JSON
[
|
|
{
|
|
"name": "list_windows",
|
|
"description": "Find the authorized application window. If its PID is unknown, call with app only, e.g. {\"app\":\"Brave\",\"on_screen_only\":true}; omit pid or use null, never 0. Use the returned matching pid/window_id for later actions. Empty filtered results are not proof that the app is unavailable; follow lookup.suggested_arguments when present.",
|
|
"inputSchema": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"app": {
|
|
"description": "Application name substring (case-insensitive). Start with this when the target PID is unknown. app and a non-null pid are combined filters; do not copy the PID of an unrelated foreground app.",
|
|
"type": [
|
|
"string",
|
|
"null"
|
|
]
|
|
},
|
|
"on_screen_only": {
|
|
"description": "When true, drop windows that aren't currently on-screen. Default false.",
|
|
"type": "boolean"
|
|
},
|
|
"pid": {
|
|
"description": "Known positive Windows process ID from a matching list_windows result. Unknown: omit or null; never 0. With app, both filters must match the same window.",
|
|
"type": [
|
|
"integer",
|
|
"null"
|
|
],
|
|
"minimum": 1,
|
|
"maximum": 4294967295
|
|
}
|
|
},
|
|
"type": "object",
|
|
"required": []
|
|
},
|
|
"annotations": {
|
|
"readOnlyHint": true,
|
|
"idempotentHint": true
|
|
}
|
|
},
|
|
{
|
|
"name": "get_window_state",
|
|
"description": "Read fresh window screenshot and compact UIA tree. Every [N] uses element_token=<snapshot_id>:N; a new snapshot invalidates old tokens. Pixel x/y are screenshot-local; use capture_id. since=latest returns changes. Keep the new snapshot_id.",
|
|
"inputSchema": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"max_elements": {
|
|
"description": "Cap on the total number of UIA nodes walked. Truncates depth-first; markdown and structured elements truncate together. Default 250 (5 000 with full_output:true); the response states when the tree was cut. Raise it to read further into a la",
|
|
"minimum": 1,
|
|
"type": "integer"
|
|
},
|
|
"max_image_dimension": {
|
|
"description": "Per-call long-edge limit for the returned screenshot, in pixels (aspect ratio preserved). Explicit values override configured behavior; 0 returns native resolution. Omit to preserve the configured default.",
|
|
"minimum": 0,
|
|
"type": "integer"
|
|
},
|
|
"pid": {
|
|
"description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.",
|
|
"type": "integer"
|
|
},
|
|
"query": {
|
|
"description": "Optional case-insensitive substring. Projects both tree_markdown and structured elements to matches plus ancestors while preserving original indices. Compare total_element_count with returned_element_count.",
|
|
"type": "string"
|
|
},
|
|
"since": {
|
|
"description": "A `snapshot_id` from an earlier get_window_state of the same window, or \"latest\" for this window's most recent read with the same query/max_elements/max_depth. Use it on every re-read after acting. Returns only what changed against that sna",
|
|
"type": "string"
|
|
},
|
|
"window_id": {
|
|
"description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.",
|
|
"type": "integer"
|
|
}
|
|
},
|
|
"required": [
|
|
"pid",
|
|
"window_id"
|
|
],
|
|
"type": "object"
|
|
},
|
|
"annotations": {
|
|
"readOnlyHint": true,
|
|
"idempotentHint": true
|
|
}
|
|
},
|
|
{
|
|
"name": "verify_state",
|
|
"description": "Read-only assertion for exact window. satisfied/unsatisfied/unknown describes the observation, not the whole task verdict.",
|
|
"inputSchema": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"expect": {
|
|
"description": "One to eight predicates, combined with logical AND.",
|
|
"items": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"element": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"enabled": {
|
|
"type": "boolean"
|
|
},
|
|
"exists": {
|
|
"description": "Assert that at least one trusted element matches the selector.\n\nElement walks are not yet exhaustive on every platform, so absence\ncannot be proven. `false` is rejected instead of returning an\nindefinitely-unknown predicate.",
|
|
"type": "boolean"
|
|
},
|
|
"selected": {
|
|
"type": "boolean"
|
|
},
|
|
"selector": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"label_contains": {
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
"role": {
|
|
"minLength": 1,
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [],
|
|
"type": "object"
|
|
},
|
|
"value_equals": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"selector"
|
|
],
|
|
"type": "object"
|
|
},
|
|
"window": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"bounds": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"height": {
|
|
"type": "number"
|
|
},
|
|
"tolerance_px": {
|
|
"maximum": 100,
|
|
"minimum": 0,
|
|
"type": "number"
|
|
},
|
|
"width": {
|
|
"type": "number"
|
|
},
|
|
"x": {
|
|
"type": "number"
|
|
},
|
|
"y": {
|
|
"type": "number"
|
|
}
|
|
},
|
|
"required": [
|
|
"x",
|
|
"y",
|
|
"width",
|
|
"height"
|
|
],
|
|
"type": "object"
|
|
},
|
|
"exists": {
|
|
"type": "boolean"
|
|
}
|
|
},
|
|
"required": [],
|
|
"type": "object"
|
|
}
|
|
},
|
|
"required": [],
|
|
"type": "object"
|
|
},
|
|
"maxItems": 8,
|
|
"minItems": 1,
|
|
"type": "array"
|
|
},
|
|
"include_screenshot": {
|
|
"description": "Return the final window screenshot as image content for a multimodal\ncaller. The driver does not interpret that image.",
|
|
"type": "boolean"
|
|
},
|
|
"pid": {
|
|
"description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.",
|
|
"minimum": 1,
|
|
"type": "integer"
|
|
},
|
|
"stable_samples": {
|
|
"default": 2,
|
|
"description": "Consecutive satisfied samples required before returning success.",
|
|
"maximum": 5,
|
|
"minimum": 1,
|
|
"type": "integer"
|
|
},
|
|
"timeout_ms": {
|
|
"default": 5000,
|
|
"description": "Bounded wait. Zero performs one sample.",
|
|
"maximum": 10000,
|
|
"minimum": 0,
|
|
"type": "integer"
|
|
},
|
|
"window_id": {
|
|
"description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.",
|
|
"type": "integer"
|
|
}
|
|
},
|
|
"required": [
|
|
"pid",
|
|
"window_id",
|
|
"expect"
|
|
],
|
|
"type": "object"
|
|
},
|
|
"annotations": {
|
|
"readOnlyHint": true,
|
|
"idempotentHint": true
|
|
}
|
|
},
|
|
{
|
|
"name": "click",
|
|
"description": "Click the latest UIA element_token. This tool has no pixel coordinates. For visual targets use click_at. Returns effect and fresh observation; old tokens become stale. For Chromium/Brave use foreground, especially after scrolling: it resolves off-screen element bounds before input. Background posted clicks may use old/off-screen positions; inspect actual evidence before retrying.",
|
|
"inputSchema": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"button": {
|
|
"description": "Mouse button. Default \"left\".",
|
|
"enum": [
|
|
"left",
|
|
"right",
|
|
"middle"
|
|
],
|
|
"type": "string"
|
|
},
|
|
"count": {
|
|
"description": "Click count — 1 (single), 2 (double), 3 (triple). Default 1.",
|
|
"maximum": 3,
|
|
"minimum": 1,
|
|
"type": "integer"
|
|
},
|
|
"delivery_mode": {
|
|
"description": "background does not activate the window. foreground uses Cua bring_to_front before input and leaves the target foreground for subsequent rendering and evidence; it does not switch back to the user's previous app after each action.",
|
|
"enum": [
|
|
"background",
|
|
"foreground"
|
|
],
|
|
"type": "string"
|
|
},
|
|
"element_token": {
|
|
"description": "Element handle `<snapshot_id>:<row>` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current sna",
|
|
"pattern": "^s[0-9a-f]{8}:[0-9]+$",
|
|
"type": "string"
|
|
},
|
|
"pid": {
|
|
"description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.",
|
|
"type": "integer"
|
|
},
|
|
"window_id": {
|
|
"description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.",
|
|
"type": "integer"
|
|
}
|
|
},
|
|
"type": "object",
|
|
"required": [
|
|
"pid",
|
|
"window_id",
|
|
"element_token"
|
|
]
|
|
},
|
|
"annotations": {
|
|
"readOnlyHint": false,
|
|
"idempotentHint": false
|
|
}
|
|
},
|
|
{
|
|
"name": "drag",
|
|
"description": "Drag between screenshot-local coordinates. Returns actual delivery effect and fresh post-action observation. Requires the latest capture_id. Stale, moved/resized or out-of-image targets are rejected before input.",
|
|
"inputSchema": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"delivery_mode": {
|
|
"description": "background does not activate the window. foreground uses Cua bring_to_front before input and leaves the target foreground for subsequent rendering and evidence; it does not switch back to the user's previous app after each action.",
|
|
"enum": [
|
|
"background",
|
|
"foreground"
|
|
],
|
|
"type": "string"
|
|
},
|
|
"duration_ms": {
|
|
"description": "Wall-clock duration of drag path. Default: 500.",
|
|
"maximum": 10000,
|
|
"minimum": 0,
|
|
"type": "integer"
|
|
},
|
|
"from_x": {
|
|
"description": "Drag-start X in window-local screenshot pixels.",
|
|
"type": "number"
|
|
},
|
|
"from_y": {
|
|
"description": "Drag-start Y in window-local screenshot pixels.",
|
|
"type": "number"
|
|
},
|
|
"pid": {
|
|
"description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.",
|
|
"type": "integer"
|
|
},
|
|
"to_x": {
|
|
"description": "Drag-end X in window-local screenshot pixels.",
|
|
"type": "number"
|
|
},
|
|
"to_y": {
|
|
"description": "Drag-end Y in window-local screenshot pixels.",
|
|
"type": "number"
|
|
},
|
|
"window_id": {
|
|
"description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.",
|
|
"type": "integer"
|
|
},
|
|
"capture_id": {
|
|
"type": "string",
|
|
"minLength": 1,
|
|
"description": "capture_id from the latest returned screenshot for this exact window. Each action returns a new screenshot; use its new ID and dimensions."
|
|
}
|
|
},
|
|
"required": [
|
|
"pid",
|
|
"window_id",
|
|
"from_x",
|
|
"from_y",
|
|
"to_x",
|
|
"to_y",
|
|
"capture_id"
|
|
],
|
|
"type": "object"
|
|
},
|
|
"annotations": {
|
|
"readOnlyHint": false,
|
|
"idempotentHint": false
|
|
}
|
|
},
|
|
{
|
|
"name": "type_text",
|
|
"description": "Type text at the current focus in the exact window. No element_token or coordinates. Focus a field with click/click_at or the address bar with Ctrl+L first; use set_value for direct UIA replacement. Chromium/Brave input requires foreground. Fresh observation follows.",
|
|
"inputSchema": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"delivery_mode": {
|
|
"description": "background does not activate the window. foreground uses Cua bring_to_front before input and leaves the target foreground for subsequent rendering and evidence; it does not switch back to the user's previous app after each action.",
|
|
"enum": [
|
|
"background",
|
|
"foreground"
|
|
],
|
|
"type": "string"
|
|
},
|
|
"pid": {
|
|
"description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.",
|
|
"type": "integer"
|
|
},
|
|
"text": {
|
|
"description": "Text to insert at the focused element's cursor.",
|
|
"type": "string"
|
|
},
|
|
"window_id": {
|
|
"description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.",
|
|
"type": "integer"
|
|
}
|
|
},
|
|
"required": [
|
|
"pid",
|
|
"window_id",
|
|
"text"
|
|
],
|
|
"type": "object"
|
|
},
|
|
"annotations": {
|
|
"readOnlyHint": false,
|
|
"idempotentHint": false
|
|
}
|
|
},
|
|
{
|
|
"name": "press_key",
|
|
"description": "Send a key and modifiers to the exact window current focus. No element_token or coordinates. For browser shortcuts (Ctrl+T, Ctrl+L, Enter), use foreground. To target a field, focus it with click/click_at first. Fresh observation follows; inspect before repeating.",
|
|
"inputSchema": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"delivery_mode": {
|
|
"description": "background does not activate the window. foreground uses Cua bring_to_front before input and leaves the target foreground for subsequent rendering and evidence; it does not switch back to the user's previous app after each action.",
|
|
"enum": [
|
|
"background",
|
|
"foreground"
|
|
],
|
|
"type": "string"
|
|
},
|
|
"key": {
|
|
"description": "Key name (return, tab, escape, up, down, left, right, space, delete, home, end, pageup, pagedown, f1-f12, letter, digit).",
|
|
"type": "string"
|
|
},
|
|
"modifiers": {
|
|
"description": "Optional modifier names held while the key is pressed (ctrl/shift/alt/win).",
|
|
"items": {
|
|
"type": "string"
|
|
},
|
|
"type": "array"
|
|
},
|
|
"pid": {
|
|
"description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.",
|
|
"type": "integer"
|
|
},
|
|
"window_id": {
|
|
"description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.",
|
|
"type": "integer"
|
|
}
|
|
},
|
|
"required": [
|
|
"pid",
|
|
"window_id",
|
|
"key"
|
|
],
|
|
"type": "object"
|
|
},
|
|
"annotations": {
|
|
"readOnlyHint": false,
|
|
"idempotentHint": false
|
|
}
|
|
},
|
|
{
|
|
"name": "set_value",
|
|
"description": "Set the latest UIA element_token value. Returns delivery effect/readback and fresh screenshot/tree.",
|
|
"inputSchema": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"delivery_mode": {
|
|
"description": "background does not activate the window. foreground uses Cua bring_to_front before input and leaves the target foreground for subsequent rendering and evidence; it does not switch back to the user's previous app after each action.",
|
|
"enum": [
|
|
"background",
|
|
"foreground"
|
|
],
|
|
"type": "string"
|
|
},
|
|
"element_token": {
|
|
"description": "Element handle `<snapshot_id>:<row>` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current sna",
|
|
"pattern": "^s[0-9a-f]{8}:[0-9]+$",
|
|
"type": "string"
|
|
},
|
|
"pid": {
|
|
"description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.",
|
|
"type": "integer"
|
|
},
|
|
"value": {
|
|
"description": "New value. UIA will coerce to the element's native type.",
|
|
"type": "string"
|
|
},
|
|
"window_id": {
|
|
"description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.",
|
|
"type": "integer"
|
|
}
|
|
},
|
|
"required": [
|
|
"pid",
|
|
"window_id",
|
|
"value",
|
|
"element_token"
|
|
],
|
|
"type": "object"
|
|
},
|
|
"annotations": {
|
|
"readOnlyHint": false,
|
|
"idempotentHint": false
|
|
}
|
|
},
|
|
{
|
|
"name": "scroll",
|
|
"description": "Scroll at x,y in the latest window screenshot. No element_token. Use a point inside the intended scroll surface, amount=1..50, by=line/page. Chromium/Brave requires foreground. Verify actual content movement in the returned observation. Requires the latest capture_id. Stale, moved/resized or out-of-image targets are rejected before input.",
|
|
"inputSchema": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"amount": {
|
|
"description": "Number of scroll ticks. Default 3.",
|
|
"maximum": 50,
|
|
"minimum": 1,
|
|
"type": "integer"
|
|
},
|
|
"by": {
|
|
"description": "Scroll granularity. Default: line.",
|
|
"enum": [
|
|
"line",
|
|
"page"
|
|
],
|
|
"type": "string"
|
|
},
|
|
"delivery_mode": {
|
|
"description": "background does not activate the window. foreground uses Cua bring_to_front before input and leaves the target foreground for subsequent rendering and evidence; it does not switch back to the user's previous app after each action.",
|
|
"enum": [
|
|
"background",
|
|
"foreground"
|
|
],
|
|
"type": "string"
|
|
},
|
|
"direction": {
|
|
"description": "Scroll direction.",
|
|
"enum": [
|
|
"up",
|
|
"down",
|
|
"left",
|
|
"right"
|
|
],
|
|
"type": "string"
|
|
},
|
|
"pid": {
|
|
"description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.",
|
|
"type": "integer"
|
|
},
|
|
"window_id": {
|
|
"description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.",
|
|
"type": "integer"
|
|
},
|
|
"x": {
|
|
"description": "X inside the scroll area in latest window-local screenshot pixels; never screen coordinates. Must be paired with the other coordinate.",
|
|
"type": "number"
|
|
},
|
|
"y": {
|
|
"description": "Y inside the scroll area in latest window-local screenshot pixels; never screen coordinates. Must be paired with the other coordinate.",
|
|
"type": "number"
|
|
},
|
|
"capture_id": {
|
|
"type": "string",
|
|
"minLength": 1,
|
|
"description": "capture_id from the latest returned screenshot for this exact window. Each action returns a new screenshot; use its new ID and dimensions."
|
|
}
|
|
},
|
|
"required": [
|
|
"pid",
|
|
"window_id",
|
|
"direction",
|
|
"x",
|
|
"y",
|
|
"capture_id"
|
|
],
|
|
"type": "object"
|
|
},
|
|
"annotations": {
|
|
"readOnlyHint": false,
|
|
"idempotentHint": false
|
|
}
|
|
},
|
|
{
|
|
"name": "save_screenshot",
|
|
"description": "Save fresh window screenshot to this task trace. Plain .png filename only; no overwrite. Returns verified attachment hash, screenshot and latest tree.",
|
|
"inputSchema": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"pid": {
|
|
"type": "integer",
|
|
"minimum": 1,
|
|
"description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID."
|
|
},
|
|
"window_id": {
|
|
"type": "integer",
|
|
"minimum": 1,
|
|
"description": "Exact HWND returned for this application by list_windows. Required and must belong to pid."
|
|
},
|
|
"filename": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"pid",
|
|
"window_id",
|
|
"filename"
|
|
]
|
|
},
|
|
"annotations": {
|
|
"readOnlyHint": false,
|
|
"idempotentHint": false
|
|
}
|
|
},
|
|
{
|
|
"name": "click_at",
|
|
"description": "Click a visual target at x,y in the latest window screenshot, with its exact capture_id. Never pass an element token. Returns effect and fresh observation. Use the latest returned image dimensions; no screen coordinates or Android normalization.",
|
|
"inputSchema": {
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"button": {
|
|
"description": "Mouse button. Default \"left\".",
|
|
"enum": [
|
|
"left",
|
|
"right",
|
|
"middle"
|
|
],
|
|
"type": "string"
|
|
},
|
|
"capture_id": {
|
|
"description": "Optional ID from get_window_state or get_desktop_state. When present, x and y are admitted against that exact capture and the ID is consumed before native dispatch.",
|
|
"minLength": 1,
|
|
"type": "string"
|
|
},
|
|
"count": {
|
|
"description": "Click count — 1 (single), 2 (double), 3 (triple). Default 1.",
|
|
"maximum": 3,
|
|
"minimum": 1,
|
|
"type": "integer"
|
|
},
|
|
"delivery_mode": {
|
|
"description": "background does not activate the window. foreground uses Cua bring_to_front before input and leaves the target foreground for subsequent rendering and evidence; it does not switch back to the user's previous app after each action.",
|
|
"enum": [
|
|
"background",
|
|
"foreground"
|
|
],
|
|
"type": "string"
|
|
},
|
|
"pid": {
|
|
"description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.",
|
|
"type": "integer"
|
|
},
|
|
"window_id": {
|
|
"description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.",
|
|
"type": "integer"
|
|
},
|
|
"x": {
|
|
"description": "X in window-local screenshot pixels — same space as the PNG get_window_state returns. Must be provided together with y.",
|
|
"type": "number"
|
|
},
|
|
"y": {
|
|
"description": "Y in window-local screenshot pixels. Must be provided together with x.",
|
|
"type": "number"
|
|
}
|
|
},
|
|
"type": "object",
|
|
"required": [
|
|
"pid",
|
|
"window_id",
|
|
"x",
|
|
"y",
|
|
"capture_id"
|
|
]
|
|
},
|
|
"annotations": {
|
|
"readOnlyHint": false,
|
|
"idempotentHint": false
|
|
}
|
|
},
|
|
{
|
|
"name": "bring_to_front",
|
|
"description": "Activate the exact authorized window and leave it foreground across observations and actions. Use when a background browser body is blank or its page tree is missing. This changes the user foreground; it does not navigate, refresh, or repeat any business action. Read the returned fresh screenshot before continuing.",
|
|
"inputSchema": {
|
|
"type": "object",
|
|
"required": [
|
|
"pid",
|
|
"window_id"
|
|
],
|
|
"properties": {
|
|
"pid": {
|
|
"type": "integer",
|
|
"minimum": 1,
|
|
"maximum": 4294967295
|
|
},
|
|
"window_id": {
|
|
"type": "integer",
|
|
"minimum": 1
|
|
}
|
|
},
|
|
"additionalProperties": false
|
|
},
|
|
"annotations": {
|
|
"readOnlyHint": false,
|
|
"destructiveHint": false,
|
|
"idempotentHint": true,
|
|
"openWorldHint": false
|
|
}
|
|
}
|
|
]
|