[ { "name": "list_windows", "description": "List visible application windows with exact pid/window_id and physical bounds. Pick only the user-authorized target.", "inputSchema": { "additionalProperties": false, "properties": { "app": { "description": "Only windows of apps whose name contains this text (case-insensitive), or whose bundle id equals it. Prefer this or `pid` over an unfiltered call, which returns every window on the host.", "type": "string" }, "on_screen_only": { "description": "When true, drop windows that aren't currently on-screen. Default false.", "type": "boolean" }, "pid": { "description": "Optional pid filter. When set, only this pid's windows are returned.", "type": "integer" } }, "type": "object", "required": [] }, "annotations": { "readOnlyHint": true, "idempotentHint": true } }, { "name": "get_window_state", "description": "Read fresh window screenshot and compact UIA tree. Every [N] uses element_token=:N; a new snapshot invalidates old tokens. Pixel x/y are screenshot-local; use capture_id. since=latest returns changes. Keep the new snapshot_id.", "inputSchema": { "additionalProperties": false, "properties": { "max_elements": { "description": "Cap on the total number of UIA nodes walked. Truncates depth-first; markdown and structured elements truncate together. Default 250 (5 000 with full_output:true); the response states when the tree was cut. Raise it to read further into a la", "minimum": 1, "type": "integer" }, "max_image_dimension": { "description": "Per-call long-edge limit for the returned screenshot, in pixels (aspect ratio preserved). Explicit values override configured behavior; 0 returns native resolution. Omit to preserve the configured default.", "minimum": 0, "type": "integer" }, "pid": { "description": "Process ID from `list_apps`.", "type": "integer" }, "query": { "description": "Optional case-insensitive substring. Projects both tree_markdown and structured elements to matches plus ancestors while preserving original indices. Compare total_element_count with returned_element_count.", "type": "string" }, "since": { "description": "A `snapshot_id` from an earlier get_window_state of the same window, or \"latest\" for this window's most recent read with the same query/max_elements/max_depth. Use it on every re-read after acting. Returns only what changed against that sna", "type": "string" }, "window_id": { "description": "HWND of the target window. Must belong to `pid`. Enumerate via `list_windows` or read from `launch_app`'s `windows` array.", "type": "integer" } }, "required": [ "pid", "window_id" ], "type": "object" }, "annotations": { "readOnlyHint": true, "idempotentHint": true } }, { "name": "verify_state", "description": "Read-only assertion for exact window. satisfied/unsatisfied/unknown describes the observation, not the whole task verdict.", "inputSchema": { "additionalProperties": false, "properties": { "expect": { "description": "One to eight predicates, combined with logical AND.", "items": { "additionalProperties": false, "properties": { "element": { "additionalProperties": false, "properties": { "enabled": { "type": "boolean" }, "exists": { "description": "Assert that at least one trusted element matches the selector.\n\nElement walks are not yet exhaustive on every platform, so absence\ncannot be proven. `false` is rejected instead of returning an\nindefinitely-unknown predicate.", "type": "boolean" }, "selected": { "type": "boolean" }, "selector": { "additionalProperties": false, "properties": { "label_contains": { "minLength": 1, "type": "string" }, "role": { "minLength": 1, "type": "string" } }, "required": [], "type": "object" }, "value_equals": { "type": "string" } }, "required": [ "selector" ], "type": "object" }, "window": { "additionalProperties": false, "properties": { "bounds": { "additionalProperties": false, "properties": { "height": { "type": "number" }, "tolerance_px": { "maximum": 100, "minimum": 0, "type": "number" }, "width": { "type": "number" }, "x": { "type": "number" }, "y": { "type": "number" } }, "required": [ "x", "y", "width", "height" ], "type": "object" }, "exists": { "type": "boolean" } }, "required": [], "type": "object" } }, "required": [], "type": "object" }, "maxItems": 8, "minItems": 1, "type": "array" }, "include_screenshot": { "description": "Return the final window screenshot as image content for a multimodal\ncaller. The driver does not interpret that image.", "type": "boolean" }, "pid": { "description": "Exact process whose window may be observed.", "minimum": 1, "type": "integer" }, "stable_samples": { "default": 2, "description": "Consecutive satisfied samples required before returning success.", "maximum": 5, "minimum": 1, "type": "integer" }, "timeout_ms": { "default": 5000, "description": "Bounded wait. Zero performs one sample.", "maximum": 10000, "minimum": 0, "type": "integer" }, "window_id": { "description": "Exact native window identifier.", "type": "integer" } }, "required": [ "pid", "window_id", "expect" ], "type": "object" }, "annotations": { "readOnlyHint": true, "idempotentHint": true } }, { "name": "click", "description": "Click the latest UIA element_token. This tool has no pixel coordinates. For visual targets use click_at. Returns effect and fresh observation; old tokens become stale.", "inputSchema": { "additionalProperties": false, "properties": { "button": { "description": "Mouse button. Default \"left\".", "enum": [ "left", "right", "middle" ], "type": "string" }, "count": { "description": "Click count — 1 (single), 2 (double), 3 (triple). Default 1.", "maximum": 3, "minimum": 1, "type": "integer" }, "delivery_mode": { "description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button", "enum": [ "background", "foreground" ], "type": "string" }, "element_token": { "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current sna", "pattern": "^s[0-9a-f]{8}:[0-9]+$", "type": "string" }, "pid": { "description": "Target process ID for window scope. Optional with element_token (the token carries the pid). Omit with scope=desktop for screen-absolute coordinates from get_desktop_state.", "type": "integer" }, "window_id": { "description": "HWND of the target window. Omit when element_token is supplied (the token carries it).", "type": "integer" } }, "type": "object", "required": [ "pid", "window_id", "element_token" ] }, "annotations": { "readOnlyHint": false, "idempotentHint": false } }, { "name": "drag", "description": "Drag between screenshot-local coordinates. Returns actual delivery effect and fresh post-action observation.", "inputSchema": { "additionalProperties": false, "properties": { "delivery_mode": { "description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button", "enum": [ "background", "foreground" ], "type": "string" }, "duration_ms": { "description": "Wall-clock duration of drag path. Default: 500.", "maximum": 10000, "minimum": 0, "type": "integer" }, "from_x": { "description": "Drag-start X in window-local screenshot pixels.", "type": "number" }, "from_y": { "description": "Drag-start Y in window-local screenshot pixels.", "type": "number" }, "pid": { "description": "Target process ID.", "type": "integer" }, "to_x": { "description": "Drag-end X in window-local screenshot pixels.", "type": "number" }, "to_y": { "description": "Drag-end Y in window-local screenshot pixels.", "type": "number" }, "window_id": { "description": "Target window handle (HWND). Optional — driver picks frontmost window of pid when omitted.", "type": "integer" } }, "required": [ "pid", "window_id", "from_x", "from_y", "to_x", "to_y" ], "type": "object" }, "annotations": { "readOnlyHint": false, "idempotentHint": false } }, { "name": "type_text", "description": "Type text into the focused target window or specified latest element. Fresh observation follows. If background is unsupported, explicitly choose foreground within task scope.", "inputSchema": { "additionalProperties": false, "properties": { "delivery_mode": { "description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button", "enum": [ "background", "foreground" ], "type": "string" }, "element_token": { "description": "Opaque element handle from get_window_state. When supplied, type_text writes through UIA ValuePattern and reads that exact element back by handle. The shared ActionResult is confirmed only when the complete expected value is synchronously v", "pattern": "^s[0-9a-f]{8}:[0-9]+$", "type": "string" }, "pid": { "description": "Target process ID.", "type": "integer" }, "text": { "description": "Text to insert at the focused element's cursor.", "type": "string" }, "window_id": { "description": "HWND of the target window. Omit when element_token is supplied (the token carries it).", "type": "integer" }, "x": { "description": "Window-local screenshot-pixel X of the field to type into — the element px action form. Pass x,y (no element_token) and the tool pixel-clicks there to establish real renderer focus, then types. Use for Chromium/Electron inputs the UIA/WM_CH", "type": "number" }, "y": { "description": "Window-local screenshot-pixel Y of the field (see x).", "type": "number" } }, "required": [ "pid", "window_id", "text" ], "type": "object" }, "annotations": { "readOnlyHint": false, "idempotentHint": false } }, { "name": "press_key", "description": "Send key and modifiers to the exact window, optionally focus element_token or x/y first. Fresh observation follows.", "inputSchema": { "additionalProperties": false, "properties": { "delivery_mode": { "description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button", "enum": [ "background", "foreground" ], "type": "string" }, "element_token": { "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current sna", "pattern": "^s[0-9a-f]{8}:[0-9]+$", "type": "string" }, "key": { "description": "Key name (return, tab, escape, up, down, left, right, space, delete, home, end, pageup, pagedown, f1-f12, letter, digit).", "type": "string" }, "modifiers": { "description": "Optional modifier names held while the key is pressed (ctrl/shift/alt/win).", "items": { "type": "string" }, "type": "array" }, "pid": { "description": "Target process ID.", "type": "integer" }, "window_id": { "description": "HWND for the target window. Omit when element_token is supplied; otherwise auto-resolves the pid's first visible window.", "type": "integer" }, "x": { "description": "Window-local screenshot-pixel X — the element px action form: pixel-click there to focus, then send the key. Use when the key must go to a Chromium/Electron surface the UIA path can't focus. Pass with y, no element_token.", "type": "number" }, "y": { "description": "Window-local screenshot-pixel Y (see x).", "type": "number" } }, "required": [ "pid", "window_id", "key" ], "type": "object" }, "annotations": { "readOnlyHint": false, "idempotentHint": false } }, { "name": "set_value", "description": "Set the latest UIA element_token value. Returns delivery effect/readback and fresh screenshot/tree.", "inputSchema": { "additionalProperties": false, "properties": { "delivery_mode": { "description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button", "enum": [ "background", "foreground" ], "type": "string" }, "element_token": { "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current sna", "pattern": "^s[0-9a-f]{8}:[0-9]+$", "type": "string" }, "pid": { "description": "Target process ID. Optional with element_token (the token carries the pid).", "type": "integer" }, "value": { "description": "New value. UIA will coerce to the element's native type.", "type": "string" }, "window_id": { "description": "HWND of the window. Omit when element_token is supplied (the token carries it).", "type": "integer" } }, "required": [ "pid", "window_id", "value" ], "type": "object" }, "annotations": { "readOnlyHint": false, "idempotentHint": false } }, { "name": "scroll", "description": "Scroll the target window/element at x,y. amount=1..50, by=line/page. Verify content changed in fresh returned observation.", "inputSchema": { "additionalProperties": false, "properties": { "amount": { "description": "Number of scroll ticks. Default 3.", "maximum": 50, "minimum": 1, "type": "integer" }, "by": { "description": "Scroll granularity. Default: line.", "enum": [ "line", "page" ], "type": "string" }, "delivery_mode": { "description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button", "enum": [ "background", "foreground" ], "type": "string" }, "direction": { "description": "Scroll direction.", "enum": [ "up", "down", "left", "right" ], "type": "string" }, "element_token": { "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current sna", "pattern": "^s[0-9a-f]{8}:[0-9]+$", "type": "string" }, "pid": { "description": "Target process ID for window scope. Optional with element_token (the token carries the pid). Omit with scope=desktop for screen-absolute coordinates from get_desktop_state.", "type": "integer" }, "window_id": { "description": "HWND of the target window. Omit when element_token is supplied; otherwise auto-resolves the pid's first visible window.", "type": "integer" }, "x": { "description": "With pid/window_id: window-local screenshot X used to target a nested scroll surface in foreground mode. Without pid/window_id: screen-absolute X for desktop scope. Must be paired with y.", "type": "number" }, "y": { "description": "With pid/window_id: window-local screenshot Y used to target a nested scroll surface in foreground mode. Without pid/window_id: screen-absolute Y for desktop scope. Must be paired with x.", "type": "number" } }, "required": [ "pid", "window_id", "direction" ], "type": "object" }, "annotations": { "readOnlyHint": false, "idempotentHint": false } }, { "name": "save_screenshot", "description": "Save fresh window screenshot to this task trace. Plain .png filename only; no overwrite. Returns verified attachment hash, screenshot and latest tree.", "inputSchema": { "type": "object", "additionalProperties": false, "properties": { "pid": { "type": "integer", "minimum": 1 }, "window_id": { "type": "integer", "minimum": 1 }, "filename": { "type": "string" } }, "required": [ "pid", "window_id", "filename" ] }, "annotations": { "readOnlyHint": false, "idempotentHint": false } }, { "name": "click_at", "description": "Click a visual target at x,y in the latest window screenshot, with its exact capture_id. Never pass an element token. Returns effect and fresh observation.", "inputSchema": { "additionalProperties": false, "properties": { "button": { "description": "Mouse button. Default \"left\".", "enum": [ "left", "right", "middle" ], "type": "string" }, "capture_id": { "description": "Optional ID from get_window_state or get_desktop_state. When present, x and y are admitted against that exact capture and the ID is consumed before native dispatch.", "minLength": 1, "type": "string" }, "count": { "description": "Click count — 1 (single), 2 (double), 3 (triple). Default 1.", "maximum": 3, "minimum": 1, "type": "integer" }, "delivery_mode": { "description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button", "enum": [ "background", "foreground" ], "type": "string" }, "pid": { "description": "Target process ID for window scope. Optional with element_token (the token carries the pid). Omit with scope=desktop for screen-absolute coordinates from get_desktop_state.", "type": "integer" }, "window_id": { "description": "HWND of the target window. Omit when element_token is supplied (the token carries it).", "type": "integer" }, "x": { "description": "X in window-local screenshot pixels — same space as the PNG get_window_state returns. Must be provided together with y.", "type": "number" }, "y": { "description": "Y in window-local screenshot pixels. Must be provided together with x.", "type": "number" } }, "type": "object", "required": [ "pid", "window_id", "x", "y", "capture_id" ] }, "annotations": { "readOnlyHint": false, "idempotentHint": false } } ]