[ { "name": "list_windows", "description": "Find the authorized application window. If its PID is unknown, call with app only, e.g. {\"app\":\"Brave\",\"on_screen_only\":true}; omit pid or use null, never 0. Use the returned matching pid/window_id for later actions. Empty filtered results are not proof that the app is unavailable; follow lookup.suggested_arguments when present.", "inputSchema": { "additionalProperties": false, "properties": { "app": { "description": "Application name substring (case-insensitive). Start with this when the target PID is unknown. app and a non-null pid are combined filters; do not copy the PID of an unrelated foreground app.", "type": [ "string", "null" ] }, "on_screen_only": { "description": "When true, drop windows that aren't currently on-screen. Default false.", "type": "boolean" }, "pid": { "description": "Known positive Windows process ID from a matching list_windows result. Unknown: omit or null; never 0. With app, both filters must match the same window.", "type": [ "integer", "null" ], "minimum": 1, "maximum": 4294967295 } }, "type": "object", "required": [] }, "annotations": { "readOnlyHint": true, "idempotentHint": true } }, { "name": "get_window_state", "description": "Read fresh window screenshot and compact UIA tree. Every [N] uses element_token=:N; a new snapshot invalidates old tokens. Pixel x/y are screenshot-local; use capture_id. since=latest returns changes. Keep the new snapshot_id.", "inputSchema": { "additionalProperties": false, "properties": { "max_elements": { "description": "Cap on the total number of UIA nodes walked. Truncates depth-first; markdown and structured elements truncate together. Default 250 (5 000 with full_output:true); the response states when the tree was cut. Raise it to read further into a la", "minimum": 1, "type": "integer" }, "max_image_dimension": { "description": "Per-call long-edge limit for the returned screenshot, in pixels (aspect ratio preserved). Explicit values override configured behavior; 0 returns native resolution. Omit to preserve the configured default.", "minimum": 0, "type": "integer" }, "pid": { "description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.", "type": "integer" }, "query": { "description": "Optional case-insensitive substring. Projects both tree_markdown and structured elements to matches plus ancestors while preserving original indices. Compare total_element_count with returned_element_count.", "type": "string" }, "since": { "description": "A `snapshot_id` from an earlier get_window_state of the same window, or \"latest\" for this window's most recent read with the same query/max_elements/max_depth. Use it on every re-read after acting. Returns only what changed against that sna", "type": "string" }, "window_id": { "description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.", "type": "integer" } }, "required": [ "pid", "window_id" ], "type": "object" }, "annotations": { "readOnlyHint": true, "idempotentHint": true } }, { "name": "verify_state", "description": "Read-only assertion for exact window. satisfied/unsatisfied/unknown describes the observation, not the whole task verdict.", "inputSchema": { "additionalProperties": false, "properties": { "expect": { "description": "One to eight predicates, combined with logical AND.", "items": { "additionalProperties": false, "properties": { "element": { "additionalProperties": false, "properties": { "enabled": { "type": "boolean" }, "exists": { "description": "Assert that at least one trusted element matches the selector.\n\nElement walks are not yet exhaustive on every platform, so absence\ncannot be proven. `false` is rejected instead of returning an\nindefinitely-unknown predicate.", "type": "boolean" }, "selected": { "type": "boolean" }, "selector": { "additionalProperties": false, "properties": { "label_contains": { "minLength": 1, "type": "string" }, "role": { "minLength": 1, "type": "string" } }, "required": [], "type": "object" }, "value_equals": { "type": "string" } }, "required": [ "selector" ], "type": "object" }, "window": { "additionalProperties": false, "properties": { "bounds": { "additionalProperties": false, "properties": { "height": { "type": "number" }, "tolerance_px": { "maximum": 100, "minimum": 0, "type": "number" }, "width": { "type": "number" }, "x": { "type": "number" }, "y": { "type": "number" } }, "required": [ "x", "y", "width", "height" ], "type": "object" }, "exists": { "type": "boolean" } }, "required": [], "type": "object" } }, "required": [], "type": "object" }, "maxItems": 8, "minItems": 1, "type": "array" }, "include_screenshot": { "description": "Return the final window screenshot as image content for a multimodal\ncaller. The driver does not interpret that image.", "type": "boolean" }, "pid": { "description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.", "minimum": 1, "type": "integer" }, "stable_samples": { "default": 2, "description": "Consecutive satisfied samples required before returning success.", "maximum": 5, "minimum": 1, "type": "integer" }, "timeout_ms": { "default": 5000, "description": "Bounded wait. Zero performs one sample.", "maximum": 10000, "minimum": 0, "type": "integer" }, "window_id": { "description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.", "type": "integer" } }, "required": [ "pid", "window_id", "expect" ], "type": "object" }, "annotations": { "readOnlyHint": true, "idempotentHint": true } }, { "name": "click", "description": "Click the latest UIA element_token. This tool has no pixel coordinates. For visual targets use click_at. Returns effect and fresh observation; old tokens become stale. For Chromium/Brave use foreground, especially after scrolling: it resolves off-screen element bounds before input. Background posted clicks may use old/off-screen positions; inspect actual evidence before retrying.", "inputSchema": { "additionalProperties": false, "properties": { "button": { "description": "Mouse button. Default \"left\".", "enum": [ "left", "right", "middle" ], "type": "string" }, "count": { "description": "Click count — 1 (single), 2 (double), 3 (triple). Default 1.", "maximum": 3, "minimum": 1, "type": "integer" }, "delivery_mode": { "description": "background (default) avoids raising the window; unsupported input is refused. Use foreground for Chromium/Brave keyboard input and scrolling within task scope. Inspect the returned state before retrying an uncertain action.", "enum": [ "background", "foreground" ], "type": "string" }, "element_token": { "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current sna", "pattern": "^s[0-9a-f]{8}:[0-9]+$", "type": "string" }, "pid": { "description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.", "type": "integer" }, "window_id": { "description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.", "type": "integer" } }, "type": "object", "required": [ "pid", "window_id", "element_token" ] }, "annotations": { "readOnlyHint": false, "idempotentHint": false } }, { "name": "drag", "description": "Drag between screenshot-local coordinates. Returns actual delivery effect and fresh post-action observation. Requires the latest capture_id. Stale, moved/resized or out-of-image targets are rejected before input.", "inputSchema": { "additionalProperties": false, "properties": { "delivery_mode": { "description": "background (default) avoids raising the window; unsupported input is refused. Use foreground for Chromium/Brave keyboard input and scrolling within task scope. Inspect the returned state before retrying an uncertain action.", "enum": [ "background", "foreground" ], "type": "string" }, "duration_ms": { "description": "Wall-clock duration of drag path. Default: 500.", "maximum": 10000, "minimum": 0, "type": "integer" }, "from_x": { "description": "Drag-start X in window-local screenshot pixels.", "type": "number" }, "from_y": { "description": "Drag-start Y in window-local screenshot pixels.", "type": "number" }, "pid": { "description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.", "type": "integer" }, "to_x": { "description": "Drag-end X in window-local screenshot pixels.", "type": "number" }, "to_y": { "description": "Drag-end Y in window-local screenshot pixels.", "type": "number" }, "window_id": { "description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.", "type": "integer" }, "capture_id": { "type": "string", "minLength": 1, "description": "capture_id from the latest returned screenshot for this exact window. Each action returns a new screenshot; use its new ID and dimensions." } }, "required": [ "pid", "window_id", "from_x", "from_y", "to_x", "to_y", "capture_id" ], "type": "object" }, "annotations": { "readOnlyHint": false, "idempotentHint": false } }, { "name": "type_text", "description": "Type text at the current focus in the exact window. No element_token or coordinates. Focus a field with click/click_at or the address bar with Ctrl+L first; use set_value for direct UIA replacement. Chromium/Brave input requires foreground. Fresh observation follows.", "inputSchema": { "additionalProperties": false, "properties": { "delivery_mode": { "description": "background (default) avoids raising the window; unsupported input is refused. Use foreground for Chromium/Brave keyboard input and scrolling within task scope. Inspect the returned state before retrying an uncertain action.", "enum": [ "background", "foreground" ], "type": "string" }, "pid": { "description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.", "type": "integer" }, "text": { "description": "Text to insert at the focused element's cursor.", "type": "string" }, "window_id": { "description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.", "type": "integer" } }, "required": [ "pid", "window_id", "text" ], "type": "object" }, "annotations": { "readOnlyHint": false, "idempotentHint": false } }, { "name": "press_key", "description": "Send a key and modifiers to the exact window current focus. No element_token or coordinates. For browser shortcuts (Ctrl+T, Ctrl+L, Enter), use foreground. To target a field, focus it with click/click_at first. Fresh observation follows; inspect before repeating.", "inputSchema": { "additionalProperties": false, "properties": { "delivery_mode": { "description": "background (default) avoids raising the window; unsupported input is refused. Use foreground for Chromium/Brave keyboard input and scrolling within task scope. Inspect the returned state before retrying an uncertain action.", "enum": [ "background", "foreground" ], "type": "string" }, "key": { "description": "Key name (return, tab, escape, up, down, left, right, space, delete, home, end, pageup, pagedown, f1-f12, letter, digit).", "type": "string" }, "modifiers": { "description": "Optional modifier names held while the key is pressed (ctrl/shift/alt/win).", "items": { "type": "string" }, "type": "array" }, "pid": { "description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.", "type": "integer" }, "window_id": { "description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.", "type": "integer" } }, "required": [ "pid", "window_id", "key" ], "type": "object" }, "annotations": { "readOnlyHint": false, "idempotentHint": false } }, { "name": "set_value", "description": "Set the latest UIA element_token value. Returns delivery effect/readback and fresh screenshot/tree.", "inputSchema": { "additionalProperties": false, "properties": { "delivery_mode": { "description": "background (default) avoids raising the window; unsupported input is refused. Use foreground for Chromium/Brave keyboard input and scrolling within task scope. Inspect the returned state before retrying an uncertain action.", "enum": [ "background", "foreground" ], "type": "string" }, "element_token": { "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current sna", "pattern": "^s[0-9a-f]{8}:[0-9]+$", "type": "string" }, "pid": { "description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.", "type": "integer" }, "value": { "description": "New value. UIA will coerce to the element's native type.", "type": "string" }, "window_id": { "description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.", "type": "integer" } }, "required": [ "pid", "window_id", "value", "element_token" ], "type": "object" }, "annotations": { "readOnlyHint": false, "idempotentHint": false } }, { "name": "scroll", "description": "Scroll at x,y in the latest window screenshot. No element_token. Use a point inside the intended scroll surface, amount=1..50, by=line/page. Chromium/Brave requires foreground. Verify actual content movement in the returned observation. Requires the latest capture_id. Stale, moved/resized or out-of-image targets are rejected before input.", "inputSchema": { "additionalProperties": false, "properties": { "amount": { "description": "Number of scroll ticks. Default 3.", "maximum": 50, "minimum": 1, "type": "integer" }, "by": { "description": "Scroll granularity. Default: line.", "enum": [ "line", "page" ], "type": "string" }, "delivery_mode": { "description": "background (default) avoids raising the window; unsupported input is refused. Use foreground for Chromium/Brave keyboard input and scrolling within task scope. Inspect the returned state before retrying an uncertain action.", "enum": [ "background", "foreground" ], "type": "string" }, "direction": { "description": "Scroll direction.", "enum": [ "up", "down", "left", "right" ], "type": "string" }, "pid": { "description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.", "type": "integer" }, "window_id": { "description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.", "type": "integer" }, "x": { "description": "X inside the scroll area in latest window-local screenshot pixels; never screen coordinates. Must be paired with the other coordinate.", "type": "number" }, "y": { "description": "Y inside the scroll area in latest window-local screenshot pixels; never screen coordinates. Must be paired with the other coordinate.", "type": "number" }, "capture_id": { "type": "string", "minLength": 1, "description": "capture_id from the latest returned screenshot for this exact window. Each action returns a new screenshot; use its new ID and dimensions." } }, "required": [ "pid", "window_id", "direction", "x", "y", "capture_id" ], "type": "object" }, "annotations": { "readOnlyHint": false, "idempotentHint": false } }, { "name": "save_screenshot", "description": "Save fresh window screenshot to this task trace. Plain .png filename only; no overwrite. Returns verified attachment hash, screenshot and latest tree.", "inputSchema": { "type": "object", "additionalProperties": false, "properties": { "pid": { "type": "integer", "minimum": 1, "description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID." }, "window_id": { "type": "integer", "minimum": 1, "description": "Exact HWND returned for this application by list_windows. Required and must belong to pid." }, "filename": { "type": "string" } }, "required": [ "pid", "window_id", "filename" ] }, "annotations": { "readOnlyHint": false, "idempotentHint": false } }, { "name": "click_at", "description": "Click a visual target at x,y in the latest window screenshot, with its exact capture_id. Never pass an element token. Returns effect and fresh observation. Use the latest returned image dimensions; no screen coordinates or Android normalization.", "inputSchema": { "additionalProperties": false, "properties": { "button": { "description": "Mouse button. Default \"left\".", "enum": [ "left", "right", "middle" ], "type": "string" }, "capture_id": { "description": "Optional ID from get_window_state or get_desktop_state. When present, x and y are admitted against that exact capture and the ID is consumed before native dispatch.", "minLength": 1, "type": "string" }, "count": { "description": "Click count — 1 (single), 2 (double), 3 (triple). Default 1.", "maximum": 3, "minimum": 1, "type": "integer" }, "delivery_mode": { "description": "background (default) avoids raising the window; unsupported input is refused. Use foreground for Chromium/Brave keyboard input and scrolling within task scope. Inspect the returned state before retrying an uncertain action.", "enum": [ "background", "foreground" ], "type": "string" }, "pid": { "description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.", "type": "integer" }, "window_id": { "description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.", "type": "integer" }, "x": { "description": "X in window-local screenshot pixels — same space as the PNG get_window_state returns. Must be provided together with y.", "type": "number" }, "y": { "description": "Y in window-local screenshot pixels. Must be provided together with x.", "type": "number" } }, "type": "object", "required": [ "pid", "window_id", "x", "y", "capture_id" ] }, "annotations": { "readOnlyHint": false, "idempotentHint": false } } ]