fix: separate Cua keyboard targeting and verify browser actions
This commit is contained in:
chenmingxuan committed
1 parent
1c1d700e9b
commit
9c98b6bf8a
10 files changed
+591
-69
No files matched your search
@@ -63,7 +63,7 @@
|
||||
"type": "string"
|
||||
},
|
||||
"window_id": {
|
||||
"description": "HWND of the target window. Must belong to `pid`. Enumerate via `list_windows` or read from `launch_app`'s `windows` array.",
|
||||
"description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
@@ -197,7 +197,7 @@
|
||||
"type": "integer"
|
||||
},
|
||||
"window_id": {
|
||||
"description": "Exact native window identifier.",
|
||||
"description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
@@ -215,7 +215,7 @@
|
||||
},
|
||||
{
|
||||
"name": "click",
|
||||
"description": "Click the latest UIA element_token. This tool has no pixel coordinates. For visual targets use click_at. Returns effect and fresh observation; old tokens become stale.",
|
||||
"description": "Click the latest UIA element_token. This tool has no pixel coordinates. For visual targets use click_at. Returns effect and fresh observation; old tokens become stale. For Chromium/Brave use foreground, especially after scrolling: it resolves off-screen element bounds before input. Background posted clicks may use old/off-screen positions; inspect actual evidence before retrying.",
|
||||
"inputSchema": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
@@ -235,7 +235,7 @@
|
||||
"type": "integer"
|
||||
},
|
||||
"delivery_mode": {
|
||||
"description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button",
|
||||
"description": "background (default) avoids raising the window; unsupported input is refused. Use foreground for Chromium/Brave keyboard input and scrolling within task scope. Inspect the returned state before retrying an uncertain action.",
|
||||
"enum": [
|
||||
"background",
|
||||
"foreground"
|
||||
@@ -252,7 +252,7 @@
|
||||
"type": "integer"
|
||||
},
|
||||
"window_id": {
|
||||
"description": "HWND of the target window. Omit when element_token is supplied (the token carries it).",
|
||||
"description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
@@ -275,7 +275,7 @@
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"delivery_mode": {
|
||||
"description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button",
|
||||
"description": "background (default) avoids raising the window; unsupported input is refused. Use foreground for Chromium/Brave keyboard input and scrolling within task scope. Inspect the returned state before retrying an uncertain action.",
|
||||
"enum": [
|
||||
"background",
|
||||
"foreground"
|
||||
@@ -309,7 +309,7 @@
|
||||
"type": "number"
|
||||
},
|
||||
"window_id": {
|
||||
"description": "Target window handle (HWND). Optional — driver picks frontmost window of pid when omitted.",
|
||||
"description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
@@ -330,23 +330,18 @@
|
||||
},
|
||||
{
|
||||
"name": "type_text",
|
||||
"description": "Type text into the focused target window or specified latest element. Fresh observation follows. If background is unsupported, explicitly choose foreground within task scope.",
|
||||
"description": "Type text at the current focus in the exact window. No element_token or coordinates. Focus a field with click/click_at or the address bar with Ctrl+L first; use set_value for direct UIA replacement. Chromium/Brave input requires foreground. Fresh observation follows.",
|
||||
"inputSchema": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"delivery_mode": {
|
||||
"description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button",
|
||||
"description": "background (default) avoids raising the window; unsupported input is refused. Use foreground for Chromium/Brave keyboard input and scrolling within task scope. Inspect the returned state before retrying an uncertain action.",
|
||||
"enum": [
|
||||
"background",
|
||||
"foreground"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"element_token": {
|
||||
"description": "Opaque element handle from get_window_state. When supplied, type_text writes through UIA ValuePattern and reads that exact element back by handle. The shared ActionResult is confirmed only when the complete expected value is synchronously v",
|
||||
"pattern": "^s[0-9a-f]{8}:[0-9]+$",
|
||||
"type": "string"
|
||||
},
|
||||
"pid": {
|
||||
"description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.",
|
||||
"type": "integer"
|
||||
@@ -356,16 +351,8 @@
|
||||
"type": "string"
|
||||
},
|
||||
"window_id": {
|
||||
"description": "HWND of the target window. Omit when element_token is supplied (the token carries it).",
|
||||
"description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.",
|
||||
"type": "integer"
|
||||
},
|
||||
"x": {
|
||||
"description": "Window-local screenshot-pixel X of the field to type into — the element px action form. Pass x,y (no element_token) and the tool pixel-clicks there to establish real renderer focus, then types. Use for Chromium/Electron inputs the UIA/WM_CH",
|
||||
"type": "number"
|
||||
},
|
||||
"y": {
|
||||
"description": "Window-local screenshot-pixel Y of the field (see x).",
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -382,23 +369,18 @@
|
||||
},
|
||||
{
|
||||
"name": "press_key",
|
||||
"description": "Send key and modifiers to the exact window, optionally focus element_token or x/y first. Fresh observation follows.",
|
||||
"description": "Send a key and modifiers to the exact window current focus. No element_token or coordinates. For browser shortcuts (Ctrl+T, Ctrl+L, Enter), use foreground. To target a field, focus it with click/click_at first. Fresh observation follows; inspect before repeating.",
|
||||
"inputSchema": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"delivery_mode": {
|
||||
"description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button",
|
||||
"description": "background (default) avoids raising the window; unsupported input is refused. Use foreground for Chromium/Brave keyboard input and scrolling within task scope. Inspect the returned state before retrying an uncertain action.",
|
||||
"enum": [
|
||||
"background",
|
||||
"foreground"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"element_token": {
|
||||
"description": "Element handle `<snapshot_id>:<row>` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current sna",
|
||||
"pattern": "^s[0-9a-f]{8}:[0-9]+$",
|
||||
"type": "string"
|
||||
},
|
||||
"key": {
|
||||
"description": "Key name (return, tab, escape, up, down, left, right, space, delete, home, end, pageup, pagedown, f1-f12, letter, digit).",
|
||||
"type": "string"
|
||||
@@ -415,16 +397,8 @@
|
||||
"type": "integer"
|
||||
},
|
||||
"window_id": {
|
||||
"description": "HWND for the target window. Omit when element_token is supplied; otherwise auto-resolves the pid's first visible window.",
|
||||
"description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.",
|
||||
"type": "integer"
|
||||
},
|
||||
"x": {
|
||||
"description": "Window-local screenshot-pixel X — the element px action form: pixel-click there to focus, then send the key. Use when the key must go to a Chromium/Electron surface the UIA path can't focus. Pass with y, no element_token.",
|
||||
"type": "number"
|
||||
},
|
||||
"y": {
|
||||
"description": "Window-local screenshot-pixel Y (see x).",
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -446,7 +420,7 @@
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"delivery_mode": {
|
||||
"description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button",
|
||||
"description": "background (default) avoids raising the window; unsupported input is refused. Use foreground for Chromium/Brave keyboard input and scrolling within task scope. Inspect the returned state before retrying an uncertain action.",
|
||||
"enum": [
|
||||
"background",
|
||||
"foreground"
|
||||
@@ -467,14 +441,15 @@
|
||||
"type": "string"
|
||||
},
|
||||
"window_id": {
|
||||
"description": "HWND of the window. Omit when element_token is supplied (the token carries it).",
|
||||
"description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"pid",
|
||||
"window_id",
|
||||
"value"
|
||||
"value",
|
||||
"element_token"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
@@ -485,7 +460,7 @@
|
||||
},
|
||||
{
|
||||
"name": "scroll",
|
||||
"description": "Scroll the target window/element at x,y. amount=1..50, by=line/page. Verify content changed in fresh returned observation.",
|
||||
"description": "Scroll at x,y in the latest window screenshot. No element_token. Use a point inside the intended scroll surface, amount=1..50, by=line/page. Chromium/Brave requires foreground. Verify actual content movement in the returned observation.",
|
||||
"inputSchema": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
@@ -504,7 +479,7 @@
|
||||
"type": "string"
|
||||
},
|
||||
"delivery_mode": {
|
||||
"description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button",
|
||||
"description": "background (default) avoids raising the window; unsupported input is refused. Use foreground for Chromium/Brave keyboard input and scrolling within task scope. Inspect the returned state before retrying an uncertain action.",
|
||||
"enum": [
|
||||
"background",
|
||||
"foreground"
|
||||
@@ -521,32 +496,29 @@
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"element_token": {
|
||||
"description": "Element handle `<snapshot_id>:<row>` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current sna",
|
||||
"pattern": "^s[0-9a-f]{8}:[0-9]+$",
|
||||
"type": "string"
|
||||
},
|
||||
"pid": {
|
||||
"description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.",
|
||||
"type": "integer"
|
||||
},
|
||||
"window_id": {
|
||||
"description": "HWND of the target window. Omit when element_token is supplied; otherwise auto-resolves the pid's first visible window.",
|
||||
"description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.",
|
||||
"type": "integer"
|
||||
},
|
||||
"x": {
|
||||
"description": "With pid/window_id: window-local screenshot X used to target a nested scroll surface in foreground mode. Without pid/window_id: screen-absolute X for desktop scope. Must be paired with y.",
|
||||
"description": "X inside the scroll area in latest window-local screenshot pixels; never screen coordinates. Must be paired with the other coordinate.",
|
||||
"type": "number"
|
||||
},
|
||||
"y": {
|
||||
"description": "With pid/window_id: window-local screenshot Y used to target a nested scroll surface in foreground mode. Without pid/window_id: screen-absolute Y for desktop scope. Must be paired with x.",
|
||||
"description": "Y inside the scroll area in latest window-local screenshot pixels; never screen coordinates. Must be paired with the other coordinate.",
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"pid",
|
||||
"window_id",
|
||||
"direction"
|
||||
"direction",
|
||||
"x",
|
||||
"y"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
@@ -569,7 +541,8 @@
|
||||
},
|
||||
"window_id": {
|
||||
"type": "integer",
|
||||
"minimum": 1
|
||||
"minimum": 1,
|
||||
"description": "Exact HWND returned for this application by list_windows. Required and must belong to pid."
|
||||
},
|
||||
"filename": {
|
||||
"type": "string"
|
||||
@@ -613,7 +586,7 @@
|
||||
"type": "integer"
|
||||
},
|
||||
"delivery_mode": {
|
||||
"description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button",
|
||||
"description": "background (default) avoids raising the window; unsupported input is refused. Use foreground for Chromium/Brave keyboard input and scrolling within task scope. Inspect the returned state before retrying an uncertain action.",
|
||||
"enum": [
|
||||
"background",
|
||||
"foreground"
|
||||
@@ -625,7 +598,7 @@
|
||||
"type": "integer"
|
||||
},
|
||||
"window_id": {
|
||||
"description": "HWND of the target window. Omit when element_token is supplied (the token carries it).",
|
||||
"description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.",
|
||||
"type": "integer"
|
||||
},
|
||||
"x": {
|
||||
|
||||
Reference in new issue
Block a user