fix: bind Cua pixel actions to current screenshot geometry

Correct native scroll scaling, reject stale or out-of-bounds pixel targets, and scope DPI awareness to geometry reads. Package adapter 0.36.0+artemis.3 and document the high-effort cross-device verification.

Validation: Go tests and vet, packaged MCP schema and handshake, Python checks, and user-run high-effort task completed with a passed verdict and explicit coverage limits.
This commit is contained in:
chenmingxuan committed 2026-10-17 19:43:22 +08:00
1 parent 9c98b6bf8a
commit 889a951c9b
13 files changed
+363 -19

No files matched your search

+17 -5
View File
@@ -270,7 +270,7 @@
},
{
"name": "drag",
"description": "Drag between screenshot-local coordinates. Returns actual delivery effect and fresh post-action observation.",
"description": "Drag between screenshot-local coordinates. Returns actual delivery effect and fresh post-action observation. Requires the latest capture_id. Stale, moved/resized or out-of-image targets are rejected before input.",
"inputSchema": {
"additionalProperties": false,
"properties": {
@@ -311,6 +311,11 @@
"window_id": {
"description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.",
"type": "integer"
},
"capture_id": {
"type": "string",
"minLength": 1,
"description": "capture_id from the latest returned screenshot for this exact window. Each action returns a new screenshot; use its new ID and dimensions."
}
},
"required": [
@@ -319,7 +324,8 @@
"from_x",
"from_y",
"to_x",
"to_y"
"to_y",
"capture_id"
],
"type": "object"
},
@@ -460,7 +466,7 @@
},
{
"name": "scroll",
"description": "Scroll at x,y in the latest window screenshot. No element_token. Use a point inside the intended scroll surface, amount=1..50, by=line/page. Chromium/Brave requires foreground. Verify actual content movement in the returned observation.",
"description": "Scroll at x,y in the latest window screenshot. No element_token. Use a point inside the intended scroll surface, amount=1..50, by=line/page. Chromium/Brave requires foreground. Verify actual content movement in the returned observation. Requires the latest capture_id. Stale, moved/resized or out-of-image targets are rejected before input.",
"inputSchema": {
"additionalProperties": false,
"properties": {
@@ -511,6 +517,11 @@
"y": {
"description": "Y inside the scroll area in latest window-local screenshot pixels; never screen coordinates. Must be paired with the other coordinate.",
"type": "number"
},
"capture_id": {
"type": "string",
"minLength": 1,
"description": "capture_id from the latest returned screenshot for this exact window. Each action returns a new screenshot; use its new ID and dimensions."
}
},
"required": [
@@ -518,7 +529,8 @@
"window_id",
"direction",
"x",
"y"
"y",
"capture_id"
],
"type": "object"
},
@@ -561,7 +573,7 @@
},
{
"name": "click_at",
"description": "Click a visual target at x,y in the latest window screenshot, with its exact capture_id. Never pass an element token. Returns effect and fresh observation.",
"description": "Click a visual target at x,y in the latest window screenshot, with its exact capture_id. Never pass an element token. Returns effect and fresh observation. Use the latest returned image dimensions; no screen coordinates or Android normalization.",
"inputSchema": {
"additionalProperties": false,
"properties": {