diff --git a/plugins/cua-driver/RELEASE.md b/plugins/cua-driver/RELEASE.md index ae43817..efef19c 100644 --- a/plugins/cua-driver/RELEASE.md +++ b/plugins/cua-driver/RELEASE.md @@ -1,8 +1,10 @@ -# Cua 电脑控制 0.36.0+artemis.2 +# Cua 电脑控制 0.36.0+artemis.3 -作为 ARTEMIS 插件市场的默认电脑控制实现，替代已下架的 Open Computer Use。此修订修复模型同时填写控件令牌与坐标，导致浏览器按键连续被拒绝的问题；上游仍是固定的 Cua Driver 0.36.0 预发布版，市场标注开发预览。 +作为 ARTEMIS 插件市场的默认电脑控制实现，替代已下架的 Open Computer Use。此修订修复缩放截图后的滚动坐标转换，加入截图编号及范围校验，保留上一修订的键盘参数修复；上游仍是固定的 Cua Driver 0.36.0 预发布版，市场标注开发预览。 -修复内容：`press_key` / `type_text` 只对目标窗口的当前焦点操作，不再暴露互斥的 element_token/x/y 参数；聚焦使用独立点击工具，控件赋值由必填 element_token 的 `set_value` 完成。`scroll` 仅按截图坐标定位滚动区域。旧格式在适配层返回明确未执行回执，不发送输入、不重新截图。补充浏览器前台输入与滚动后点击规则，不自动重试结果不确定的写操作。保留上一修订的应用名发现和无效 PID 修复。 +本修订：上游 Windows 0.36.0 的滚动路径没有应用截图缩放比，适配器现按实际原始截图尺寸转换；每次滚动前仅内部读取原始截图（不读取 UIA、不发送给模型、不增加模型调用），避免推算 DPI 或假定固定裁边。点击和拖拽继续由上游转换，避免重复缩放。所有像素操作要求最新 capture_id，过期、越界或窗口移动/缩放时拒绝派发。截图编号和尺寸置于文字结果开头，避免被长控件树挤出上下文。Windows 几何读取在锁定 OS 线程后设置并还原 DPI 上下文。 + +上一修订内容：`press_key` / `type_text` 只对目标窗口的当前焦点操作，不再暴露互斥的 element_token/x/y 参数；聚焦使用独立点击工具，控件赋值由必填 element_token 的 `set_value` 完成。`scroll` 仅按截图坐标定位滚动区域。旧格式在适配层返回明确未执行回执，不发送输入、不重新截图。补充浏览器前台输入与滚动后点击规则，不自动重试结果不确定的写操作。保留上一修订的应用名发现和无效 PID 修复。 上游提交：`6d6c734c127025fb39ad5d585a10382808b39dc1`。Windows x64 原始 ZIP SHA256：`f8c47024b626f4e0e70f8428d2998df39e4879110c5722a29d5fd03e367353ad`。采用 MIT Driver，许可证随包提供；不包含 Cua Spaces、云环境或模型服务。 diff --git a/plugins/cua-driver/SKILL.md b/plugins/cua-driver/SKILL.md index bfcc545..449883a 100644 --- a/plugins/cua-driver/SKILL.md +++ b/plugins/cua-driver/SKILL.md @@ -11,3 +11,9 @@ 7. `save_screenshot` 保存最新窗口 PNG 到当前任务 trace，filename 仅是文件名，不能传路径，不覆盖已有文件。实际成功返回 SHA256 和附件；不得把工具内存截图当成已保存文件。 不提供 Cua 自带调度、录屏、回放、配置修改、安装器、任意脚本或自动升级接口。浏览器本轮使用窗口视觉/UIA 通路；未接入 DOM 或已有登录浏览器调试权限。此包基于上游 0.36.0 预发布版，通过已列出的本机验证，其他场景的支持范围以 VERIFICATION.md 为准。 + +## 截图坐标与证据 + +- `click_at`、`scroll`、`drag` 都携带最新返回截图的 `capture_id`，坐标取自同一张图的实际宽高。动作返回的新截图可能改变分辨率；旧编号、旧坐标不可沿用。窗口移动、缩放或坐标越界被拒绝后重新观察，不反复重试原参数。 +- 缩放转换由适配层处理，不自行乘 DPI 或加窗口偏移。优先用最新 `element_token` 点击可访问控件；UIA 的屏幕 `frame` 不能作为截图坐标。 +- 操作及保存截图前，从已有观察核对应用、窗口标题和页面身份（浏览器结合地址栏、选中标签及页面内容）。身份不符时定位正确页面再操作。无位移的滚动不能证明到底；先检查遮挡、焦点和滚动容器，再决定补一次边界验证或换路径。 diff --git a/plugins/cua-driver/VERIFICATION.md b/plugins/cua-driver/VERIFICATION.md index d419446..0d0779d 100644 --- a/plugins/cua-driver/VERIFICATION.md +++ b/plugins/cua-driver/VERIFICATION.md @@ -60,3 +60,11 @@ Cua 第三轮虽然独立验收通过，ARTEMIS 最终核验却没有生成结 `tests/cua_browser_smoke.py` 新增独立 Brave profile 和本地表单，全部操作经过打包后的 Cua MCP，未使用 CDP/DOM 控制。实测通过 Ctrl+T 新建标签、Ctrl+L 地址栏导航、UIA set_value、聚焦后的中文 type_text、表单恰好提交一次并读回、实际滚动位移、底部按钮及 PNG SHA256。测试窗口退出，不操作用户原有标签页。 实测还发现 Chromium 后台点击可能沿用滚动前的控件位置；前台路径会解析可见位置，已验证滚动后点击成功，schema/Skill 明确要求浏览器使用 foreground。滚轮 amount/by 不是精确滚动像素或页数，测试按页面滚动事件决定是否继续。不能据小样本宣称所有浏览器、DPI、跨端业务都通过；原业务完整任务未重跑，移动端问题不在本修订范围。 + +## 截图坐标修订 0.36.0+artemis.3 + +对照实际任务中 2558×1538 原图、1280×770 后续截图，以及超界 (1700, 1000) 滚动仍被派发的回执，核对固定上游提交的 Windows ScrollTool：其前台路径直接 bitmap_to_screen，而点击/拖拽使用 capture 映射或 snapshot 缩放。修复仅对滚动补映射，不重复缩放点击和拖拽。 + +Go 确定性回归覆盖上述尺寸、原始尺寸无二次缩放、不同裁边、负窗口原点、非整数点、缺失/过期编号、非数值/越界、拖拽两端、窗口移动/缩放及原始截图读取失败。Go test、vet 与构建通过。本修订没有重跑付费模型业务任务，也未声称真实多显示器、多 DPI 或跨端成功率已验证；待使用新插件进行业务复测。 + +补充用户业务复测：2026-10-17 使用 `gpt-6-luna / high` 与 `0.36.0+artemis.3` 完成手机 Chrome、电脑 Brave 的混合页面探索，33 步、约 410 秒，执行状态为 completed，最终核验为 passed。记录确认移动端 Desktop site 未勾选、两端页底观察及追加滚动；报告明确保留 PC 中段局部未逐屏覆盖和文字证据冲突。此通过仅针对本次探索与报告要求，不表示页面所有区域无缺陷，也不外推多显示器、多 DPI 或整体成功率。 diff --git a/plugins/cua-driver/coordinates.go b/plugins/cua-driver/coordinates.go new file mode 100644 index 0000000..13e1959 --- /dev/null +++ b/plugins/cua-driver/coordinates.go @@ -0,0 +1,105 @@ +package main + +import ( + "errors" + "fmt" + "math" +) + +// A frame belongs to one adapter session and one concrete window. Encoded +// pixels, DWM frame dimensions and Windows logical pixels are distinct spaces. +type pointerFrame struct { + id string + width, height int + window obj +} + +func targetKey(args obj) string { + return fmt.Sprintf("%d:%d", num(args["pid"]), num(args["window_id"])) +} +func prependFrameHint(r obj, f pointerFrame) { + // Keep coordinate metadata before long trees so bounded model context cannot + // silently drop the only capture ID while retaining the screenshot. + r["content"] = append([]any{textBlock(fmt.Sprintf("Screenshot capture_id=%s, size=%dx%d; pixel x=0..%d, y=0..%d. Use this new ID for click_at/scroll/drag.", f.id, f.width, f.height, f.width-1, f.height-1))}, items(r["content"])...) +} +func isPointerTool(name string) bool { return name == "click_at" || name == "drag" || name == "scroll" } +func copyArgs(args obj) obj { + out := obj{} + for k, v := range args { + out[k] = v + } + return out +} +func capturedFrame(r, src obj) (pointerFrame, bool) { + w, h := imageSize(r) + id := str(object(r["structuredContent"])["capture_id"]) + if src == nil || id == "" || w <= 0 || h <= 0 { + return pointerFrame{}, false + } + return pointerFrame{id, w, h, copyArgs(object(src["window"]))}, true +} +func coordinate(v any) (float64, bool) { + var n float64 + switch x := v.(type) { + case float64: + n = x + case int: + n = float64(x) + case int64: + n = float64(x) + default: + return 0, false + } + return n, !math.IsNaN(n) && !math.IsInf(n, 0) +} +func (f pointerFrame) validate(name string, args obj) error { + if f.id == "" || str(args["capture_id"]) != f.id { + return errors.New("capture_id is missing or stale; use the latest screenshot for this window") + } + pairs := [][2]string{{"x", "y"}} + if name == "drag" { + pairs = [][2]string{{"from_x", "from_y"}, {"to_x", "to_y"}} + } + for _, pair := range pairs { + x, xok := coordinate(args[pair[0]]) + y, yok := coordinate(args[pair[1]]) + if !xok || !yok || x < 0 || y < 0 || x > float64(f.width-1) || y > float64(f.height-1) { + return fmt.Errorf("%s/%s must be inside screenshot %dx%d (0..%d, 0..%d); no screen or Android normalized coordinates", pair[0], pair[1], f.width, f.height, f.width-1, f.height-1) + } + } + return nil +} +func sameWindow(a, b obj) bool { + for _, k := range []string{"left", "top", "width", "height"} { + if num(a[k]) != num(b[k]) { + return false + } + } + return num(a["width"]) > 0 && num(a["height"]) > 0 +} + +// Only scroll needs pre-scaling in the pinned driver. click_at is capture-bound +// upstream and drag uses the driver's snapshot ratio; never scale either twice. +func nativeScrollArgs(f pointerFrame, args obj, readNative func() (pointerFrame, error)) (obj, error) { + if err := f.validate("scroll", args); err != nil { + return nil, err + } + native, err := readNative() + if err != nil { + return nil, err + } + if !sameWindow(f.window, native.window) || native.width < f.width || native.height < f.height { + return nil, errors.New("capture geometry changed; get_window_state again before scrolling") + } + // Reject a different crop/aspect ratio. One-pixel resize rounding is normal. + if math.Abs(float64(f.height)*float64(native.width)/float64(f.width)-float64(native.height)) > float64(native.width)/float64(f.width)+1 { + return nil, errors.New("capture aspect ratio changed; get_window_state again before scrolling") + } + x, _ := coordinate(args["x"]) + y, _ := coordinate(args["y"]) + out := copyArgs(args) + out["x"] = x * float64(native.width) / float64(f.width) + out["y"] = y * float64(native.height) / float64(f.height) + delete(out, "capture_id") // v0.36.0 scroll does not support capture binding. + return out, nil +} diff --git a/plugins/cua-driver/coordinates_test.go b/plugins/cua-driver/coordinates_test.go new file mode 100644 index 0000000..abbd317 --- /dev/null +++ b/plugins/cua-driver/coordinates_test.go @@ -0,0 +1,137 @@ +package main + +import ( + "errors" + "math" + "testing" +) + +func fixtureFrame(w, h int) pointerFrame { + return pointerFrame{"current", w, h, obj{"left": -2560, "top": 120, "width": 2560, "height": 1540}} +} + +func TestScrollMapsActualNativeCaptureWithoutDPIOrWindowOffset(t *testing.T) { + // Reproduce the task's 2558x1538 -> 1280x770 image transition. Window + // position and OS logical-screen dimensions must never affect this ratio. + f := fixtureFrame(1280, 770) + args := obj{"pid": 1, "window_id": 2, "capture_id": "current", "x": 900, "y": 600, "direction": "down"} + got, err := nativeScrollArgs(f, args, func() (pointerFrame, error) { return fixtureFrame(2558, 1538), nil }) + if err != nil { + t.Fatal(err) + } + if math.Abs(got["x"].(float64)-1798.59375) > 0.0001 || math.Abs(got["y"].(float64)-1198.441558) > 0.0001 { + t.Fatalf("incorrect mapping: %v", got) + } + if _, ok := got["capture_id"]; ok { + t.Fatal("upstream scroll does not support capture_id") + } + if args["x"] != 900 || args["capture_id"] != "current" { + t.Fatal("input evidence mutated") + } + if got["pid"] != 1 || got["window_id"] != 2 || got["direction"] != "down" { + t.Fatal("target changed") + } +} + +func TestNativeScrollAndNonCroppedCapture(t *testing.T) { + for _, size := range [][2]int{{2558, 1538}, {2560, 1540}, {1280, 770}} { + f := fixtureFrame(size[0], size[1]) + args := obj{"capture_id": "current", "x": 200.5, "y": 300.25} + got, err := nativeScrollArgs(f, args, func() (pointerFrame, error) { return f, nil }) + if err != nil || got["x"] != 200.5 || got["y"] != 300.25 { + t.Fatalf("native coordinates changed: %v %v", got, err) + } + } + // WGC can have a different native extent than a one-pixel DWM crop. + f := fixtureFrame(1280, 770) + got, err := nativeScrollArgs(f, obj{"capture_id": "current", "x": 640, "y": 385}, func() (pointerFrame, error) { return fixtureFrame(2560, 1540), nil }) + if err != nil || got["x"] != float64(1280) || got["y"] != float64(770) { + t.Fatalf("assumed DWM crop: %v %v", got, err) + } +} + +func TestInvalidCoordinatesNeverReachNativeProbe(t *testing.T) { + f := fixtureFrame(1280, 770) + for _, bad := range []obj{ + {"capture_id": "current", "x": 1700, "y": 1000}, // Actual failing task. + {"capture_id": "old", "x": 900, "y": 600}, + {"x": 900, "y": 600}, + {"capture_id": "current", "x": -1, "y": 0}, + {"capture_id": "current", "x": 1280, "y": 0}, + {"capture_id": "current", "x": 0, "y": 770}, + {"capture_id": "current", "x": "900", "y": 600}, + {"capture_id": "current", "x": math.NaN(), "y": 600}, + {"capture_id": "current", "x": 900, "y": math.Inf(1)}, + } { + _, err := nativeScrollArgs(f, bad, func() (pointerFrame, error) { t.Fatal("invalid coordinates reached probe"); return f, nil }) + if err == nil { + t.Fatalf("accepted %v", bad) + } + } +} + +func TestScrollRejectsMovedResizedOrDifferentCrop(t *testing.T) { + f := fixtureFrame(1280, 770) + for _, kind := range []string{"moved", "resized", "crop", "missing"} { + native := fixtureFrame(2558, 1538) + switch kind { + case "moved": + native.window["left"] = 0 + case "resized": + native.window["width"] = 1920 + case "crop": + native.height = 1000 + case "missing": + native.width = 0 + } + _, err := nativeScrollArgs(f, obj{"capture_id": "current", "x": 900, "y": 600}, func() (pointerFrame, error) { return native, nil }) + if err == nil { + t.Fatal(kind) + } + } + _, err := nativeScrollArgs(f, obj{"capture_id": "current", "x": 900, "y": 600}, func() (pointerFrame, error) { return pointerFrame{}, errors.New("capture disconnected") }) + if err == nil { + t.Fatal("missing capture was accepted") + } +} + +func TestClickAndBothDragEndpointsUseLatestFrame(t *testing.T) { + f := fixtureFrame(1280, 770) + click := obj{"capture_id": "current", "x": 1279, "y": 769} + if err := f.validate("click_at", click); err != nil { + t.Fatal(err) + } + if click["x"] != 1279 || click["y"] != 769 { + t.Fatal("click scaled twice") + } + drag := obj{"capture_id": "current", "from_x": 0, "from_y": 0, "to_x": 1279, "to_y": 769} + if err := f.validate("drag", drag); err != nil { + t.Fatal(err) + } + for _, key := range []string{"from_x", "from_y", "to_x", "to_y"} { + bad := copyArgs(drag) + bad[key] = 3000 + if f.validate("drag", bad) == nil { + t.Fatal("accepted out of bounds " + key) + } + } + drag["capture_id"] = "old" + if f.validate("drag", drag) == nil { + t.Fatal("accepted stale drag") + } +} + +func TestAllPixelSchemasRequireCapture(t *testing.T) { + a := newAdapter(nil) + for _, name := range []string{"click_at", "scroll", "drag"} { + found := false + for _, key := range items(a.schemas[name]["required"]) { + if key == "capture_id" { + found = true + } + } + if !found { + t.Fatal(name + " lacks capture binding") + } + } +} diff --git a/plugins/cua-driver/main.go b/plugins/cua-driver/main.go index 7cff8ee..cedd7b1 100644 --- a/plugins/cua-driver/main.go +++ b/plugins/cua-driver/main.go @@ -25,7 +25,7 @@ import ( //go:embed tools.json var toolsJSON []byte -const adapterVersion = "0.36.0+artemis.2" +const adapterVersion = "0.36.0+artemis.3" type obj = map[string]any type request struct { @@ -172,13 +172,14 @@ func (p *peer) tool(name string, args obj) (obj, error) { } type adapter struct { - peer *peer - tools []any - schemas map[string]obj + peer *peer + tools []any + schemas map[string]obj + captures map[string]pointerFrame } func newAdapter(p *peer) *adapter { - a := &adapter{peer: p, schemas: map[string]obj{}} + a := &adapter{peer: p, schemas: map[string]obj{}, captures: map[string]pointerFrame{}} _ = json.Unmarshal(toolsJSON, &a.tools) for _, v := range a.tools { t := object(v) @@ -287,12 +288,18 @@ func source(r obj, args obj, started float64) obj { return obj{"kind": "computer", "target_id": fmt.Sprint(num(args["window_id"])), "label": s["window_title"], "captured_at": started, "width": w, "height": h, "window": rect, "screen": screen} } func (a *adapter) observe(args obj) (obj, obj, error) { + delete(a.captures, targetKey(args)) started := float64(time.Now().UnixMilli()) / 1000 r, err := a.peer.tool("get_window_state", args) if err != nil { return nil, nil, err } - return r, source(r, args, started), nil + src := source(r, args, started) + if frame, ok := capturedFrame(r, src); ok { + a.captures[targetKey(args)] = frame + prependFrameHint(r, frame) + } + return r, src, nil } func observationArgs(args obj) obj { return obj{"pid": args["pid"], "window_id": args["window_id"], "max_image_dimension": 1280, "max_elements": 250, "tree_format": "markdown"} @@ -325,10 +332,58 @@ func (a *adapter) handle(name string, args obj) obj { r = windowListing(r, query) } } else { + if isPointerTool(name) { + frame, ok := a.captures[targetKey(args)] + if !ok { + return failure("No current screenshot; get_window_state before a pixel action", "not_started", "none", "invalid_coordinates") + } + if err := frame.validate(name, args); err != nil { + return failure(err.Error(), "not_started", "none", "invalid_coordinates") + } + if _, ok := verifyWindow(num(args["pid"]), num(args["window_id"]), frame.window); !ok { + delete(a.captures, targetKey(args)) + return failure("Window moved or resized; get_window_state again", "not_started", "none", "stale_capture") + } + if name == "scroll" { + // v0.36.0 scroll omits screenshot scaling. Measure actual native + // capture pixels (not DWM/DPI estimates) without another model call. + // Do not expose this internal frame or reuse its tokens. + delete(a.captures, targetKey(args)) + prepared, e := nativeScrollArgs(frame, args, func() (pointerFrame, error) { + nativeArgs := observationArgs(args) + nativeArgs["max_image_dimension"] = 0 + nativeArgs["include_accessibility_tree"] = false + native, err := a.peer.tool("get_window_state", nativeArgs) + if err != nil { + return pointerFrame{}, err + } + src := source(native, args, float64(time.Now().UnixMilli())/1000) + f, ok := capturedFrame(native, src) + if !ok { + return pointerFrame{}, errors.New("native capture unavailable; get_window_state again") + } + return f, nil + }) + if e != nil { + return failure(e.Error(), "not_started", "none", "invalid_coordinates") + } + args = prepared + } else if name == "drag" { + // Upstream drag already scales using its current screenshot. + args = copyArgs(args) + delete(args, "capture_id") + } + } upstream := name if name == "click_at" { upstream = "click" } + if name == "verify_state" { + delete(a.captures, targetKey(args)) + } + if writeTool(name) { + delete(a.captures, targetKey(args)) + } r, err = a.peer.tool(upstream, args) } if err != nil { @@ -353,6 +408,10 @@ func (a *adapter) handle(name string, args obj) obj { } if name == "verify_state" { // Read-only verification may include a screenshot; it is not a task verdict. src = source(r, args, float64(time.Now().UnixMilli())/1000) + if frame, ok := capturedFrame(r, src); ok { + a.captures[targetKey(args)] = frame + prependFrameHint(r, frame) + } } if src != nil { rec["source"] = src diff --git a/plugins/cua-driver/main_test.go b/plugins/cua-driver/main_test.go index 36698d1..558c143 100644 --- a/plugins/cua-driver/main_test.go +++ b/plugins/cua-driver/main_test.go @@ -101,7 +101,7 @@ func TestInputContractsHaveOneTargetingMode(t *testing.T) { "press_key": {"pid": 1, "window_id": 2, "key": "t", "modifiers": []any{"ctrl"}, "delivery_mode": "foreground"}, "type_text": {"pid": 1, "window_id": 2, "text": "中文"}, "set_value": {"pid": 1, "window_id": 2, "element_token": "s00000001:9", "value": "value"}, - "scroll": {"pid": 1, "window_id": 2, "direction": "down", "x": 100, "y": 100}, + "scroll": {"pid": 1, "window_id": 2, "direction": "down", "x": 100, "y": 100, "capture_id": "current"}, } { if err := a.validate(name, args); err != nil { t.Fatalf("valid %s: %v", name, err) diff --git a/plugins/cua-driver/plugin.json b/plugins/cua-driver/plugin.json index 3914f23..3d6e8d9 100644 --- a/plugins/cua-driver/plugin.json +++ b/plugins/cua-driver/plugin.json @@ -2,7 +2,7 @@ "schema_version": 2, "id": "cua-driver", "name": "电脑控制 · Cua", - "version": "0.36.0+artemis.2", + "version": "0.36.0+artemis.3", "description": "通过 Cua Driver 的窗口截图与可访问性树操作 Windows 应用及浏览器窗口；复用 ARTEMIS 的任务调度、步骤证据与录像。", "artemis_version": ">=2.0.0,<3.0.0", "platforms": [ diff --git a/plugins/cua-driver/tools.json b/plugins/cua-driver/tools.json index 175735a..fcd085c 100644 --- a/plugins/cua-driver/tools.json +++ b/plugins/cua-driver/tools.json @@ -270,7 +270,7 @@ }, { "name": "drag", - "description": "Drag between screenshot-local coordinates. Returns actual delivery effect and fresh post-action observation.", + "description": "Drag between screenshot-local coordinates. Returns actual delivery effect and fresh post-action observation. Requires the latest capture_id. Stale, moved/resized or out-of-image targets are rejected before input.", "inputSchema": { "additionalProperties": false, "properties": { @@ -311,6 +311,11 @@ "window_id": { "description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.", "type": "integer" + }, + "capture_id": { + "type": "string", + "minLength": 1, + "description": "capture_id from the latest returned screenshot for this exact window. Each action returns a new screenshot; use its new ID and dimensions." } }, "required": [ @@ -319,7 +324,8 @@ "from_x", "from_y", "to_x", - "to_y" + "to_y", + "capture_id" ], "type": "object" }, @@ -460,7 +466,7 @@ }, { "name": "scroll", - "description": "Scroll at x,y in the latest window screenshot. No element_token. Use a point inside the intended scroll surface, amount=1..50, by=line/page. Chromium/Brave requires foreground. Verify actual content movement in the returned observation.", + "description": "Scroll at x,y in the latest window screenshot. No element_token. Use a point inside the intended scroll surface, amount=1..50, by=line/page. Chromium/Brave requires foreground. Verify actual content movement in the returned observation. Requires the latest capture_id. Stale, moved/resized or out-of-image targets are rejected before input.", "inputSchema": { "additionalProperties": false, "properties": { @@ -511,6 +517,11 @@ "y": { "description": "Y inside the scroll area in latest window-local screenshot pixels; never screen coordinates. Must be paired with the other coordinate.", "type": "number" + }, + "capture_id": { + "type": "string", + "minLength": 1, + "description": "capture_id from the latest returned screenshot for this exact window. Each action returns a new screenshot; use its new ID and dimensions." } }, "required": [ @@ -518,7 +529,8 @@ "window_id", "direction", "x", - "y" + "y", + "capture_id" ], "type": "object" }, @@ -561,7 +573,7 @@ }, { "name": "click_at", - "description": "Click a visual target at x,y in the latest window screenshot, with its exact capture_id. Never pass an element token. Returns effect and fresh observation.", + "description": "Click a visual target at x,y in the latest window screenshot, with its exact capture_id. Never pass an element token. Returns effect and fresh observation. Use the latest returned image dimensions; no screen coordinates or Android normalization.", "inputSchema": { "additionalProperties": false, "properties": { diff --git a/plugins/cua-driver/windows.go b/plugins/cua-driver/windows.go index ed2d5c8..79f3faf 100644 --- a/plugins/cua-driver/windows.go +++ b/plugins/cua-driver/windows.go @@ -3,6 +3,7 @@ package main import ( "fmt" "os/exec" + "runtime" "syscall" "unsafe" ) @@ -12,7 +13,6 @@ var kernel32 = syscall.NewLazyDLL("kernel32.dll") func configureProcess(c *exec.Cmd) { c.SysProcAttr = &syscall.SysProcAttr{HideWindow: true} - user32.NewProc("SetThreadDpiAwarenessContext").Call(^uintptr(3)) } func windowOwned(pid, hwnd int64) bool { var actual uint32 @@ -24,6 +24,17 @@ func windowOwned(pid, hwnd int64) bool { return int64(actual) == pid } func verifyWindow(pid, hwnd int64, b obj) (obj, bool) { + // DPI context belongs to an OS thread, not a Go goroutine. Apply it around + // the actual geometry reads; setting it once while starting a child leaks + // context and later GetSystemMetrics can return virtualized dimensions. + runtime.LockOSThread() + defer runtime.UnlockOSThread() + setDPI := user32.NewProc("SetThreadDpiAwarenessContext") + previous, _, _ := setDPI.Call(^uintptr(3)) + if previous == 0 { + return nil, false + } + defer setDPI.Call(previous) if !windowOwned(pid, hwnd) { return nil, false } diff --git a/scripts/build_cua_driver.py b/scripts/build_cua_driver.py index 7a46ba7..84d0d82 100644 --- a/scripts/build_cua_driver.py +++ b/scripts/build_cua_driver.py @@ -99,7 +99,7 @@ def main(): "source_commit": COMMIT, "upstream_version": UPSTREAM_VERSION, "plugin_version": version, - "adapter_revision": 3, + "adapter_revision": 4, "prerelease": True, "url": URL, "sha256": DIGEST, diff --git a/tests/cua_browser_smoke.py b/tests/cua_browser_smoke.py index 62737e5..f478b3c 100644 --- a/tests/cua_browser_smoke.py +++ b/tests/cua_browser_smoke.py @@ -54,7 +54,7 @@ async def model_task(base, pid, url, output, events, traces, model): { "id": "cua-driver", "profile": "benchmark", - "version": "0.36.0+artemis.2", + "version": "0.36.0+artemis.3", } ], "model": model, @@ -353,6 +353,7 @@ async def run( "by": "page", "x": source["width"] // 2, "y": source["height"] // 2, + "capture_id": state["capture_id"], "delivery_mode": "foreground", }, ) diff --git a/tests/cua_fixture_validation.py b/tests/cua_fixture_validation.py index 86898b8..3c96067 100644 --- a/tests/cua_fixture_validation.py +++ b/tests/cua_fixture_validation.py @@ -79,12 +79,14 @@ async def main(executable): "by": "page", "x": 400, "y": 470, + "capture_id": pixel.structuredContent["observation"]["capture_id"], }, ) if scrolled.isError: assert ( scrolled.structuredContent["artemis"]["execution"] == "not_started" ), scrolled + state = await session.call("get_window_state", target) scrolled = await session.call( "scroll", { @@ -94,6 +96,7 @@ async def main(executable): "by": "page", "x": 400, "y": 470, + "capture_id": state.structuredContent["capture_id"], "delivery_mode": "foreground", }, )