package main import ( "errors" "fmt" "math" ) // A frame belongs to one adapter session and one concrete window. Encoded // pixels, DWM frame dimensions and Windows logical pixels are distinct spaces. type pointerFrame struct { id string width, height int window obj } func targetKey(args obj) string { return fmt.Sprintf("%d:%d", num(args["pid"]), num(args["window_id"])) } func prependFrameHint(r obj, f pointerFrame) { // Keep coordinate metadata before long trees so bounded model context cannot // silently drop the only capture ID while retaining the screenshot. r["content"] = append([]any{textBlock(fmt.Sprintf("Screenshot capture_id=%s, size=%dx%d; pixel x=0..%d, y=0..%d. Use this new ID for click_at/scroll/drag.", f.id, f.width, f.height, f.width-1, f.height-1))}, items(r["content"])...) } func isPointerTool(name string) bool { return name == "click_at" || name == "drag" || name == "scroll" } func copyArgs(args obj) obj { out := obj{} for k, v := range args { out[k] = v } return out } func capturedFrame(r, src obj) (pointerFrame, bool) { w, h := imageSize(r) id := str(object(r["structuredContent"])["capture_id"]) if src == nil || id == "" || w <= 0 || h <= 0 { return pointerFrame{}, false } return pointerFrame{id, w, h, copyArgs(object(src["window"]))}, true } func coordinate(v any) (float64, bool) { var n float64 switch x := v.(type) { case float64: n = x case int: n = float64(x) case int64: n = float64(x) default: return 0, false } return n, !math.IsNaN(n) && !math.IsInf(n, 0) } func (f pointerFrame) validate(name string, args obj) error { if f.id == "" || str(args["capture_id"]) != f.id { return errors.New("capture_id is missing or stale; use the latest screenshot for this window") } pairs := [][2]string{{"x", "y"}} if name == "drag" { pairs = [][2]string{{"from_x", "from_y"}, {"to_x", "to_y"}} } for _, pair := range pairs { x, xok := coordinate(args[pair[0]]) y, yok := coordinate(args[pair[1]]) if !xok || !yok || x < 0 || y < 0 || x > float64(f.width-1) || y > float64(f.height-1) { return fmt.Errorf("%s/%s must be inside screenshot %dx%d (0..%d, 0..%d); no screen or Android normalized coordinates", pair[0], pair[1], f.width, f.height, f.width-1, f.height-1) } } return nil } func sameWindow(a, b obj) bool { for _, k := range []string{"left", "top", "width", "height"} { if num(a[k]) != num(b[k]) { return false } } return num(a["width"]) > 0 && num(a["height"]) > 0 } // Only scroll needs pre-scaling in the pinned driver. click_at is capture-bound // upstream and drag uses the driver's snapshot ratio; never scale either twice. func nativeScrollArgs(f pointerFrame, args obj, readNative func() (pointerFrame, error)) (obj, error) { if err := f.validate("scroll", args); err != nil { return nil, err } native, err := readNative() if err != nil { return nil, err } if !sameWindow(f.window, native.window) || native.width < f.width || native.height < f.height { return nil, errors.New("capture geometry changed; get_window_state again before scrolling") } // Reject a different crop/aspect ratio. One-pixel resize rounding is normal. if math.Abs(float64(f.height)*float64(native.width)/float64(f.width)-float64(native.height)) > float64(native.width)/float64(f.width)+1 { return nil, errors.New("capture aspect ratio changed; get_window_state again before scrolling") } x, _ := coordinate(args["x"]) y, _ := coordinate(args["y"]) out := copyArgs(args) out["x"] = x * float64(native.width) / float64(f.width) out["y"] = y * float64(native.height) / float64(f.height) delete(out, "capture_id") // v0.36.0 scroll does not support capture binding. return out, nil }