package main import ( "errors" "fmt" "math" ) // A frame belongs to one adapter session and one concrete window. Encoded // pixels, DWM frame dimensions and Windows logical pixels are distinct spaces. type pointerFrame struct { id string width, height int window obj } func targetKey(args obj) string { return fmt.Sprintf("%d:%d", num(args["pid"]), num(args["window_id"])) } func prependFrameHint(r obj, f pointerFrame) { // Keep coordinate metadata before long trees so bounded model context cannot // silently drop the only capture ID while retaining the screenshot. r["content"] = append([]any{textBlock(fmt.Sprintf("Screenshot capture_id=%s, size=%dx%d; pixel x=0..%d, y=0..%d. Use this new ID for click_at/scroll/drag.", f.id, f.width, f.height, f.width-1, f.height-1))}, items(r["content"])...) } func isPointerTool(name string) bool { return name == "click_at" || name == "drag" || name == "scroll" } func copyArgs(args obj) obj { out := obj{} for k, v := range args { out[k] = v } return out } func capturedFrame(r, src obj) (pointerFrame, bool) { w, h := imageSize(r) id := str(object(r["structuredContent"])["capture_id"]) if src == nil || id == "" || w <= 0 || h <= 0 { return pointerFrame{}, false } return pointerFrame{id, w, h, copyArgs(object(src["window"]))}, true } func coordinate(v any) (float64, bool) { var n float64 switch x := v.(type) { case float64: n = x case int: n = float64(x) case int64: n = float64(x) default: return 0, false } return n, !math.IsNaN(n) && !math.IsInf(n, 0) } func (f pointerFrame) validate(name string, args obj) error { if f.id == "" || str(args["capture_id"]) != f.id { return errors.New("capture_id is missing or stale; use the latest screenshot for this window") } pairs := [][2]string{{"x", "y"}} if name == "drag" { pairs = [][2]string{{"from_x", "from_y"}, {"to_x", "to_y"}} } for _, pair := range pairs { x, xok := coordinate(args[pair[0]]) y, yok := coordinate(args[pair[1]]) if !xok || !yok || x < 0 || y < 0 || x > float64(f.width-1) || y > float64(f.height-1) { return fmt.Errorf("%s/%s must be inside screenshot %dx%d (0..%d, 0..%d); no screen or Android normalized coordinates", pair[0], pair[1], f.width, f.height, f.width-1, f.height-1) } } return nil } func sameWindow(a, b obj) bool { for _, k := range []string{"left", "top", "width", "height"} { if num(a[k]) != num(b[k]) { return false } } return num(a["width"]) > 0 && num(a["height"]) > 0 } // Only scroll needs pre-scaling in the pinned driver. click_at is capture-bound // upstream and drag uses the driver's snapshot ratio; never scale either twice. // v0.38.0 scroll and drag use their latest window screenshot context. // Admission stays host-bound to the exact capture; the upstream schema does // not accept capture_id for these two tools. Do not rescale x/y here. func withoutCaptureBinding(args obj) obj { out := copyArgs(args) delete(out, "capture_id") return out }