chenmingxuan
889a951c9b
Correct native scroll scaling, reject stale or out-of-bounds pixel targets, and scope DPI awareness to geometry reads. Package adapter 0.36.0+artemis.3 and document the high-effort cross-device verification. Validation: Go tests and vet, packaged MCP schema and handshake, Python checks, and user-run high-effort task completed with a passed verdict and explicit coverage limits.
106 lines
3.6 KiB
Go
106 lines
3.6 KiB
Go
package main
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"math"
|
|
)
|
|
|
|
// A frame belongs to one adapter session and one concrete window. Encoded
|
|
// pixels, DWM frame dimensions and Windows logical pixels are distinct spaces.
|
|
type pointerFrame struct {
|
|
id string
|
|
width, height int
|
|
window obj
|
|
}
|
|
|
|
func targetKey(args obj) string {
|
|
return fmt.Sprintf("%d:%d", num(args["pid"]), num(args["window_id"]))
|
|
}
|
|
func prependFrameHint(r obj, f pointerFrame) {
|
|
// Keep coordinate metadata before long trees so bounded model context cannot
|
|
// silently drop the only capture ID while retaining the screenshot.
|
|
r["content"] = append([]any{textBlock(fmt.Sprintf("Screenshot capture_id=%s, size=%dx%d; pixel x=0..%d, y=0..%d. Use this new ID for click_at/scroll/drag.", f.id, f.width, f.height, f.width-1, f.height-1))}, items(r["content"])...)
|
|
}
|
|
func isPointerTool(name string) bool { return name == "click_at" || name == "drag" || name == "scroll" }
|
|
func copyArgs(args obj) obj {
|
|
out := obj{}
|
|
for k, v := range args {
|
|
out[k] = v
|
|
}
|
|
return out
|
|
}
|
|
func capturedFrame(r, src obj) (pointerFrame, bool) {
|
|
w, h := imageSize(r)
|
|
id := str(object(r["structuredContent"])["capture_id"])
|
|
if src == nil || id == "" || w <= 0 || h <= 0 {
|
|
return pointerFrame{}, false
|
|
}
|
|
return pointerFrame{id, w, h, copyArgs(object(src["window"]))}, true
|
|
}
|
|
func coordinate(v any) (float64, bool) {
|
|
var n float64
|
|
switch x := v.(type) {
|
|
case float64:
|
|
n = x
|
|
case int:
|
|
n = float64(x)
|
|
case int64:
|
|
n = float64(x)
|
|
default:
|
|
return 0, false
|
|
}
|
|
return n, !math.IsNaN(n) && !math.IsInf(n, 0)
|
|
}
|
|
func (f pointerFrame) validate(name string, args obj) error {
|
|
if f.id == "" || str(args["capture_id"]) != f.id {
|
|
return errors.New("capture_id is missing or stale; use the latest screenshot for this window")
|
|
}
|
|
pairs := [][2]string{{"x", "y"}}
|
|
if name == "drag" {
|
|
pairs = [][2]string{{"from_x", "from_y"}, {"to_x", "to_y"}}
|
|
}
|
|
for _, pair := range pairs {
|
|
x, xok := coordinate(args[pair[0]])
|
|
y, yok := coordinate(args[pair[1]])
|
|
if !xok || !yok || x < 0 || y < 0 || x > float64(f.width-1) || y > float64(f.height-1) {
|
|
return fmt.Errorf("%s/%s must be inside screenshot %dx%d (0..%d, 0..%d); no screen or Android normalized coordinates", pair[0], pair[1], f.width, f.height, f.width-1, f.height-1)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
func sameWindow(a, b obj) bool {
|
|
for _, k := range []string{"left", "top", "width", "height"} {
|
|
if num(a[k]) != num(b[k]) {
|
|
return false
|
|
}
|
|
}
|
|
return num(a["width"]) > 0 && num(a["height"]) > 0
|
|
}
|
|
|
|
// Only scroll needs pre-scaling in the pinned driver. click_at is capture-bound
|
|
// upstream and drag uses the driver's snapshot ratio; never scale either twice.
|
|
func nativeScrollArgs(f pointerFrame, args obj, readNative func() (pointerFrame, error)) (obj, error) {
|
|
if err := f.validate("scroll", args); err != nil {
|
|
return nil, err
|
|
}
|
|
native, err := readNative()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !sameWindow(f.window, native.window) || native.width < f.width || native.height < f.height {
|
|
return nil, errors.New("capture geometry changed; get_window_state again before scrolling")
|
|
}
|
|
// Reject a different crop/aspect ratio. One-pixel resize rounding is normal.
|
|
if math.Abs(float64(f.height)*float64(native.width)/float64(f.width)-float64(native.height)) > float64(native.width)/float64(f.width)+1 {
|
|
return nil, errors.New("capture aspect ratio changed; get_window_state again before scrolling")
|
|
}
|
|
x, _ := coordinate(args["x"])
|
|
y, _ := coordinate(args["y"])
|
|
out := copyArgs(args)
|
|
out["x"] = x * float64(native.width) / float64(f.width)
|
|
out["y"] = y * float64(native.height) / float64(f.height)
|
|
delete(out, "capture_id") // v0.36.0 scroll does not support capture binding.
|
|
return out, nil
|
|
}
|