feat(cua): release 0.38.0+artemis.1 with verified shell observation

Upgrade the pinned driver, validate menu and taskbar focus ownership, patch untitled Shell resolution, and remove duplicate scroll scaling. Publish package provenance, verification boundaries and matching marketplace checksum. Go tests, vet, build and byte-level package verification passed.
This commit is contained in:
chenmingxuan committed 2026-10-27 01:13:43 +08:00
1 parent dccddaa7e8
commit b780a60e8d
20 files changed
+759 -149

No files matched your search

+35 -30
View File
@@ -25,7 +25,7 @@ import (
//go:embed tools.json
var toolsJSON []byte
const adapterVersion = "0.36.0+artemis.4"
const adapterVersion = "0.38.0+artemis.1"
type obj = map[string]any
type request struct {
@@ -291,11 +291,13 @@ func (a *adapter) observe(args obj) (obj, obj, error) {
delete(a.captures, targetKey(args))
started := float64(time.Now().UnixMilli()) / 1000
before := foregroundWindow()
focusBefore := keyboardFocus()
r, err := a.peer.tool("get_window_state", args)
if err != nil {
return nil, nil, err
}
annotateCaptureFocus(r, num(args["window_id"]), before, foregroundWindow())
object(r["structuredContent"])["keyboard_focus"] = obj{"before": focusBefore, "after": keyboardFocus()}
src := source(r, args, started)
if frame, ok := capturedFrame(r, src); ok {
a.captures[targetKey(args)] = frame
@@ -331,6 +333,7 @@ func (a *adapter) handle(name string, args obj) obj {
query, _ := windowQuery(args) // Already validated above.
r, err = a.peer.tool(name, query)
if err == nil {
r = supplementShellWindows(r, query, nativeShellWindows())
r = windowListing(r, query)
}
} else {
@@ -346,34 +349,9 @@ func (a *adapter) handle(name string, args obj) obj {
delete(a.captures, targetKey(args))
return failure("Window moved or resized; get_window_state again", "not_started", "none", "stale_capture")
}
if name == "scroll" {
// v0.36.0 scroll omits screenshot scaling. Measure actual native
// capture pixels (not DWM/DPI estimates) without another model call.
// Do not expose this internal frame or reuse its tokens.
delete(a.captures, targetKey(args))
prepared, e := nativeScrollArgs(frame, args, func() (pointerFrame, error) {
nativeArgs := observationArgs(args)
nativeArgs["max_image_dimension"] = 0
nativeArgs["include_accessibility_tree"] = false
native, err := a.peer.tool("get_window_state", nativeArgs)
if err != nil {
return pointerFrame{}, err
}
src := source(native, args, float64(time.Now().UnixMilli())/1000)
f, ok := capturedFrame(native, src)
if !ok {
return pointerFrame{}, errors.New("native capture unavailable; get_window_state again")
}
return f, nil
})
if e != nil {
return failure(e.Error(), "not_started", "none", "invalid_coordinates")
}
args = prepared
} else if name == "drag" {
// Upstream drag already scales using its current screenshot.
args = copyArgs(args)
delete(args, "capture_id")
if name == "scroll" || name == "drag" {
// v0.38.0 maps screenshot pixels itself. Keep x/y unchanged.
args = withoutCaptureBinding(args)
}
}
upstream := name
@@ -402,13 +380,40 @@ func (a *adapter) handle(name string, args obj) obj {
}
if writeTool(name) && rec["execution"] != "not_started" {
// Observe once after delivery; NEVER repeat an uncertain write.
observation, origin, e := a.observe(observationArgs(args))
focus := keyboardFocus()
next, reason := postActionTarget(args, nativeWindowInfo(num(args["window_id"])), focus, nativeShellWindows())
structured["post_action_focus"] = focus
structured["observation_target"] = obj{"requested": obj{"pid": args["pid"], "window_id": args["window_id"]},
"observed": obj{"pid": next["pid"], "window_id": next["window_id"]}, "reason": reason}
observation, origin, e := a.observe(next)
if e == nil && origin != nil {
src = origin
structured["observation"] = observation["structuredContent"]
r["content"] = append(items(r["content"]), items(observation["content"])...)
} else {
structured["observation_error"] = "Post-action evidence unavailable; inspect before another action."
if e != nil {
structured["observation_error"] = e.Error()
} else if observation != nil {
var errors []string
if observation["isError"] == true {
for _, item := range items(observation["content"]) {
block := object(item)
if block["type"] == "text" {
text := []rune(str(block["text"]))
if len(text) > 2000 {
text = text[:2000]
}
errors = append(errors, string(text))
}
}
}
// Failed identity/geometry validation must never forward its image
// or full tree as task evidence, even inside diagnostics.
structured["observation_diagnostics"] = obj{"driver_errors": errors,
"native_target": nativeWindowInfo(num(next["window_id"])), "isError": observation["isError"],
"image_size": func() []int { w, h := imageSize(observation); return []int{w, h} }()}
}
}
}
if name == "verify_state" { // Read-only verification may include a screenshot; it is not a task verdict.