diff --git a/catalog.json b/catalog.json index 5e8fc0c..426925a 100644 --- a/catalog.json +++ b/catalog.json @@ -6,7 +6,7 @@ "schema_version": 2, "id": "cua-driver", "name": "电脑控制 · Cua", - "version": "0.36.0+artemis.1", + "version": "0.36.0+artemis.2", "description": "通过 Cua Driver 的窗口截图与可访问性树操作 Windows 应用及浏览器窗口；复用 ARTEMIS 的任务调度、步骤证据与录像。", "artemis_version": ">=2.0.0,<3.0.0", "platforms": [ @@ -172,9 +172,9 @@ } }, "release_stage": "preview", - "archive_url": "https://git.playones.com/chenmingxuan/Artemis-Plugins/releases/download/cua-driver-v0.36.0+artemis.1/cua-driver-0.36.0+artemis.1-windows-amd64.zip", - "sha256": "43a90d514e0d148e4476ca8508e22a2bd1f60d8af97cb5393a7a2034e2af6aa6", - "verification": "Windows x64 / ARTEMIS 2.0.0 开发版。固定 Cua Driver 0.36.0 预发布版。已验证原生窗口表单、控件与像素点击、34 像素目标、滚动、trace 截图及哈希、ARTEMIS PC 窗口录像、子进程退出；同一复合用例重复 3 次，独立验收 3/3，AI 裁定 2 次通过、1 次未判定。0.36.0 沿用已测 0.1.2 动作实现，取代 OCU 市场入口。未覆盖复杂浏览器控制台、多 DPI、多显示器和跨端长任务。完整验证与已知限制见仓库 plugins/cua-driver/VERIFICATION.md。 本修订实测补充：Brave 按应用名发现、无效/null PID、冲突 PID 纠正、实际截图及 UIA 树通过；未重跑完整跨端业务任务。" + "archive_url": "https://git.playones.com/chenmingxuan/Artemis-Plugins/releases/download/cua-driver-v0.36.0+artemis.2/cua-driver-0.36.0+artemis.2-windows-amd64.zip", + "sha256": "da78bc2fad9c64222e69c717735514da0e644f59d778bfdeb562c02a16cb7f66", + "verification": "Windows x64 / ARTEMIS 2.0.0 开发版。固定 Cua Driver 0.36.0 预发布版。已验证原生窗口表单、控件与像素点击、34 像素目标、滚动、trace 截图及哈希、ARTEMIS PC 窗口录像、子进程退出；同一复合用例重复 3 次，独立验收 3/3，AI 裁定 2 次通过、1 次未判定。0.36.0 沿用已测 0.1.2 动作实现，取代 OCU 市场入口。未覆盖复杂浏览器控制台、多 DPI、多显示器和跨端长任务。完整验证与已知限制见仓库 plugins/cua-driver/VERIFICATION.md。 本修订实测补充：Brave 按应用名发现、无效/null PID、冲突 PID 纠正、实际截图及 UIA 树通过；未重跑完整跨端业务任务。 0.36.0+artemis.2 补充独立 Brave 实测：新标签、地址导航、UIA 赋值、中文输入、表单单次提交、滚动后前台点击和 PNG 哈希通过；互斥输入参数在执行前拒绝。 补充 Flash / gpt-6.1-sol 自主浏览器任务：表单单次提交、滚动后点击与 trace 截图独立验收通过；不代表原移动端/PC完整业务任务已通过。" } ] } diff --git a/plugins/cua-driver/RELEASE.md b/plugins/cua-driver/RELEASE.md index 5d6f3b4..ae43817 100644 --- a/plugins/cua-driver/RELEASE.md +++ b/plugins/cua-driver/RELEASE.md @@ -1,8 +1,8 @@ -# Cua 电脑控制 0.36.0+artemis.1 +# Cua 电脑控制 0.36.0+artemis.2 -作为 ARTEMIS 插件市场的默认电脑控制实现，替代已下架的 Open Computer Use。此修订修复按应用名查找窗口时，被无效或冲突 PID 筛选为零窗口并误判浏览器不可用的问题；上游仍是固定的 Cua Driver 0.36.0 预发布版，市场标注开发预览。 +作为 ARTEMIS 插件市场的默认电脑控制实现，替代已下架的 Open Computer Use。此修订修复模型同时填写控件令牌与坐标，导致浏览器按键连续被拒绝的问题；上游仍是固定的 Cua Driver 0.36.0 预发布版，市场标注开发预览。 -修复内容：未知 PID 可省略或填 `null`；`0`、负数、非整数等无效 PID 明确拒绝。零窗口结果保留实际筛选条件，并给出同应用重新查找或检查最小化窗口的建议；去掉无关前台应用 PID 提示，不自动扩大查询范围或替用户选择其他应用。 +修复内容：`press_key` / `type_text` 只对目标窗口的当前焦点操作，不再暴露互斥的 element_token/x/y 参数；聚焦使用独立点击工具，控件赋值由必填 element_token 的 `set_value` 完成。`scroll` 仅按截图坐标定位滚动区域。旧格式在适配层返回明确未执行回执，不发送输入、不重新截图。补充浏览器前台输入与滚动后点击规则，不自动重试结果不确定的写操作。保留上一修订的应用名发现和无效 PID 修复。 上游提交：`6d6c734c127025fb39ad5d585a10382808b39dc1`。Windows x64 原始 ZIP SHA256：`f8c47024b626f4e0e70f8428d2998df39e4879110c5722a29d5fd03e367353ad`。采用 MIT Driver，许可证随包提供；不包含 Cua Spaces、云环境或模型服务。 diff --git a/plugins/cua-driver/SKILL.md b/plugins/cua-driver/SKILL.md index e8dfd17..bfcc545 100644 --- a/plugins/cua-driver/SKILL.md +++ b/plugins/cua-driver/SKILL.md @@ -4,10 +4,10 @@ 1. 先按应用名发现窗口，例如 `list_windows({"app":"Brave","on_screen_only":true})`；未知 `pid` 必须省略或填 `null`，不能填 `0`。`app` 与非空 `pid` 同时提供时会取交集，只有该应用实际返回的 PID 才能用于后续调用。空结果时先按 `lookup.suggested_arguments` 纠正筛选（去掉错误 PID 或检查最小化窗口），不能直接宣称浏览器不可用，也不能把无关前台进程号当成目标 PID。取得目标的 `pid`、`window_id` 后读取真实画面；只操作任务要求的窗口，不关闭其他应用。 2. `get_window_state` 同时读取截图和 UIA 控件树。最新 `snapshot_id` 加 `:N` 即控件 `element_token`。优先用 `click` 点击最新控件；视觉目标用 `click_at`，坐标仅取本次截图上的窗口局部像素，不能用屏幕坐标，需携带本次 `capture_id`。两种点击参数互斥，不混用。 -3. 所有窗口操作明确传 `pid`、`window_id`。`set_value` 适合直接设置输入框；不支持时可明确聚焦后 `type_text`。观察控件支持的动作，不猜元素编号。`press_key` 传单键及 `modifiers` 数组。 +3. 所有窗口操作明确传 `pid`、`window_id`。`set_value` 必须带最新 `element_token`，用于设置具有 set_value 动作的输入框；同名 Text 标签不能当成 Edit 输入框。`press_key` 和 `type_text` 只操作当前焦点，不能传 `element_token`、`x`、`y`（也不要补 0/null）。需要聚焦控件时先 `click` / `click_at`，再输入。`press_key` 传单键及 `modifiers` 数组。 4. 每个动作后插件自动返回一次新截图和控件树，新 snapshot 替换旧 token。接着用新 token，无需重复获取相同状态。仅当画面确实还在变化或需要核验时再次观察。一次调用只执行一次动作，超时不自动重发。 -5. 默认后台操作。`effect=unverifiable/suspected_noop` 仅代表投递/效果未确认，必须检查返回画面和读回值；`refused` 是未执行。后台无效时，可在任务授权范围内显式 `delivery_mode=foreground` 后再次检查。不要仅凭工具成功声称测试通过。 -6. `scroll` 需要最新窗口/滚动区域，使用 up/down/left/right、amount、by=line/page；先观察是否真的滚动。`get_window_state(since="latest")` 可在同一窗口读取差异，但新 snapshot_id 仍替换旧 token。 +5. 默认后台操作；Brave/Chrome/Edge 等 Chromium 浏览器的点击（尤其滚动后）、快捷键、文字输入和滚动直接使用 `delivery_mode=foreground`。例如新标签页：`press_key(key="t",modifiers=["ctrl"],delivery_mode="foreground")`；地址导航：Ctrl+L → `type_text(text=网址,delivery_mode="foreground")` → Enter，各调用均带同一目标 pid/window_id。检查新页面再继续。`effect=unverifiable/suspected_noop` 仅代表投递/效果未确认，必须检查返回画面和读回值；`refused` 是未执行。其他应用后台无效时，可在任务授权范围内显式 foreground 后再次检查。不要仅凭工具成功声称测试通过。 +6. `scroll` 必须带最新截图中滚动区域内的 `x,y`，不能传 `element_token`。使用 up/down/left/right、amount、by=line/page；滚轮刻度不保证固定像素或整页距离，根据返回画面确认实际位移再继续。`get_window_state(since="latest")` 可在同一窗口读取差异，但新 snapshot_id 仍替换旧 token。 7. `save_screenshot` 保存最新窗口 PNG 到当前任务 trace，filename 仅是文件名，不能传路径，不覆盖已有文件。实际成功返回 SHA256 和附件；不得把工具内存截图当成已保存文件。 不提供 Cua 自带调度、录屏、回放、配置修改、安装器、任意脚本或自动升级接口。浏览器本轮使用窗口视觉/UIA 通路；未接入 DOM 或已有登录浏览器调试权限。此包基于上游 0.36.0 预发布版，通过已列出的本机验证，其他场景的支持范围以 VERIFICATION.md 为准。 diff --git a/plugins/cua-driver/VERIFICATION.md b/plugins/cua-driver/VERIFICATION.md index 1bc7d62..d419446 100644 --- a/plugins/cua-driver/VERIFICATION.md +++ b/plugins/cua-driver/VERIFICATION.md @@ -50,3 +50,13 @@ Cua 第三轮虽然独立验收通过，ARTEMIS 最终核验却没有生成结 新增 Go 回归覆盖无效 PID 在调用驱动前拒绝、null 省略、冲突筛选保持原范围并提示同应用重查、可见窗口为空时保留目标检查最小化窗口，以及驱动错误不得伪装成空列表。`tests/cua_discovery_smoke.py` 对已打开的指定应用只读验证主机 schema、适配器、应用名发现、null PID、冲突 PID 恢复，以及实际截图和 UIA 树；不点击、不导航、不修改用户页面。 本修订实测通过：真实 Brave 应用名枚举、null PID、省略 PID、冲突 PID 后按建议恢复，并取得真实截图及 UIA 树。原生窗口表单、34 像素目标、滚动、PNG 附件校验和窗口录像回归通过；Go 测试、vet 与 Python Ruff 检查通过。未重跑原业务的整段移动端/PC布局验收，不将窗口发现修复等同于原任务通过。 + +## 输入参数修订 0.36.0+artemis.2 + +任务 f1928dc9 的窗口发现、截图和 UIA 树已经成功，但后续三次 Ctrl+T 同时传 element_token 与 x/y，上游在执行前拒绝，未完成导航。旧适配层把此错误归为不确定并刷新截图，产生新 token 后仍重复相同冲突。 + +修订在工具 schema 中分开聚焦和键盘输入：按键/打字只接受窗口当前焦点，set_value 必须指定控件，scroll 仅接受成对坐标。Go 回归覆盖真实冲突参数（包括 x=0,y=0）、明确 not_started/none 回执，以及在接触 HWND/驱动前拒绝，不自动重试写动作。 + +`tests/cua_browser_smoke.py` 新增独立 Brave profile 和本地表单，全部操作经过打包后的 Cua MCP，未使用 CDP/DOM 控制。实测通过 Ctrl+T 新建标签、Ctrl+L 地址栏导航、UIA set_value、聚焦后的中文 type_text、表单恰好提交一次并读回、实际滚动位移、底部按钮及 PNG SHA256。测试窗口退出，不操作用户原有标签页。 + +实测还发现 Chromium 后台点击可能沿用滚动前的控件位置；前台路径会解析可见位置，已验证滚动后点击成功，schema/Skill 明确要求浏览器使用 foreground。滚轮 amount/by 不是精确滚动像素或页数，测试按页面滚动事件决定是否继续。不能据小样本宣称所有浏览器、DPI、跨端业务都通过；原业务完整任务未重跑，移动端问题不在本修订范围。 diff --git a/plugins/cua-driver/main.go b/plugins/cua-driver/main.go index 1ec7b2b..7cff8ee 100644 --- a/plugins/cua-driver/main.go +++ b/plugins/cua-driver/main.go @@ -25,7 +25,7 @@ import ( //go:embed tools.json var toolsJSON []byte -const adapterVersion = "0.36.0+artemis.1" +const adapterVersion = "0.36.0+artemis.2" type obj = map[string]any type request struct { @@ -191,6 +191,13 @@ func (a *adapter) validate(name string, args obj) error { if !ok { return errors.New("tool is not exposed") } + if name == "press_key" || name == "type_text" { + for _, key := range []string{"element_token", "x", "y"} { + if _, supplied := args[key]; supplied { + return errors.New(name + " uses the window's current focus; omit element_token/x/y. Focus with click/click_at first, or use set_value for a UIA field") + } + } + } props := object(schema["properties"]) for key := range args { if _, ok := props[key]; !ok { @@ -209,6 +216,9 @@ func (a *adapter) validate(name string, args obj) error { if num(args["pid"]) <= 0 || num(args["window_id"]) <= 0 { return errors.New("a concrete pid and window_id are required") } + if name == "set_value" && strings.TrimSpace(str(args["element_token"])) == "" { + return errors.New("set_value requires the latest element_token; use type_text for the current focus") + } if name == "save_screenshot" && !validFilename(str(args["filename"])) { return errors.New("filename must be a new plain PNG filename") } diff --git a/plugins/cua-driver/main_test.go b/plugins/cua-driver/main_test.go index 6a8896b..36698d1 100644 --- a/plugins/cua-driver/main_test.go +++ b/plugins/cua-driver/main_test.go @@ -59,3 +59,69 @@ func TestClickModesAreDisjoint(t *testing.T) { t.Fatal("pixel action lacks capture") } } + +func TestKeyboardRejectsAmbiguousTargetsBeforeWindowOrDriverAccess(t *testing.T) { + a := newAdapter(nil) // Any upstream access would panic. + for _, name := range []string{"press_key", "type_text"} { + for _, extra := range []obj{ + {"element_token": "s00000001:9", "x": 576, "y": 25}, + {"element_token": "s00000002:9", "x": 0, "y": 0}, + {"x": 0, "y": 0}, {"element_token": nil}, + } { + args := obj{"pid": 1, "window_id": 2} + if name == "press_key" { + args["key"] = "t" + args["modifiers"] = []any{"ctrl"} + } else { + args["text"] = "test" + } + for k, v := range extra { + args[k] = v + } + r := a.handle(name, args) + rec := object(object(r["structuredContent"])["artemis"]) + if r["isError"] != true || rec["execution"] != "not_started" || rec["side_effect"] != "none" || rec["error_code"] != "invalid_input" { + t.Fatalf("%s ambiguous action was not rejected before delivery: %v", name, r) + } + } + } +} + +func TestInputContractsHaveOneTargetingMode(t *testing.T) { + a := newAdapter(nil) + for _, name := range []string{"press_key", "type_text"} { + props := object(a.schemas[name]["properties"]) + for _, key := range []string{"element_token", "x", "y"} { + if _, exists := props[key]; exists { + t.Fatalf("%s exposes ambiguous %s", name, key) + } + } + } + for name, args := range map[string]obj{ + "press_key": {"pid": 1, "window_id": 2, "key": "t", "modifiers": []any{"ctrl"}, "delivery_mode": "foreground"}, + "type_text": {"pid": 1, "window_id": 2, "text": "中文"}, + "set_value": {"pid": 1, "window_id": 2, "element_token": "s00000001:9", "value": "value"}, + "scroll": {"pid": 1, "window_id": 2, "direction": "down", "x": 100, "y": 100}, + } { + if err := a.validate(name, args); err != nil { + t.Fatalf("valid %s: %v", name, err) + } + } + for _, token := range []any{nil, "", " "} { + if a.validate("set_value", obj{"pid": 1, "window_id": 2, "value": "v", "element_token": token}) == nil { + t.Fatal("set_value must target a concrete element") + } + } + if a.validate("set_value", obj{"pid": 1, "window_id": 2, "value": "v"}) == nil { + t.Fatal("set_value requires an element") + } + for _, extra := range []obj{{"x": 1}, {"x": 1, "y": 1, "element_token": "s00000001:9"}} { + args := obj{"pid": 1, "window_id": 2, "direction": "down"} + for k, v := range extra { + args[k] = v + } + if a.validate("scroll", args) == nil { + t.Fatal("ambiguous scroll accepted") + } + } +} diff --git a/plugins/cua-driver/plugin.json b/plugins/cua-driver/plugin.json index d75b48d..3914f23 100644 --- a/plugins/cua-driver/plugin.json +++ b/plugins/cua-driver/plugin.json @@ -2,7 +2,7 @@ "schema_version": 2, "id": "cua-driver", "name": "电脑控制 · Cua", - "version": "0.36.0+artemis.1", + "version": "0.36.0+artemis.2", "description": "通过 Cua Driver 的窗口截图与可访问性树操作 Windows 应用及浏览器窗口；复用 ARTEMIS 的任务调度、步骤证据与录像。", "artemis_version": ">=2.0.0,<3.0.0", "platforms": [ diff --git a/plugins/cua-driver/tools.json b/plugins/cua-driver/tools.json index f391774..175735a 100644 --- a/plugins/cua-driver/tools.json +++ b/plugins/cua-driver/tools.json @@ -63,7 +63,7 @@ "type": "string" }, "window_id": { - "description": "HWND of the target window. Must belong to `pid`. Enumerate via `list_windows` or read from `launch_app`'s `windows` array.", + "description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.", "type": "integer" } }, @@ -197,7 +197,7 @@ "type": "integer" }, "window_id": { - "description": "Exact native window identifier.", + "description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.", "type": "integer" } }, @@ -215,7 +215,7 @@ }, { "name": "click", - "description": "Click the latest UIA element_token. This tool has no pixel coordinates. For visual targets use click_at. Returns effect and fresh observation; old tokens become stale.", + "description": "Click the latest UIA element_token. This tool has no pixel coordinates. For visual targets use click_at. Returns effect and fresh observation; old tokens become stale. For Chromium/Brave use foreground, especially after scrolling: it resolves off-screen element bounds before input. Background posted clicks may use old/off-screen positions; inspect actual evidence before retrying.", "inputSchema": { "additionalProperties": false, "properties": { @@ -235,7 +235,7 @@ "type": "integer" }, "delivery_mode": { - "description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button", + "description": "background (default) avoids raising the window; unsupported input is refused. Use foreground for Chromium/Brave keyboard input and scrolling within task scope. Inspect the returned state before retrying an uncertain action.", "enum": [ "background", "foreground" @@ -252,7 +252,7 @@ "type": "integer" }, "window_id": { - "description": "HWND of the target window. Omit when element_token is supplied (the token carries it).", + "description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.", "type": "integer" } }, @@ -275,7 +275,7 @@ "additionalProperties": false, "properties": { "delivery_mode": { - "description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button", + "description": "background (default) avoids raising the window; unsupported input is refused. Use foreground for Chromium/Brave keyboard input and scrolling within task scope. Inspect the returned state before retrying an uncertain action.", "enum": [ "background", "foreground" @@ -309,7 +309,7 @@ "type": "number" }, "window_id": { - "description": "Target window handle (HWND). Optional — driver picks frontmost window of pid when omitted.", + "description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.", "type": "integer" } }, @@ -330,23 +330,18 @@ }, { "name": "type_text", - "description": "Type text into the focused target window or specified latest element. Fresh observation follows. If background is unsupported, explicitly choose foreground within task scope.", + "description": "Type text at the current focus in the exact window. No element_token or coordinates. Focus a field with click/click_at or the address bar with Ctrl+L first; use set_value for direct UIA replacement. Chromium/Brave input requires foreground. Fresh observation follows.", "inputSchema": { "additionalProperties": false, "properties": { "delivery_mode": { - "description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button", + "description": "background (default) avoids raising the window; unsupported input is refused. Use foreground for Chromium/Brave keyboard input and scrolling within task scope. Inspect the returned state before retrying an uncertain action.", "enum": [ "background", "foreground" ], "type": "string" }, - "element_token": { - "description": "Opaque element handle from get_window_state. When supplied, type_text writes through UIA ValuePattern and reads that exact element back by handle. The shared ActionResult is confirmed only when the complete expected value is synchronously v", - "pattern": "^s[0-9a-f]{8}:[0-9]+$", - "type": "string" - }, "pid": { "description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.", "type": "integer" @@ -356,16 +351,8 @@ "type": "string" }, "window_id": { - "description": "HWND of the target window. Omit when element_token is supplied (the token carries it).", + "description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.", "type": "integer" - }, - "x": { - "description": "Window-local screenshot-pixel X of the field to type into — the element px action form. Pass x,y (no element_token) and the tool pixel-clicks there to establish real renderer focus, then types. Use for Chromium/Electron inputs the UIA/WM_CH", - "type": "number" - }, - "y": { - "description": "Window-local screenshot-pixel Y of the field (see x).", - "type": "number" } }, "required": [ @@ -382,23 +369,18 @@ }, { "name": "press_key", - "description": "Send key and modifiers to the exact window, optionally focus element_token or x/y first. Fresh observation follows.", + "description": "Send a key and modifiers to the exact window current focus. No element_token or coordinates. For browser shortcuts (Ctrl+T, Ctrl+L, Enter), use foreground. To target a field, focus it with click/click_at first. Fresh observation follows; inspect before repeating.", "inputSchema": { "additionalProperties": false, "properties": { "delivery_mode": { - "description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button", + "description": "background (default) avoids raising the window; unsupported input is refused. Use foreground for Chromium/Brave keyboard input and scrolling within task scope. Inspect the returned state before retrying an uncertain action.", "enum": [ "background", "foreground" ], "type": "string" }, - "element_token": { - "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current sna", - "pattern": "^s[0-9a-f]{8}:[0-9]+$", - "type": "string" - }, "key": { "description": "Key name (return, tab, escape, up, down, left, right, space, delete, home, end, pageup, pagedown, f1-f12, letter, digit).", "type": "string" @@ -415,16 +397,8 @@ "type": "integer" }, "window_id": { - "description": "HWND for the target window. Omit when element_token is supplied; otherwise auto-resolves the pid's first visible window.", + "description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.", "type": "integer" - }, - "x": { - "description": "Window-local screenshot-pixel X — the element px action form: pixel-click there to focus, then send the key. Use when the key must go to a Chromium/Electron surface the UIA path can't focus. Pass with y, no element_token.", - "type": "number" - }, - "y": { - "description": "Window-local screenshot-pixel Y (see x).", - "type": "number" } }, "required": [ @@ -446,7 +420,7 @@ "additionalProperties": false, "properties": { "delivery_mode": { - "description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button", + "description": "background (default) avoids raising the window; unsupported input is refused. Use foreground for Chromium/Brave keyboard input and scrolling within task scope. Inspect the returned state before retrying an uncertain action.", "enum": [ "background", "foreground" @@ -467,14 +441,15 @@ "type": "string" }, "window_id": { - "description": "HWND of the window. Omit when element_token is supplied (the token carries it).", + "description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.", "type": "integer" } }, "required": [ "pid", "window_id", - "value" + "value", + "element_token" ], "type": "object" }, @@ -485,7 +460,7 @@ }, { "name": "scroll", - "description": "Scroll the target window/element at x,y. amount=1..50, by=line/page. Verify content changed in fresh returned observation.", + "description": "Scroll at x,y in the latest window screenshot. No element_token. Use a point inside the intended scroll surface, amount=1..50, by=line/page. Chromium/Brave requires foreground. Verify actual content movement in the returned observation.", "inputSchema": { "additionalProperties": false, "properties": { @@ -504,7 +479,7 @@ "type": "string" }, "delivery_mode": { - "description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button", + "description": "background (default) avoids raising the window; unsupported input is refused. Use foreground for Chromium/Brave keyboard input and scrolling within task scope. Inspect the returned state before retrying an uncertain action.", "enum": [ "background", "foreground" @@ -521,32 +496,29 @@ ], "type": "string" }, - "element_token": { - "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current sna", - "pattern": "^s[0-9a-f]{8}:[0-9]+$", - "type": "string" - }, "pid": { "description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.", "type": "integer" }, "window_id": { - "description": "HWND of the target window. Omit when element_token is supplied; otherwise auto-resolves the pid's first visible window.", + "description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.", "type": "integer" }, "x": { - "description": "With pid/window_id: window-local screenshot X used to target a nested scroll surface in foreground mode. Without pid/window_id: screen-absolute X for desktop scope. Must be paired with y.", + "description": "X inside the scroll area in latest window-local screenshot pixels; never screen coordinates. Must be paired with the other coordinate.", "type": "number" }, "y": { - "description": "With pid/window_id: window-local screenshot Y used to target a nested scroll surface in foreground mode. Without pid/window_id: screen-absolute Y for desktop scope. Must be paired with x.", + "description": "Y inside the scroll area in latest window-local screenshot pixels; never screen coordinates. Must be paired with the other coordinate.", "type": "number" } }, "required": [ "pid", "window_id", - "direction" + "direction", + "x", + "y" ], "type": "object" }, @@ -569,7 +541,8 @@ }, "window_id": { "type": "integer", - "minimum": 1 + "minimum": 1, + "description": "Exact HWND returned for this application by list_windows. Required and must belong to pid." }, "filename": { "type": "string" @@ -613,7 +586,7 @@ "type": "integer" }, "delivery_mode": { - "description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button", + "description": "background (default) avoids raising the window; unsupported input is refused. Use foreground for Chromium/Brave keyboard input and scrolling within task scope. Inspect the returned state before retrying an uncertain action.", "enum": [ "background", "foreground" @@ -625,7 +598,7 @@ "type": "integer" }, "window_id": { - "description": "HWND of the target window. Omit when element_token is supplied (the token carries it).", + "description": "Exact HWND returned for this application by list_windows. Required and must belong to pid.", "type": "integer" }, "x": { diff --git a/scripts/build_cua_driver.py b/scripts/build_cua_driver.py index 96c7d51..7a46ba7 100644 --- a/scripts/build_cua_driver.py +++ b/scripts/build_cua_driver.py @@ -99,7 +99,7 @@ def main(): "source_commit": COMMIT, "upstream_version": UPSTREAM_VERSION, "plugin_version": version, - "adapter_revision": 2, + "adapter_revision": 3, "prerelease": True, "url": URL, "sha256": DIGEST, diff --git a/tests/cua_browser_smoke.py b/tests/cua_browser_smoke.py new file mode 100644 index 0000000..62737e5 --- /dev/null +++ b/tests/cua_browser_smoke.py @@ -0,0 +1,463 @@ +"""Opt-in Cua browser regression: real keyboard, form, scrolling and evidence. + +Uses a disposable browser profile and a loopback event oracle, no CDP/DOM control. +Only the launched browser process tree is cleaned up. No user tabs are changed. +""" + +import argparse +import asyncio +import hashlib +import json +import re +import subprocess +import tempfile +import threading +import time +import uuid +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import httpx +import jsonschema +import psutil +from artemis.plugins.models import ServerConfig +from artemis.plugins.session import PluginSession + + +async def model_task(base, pid, url, output, events, traces, model): + """Opt-in paid model run through the user's existing idle ARTEMIS service.""" + sid = str(uuid.uuid4()) + submitted = done = False + async with httpx.AsyncClient(base_url=base, trust_env=False, timeout=30) as client: + + async def api(path, body=None): + r = ( + await client.get(path) + if body is None + else await client.post(path, json=body) + ) + r.raise_for_status() + return r.json() + + status = await api("/api/status") + assert not status.get("active_tasks") and not status.get("queue"), ( + "ARTEMIS is busy" + ) + body = { + "goal": f"仅操作 PID={pid} 的独立 Brave 测试窗口，不操作其他浏览器窗口或手机。新建一个标签页并通过地址栏打开 {url}/form。将 Test parameter 输入框填写为 ARTEMIS 中文参数，点击 Apply parameter 一次，核对页面显示 Saved: ARTEMIS 中文参数。然后向下滚动到页面底部，点击 Finish scrolling 一次，确认显示 Finished。把最终窗口截图保存到任务 trace，文件名 browser-model-proof.png。以实际页面证据判定结果。", + "session_id": sid, + "profile": "flash", + "ingress": "client", + "runtime_overrides": { + "execution_target": "computer", + "plugins": [ + { + "id": "cua-driver", + "profile": "benchmark", + "version": "0.36.0+artemis.2", + } + ], + "model": model, + "reasoning_effort": "high", + "timeout_seconds": 300, + "request_timeout_seconds": 90, + "max_retries": 1, + "record_video": False, + "analyze_video": False, + }, + } + (output / "request.json").write_text( + json.dumps(body, ensure_ascii=False, indent=2), "utf-8" + ) + try: + reply = await api("/api/run", body) + assert reply.get("status") not in {"rejected", "error"}, reply + submitted = True + print(json.dumps({"trace_id": sid, "submission": reply}), flush=True) + async with asyncio.timeout(420): + while True: + await asyncio.sleep(10) + status = await api(f"/api/sessions/{sid}") + print( + json.dumps( + { + "trace_id": sid, + "status": status["status"], + "saved": any(e["path"] == "/save" for e in events), + "finished": any(e["path"] == "/finish" for e in events), + } + ), + flush=True, + ) + if status["status"] not in { + "running", + "pending", + "queued", + "stopping", + "starting", + }: + done = True + break + proof = ( + traces / sid / "artifacts/plugins/cua-driver/browser-model-proof.png" + ) + report = { + "trace_id": sid, + "status": status["status"], + "events": list(events), + "actual_pass": ( + [e["body"] for e in events if e["path"] == "/save"] + == ["ARTEMIS 中文参数"] + and len([e for e in events if e["path"] == "/finish"]) == 1 + and any( + e["path"] == "/scroll" and float(e["body"]) > 1000 + for e in events + ) + and proof.is_file() + ), + } + (output / "model-report.json").write_text( + json.dumps(report, ensure_ascii=False, indent=2), "utf-8" + ) + assert report["actual_pass"], report + print(json.dumps({"trace_id": sid, "actual_pass": True}), flush=True) + finally: + if submitted and not done: + await api("/api/stop", {"session_id": sid}) + + +async def run( + executable: Path, + browser: Path, + output: Path, + base=None, + traces=None, + model="gpt-6.1-sol", +): + output.mkdir(parents=True, exist_ok=True) + events = [] + calls = [] + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_GET(self): + events.append({"path": self.path}) + if self.path == "/form": + body = """ + ARTEMIS Cua Browser Form + +

ARTEMIS Cua Browser Form

+ + +
Pending
+
Scroll down to finish
+ + + """ + else: + body = "ARTEMIS Cua Browser Start

Isolated browser test

" + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.end_headers() + self.wfile.write(body.encode()) + + def do_POST(self): + events.append( + { + "path": self.path, + "body": self.rfile.read( + int(self.headers["Content-Length"]) + ).decode(), + } + ) + self.send_response(200) + self.end_headers() + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + url = f"http://127.0.0.1:{server.server_port}" + session = None + process = None + try: + with tempfile.TemporaryDirectory(prefix="artemis-cua-browser-") as profile: + try: + process = await asyncio.create_subprocess_exec( + str(browser), + "--user-data-dir=" + profile, + "--no-first-run", + "--no-default-browser-check", + "--disable-extensions", + "--disable-background-mode", + "--force-renderer-accessibility", + "--window-size=1000,800", + "--window-position=100,100", + url + "/start", + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + creationflags=subprocess.CREATE_NO_WINDOW, + ) + if base: + await model_task( + base, process.pid, url, output, events, traces, model + ) + return + session = await PluginSession( + ServerConfig( + command=str(executable), + env={"ARTEMIS_ARTIFACTS_DIR": str(output / "attachments")}, + ), + executable.parent, + ).start() + schemas = {t.name: t.inputSchema for t in session.tools} + + async def call(name, args, *, invalid=False): + if not invalid: + jsonschema.validate(args, schemas[name]) + started = time.monotonic() + result = await session.call(name, args) + data = result.structuredContent or {} + calls.append( + { + "tool": name, + "arguments": args, + "seconds": round(time.monotonic() - started, 3), + "is_error": result.isError, + "result": data, + "text": [ + b.text for b in result.content if b.type == "text" + ], + } + ) + (output / "calls.json").write_text( + json.dumps(calls, ensure_ascii=False, indent=2), "utf-8" + ) + if not invalid: + assert not result.isError, calls[-1] + return data + + async with asyncio.timeout(20): + while True: + listing = await call("list_windows", {"pid": process.pid}) + own = listing["windows"] + if own: + break + await asyncio.sleep(0.2) + target = {"pid": process.pid, "window_id": own[0]["window_id"]} + + async def observe(): + return await call("get_window_state", target) + + async def wait_tree(text): + async with asyncio.timeout(10): + while True: + state = await observe() + if text in state["tree_markdown"]: + return state + await asyncio.sleep(0.2) + + async def key(key, modifiers=None): + return await call( + "press_key", + { + **target, + "key": key, + "modifiers": modifiers or [], + "delivery_mode": "foreground", + }, + ) + + def token(state, label, role="Button"): + match = re.search( + r"\[(\d+)\] " + re.escape(role) + ' "' + re.escape(label) + '"', + state["tree_markdown"], + ) + assert match, state["tree_markdown"] + return state["snapshot_id"] + ":" + match[1] + + await wait_tree("ARTEMIS Cua Browser Start") + # Reproduce task f1928... invalid form before any keyboard dispatch. + bad = await call( + "press_key", + { + **target, + "key": "t", + "modifiers": ["ctrl"], + "element_token": "s00000001:9", + "x": 0, + "y": 0, + }, + invalid=True, + ) + assert bad["artemis"]["execution"] == "not_started" + assert bad["artemis"]["side_effect"] == "none" + assert "observation" not in bad + await key("t", ["ctrl"]) + state = await observe() + assert "ARTEMIS Cua Browser Start" not in state.get("window_title", "") + await key("l", ["ctrl"]) + await call( + "type_text", + {**target, "text": url + "/form", "delivery_mode": "foreground"}, + ) + await key("return") + state = await wait_tree("Test parameter") + # Direct UIA replacement is a separate, unambiguous tool. + await call( + "set_value", + { + **target, + "element_token": token(state, "Test parameter", "Edit"), + "value": "seed", + }, + ) + state = await observe() + await call( + "click", + { + **target, + "element_token": token(state, "Test parameter", "Edit"), + "delivery_mode": "foreground", + }, + ) + await key("a", ["ctrl"]) + await call( + "type_text", + { + **target, + "text": "ARTEMIS 中文参数", + "delivery_mode": "foreground", + }, + ) + state = await wait_tree("ARTEMIS 中文参数") + await call( + "click", + {**target, "element_token": token(state, "Apply parameter")}, + ) + state = await wait_tree("Saved: ARTEMIS 中文参数") + assert [e["body"] for e in events if e["path"] == "/save"] == [ + "ARTEMIS 中文参数" + ] + source = state["artemis"]["source"] + # Wheel ticks do not guarantee an exact pixel/page distance. + # Observe each actual movement before another bounded scroll. + for _ in range(3): + before = sum(e["path"] == "/scroll" for e in events) + await call( + "scroll", + { + **target, + "direction": "down", + "amount": 8, + "by": "page", + "x": source["width"] // 2, + "y": source["height"] // 2, + "delivery_mode": "foreground", + }, + ) + async with asyncio.timeout(5): + while sum(e["path"] == "/scroll" for e in events) <= before: + await asyncio.sleep(0.1) + state = await observe() + source = state["artemis"]["source"] + if any( + e["path"] == "/scroll" and float(e["body"]) > 1000 + for e in events + ): + break + assert any( + e["path"] == "/scroll" and float(e["body"]) > 1000 for e in events + ) + state = await observe() + await call( + "click", + { + **target, + "element_token": token(state, "Finish scrolling"), + "delivery_mode": "foreground", + }, + ) + await wait_tree("Finished") + assert len([e for e in events if e["path"] == "/finish"]) == 1 + saved = await call( + "save_screenshot", {**target, "filename": "browser-proof.png"} + ) + attachment = saved["artemis"]["attachments"][0] + assert ( + hashlib.sha256( + (output / "attachments" / attachment["name"]).read_bytes() + ).hexdigest() + == attachment["sha256"] + ) + report = { + "passed": True, + "checks": [ + "mixed parameters refused before dispatch", + "Ctrl+T new tab", + "Ctrl+L URL navigation", + "UIA set_value", + "focused Unicode text input", + "single form submission readback", + "actual page scroll", + "finish button", + "PNG hash", + ], + "events": events, + "calls": len(calls), + } + (output / "report.json").write_text( + json.dumps(report, ensure_ascii=False, indent=2), "utf-8" + ) + print(json.dumps(report, ensure_ascii=False)) + finally: + (output / "events.json").write_text( + json.dumps(events, ensure_ascii=False, indent=2), "utf-8" + ) + if session: + await session.aclose() + if process: + try: + parent = psutil.Process(process.pid) + owned = [*parent.children(recursive=True), parent] + for proc in owned: + try: + proc.terminate() + except psutil.NoSuchProcess: + pass + _, alive = psutil.wait_procs(owned, timeout=5) + for proc in alive: + proc.kill() + psutil.wait_procs(alive, timeout=5) + except psutil.NoSuchProcess: + pass + finally: + server.shutdown() + server.server_close() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(__doc__) + parser.add_argument("executable", type=Path) + parser.add_argument("--browser", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument( + "--model-base", help="Opt-in paid model test through an existing ARTEMIS server" + ) + parser.add_argument( + "--traces", type=Path, help="That server's traces directory (model test only)" + ) + parser.add_argument("--model", default="gpt-6.1-sol") + args = parser.parse_args() + if args.model_base and not args.traces: + parser.error("--model-base requires --traces") + asyncio.run( + run( + args.executable.resolve(), + args.browser, + args.output.resolve(), + args.model_base, + args.traces, + args.model, + ) + )