From 1c1d700e9b380d9de2dc2f23dc27a4a170ecb27b Mon Sep 17 00:00:00 2001 From: chenmingxuan Date: Sat, 17 Oct 2026 13:23:56 +0800 Subject: [PATCH] fix: validate Cua window discovery filters and guide recovery --- README.md | 7 +- catalog.json | 8 +-- plugins/cua-driver/RELEASE.md | 6 +- plugins/cua-driver/SKILL.md | 2 +- plugins/cua-driver/VERIFICATION.md | 8 +++ plugins/cua-driver/main.go | 18 ++++- plugins/cua-driver/plugin.json | 2 +- plugins/cua-driver/tools.json | 41 ++++++----- plugins/cua-driver/window_lookup.go | 92 ++++++++++++++++++++++++ plugins/cua-driver/window_lookup_test.go | 88 +++++++++++++++++++++++ scripts/build_cua_driver.py | 8 ++- tests/cua_discovery_smoke.py | 88 +++++++++++++++++++++++ 12 files changed, 337 insertions(+), 31 deletions(-) create mode 100644 plugins/cua-driver/window_lookup.go create mode 100644 plugins/cua-driver/window_lookup_test.go create mode 100644 tests/cua_discovery_smoke.py diff --git a/README.md b/README.md index 6ca7660..7ff0ed8 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Artemis Plugins -ARTEMIS 2.0.0 的插件市场。当前电脑控制插件为 **Cua 0.36.0**，支持 Windows x64，通过窗口截图与可访问性树操作电脑应用和浏览器窗口，复用 ARTEMIS 的调度、资源互斥、步骤、截图附件和录像。 +ARTEMIS 2.0.0 的插件市场。当前电脑控制插件为 **Cua 0.36.0+artemis.1**，支持 Windows x64，通过窗口截图与可访问性树操作电脑应用和浏览器窗口，复用 ARTEMIS 的调度、资源互斥、步骤、截图附件和录像。 在 ARTEMIS 的「插件」页面安装「电脑控制 · Cua」，授权并启用即可。安卓任务可根据提示词按需调用电脑控制，不需要单独配置 Codex MCP 或安装 Go、Node、Python。 @@ -24,12 +24,13 @@ Open Computer Use（OCU）已从市场索引下架，当前市场只提供 Cua python scripts/build_cua_driver.py <上游ZIP> --go python tests/cua_smoke.py <安装后的artemis-cua-adapter.exe> artifacts/cua-smoke python tests/cua_fixture_validation.py <安装后的artemis-cua-adapter.exe> +python tests/cua_discovery_smoke.py <安装后的artemis-cua-adapter.exe> --app Brave --output artifacts/cua-discovery.json ``` -验证脚本使用已安装 ARTEMIS 依赖的 Python，并将 ARTEMIS 源码目录放入 `PYTHONPATH`。它们只操作自行创建的测试窗口。历史 A/B 脚本仍固定 OCU 1.1.0 与 Cua 0.1.2，便于复现已有测量；它不是安装后必须执行的步骤。 +验证脚本使用已安装 ARTEMIS 依赖的 Python，并将 ARTEMIS 源码目录放入 `PYTHONPATH`。前两项实测只操作自行创建的测试窗口；窗口发现实测只读取指定的已打开应用，不改变其页面。历史 A/B 脚本仍固定 OCU 1.1.0 与 Cua 0.1.2，便于复现已有测量；它不是安装后必须执行的步骤。 `catalog.json` 是市场索引，包含清单、HTTPS 安装包地址、SHA256 和验证说明。`artifacts/` 保存本机安装包与测试资料，不进入 Git。版本文件不可原地覆盖，任务固定使用其启动时的版本。 -插件版本跟随所用 Cua Driver 上游版本：当前为 `0.36.0`，对应上游标签 `cua-driver-rs-v0.36.0`，本仓库发行标签为 `cua-driver-v0.36.0`。只修改 ARTEMIS 适配而未升级上游时使用 `+artemis.N` 后缀区分不可变安装包；包内 `provenance.json` 记录上游提交、原包校验值和适配修订，构建会检查版本对应关系。 +插件版本跟随所用 Cua Driver 上游版本：当前为 `0.36.0`，对应上游标签 `cua-driver-rs-v0.36.0`，当前适配修订为 `0.36.0+artemis.1`，本仓库发行标签为 `cua-driver-v0.36.0+artemis.1`。只修改 ARTEMIS 适配而未升级上游时使用 `+artemis.N` 后缀区分不可变安装包；包内 `provenance.json` 记录上游提交、原包校验值和适配修订，构建会检查版本对应关系。 [发行说明](plugins/cua-driver/RELEASE.md) · [ARTEMIS 主项目](https://git.playones.com/chenmingxuan/Artemis-Eapil) diff --git a/catalog.json b/catalog.json index 677c6c8..5e8fc0c 100644 --- a/catalog.json +++ b/catalog.json @@ -6,7 +6,7 @@ "schema_version": 2, "id": "cua-driver", "name": "电脑控制 · Cua", - "version": "0.36.0", + "version": "0.36.0+artemis.1", "description": "通过 Cua Driver 的窗口截图与可访问性树操作 Windows 应用及浏览器窗口；复用 ARTEMIS 的任务调度、步骤证据与录像。", "artemis_version": ">=2.0.0,<3.0.0", "platforms": [ @@ -172,9 +172,9 @@ } }, "release_stage": "preview", - "archive_url": "https://git.playones.com/chenmingxuan/Artemis-Plugins/releases/download/cua-driver-v0.36.0/cua-driver-0.36.0-windows-amd64.zip", - "sha256": "dea40dce182ac6b982c7da17d3610d5db87e2639205ca32f179c55ca1d7833ce", - "verification": "Windows x64 / ARTEMIS 2.0.0 开发版。固定 Cua Driver 0.36.0 预发布版。已验证原生窗口表单、控件与像素点击、34 像素目标、滚动、trace 截图及哈希、ARTEMIS PC 窗口录像、子进程退出；同一复合用例重复 3 次，独立验收 3/3，AI 裁定 2 次通过、1 次未判定。0.36.0 沿用已测 0.1.2 动作实现，取代 OCU 市场入口。未覆盖复杂浏览器控制台、多 DPI、多显示器和跨端长任务。完整验证与已知限制见仓库 plugins/cua-driver/VERIFICATION.md。" + "archive_url": "https://git.playones.com/chenmingxuan/Artemis-Plugins/releases/download/cua-driver-v0.36.0+artemis.1/cua-driver-0.36.0+artemis.1-windows-amd64.zip", + "sha256": "43a90d514e0d148e4476ca8508e22a2bd1f60d8af97cb5393a7a2034e2af6aa6", + "verification": "Windows x64 / ARTEMIS 2.0.0 开发版。固定 Cua Driver 0.36.0 预发布版。已验证原生窗口表单、控件与像素点击、34 像素目标、滚动、trace 截图及哈希、ARTEMIS PC 窗口录像、子进程退出；同一复合用例重复 3 次，独立验收 3/3，AI 裁定 2 次通过、1 次未判定。0.36.0 沿用已测 0.1.2 动作实现，取代 OCU 市场入口。未覆盖复杂浏览器控制台、多 DPI、多显示器和跨端长任务。完整验证与已知限制见仓库 plugins/cua-driver/VERIFICATION.md。 本修订实测补充：Brave 按应用名发现、无效/null PID、冲突 PID 纠正、实际截图及 UIA 树通过；未重跑完整跨端业务任务。" } ] } diff --git a/plugins/cua-driver/RELEASE.md b/plugins/cua-driver/RELEASE.md index 9bf8a2f..5d6f3b4 100644 --- a/plugins/cua-driver/RELEASE.md +++ b/plugins/cua-driver/RELEASE.md @@ -1,6 +1,8 @@ -# Cua 电脑控制 0.36.0 +# Cua 电脑控制 0.36.0+artemis.1 -作为 ARTEMIS 插件市场的默认电脑控制实现，替代已下架的 Open Computer Use。此版本沿用候选 0.1.2 的驱动和动作逻辑，更新版本标识、名称和分发文档；上游仍是固定的 Cua Driver 0.36.0 预发布版，市场标注开发预览。 +作为 ARTEMIS 插件市场的默认电脑控制实现，替代已下架的 Open Computer Use。此修订修复按应用名查找窗口时，被无效或冲突 PID 筛选为零窗口并误判浏览器不可用的问题；上游仍是固定的 Cua Driver 0.36.0 预发布版，市场标注开发预览。 + +修复内容：未知 PID 可省略或填 `null`；`0`、负数、非整数等无效 PID 明确拒绝。零窗口结果保留实际筛选条件，并给出同应用重新查找或检查最小化窗口的建议；去掉无关前台应用 PID 提示，不自动扩大查询范围或替用户选择其他应用。 上游提交：`6d6c734c127025fb39ad5d585a10382808b39dc1`。Windows x64 原始 ZIP SHA256：`f8c47024b626f4e0e70f8428d2998df39e4879110c5722a29d5fd03e367353ad`。采用 MIT Driver，许可证随包提供；不包含 Cua Spaces、云环境或模型服务。 diff --git a/plugins/cua-driver/SKILL.md b/plugins/cua-driver/SKILL.md index 1077b66..e8dfd17 100644 --- a/plugins/cua-driver/SKILL.md +++ b/plugins/cua-driver/SKILL.md @@ -2,7 +2,7 @@ 通过本地 Cua Driver 操作用户授权的 Windows 窗口。调度、资源互斥、模型、步骤、截图和录像由 ARTEMIS 提供；插件本身不运行 AI，不另起后台服务。 -1. `list_windows` 找到目标的 `pid` 和 `window_id`；只操作任务要求的窗口，不关闭其他应用。 +1. 先按应用名发现窗口，例如 `list_windows({"app":"Brave","on_screen_only":true})`；未知 `pid` 必须省略或填 `null`，不能填 `0`。`app` 与非空 `pid` 同时提供时会取交集，只有该应用实际返回的 PID 才能用于后续调用。空结果时先按 `lookup.suggested_arguments` 纠正筛选（去掉错误 PID 或检查最小化窗口），不能直接宣称浏览器不可用，也不能把无关前台进程号当成目标 PID。取得目标的 `pid`、`window_id` 后读取真实画面；只操作任务要求的窗口，不关闭其他应用。 2. `get_window_state` 同时读取截图和 UIA 控件树。最新 `snapshot_id` 加 `:N` 即控件 `element_token`。优先用 `click` 点击最新控件；视觉目标用 `click_at`，坐标仅取本次截图上的窗口局部像素，不能用屏幕坐标，需携带本次 `capture_id`。两种点击参数互斥，不混用。 3. 所有窗口操作明确传 `pid`、`window_id`。`set_value` 适合直接设置输入框；不支持时可明确聚焦后 `type_text`。观察控件支持的动作，不猜元素编号。`press_key` 传单键及 `modifiers` 数组。 4. 每个动作后插件自动返回一次新截图和控件树，新 snapshot 替换旧 token。接着用新 token，无需重复获取相同状态。仅当画面确实还在变化或需要核验时再次观察。一次调用只执行一次动作，超时不自动重发。 diff --git a/plugins/cua-driver/VERIFICATION.md b/plugins/cua-driver/VERIFICATION.md index 191d754..1bc7d62 100644 --- a/plugins/cua-driver/VERIFICATION.md +++ b/plugins/cua-driver/VERIFICATION.md @@ -42,3 +42,11 @@ Cua 第三轮虽然独立验收通过，ARTEMIS 最终核验却没有生成结 ## 市场替换版本 0.36.0 0.36.0 沿用 0.1.2 已测驱动和动作实现，仅更新发行版本、名称及文档，作为市场默认电脑控制插件取代 OCU。上游仍固定 0.36.0 预发布版，市场继续标注开发预览；上文的未测范围和 AI 裁定缺失情况仍然有效。旧 OCU 从 catalog.json 下架，既有源码、发行附件和任务证据保留。已安装 OCU 的用户应安装并启用 Cua，再停用 OCU；不会静默为其他用户授权新代码。 + +## 窗口发现修订 0.36.0+artemis.1 + +修复复跑任务在按应用名找 Brave 时填入 PID 0，随后误用无关前台应用 PID 的失败链路。原生窗口基准没有覆盖这一入口，原有 A/B 数据不作为浏览器发现能力的验证。 + +新增 Go 回归覆盖无效 PID 在调用驱动前拒绝、null 省略、冲突筛选保持原范围并提示同应用重查、可见窗口为空时保留目标检查最小化窗口，以及驱动错误不得伪装成空列表。`tests/cua_discovery_smoke.py` 对已打开的指定应用只读验证主机 schema、适配器、应用名发现、null PID、冲突 PID 恢复，以及实际截图和 UIA 树；不点击、不导航、不修改用户页面。 + +本修订实测通过：真实 Brave 应用名枚举、null PID、省略 PID、冲突 PID 后按建议恢复，并取得真实截图及 UIA 树。原生窗口表单、34 像素目标、滚动、PNG 附件校验和窗口录像回归通过；Go 测试、vet 与 Python Ruff 检查通过。未重跑原业务的整段移动端/PC布局验收，不将窗口发现修复等同于原任务通过。 diff --git a/plugins/cua-driver/main.go b/plugins/cua-driver/main.go index 8a86938..1ec7b2b 100644 --- a/plugins/cua-driver/main.go +++ b/plugins/cua-driver/main.go @@ -25,6 +25,8 @@ import ( //go:embed tools.json var toolsJSON []byte +const adapterVersion = "0.36.0+artemis.1" + type obj = map[string]any type request struct { JSONRPC string `json:"jsonrpc"` @@ -104,7 +106,7 @@ func startPeer(exe string) (*peer, error) { p.lines <- append([]byte(nil), s.Bytes()...) } }() - _, err = p.call("initialize", obj{"protocolVersion": "2024-11-05", "clientInfo": obj{"name": "artemis-cua-adapter", "version": "0.36.0"}, "capabilities": obj{}}) + _, err = p.call("initialize", obj{"protocolVersion": "2024-11-05", "clientInfo": obj{"name": "artemis-cua-adapter", "version": adapterVersion}, "capabilities": obj{}}) if err != nil { p.stop() return nil, err @@ -200,7 +202,11 @@ func (a *adapter) validate(name string, args obj) error { return fmt.Errorf("missing argument: %s", str(key)) } } - if name != "list_windows" && (num(args["pid"]) <= 0 || num(args["window_id"]) <= 0) { + if name == "list_windows" { + _, err := windowQuery(args) + return err + } + if num(args["pid"]) <= 0 || num(args["window_id"]) <= 0 { return errors.New("a concrete pid and window_id are required") } if name == "save_screenshot" && !validFilename(str(args["filename"])) { @@ -302,6 +308,12 @@ func (a *adapter) handle(name string, args obj) obj { } } r, src, err = a.observe(args) + } else if name == "list_windows" { + query, _ := windowQuery(args) // Already validated above. + r, err = a.peer.tool(name, query) + if err == nil { + r = windowListing(r, query) + } } else { upstream := name if name == "click_at" { @@ -412,7 +424,7 @@ func serve(a *adapter, in io.Reader, out io.Writer) { var rpcErr any switch q.Method { case "initialize": - result = obj{"protocolVersion": "2024-11-05", "capabilities": obj{"tools": obj{}}, "serverInfo": obj{"name": "artemis-cua-driver", "version": "0.36.0"}} + result = obj{"protocolVersion": "2024-11-05", "capabilities": obj{"tools": obj{}}, "serverInfo": obj{"name": "artemis-cua-driver", "version": adapterVersion}} case "ping": result = obj{} case "tools/list": diff --git a/plugins/cua-driver/plugin.json b/plugins/cua-driver/plugin.json index 30d9ba8..d75b48d 100644 --- a/plugins/cua-driver/plugin.json +++ b/plugins/cua-driver/plugin.json @@ -2,7 +2,7 @@ "schema_version": 2, "id": "cua-driver", "name": "电脑控制 · Cua", - "version": "0.36.0", + "version": "0.36.0+artemis.1", "description": "通过 Cua Driver 的窗口截图与可访问性树操作 Windows 应用及浏览器窗口；复用 ARTEMIS 的任务调度、步骤证据与录像。", "artemis_version": ">=2.0.0,<3.0.0", "platforms": [ diff --git a/plugins/cua-driver/tools.json b/plugins/cua-driver/tools.json index cb6dcf5..f391774 100644 --- a/plugins/cua-driver/tools.json +++ b/plugins/cua-driver/tools.json @@ -1,21 +1,29 @@ [ { "name": "list_windows", - "description": "List visible application windows with exact pid/window_id and physical bounds. Pick only the user-authorized target.", + "description": "Find the authorized application window. If its PID is unknown, call with app only, e.g. {\"app\":\"Brave\",\"on_screen_only\":true}; omit pid or use null, never 0. Use the returned matching pid/window_id for later actions. Empty filtered results are not proof that the app is unavailable; follow lookup.suggested_arguments when present.", "inputSchema": { "additionalProperties": false, "properties": { "app": { - "description": "Only windows of apps whose name contains this text (case-insensitive), or whose bundle id equals it. Prefer this or `pid` over an unfiltered call, which returns every window on the host.", - "type": "string" + "description": "Application name substring (case-insensitive). Start with this when the target PID is unknown. app and a non-null pid are combined filters; do not copy the PID of an unrelated foreground app.", + "type": [ + "string", + "null" + ] }, "on_screen_only": { "description": "When true, drop windows that aren't currently on-screen. Default false.", "type": "boolean" }, "pid": { - "description": "Optional pid filter. When set, only this pid's windows are returned.", - "type": "integer" + "description": "Known positive Windows process ID from a matching list_windows result. Unknown: omit or null; never 0. With app, both filters must match the same window.", + "type": [ + "integer", + "null" + ], + "minimum": 1, + "maximum": 4294967295 } }, "type": "object", @@ -43,7 +51,7 @@ "type": "integer" }, "pid": { - "description": "Process ID from `list_apps`.", + "description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.", "type": "integer" }, "query": { @@ -170,7 +178,7 @@ "type": "boolean" }, "pid": { - "description": "Exact process whose window may be observed.", + "description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.", "minimum": 1, "type": "integer" }, @@ -240,7 +248,7 @@ "type": "string" }, "pid": { - "description": "Target process ID for window scope. Optional with element_token (the token carries the pid). Omit with scope=desktop for screen-absolute coordinates from get_desktop_state.", + "description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.", "type": "integer" }, "window_id": { @@ -289,7 +297,7 @@ "type": "number" }, "pid": { - "description": "Target process ID.", + "description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.", "type": "integer" }, "to_x": { @@ -340,7 +348,7 @@ "type": "string" }, "pid": { - "description": "Target process ID.", + "description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.", "type": "integer" }, "text": { @@ -403,7 +411,7 @@ "type": "array" }, "pid": { - "description": "Target process ID.", + "description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.", "type": "integer" }, "window_id": { @@ -451,7 +459,7 @@ "type": "string" }, "pid": { - "description": "Target process ID. Optional with element_token (the token carries the pid).", + "description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.", "type": "integer" }, "value": { @@ -519,7 +527,7 @@ "type": "string" }, "pid": { - "description": "Target process ID for window scope. Optional with element_token (the token carries the pid). Omit with scope=desktop for screen-absolute coordinates from get_desktop_state.", + "description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.", "type": "integer" }, "window_id": { @@ -556,7 +564,8 @@ "properties": { "pid": { "type": "integer", - "minimum": 1 + "minimum": 1, + "description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID." }, "window_id": { "type": "integer", @@ -612,7 +621,7 @@ "type": "string" }, "pid": { - "description": "Target process ID for window scope. Optional with element_token (the token carries the pid). Omit with scope=desktop for screen-absolute coordinates from get_desktop_state.", + "description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.", "type": "integer" }, "window_id": { @@ -642,4 +651,4 @@ "idempotentHint": false } } -] \ No newline at end of file +] diff --git a/plugins/cua-driver/window_lookup.go b/plugins/cua-driver/window_lookup.go new file mode 100644 index 0000000..d639e7c --- /dev/null +++ b/plugins/cua-driver/window_lookup.go @@ -0,0 +1,92 @@ +package main + +import ( + "errors" + "fmt" + "math" + "strings" +) + +// Optional filters must remain optional across JSON-schema clients. In +// particular, null means unspecified; zero is not a wildcard process ID. +func windowQuery(args obj) (obj, error) { + query := obj{} + for key, value := range args { + if value != nil { + query[key] = value + } + } + if value, ok := query["app"]; ok { + app, valid := value.(string) + if !valid { + return nil, errors.New("app must be an application name or null") + } + if app = strings.TrimSpace(app); app == "" { + delete(query, "app") + } else { + query["app"] = app + } + } + if value, ok := query["pid"]; ok { + valid := num(value) > 0 && num(value) <= math.MaxUint32 + switch n := value.(type) { + case float64: + valid = valid && n == math.Trunc(n) + case int, int64: + default: + valid = false + } + if !valid { + return nil, errors.New("pid must be a positive Windows process ID from a matching list_windows result. If unknown, omit pid or use null and search by app, e.g. {\"app\":\"Brave\",\"on_screen_only\":true}. Never use 0 or a different app's foreground PID") + } + } + return query, nil +} + +// Upstream's empty-result hint names the unrelated foreground process. Do not +// present that PID as a recovery candidate. Keep the exact filters and provide +// a separate suggested query; never silently widen the caller's PID scope. +func windowListing(result obj, query obj) obj { + if result["isError"] == true { + return result + } + data := object(result["structuredContent"]) + windows, ok := data["windows"].([]any) + if !ok { + return failure("Driver returned no window list; this is an observation error, not evidence that the application is absent", "not_started", "none", "invalid_window_list") + } + lookup := obj{"status": "matched", "filters": query, "count": len(windows)} + lines := []string{fmt.Sprintf("Found %d matching window(s).", len(windows))} + for _, value := range windows { + window := object(value) + lines = append(lines, fmt.Sprintf("- %s (pid %d), window_id %d, title %q", str(window["app_name"]), num(window["pid"]), num(window["window_id"]), str(window["title"]))) + } + if len(windows) == 0 { + lookup["status"] = "no_match" + lines = append(lines, "No windows match these filters. This alone does not prove the application is unavailable or screenshot capture is broken. Do not substitute another application's foreground PID.") + if str(query["app"]) != "" && query["pid"] != nil { + next := obj{} + for key, value := range query { + if key != "pid" { + next[key] = value + } + } + lookup["suggested_arguments"] = next + lines = append(lines, "app and pid are combined filters. Retry list_windows for the same app with pid omitted; use only a PID returned for that app.") + } else if query["on_screen_only"] == true { + next := obj{} + for key, value := range query { + next[key] = value + } + next["on_screen_only"] = false + lookup["suggested_arguments"] = next + lines = append(lines, "Retry the same filters with on_screen_only=false to check for minimized/off-screen windows. A listed window still needs a fresh get_window_state before interaction.") + } else { + lines = append(lines, "Verify the requested application name or PID. When the PID is unknown, search by the known app name with pid omitted. Do not guess a PID or operate an unrelated window.") + } + } + data["lookup"] = lookup + result["structuredContent"] = data + result["content"] = []any{textBlock(strings.Join(lines, "\n"))} + return result +} diff --git a/plugins/cua-driver/window_lookup_test.go b/plugins/cua-driver/window_lookup_test.go new file mode 100644 index 0000000..d0096ea --- /dev/null +++ b/plugins/cua-driver/window_lookup_test.go @@ -0,0 +1,88 @@ +package main + +import ( + "bytes" + "encoding/json" + "strings" + "testing" +) + +type lookupInput struct{ bytes.Buffer } + +func (*lookupInput) Close() error { return nil } + +func TestInvalidDiscoveryPIDStopsBeforeDriver(t *testing.T) { + for _, pid := range []any{0, -1, 1.5, "123", true, float64(4294967296)} { + result := newAdapter(nil).handle("list_windows", obj{"app": "Brave", "pid": pid}) + if result["isError"] != true || object(object(result["structuredContent"])["artemis"])["execution"] != "not_started" { + t.Fatalf("invalid PID %v reached driver or was reported as no windows: %v", pid, result) + } + if !strings.Contains(str(object(items(result["content"])[0])["text"]), "omit pid") { + t.Fatalf("missing recovery instruction for %v", pid) + } + } +} + +func TestAppDiscoveryOmitsNullPIDAtDriverBoundary(t *testing.T) { + input := &lookupInput{} + lines := make(chan []byte, 1) + lines <- []byte(`{"id":1,"result":{"content":[],"structuredContent":{"windows":[{"app_name":"brave.exe","pid":72,"window_id":81,"title":"Browser"}]}}}`) + a := newAdapter(&peer{input: input, lines: lines}) + args := obj{"app": " Brave ", "pid": nil, "on_screen_only": true} + result := a.handle("list_windows", args) + if result["isError"] == true || len(items(object(result["structuredContent"])["windows"])) != 1 { + t.Fatalf("failed app-only discovery: %v", result) + } + var sent obj + if err := json.Unmarshal(input.Bytes(), &sent); err != nil { + t.Fatal(err) + } + forwarded := object(object(sent["params"])["arguments"]) + if _, present := forwarded["pid"]; present || forwarded["app"] != "Brave" || forwarded["on_screen_only"] != true { + t.Fatalf("incorrect upstream query: %v", forwarded) + } + if args["app"] != " Brave " || len(args) != 3 { + t.Fatal("changed original arguments used in trace") + } +} + +func TestConflictingFiltersPreserveScopeAndRemoveForegroundHint(t *testing.T) { + query := obj{"app": "Brave", "pid": 14492, "on_screen_only": true} + result := windowListing(obj{ + "content": []any{textBlock(`No windows. Current frontmost app is "studio64.exe" (pid 14492).`)}, + "structuredContent": obj{"windows": []any{}}, + }, query) + data := object(result["structuredContent"]) + lookup := object(data["lookup"]) + next := object(lookup["suggested_arguments"]) + if len(items(data["windows"])) != 0 || lookup["status"] != "no_match" || object(lookup["filters"])["pid"] != 14492 { + t.Fatal("original filter or no-match result was hidden") + } + if _, exists := next["pid"]; exists || next["app"] != "Brave" || next["on_screen_only"] != true { + t.Fatalf("incorrect app-only recovery: %v", next) + } + text := str(object(items(result["content"])[0])["text"]) + if strings.Contains(text, "studio64.exe") || !strings.Contains(text, "pid omitted") { + t.Fatalf("misleading recovery: %s", text) + } +} + +func TestEmptyVisibleQuerySuggestsSameTargetOffScreen(t *testing.T) { + query := obj{"pid": 12, "on_screen_only": true} + result := windowListing(obj{"structuredContent": obj{"windows": []any{}}}, query) + next := object(object(object(result["structuredContent"])["lookup"])["suggested_arguments"]) + if next["pid"] != 12 || next["on_screen_only"] != false || query["on_screen_only"] != true { + t.Fatalf("changed target or original query: %v", next) + } +} + +func TestLookupErrorsNeverBecomeEmptySuccess(t *testing.T) { + for _, result := range []obj{ + {"isError": true, "content": []any{textBlock("driver unavailable")}}, + {"structuredContent": obj{}}, + } { + if windowListing(result, obj{"app": "Brave"})["isError"] != true { + t.Fatal("observation failure became a successful empty list") + } + } +} diff --git a/scripts/build_cua_driver.py b/scripts/build_cua_driver.py index 59840bd..96c7d51 100644 --- a/scripts/build_cua_driver.py +++ b/scripts/build_cua_driver.py @@ -4,6 +4,7 @@ import argparse import hashlib import json import os +import re import subprocess import urllib.request import zipfile @@ -26,6 +27,11 @@ def main(): version = json.loads((SOURCE / "plugin.json").read_text("utf-8"))["version"] if version.split("+", 1)[0] != UPSTREAM_VERSION: raise ValueError("Plugin version must match the pinned Cua Driver version") + match = re.search( + r'const adapterVersion = "([^"]+)"', (SOURCE / "main.go").read_text("utf-8") + ) + if not match or match[1] != version: + raise ValueError("Adapter MCP version must match plugin.json") if hashlib.sha256(args.upstream_zip.read_bytes()).hexdigest() != DIGEST: raise ValueError("Cua release archive hash mismatch") output = args.output.resolve() @@ -93,7 +99,7 @@ def main(): "source_commit": COMMIT, "upstream_version": UPSTREAM_VERSION, "plugin_version": version, - "adapter_revision": 1, + "adapter_revision": 2, "prerelease": True, "url": URL, "sha256": DIGEST, diff --git a/tests/cua_discovery_smoke.py b/tests/cua_discovery_smoke.py new file mode 100644 index 0000000..a53f4cc --- /dev/null +++ b/tests/cua_discovery_smoke.py @@ -0,0 +1,88 @@ +"""Opt-in read-only app discovery against an already open, named application.""" + +import argparse +import asyncio +import json +import os +from pathlib import Path + +import jsonschema +from artemis.plugins.models import ServerConfig +from artemis.plugins.session import PluginSession + + +async def run(executable: Path, app: str, output: Path): + session = await PluginSession( + ServerConfig(command=str(executable)), executable.parent + ).start() + try: + schema = next(t.inputSchema for t in session.tools if t.name == "list_windows") + query = {"app": app, "on_screen_only": True} + bad_query = {**query, "pid": 0} + try: + jsonschema.validate(bad_query, schema) + except jsonschema.ValidationError: + pass + else: + raise AssertionError("Host schema still accepts PID 0") + bad = await session.call("list_windows", bad_query) + assert bad.isError + assert bad.structuredContent["artemis"]["execution"] == "not_started" + assert "omit pid" in "\n".join(b.text for b in bad.content if b.type == "text") + + jsonschema.validate(query, schema) + discovered = await session.call("list_windows", query) + assert not discovered.isError + windows = discovered.structuredContent["windows"] + assert windows, "Open the intended app before running this opt-in probe" + + nullable = {**query, "pid": None} + jsonschema.validate(nullable, schema) + found = await session.call("list_windows", nullable) + assert not found.isError and found.structuredContent["windows"] + assert "pid" not in found.structuredContent["lookup"]["filters"] + + # This Python test process is not the requested application. A conflict + # must stay empty and suggest a new app-only query, without a guessed PID. + conflict = await session.call("list_windows", {**query, "pid": os.getpid()}) + assert not conflict.isError and conflict.structuredContent["windows"] == [] + next_query = conflict.structuredContent["lookup"]["suggested_arguments"] + assert next_query == query + assert "frontmost" not in "\n".join( + b.text for b in conflict.content if b.type == "text" + ) + recovered = await session.call("list_windows", next_query) + assert not recovered.isError and recovered.structuredContent["windows"] + + window = recovered.structuredContent["windows"][0] + target = {k: window[k] for k in ("pid", "window_id")} + state = await session.call("get_window_state", {**target, "max_elements": 100}) + assert not state.isError, state + assert any(b.type == "image" for b in state.content) + assert state.structuredContent["snapshot_id"] + assert state.structuredContent["tree_markdown"] + assert state.structuredContent["artemis"]["source"]["target_id"] == str( + target["window_id"] + ) + report = { + "app": app, + "zero_pid_rejected_by_host_and_adapter": True, + "app_only_window_count": len(windows), + "null_pid_omitted": True, + "conflicting_pid_recovered_without_substitution": True, + "screenshot_and_tree": True, + } + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(report, ensure_ascii=False, indent=2), "utf-8") + print(json.dumps(report, ensure_ascii=False)) + finally: + await session.aclose() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(__doc__) + parser.add_argument("executable", type=Path) + parser.add_argument("--app", required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + asyncio.run(run(args.executable.resolve(), args.app, args.output))