diff --git a/DESIGN.md b/DESIGN.md index 8a3acba..dfa324f 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -2,7 +2,7 @@ ## Overview -本仓库维护 ARTEMIS 插件适配与市场分发。当前发布基线是 Cua Driver 0.36.0 的 Windows x64 适配,市场版本为 `0.36.0+artemis.4`,保留预览标识。后续版本以目标分支的清单、市场索引、来源记录和发行说明为准。 +本仓库维护 ARTEMIS 插件适配与市场分发。当前发布基线是 Cua Driver 0.38.0 的 Windows x64 适配,市场版本为 `0.38.0+artemis.1`,保留预览标识。后续版本以目标分支的清单、市场索引、来源记录和发行说明为准。 这是适配器架构与行为设计,插件没有独立产品网页或通用配色。主程序网页的视觉规则由其根目录 DESIGN.md 管理;包内 [SKILL.md](plugins/cua-driver/SKILL.md) 指导调用方如何使用实际工具,不代替宿主权限或证据校验。 diff --git a/README.md b/README.md index cc6ba91..2e2958b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Artemis Plugins -ARTEMIS 2.0.0 的插件市场。当前电脑控制插件为 **Cua 0.36.0+artemis.4**,支持 Windows x64,通过窗口截图与可访问性树操作电脑应用和浏览器窗口,复用 ARTEMIS 的调度、资源互斥、步骤、截图附件和录像。 +ARTEMIS 的插件市场。配套 ARTEMIS 2.2.0 发布的电脑控制插件为 **Cua 0.38.0+artemis.1**,支持 Windows x64,通过窗口截图与可访问性树操作电脑应用和浏览器窗口,复用 ARTEMIS 的调度、资源互斥、步骤、截图附件和录像。 在 ARTEMIS 的「插件」页面安装「电脑控制 · Cua」,授权并启用即可。安卓任务可根据提示词按需调用电脑控制,不需要单独配置 Codex MCP 或安装 Go、Node、Python。 @@ -12,18 +12,18 @@ Open Computer Use(OCU)已从市场索引下架,当前市场只提供 Cua ## 验证范围 -本版本沿用 Cua 0.1.2 候选的驱动与动作实现,固定上游 [Cua Driver 0.36.0](https://github.com/trycua/cua/releases/tag/cua-driver-rs-v0.36.0) 预发布版,继续标注开发预览。 +本版本固定上游 [Cua Driver 0.38.0](https://github.com/trycua/cua/releases/tag/cua-driver-rs-v0.38.0),增加 ARTEMIS 窗口归属与焦点适配、已知无标题 Shell/菜单兼容补丁,修正滚动重复缩放。插件继续标注开发预览,升级内容见[发行说明](plugins/cua-driver/RELEASE.md)。 -已测试原生窗口表单、34 像素视觉目标、滚动、截图保存、ARTEMIS 窗口录像和子进程退出。对照测试独立验收 3/3 完成;ARTEMIS AI 裁定为 2 次通过、1 次未判定。浏览器复杂控制台、多 DPI、多显示器与安卓混合长程任务仍需进一步验证,详见 [验证记录](plugins/cua-driver/VERIFICATION.md)。 +此版本通过 Go 检查与构建、菜单/任务栏 9 次调用、隔离 Brave 表单 38 次调用验证,并参与 2.2.0 仅 PC 与跨端集中回归。旧版窗口录像、对照与性能记录保留各自版本范围,不计为新版重测。复杂控制台、多 DPI、多显示器与混合长程任务仍需进一步验证,详见[验证记录](plugins/cua-driver/VERIFICATION.md)。 适配修订另覆盖应用名发现、浏览器键盘与中文输入、截图缩放与坐标校验,以及前台激活后保持目标可见。Cua 需要前台输入时会占用用户桌面,不提供与用户操作互不干扰的执行环境;复杂业务是否通过仍以任务实际证据为准。 ## 构建与验证 -构建机需要 Python 和 Go;最终用户只安装 ZIP。从固定上游发行下载 `cua-driver-rs-0.36.0-windows-x86_64-binary.zip`,构建脚本校验 SHA256 后打包,不执行全局安装器。 +当前发布源码为 `0.38.0+artemis.1`。构建机需要 Python、Go、Rust 与 Windows C++/Spectre 依赖;最终用户只安装 ZIP。准备固定上游源码并应用包内兼容补丁后,从官方发行下载 `cua-driver-rs-0.38.0-windows-x86_64-binary.zip`;脚本校验来源后构建,不执行全局安装器,详见[构建说明](plugins/cua-driver/BUILD.md)。 ```sh -python scripts/build_cua_driver.py <上游ZIP> --go +python scripts/build_cua_driver.py <上游ZIP> --upstream-source <已应用补丁的上游源码目录> --go python tests/cua_smoke.py <安装后的artemis-cua-adapter.exe> artifacts/cua-smoke python tests/cua_fixture_validation.py <安装后的artemis-cua-adapter.exe> python tests/cua_discovery_smoke.py <安装后的artemis-cua-adapter.exe> --app Brave --output artifacts/cua-discovery.json @@ -33,6 +33,6 @@ python tests/cua_discovery_smoke.py <安装后的artemis-cua-adapter.exe> --app `catalog.json` 是市场索引,包含清单、HTTPS 安装包地址、SHA256 和验证说明。`artifacts/` 保存本机安装包与测试资料,不进入 Git。版本文件不可原地覆盖,任务固定使用其启动时的版本。 -插件版本跟随所用 Cua Driver 上游版本:当前为 `0.36.0`,对应上游标签 `cua-driver-rs-v0.36.0`,当前适配修订为 `0.36.0+artemis.4`,本仓库发行标签为 `cua-driver-v0.36.0+artemis.4`。只修改 ARTEMIS 适配而未升级上游时使用 `+artemis.N` 后缀区分不可变安装包;包内 `provenance.json` 记录上游提交、原包校验值和适配修订,构建会检查版本对应关系。 +插件版本跟随所用 Cua Driver 上游版本。市场已发布适配版为 `0.36.0+artemis.4`,发行标签 `cua-driver-v0.36.0+artemis.4`;当前源码候选为 `0.38.0+artemis.1`,对应上游标签 `cua-driver-rs-v0.38.0`。只修改 ARTEMIS 适配而未升级上游时使用 `+artemis.N` 后缀区分不可变安装包;包内 `provenance.json` 记录上游提交、原包校验值和适配修订,构建会检查版本对应关系。 [发行说明](plugins/cua-driver/RELEASE.md) · [ARTEMIS 主项目](https://git.playones.com/chenmingxuan/Artemis-Eapil) diff --git a/catalog.json b/catalog.json index 3d187fa..4a93fce 100644 --- a/catalog.json +++ b/catalog.json @@ -6,7 +6,7 @@ "schema_version": 2, "id": "cua-driver", "name": "电脑控制 · Cua", - "version": "0.36.0+artemis.4", + "version": "0.38.0+artemis.1", "description": "通过 Cua Driver 的窗口截图与可访问性树操作 Windows 应用及浏览器窗口;复用 ARTEMIS 的任务调度、步骤证据与录像。", "artemis_version": ">=2.0.0,<3.0.0", "platforms": [ @@ -184,9 +184,9 @@ } }, "release_stage": "preview", - "archive_url": "https://git.playones.com/chenmingxuan/Artemis-Plugins/releases/download/cua-driver-v0.36.0+artemis.4/cua-driver-0.36.0+artemis.4-windows-amd64.zip", - "sha256": "5bf02a9ff4b1fddb039a081c636c4b9d053efdf6673cce2830dc849b20abe8b3", - "verification": "Windows x64 / ARTEMIS 2.0.0。固定 Cua Driver 0.36.0 预发布版。已验证原生窗口表单、控件与像素点击、34 像素目标、滚动、trace 截图及哈希、ARTEMIS PC 窗口录像、子进程退出;同一复合用例重复 3 次,独立验收 3/3,AI 裁定 2 次通过、1 次未判定。0.36.0 沿用已测 0.1.2 动作实现,取代 OCU 市场入口。未覆盖复杂浏览器控制台、多 DPI、多显示器和跨端长任务。完整验证与已知限制见仓库 plugins/cua-driver/VERIFICATION.md。 本修订实测补充:Brave 按应用名发现、无效/null PID、冲突 PID 纠正、实际截图及 UIA 树通过;未重跑完整跨端业务任务。 0.36.0+artemis.2 补充独立 Brave 实测:新标签、地址导航、UIA 赋值、中文输入、表单单次提交、滚动后前台点击和 PNG 哈希通过;互斥输入参数在执行前拒绝。 补充 Flash / gpt-6.1-sol 自主浏览器任务:表单单次提交、滚动后点击与 trace 截图独立验收通过;不代表原移动端/PC完整业务任务已通过。 0.36.0+artemis.3/.4 补充:截图坐标缩放、capture_id、越界与窗口移动校验通过 Go 回归;前台交接、5 秒后保持目标前台、只读不抢焦点及恢复激活经过 38 次 Brave 工具调用验证。曾有一条 gpt-6-luna/high 跨端探索通过,仅代表该任务要求。未复测原失败业务全流程、Codex 作为前台窗口、多屏 DPI 或复杂控制台;前台输入会占用用户桌面,不承诺无干扰后台执行。" + "archive_url": "https://git.playones.com/chenmingxuan/Artemis-Plugins/releases/download/cua-driver-v0.38.0+artemis.1/cua-driver-0.38.0+artemis.1-windows-amd64.zip", + "sha256": "c007da16d58c4165a1a344113e650e68bd92a69ad3ce4ca8285818c5426dc0f4", + "verification": "Windows x64 预览,配套 ARTEMIS 2.2.0。固定 Driver 0.38.0、源码提交和官方 ZIP 校验值;包含已知无标题 Shell/菜单兼容补丁及重新构建运行库。Go 测试、vet、构建及包内源码/二进制校验通过;菜单/任务栏 9 次调用、隔离 Brave 表单 38 次调用通过,并参与仅 PC 与跨端集中回归。前台输入占用桌面;显示光标与系统鼠标偶发分离待复现,未全面覆盖多屏/DPI、长时间压力、其他托盘菜单及原失败业务全流程。详情见 plugins/cua-driver/VERIFICATION.md。" } ] } diff --git a/plugins/cua-driver/BUILD.md b/plugins/cua-driver/BUILD.md new file mode 100644 index 0000000..000846a --- /dev/null +++ b/plugins/cua-driver/BUILD.md @@ -0,0 +1,15 @@ +# 构建 0.38.0+artemis.1 + +开发机需要 Python、Go、上游固定的 Rust 工具链、Windows C++ x64 工具链及对应 Spectre 运行库。用户安装插件 ZIP 不需要这些工具。不会运行 Cua 全局安装器。 + +1. 获取官方 `cua-driver-rs-v0.38.0` 源码,核对 HEAD 为 `85e2b1318bd77ccf3cfa88ccdf875a4bb7eaed19`。保留完整源码;稀疏检出至少包括 `libs/cua-driver/rust`、`libs/cua`、`libs/images/sandbox-images.json`。 +2. 在上游源码根目录应用本插件的 `patches/cua-driver-0.38.0-untitled-shell.patch`。补丁只修改 Windows 精确窗口解析,不放宽普通窗口发现。 +3. 下载官方 Windows x64 ZIP。在具备上述编译依赖的终端,从 Artemis-Plugins 根目录执行: + +```sh +python scripts/build_cua_driver.py <官方ZIP> --upstream-source <上游源码根目录> --go +``` + +构建脚本校验官方 ZIP SHA256、上游 HEAD、唯一修改文件与补丁内容,使用 `cargo build --release --locked` 重新构建 Driver、UIA helper 和 SDK DLL,再运行 Go 测试并构建适配器。仅游标主题工具沿用官方原包二进制。 + +`provenance.json` 区分官方 ZIP 来源、兼容补丁及重新构建文件,记录各自 SHA256。打包后先运行确定性测试及授权的隔离窗口回归,再安装到本机或发布。市场索引与发布附件需另行更新,不由构建脚本自动修改。 diff --git a/plugins/cua-driver/RELEASE.md b/plugins/cua-driver/RELEASE.md index 23b0640..53b5a53 100644 --- a/plugins/cua-driver/RELEASE.md +++ b/plugins/cua-driver/RELEASE.md @@ -1,17 +1,17 @@ -# Cua 电脑控制 0.36.0+artemis.4 +# Cua 电脑控制 0.38.0+artemis.1 -此修订修复前台操作完成后立即恢复其他应用、后续浏览器观察退回后台的问题。上游仍固定 Cua Driver 0.36.0 预发布版;仅增加 ARTEMIS 适配修订,没有接入独立环境、DOM 控制或新的后台通道。 +2026-10-27,与 ARTEMIS 2.2.0 配套发布。同步官方 [Cua Driver 0.38.0](https://github.com/trycua/cua/releases/tag/cua-driver-rs-v0.38.0),替换原 0.36.0 驱动。插件仍为 Windows x64 开发预览。既有任务与旧安装包保留原版本,主程序升级不会自动更新或授权插件;当前任务结束后在插件市场单独更新。 -明确使用 foreground 的操作先调用上游自带 bring_to_front,核实目标激活后再发送一次输入。目标保持前台供后续渲染、观察和录像,不逐次切回原应用。激活失败或结果未知时不发送业务输入;写操作结果未知时不重发。独立 bring_to_front 工具可恢复被遮挡的窗口,不必刷新页面或重复业务动作。只读观察不抢焦点,capture_focus 记录采集前后的前台 HWND 和目标状态,供识别灰屏与焦点切换。PNG 保存成功仍不代表网页验收通过。 +补充可见的任务栏、托盘及原生菜单窗口发现。操作后读取实际键盘焦点,只在 PID、线程、拥有关系和窗口类型验证通过时,将观察目标跟随到相关菜单或 Shell 窗口;无关应用、同进程的其他普通窗口及归属不明的菜单不会自动接管。返回 keyboard_focus、observation_target 和实际截图来源,后续控件 token 和坐标绑定实际观察窗口。 -保留前一修订的滚动缩放、capture_id、范围与窗口几何校验,以及键盘参数修复。原生后台操作继续遵循 Cua 支持范围;foreground 会占用用户前台,不承诺用户与模型可同时操作同一桌面。 +官方驱动按精确 HWND 解析窗口时仍过滤空标题,导致已找到的原生菜单被报为不存在。此包对固定源码增加一处兼容补丁,仅放行已知 Shell/菜单类型的无标题窗口,保留可见性和 PID/HWND 校验;普通应用发现与叠加层过滤保持原逻辑。Driver、UIA helper 与 SDK DLL 从带补丁的源码重新构建,包内记录补丁和二进制 SHA256,详见 [构建说明](BUILD.md)。 -上一修订内容:`press_key` / `type_text` 只对目标窗口的当前焦点操作,不再暴露互斥的 element_token/x/y 参数;聚焦使用独立点击工具,控件赋值由必填 element_token 的 `set_value` 完成。`scroll` 仅按截图坐标定位滚动区域。旧格式在适配层返回明确未执行回执,不发送输入、不重新截图。补充浏览器前台输入与滚动后点击规则,不自动重试结果不确定的写操作。保留上一修订的应用名发现和无效 PID 修复。 +0.38.0 已在驱动内映射截图坐标,因此移除旧滚动预缩放,避免重复转换。适配器继续校验最新 capture_id、窗口归属、物理边界、尺寸和坐标范围。未知写入结果不会重发;取证失败不转发未经窗口校验的图片。 -上游提交:`6d6c734c127025fb39ad5d585a10382808b39dc1`。Windows x64 原始 ZIP SHA256:`f8c47024b626f4e0e70f8428d2998df39e4879110c5722a29d5fd03e367353ad`。采用 MIT Driver,许可证随包提供;不包含 Cua Spaces、云环境或模型服务。 +前台动作沿用先激活并核实目标、再投递一次输入的路径;只读观察不抢焦点。截图成功、动作投递与业务验证分别记录。共享 ARTEMIS 的 desktop:local 锁、调度、证据和录像,不新增控制通道或模型循环。 -提供 12 个窗口激活、观察、控件/像素点击、输入、按键、拖动、滚动及截图工具。操作后返回真实窗口观察,保留动作回执,截图校验 PID/HWND 与物理边界,生成文件保存到当前任务 trace。共享 ARTEMIS 的 `desktop:local` 资源、任务固定版本、步骤和录像,不另建调度器。 +上游提交:`85e2b1318bd77ccf3cfa88ccdf875a4bb7eaed19`。官方 Windows x64 ZIP SHA256:`5edaa74d0afa1b55a9668dbc4f32736507bd2bd4d75b18ca1dc8881bb12402b5`。MIT 许可证随包保留;不包含 Cua Spaces 或云环境。用户安装 ZIP 即可,构建依赖仅供开发者使用。实测结果与未测范围见 [验证记录](VERIFICATION.md)。 -适配器与依赖随包分发,无需额外安装 Go、Python、Node 或另一份 Codex MCP。关闭、取消或宿主退出后通过 Windows Job Object 回收 Driver/UIA 子进程;仅在该子进程环境关闭遥测。 +发布前 Go 测试、vet 与构建通过,核对包内源码、补丁和运行库校验值。此版本已有菜单/任务栏 9 次调用、隔离 Brave 表单 38 次调用验证,并参与 ARTEMIS 2.2.0 的仅 PC 与跨端集中回归。多屏/DPI、长时间压力和原失败业务全流程未全面覆盖;显示光标与系统鼠标偶发分离仍待复现,不宣称本次已修复。 -升级:任务结束后,在插件页面安装并授权启用 Cua,再停用 OCU。下架不删除旧包、历史报告或用户配置,不自动替其他用户授权。完整实测结果与尚未验证的范围见 [VERIFICATION.md](VERIFICATION.md)。 +发布附件包含 Windows amd64 ZIP 与 SHA256SUMS.txt;包内 provenance.json 记录固定上游、兼容补丁和重新构建运行库的校验值。 diff --git a/plugins/cua-driver/SKILL.md b/plugins/cua-driver/SKILL.md index 15f1384..8589982 100644 --- a/plugins/cua-driver/SKILL.md +++ b/plugins/cua-driver/SKILL.md @@ -5,15 +5,15 @@ 1. 先按应用名发现窗口,例如 `list_windows({"app":"Brave","on_screen_only":true})`;未知 `pid` 必须省略或填 `null`,不能填 `0`。`app` 与非空 `pid` 同时提供时会取交集,只有该应用实际返回的 PID 才能用于后续调用。空结果时先按 `lookup.suggested_arguments` 纠正筛选(去掉错误 PID 或检查最小化窗口),不能直接宣称浏览器不可用,也不能把无关前台进程号当成目标 PID。取得目标的 `pid`、`window_id` 后读取真实画面;只操作任务要求的窗口,不关闭其他应用。 2. `get_window_state` 同时读取截图和 UIA 控件树。最新 `snapshot_id` 加 `:N` 即控件 `element_token`。优先用 `click` 点击最新控件;视觉目标用 `click_at`,坐标仅取本次截图上的窗口局部像素,不能用屏幕坐标,需携带本次 `capture_id`。两种点击参数互斥,不混用。 3. 所有窗口操作明确传 `pid`、`window_id`。`set_value` 必须带最新 `element_token`,用于设置具有 set_value 动作的输入框;同名 Text 标签不能当成 Edit 输入框。`press_key` 和 `type_text` 只操作当前焦点,不能传 `element_token`、`x`、`y`(也不要补 0/null)。需要聚焦控件时先 `click` / `click_at`,再输入。`press_key` 传单键及 `modifiers` 数组。 -4. 每个动作后插件自动返回一次新截图和控件树,新 snapshot 替换旧 token。接着用新 token,无需重复获取相同状态。仅当画面确实还在变化或需要核验时再次观察。一次调用只执行一次动作,超时不自动重发。 +4. 每个动作后插件自动返回一次新截图和控件树,新 snapshot 替换旧 token。若操作打开原生菜单、托盘或把焦点切到任务栏,检查 `observation_target.observed` 和 `artemis.source.target_id`,后续使用实际观察目标的 PID/HWND 与新 token;旧窗口编号不代表当前焦点。`keyboard_focus` 与 `post_action_focus` 记录真实焦点。归属不明或取证失败时先只读核对,不重复业务动作。原生菜单可向其拥有者窗口发送 Escape 关闭。仅在画面变化或需要核验时再次观察。 5. 默认后台操作;明确指定 foreground 时,适配器先使用 Cua 自带 bring_to_front 激活并核实目标,随后保持目标在前台,供页面渲染和连续观察;不会每次动作后切回原应用,也不能承诺不影响用户前台工作。Brave/Chrome/Edge 等 Chromium 浏览器的点击(尤其滚动后)、快捷键、文字输入和滚动直接使用 `delivery_mode=foreground`。例如新标签页:`press_key(key="t",modifiers=["ctrl"],delivery_mode="foreground")`;地址导航:Ctrl+L → `type_text(text=网址,delivery_mode="foreground")` → Enter,各调用均带同一目标 pid/window_id。检查新页面再继续。只读观察/保存截图不会抢焦点;若正文灰屏或控件树缺少网页正文,检查 capture_focus,调用同一 pid/window_id 的 bring_to_front,再观察页面;不要靠重复刷新或业务点击恢复。用户切换窗口后需要重新确认前台,不能拿灰屏判网站失败。`effect=unverifiable/suspected_noop` 仅代表投递/效果未确认,必须检查返回画面和读回值;`refused` 是未执行。其他应用后台无效时,可在任务授权范围内显式 foreground 后再次检查。不要仅凭工具成功声称测试通过。 6. `scroll` 必须带最新截图中滚动区域内的 `x,y`,不能传 `element_token`。使用 up/down/left/right、amount、by=line/page;滚轮刻度不保证固定像素或整页距离,根据返回画面确认实际位移再继续。`get_window_state(since="latest")` 可在同一窗口读取差异,但新 snapshot_id 仍替换旧 token。 7. `save_screenshot` 保存最新窗口 PNG 到当前任务 trace,filename 仅是文件名,不能传路径,不覆盖已有文件。实际成功返回 SHA256 和附件;不得把工具内存截图当成已保存文件。 -不提供 Cua 自带调度、录屏、回放、配置修改、安装器、任意脚本或自动升级接口。浏览器本轮使用窗口视觉/UIA 通路;未接入 DOM 或已有登录浏览器调试权限。此包基于上游 0.36.0 预发布版,通过已列出的本机验证,其他场景的支持范围以 VERIFICATION.md 为准。 +不提供 Cua 自带调度、录屏、回放、配置修改、安装器、任意脚本或自动升级接口。浏览器本轮使用窗口视觉/UIA 通路;未接入 DOM 或已有登录浏览器调试权限。此包基于官方 0.38.0,另含有明确来源记录的无标题 Shell 窗口兼容补丁,构建方式见 BUILD.md,实测范围见 VERIFICATION.md。 ## 截图坐标与证据 - `click_at`、`scroll`、`drag` 都携带最新返回截图的 `capture_id`,坐标取自同一张图的实际宽高。动作返回的新截图可能改变分辨率;旧编号、旧坐标不可沿用。窗口移动、缩放或坐标越界被拒绝后重新观察,不反复重试原参数。 -- 缩放转换由适配层处理,不自行乘 DPI 或加窗口偏移。优先用最新 `element_token` 点击可访问控件;UIA 的屏幕 `frame` 不能作为截图坐标。 +- 缩放转换由 Cua Driver 处理,适配器校验截图绑定与范围,不自行乘 DPI 或加窗口偏移。优先用最新 `element_token` 点击可访问控件;UIA 的屏幕 `frame` 不能作为截图坐标。 - 操作及保存截图前,从已有观察核对应用、窗口标题和页面身份(浏览器结合地址栏、选中标签及页面内容)。身份不符时定位正确页面再操作。无位移的滚动不能证明到底;先检查遮挡、焦点和滚动容器,再决定补一次边界验证或换路径。 diff --git a/plugins/cua-driver/VERIFICATION.md b/plugins/cua-driver/VERIFICATION.md index 453c0e6..828f0c3 100644 --- a/plugins/cua-driver/VERIFICATION.md +++ b/plugins/cua-driver/VERIFICATION.md @@ -1,5 +1,26 @@ # Cua 验证记录 +## 0.38.0+artemis.1 发布核对(2026-10-27) + +重新执行 Go 测试、vet 和构建,核对最终包内源码、兼容补丁及运行库 SHA256。10 月 26 日主程序集中回归中,此插件参与仅 PC 输入、双击、清空和两轮 PC 到 Android 的校验码传递,均有两端独立证据;该组正式任务共 9 项通过,另有准备探索超出指定页面范围而失败,未计入通过数。模型统一 gpt-6-luna/high,未将历史 OCU 对照性能数据作为此次发行性能结论。 + +下述 10 月 25 日的菜单/任务栏及浏览器验证仍对应同一动作实现。此次发布主要补全发行文档、构建来源、校验值和市场索引,不操作用户桌面。显示光标与系统鼠标偶发分离尚待复现;长期压力、全部多屏/DPI、其他托盘应用和原失败业务全流程仍未全面覆盖。 + +## 0.38.0+artemis.1 本地升级验证(2026-10-25) + +官方 Driver 0.38.0、固定提交及 Windows x64 ZIP SHA256 均已核对。Driver、UIA helper 和 SDK DLL 使用原锁文件从固定源码构建,唯一源码差异为精确 PID/HWND 解析允许已知 Shell/菜单类型空标题;补丁及重新构建文件 SHA256 随包保存。未运行全局安装器,未修改市场索引。 + +Go 测试、vet、适配器与运行库构建通过。新增确定性回归覆盖:补充 Shell 窗口时保留应用/PID 筛选与去重、真实键盘焦点选择、拒绝外部应用或无关同进程窗口、无 owner 的原生菜单按已验证菜单拥有者与线程归属定位,以及滚动/拖拽不重复缩放。Python 构建与测试脚本通过 Ruff 检查。 + +在 ARTEMIS `desktop:local` 锁内完成两组隔离桌面验证,未调用付费模型或重跑用户业务任务: + +- `tests/cua_shell_smoke.py --taskbar`:9 次调用。自有 WinForms 窗口的菜单按钮只派发一次,发现并获取无标题 `#32768` 菜单的 PNG 与控件树;来源匹配实际菜单 HWND,窗口枚举可再次找到菜单。Escape 关闭菜单,独立进程计数确认按钮触发 1 次、菜单业务项触发 0 次。Win+B 后焦点及截图目标跟随 `Shell_TrayWnd`。 +- `tests/cua_browser_smoke.py --foreground-check`:38 次调用。临时 Brave profile 和本地表单验证 Ctrl+T、新标签导航、UIA 值设置、中文真实输入、表单恰好提交一次、实际滚动、页底按钮与 PNG 哈希。普通测试窗口遮挡后的显式前台切换、5 秒后焦点保持、只读观察不抢焦点和显式恢复均通过。 + +首次菜单测试出现前台 HWND 为空且按钮计数为 0,用户反馈可能手动点走窗口;该次保留失败证据,不能用作已确认的驱动故障。随后完整菜单与浏览器验证通过。记录保存在主程序的本地 `artifacts/validation/2026-10-25/cua-shell-038-repeat` 与 `cua-browser-038`。 + +本轮未验证真实 Codex/CodexM 退出、其他托盘应用菜单、全部多屏/DPI 组合或长程业务成功率。旧版本的模型、性能及录像数据仍属于下列历史记录,不计入本次验证。 + ## 候选阶段记录 以下记录对应市场替换前的候选测试。适配版本 0.1.2;上游 Cua Driver 0.36.0(预发布),仅 Windows x64。默认模型和默认插件保持原样。 diff --git a/plugins/cua-driver/coordinates.go b/plugins/cua-driver/coordinates.go index 13e1959..f2a80f2 100644 --- a/plugins/cua-driver/coordinates.go +++ b/plugins/cua-driver/coordinates.go @@ -80,26 +80,11 @@ func sameWindow(a, b obj) bool { // Only scroll needs pre-scaling in the pinned driver. click_at is capture-bound // upstream and drag uses the driver's snapshot ratio; never scale either twice. -func nativeScrollArgs(f pointerFrame, args obj, readNative func() (pointerFrame, error)) (obj, error) { - if err := f.validate("scroll", args); err != nil { - return nil, err - } - native, err := readNative() - if err != nil { - return nil, err - } - if !sameWindow(f.window, native.window) || native.width < f.width || native.height < f.height { - return nil, errors.New("capture geometry changed; get_window_state again before scrolling") - } - // Reject a different crop/aspect ratio. One-pixel resize rounding is normal. - if math.Abs(float64(f.height)*float64(native.width)/float64(f.width)-float64(native.height)) > float64(native.width)/float64(f.width)+1 { - return nil, errors.New("capture aspect ratio changed; get_window_state again before scrolling") - } - x, _ := coordinate(args["x"]) - y, _ := coordinate(args["y"]) +// v0.38.0 scroll and drag use their latest window screenshot context. +// Admission stays host-bound to the exact capture; the upstream schema does +// not accept capture_id for these two tools. Do not rescale x/y here. +func withoutCaptureBinding(args obj) obj { out := copyArgs(args) - out["x"] = x * float64(native.width) / float64(f.width) - out["y"] = y * float64(native.height) / float64(f.height) - delete(out, "capture_id") // v0.36.0 scroll does not support capture binding. - return out, nil + delete(out, "capture_id") + return out } diff --git a/plugins/cua-driver/coordinates_test.go b/plugins/cua-driver/coordinates_test.go index abbd317..c02f6ff 100644 --- a/plugins/cua-driver/coordinates_test.go +++ b/plugins/cua-driver/coordinates_test.go @@ -1,7 +1,6 @@ package main import ( - "errors" "math" "testing" ) @@ -10,50 +9,30 @@ func fixtureFrame(w, h int) pointerFrame { return pointerFrame{"current", w, h, obj{"left": -2560, "top": 120, "width": 2560, "height": 1540}} } -func TestScrollMapsActualNativeCaptureWithoutDPIOrWindowOffset(t *testing.T) { - // Reproduce the task's 2558x1538 -> 1280x770 image transition. Window - // position and OS logical-screen dimensions must never affect this ratio. - f := fixtureFrame(1280, 770) - args := obj{"pid": 1, "window_id": 2, "capture_id": "current", "x": 900, "y": 600, "direction": "down"} - got, err := nativeScrollArgs(f, args, func() (pointerFrame, error) { return fixtureFrame(2558, 1538), nil }) - if err != nil { - t.Fatal(err) - } - if math.Abs(got["x"].(float64)-1798.59375) > 0.0001 || math.Abs(got["y"].(float64)-1198.441558) > 0.0001 { - t.Fatalf("incorrect mapping: %v", got) - } - if _, ok := got["capture_id"]; ok { - t.Fatal("upstream scroll does not support capture_id") - } - if args["x"] != 900 || args["capture_id"] != "current" { - t.Fatal("input evidence mutated") - } - if got["pid"] != 1 || got["window_id"] != 2 || got["direction"] != "down" { - t.Fatal("target changed") - } -} - -func TestNativeScrollAndNonCroppedCapture(t *testing.T) { - for _, size := range [][2]int{{2558, 1538}, {2560, 1540}, {1280, 770}} { - f := fixtureFrame(size[0], size[1]) - args := obj{"capture_id": "current", "x": 200.5, "y": 300.25} - got, err := nativeScrollArgs(f, args, func() (pointerFrame, error) { return f, nil }) - if err != nil || got["x"] != 200.5 || got["y"] != 300.25 { - t.Fatalf("native coordinates changed: %v %v", got, err) +func TestScrollPixelsReachUpstreamWithoutDoubleScaling(t *testing.T) { + for _, size := range [][2]int{{1280, 770}, {2558, 1538}, {2560, 1540}} { + frame := fixtureFrame(size[0], size[1]) + args := obj{"pid": 1, "window_id": 2, "capture_id": "current", "x": 900.5, "y": 600.25, "direction": "down"} + if err := frame.validate("scroll", args); err != nil { + t.Fatal(err) + } + got := withoutCaptureBinding(args) + if got["x"] != 900.5 || got["y"] != 600.25 || got["pid"] != 1 || got["window_id"] != 2 { + t.Fatal(got) + } + if _, ok := got["capture_id"]; ok { + t.Fatal("upstream scroll rejects capture_id") + } + if args["capture_id"] != "current" { + t.Fatal("input evidence mutated") } } - // WGC can have a different native extent than a one-pixel DWM crop. - f := fixtureFrame(1280, 770) - got, err := nativeScrollArgs(f, obj{"capture_id": "current", "x": 640, "y": 385}, func() (pointerFrame, error) { return fixtureFrame(2560, 1540), nil }) - if err != nil || got["x"] != float64(1280) || got["y"] != float64(770) { - t.Fatalf("assumed DWM crop: %v %v", got, err) - } } -func TestInvalidCoordinatesNeverReachNativeProbe(t *testing.T) { - f := fixtureFrame(1280, 770) +func TestInvalidScrollCoordinatesAreRejectedBeforeDispatch(t *testing.T) { + frame := fixtureFrame(1280, 770) for _, bad := range []obj{ - {"capture_id": "current", "x": 1700, "y": 1000}, // Actual failing task. + {"capture_id": "current", "x": 1700, "y": 1000}, {"capture_id": "old", "x": 900, "y": 600}, {"x": 900, "y": 600}, {"capture_id": "current", "x": -1, "y": 0}, @@ -63,35 +42,20 @@ func TestInvalidCoordinatesNeverReachNativeProbe(t *testing.T) { {"capture_id": "current", "x": math.NaN(), "y": 600}, {"capture_id": "current", "x": 900, "y": math.Inf(1)}, } { - _, err := nativeScrollArgs(f, bad, func() (pointerFrame, error) { t.Fatal("invalid coordinates reached probe"); return f, nil }) - if err == nil { + if frame.validate("scroll", bad) == nil { t.Fatalf("accepted %v", bad) } } } -func TestScrollRejectsMovedResizedOrDifferentCrop(t *testing.T) { - f := fixtureFrame(1280, 770) - for _, kind := range []string{"moved", "resized", "crop", "missing"} { - native := fixtureFrame(2558, 1538) - switch kind { - case "moved": - native.window["left"] = 0 - case "resized": - native.window["width"] = 1920 - case "crop": - native.height = 1000 - case "missing": - native.width = 0 +func TestMovedOrResizedWindowInvalidatesFrame(t *testing.T) { + frame := fixtureFrame(1280, 770) + for _, key := range []string{"left", "top", "width", "height"} { + changed := copyArgs(frame.window) + changed[key] = num(changed[key]) + 1 + if sameWindow(frame.window, changed) { + t.Fatal("accepted changed " + key) } - _, err := nativeScrollArgs(f, obj{"capture_id": "current", "x": 900, "y": 600}, func() (pointerFrame, error) { return native, nil }) - if err == nil { - t.Fatal(kind) - } - } - _, err := nativeScrollArgs(f, obj{"capture_id": "current", "x": 900, "y": 600}, func() (pointerFrame, error) { return pointerFrame{}, errors.New("capture disconnected") }) - if err == nil { - t.Fatal("missing capture was accepted") } } diff --git a/plugins/cua-driver/main.go b/plugins/cua-driver/main.go index 3ce0623..dc5ea41 100644 --- a/plugins/cua-driver/main.go +++ b/plugins/cua-driver/main.go @@ -25,7 +25,7 @@ import ( //go:embed tools.json var toolsJSON []byte -const adapterVersion = "0.36.0+artemis.4" +const adapterVersion = "0.38.0+artemis.1" type obj = map[string]any type request struct { @@ -291,11 +291,13 @@ func (a *adapter) observe(args obj) (obj, obj, error) { delete(a.captures, targetKey(args)) started := float64(time.Now().UnixMilli()) / 1000 before := foregroundWindow() + focusBefore := keyboardFocus() r, err := a.peer.tool("get_window_state", args) if err != nil { return nil, nil, err } annotateCaptureFocus(r, num(args["window_id"]), before, foregroundWindow()) + object(r["structuredContent"])["keyboard_focus"] = obj{"before": focusBefore, "after": keyboardFocus()} src := source(r, args, started) if frame, ok := capturedFrame(r, src); ok { a.captures[targetKey(args)] = frame @@ -331,6 +333,7 @@ func (a *adapter) handle(name string, args obj) obj { query, _ := windowQuery(args) // Already validated above. r, err = a.peer.tool(name, query) if err == nil { + r = supplementShellWindows(r, query, nativeShellWindows()) r = windowListing(r, query) } } else { @@ -346,34 +349,9 @@ func (a *adapter) handle(name string, args obj) obj { delete(a.captures, targetKey(args)) return failure("Window moved or resized; get_window_state again", "not_started", "none", "stale_capture") } - if name == "scroll" { - // v0.36.0 scroll omits screenshot scaling. Measure actual native - // capture pixels (not DWM/DPI estimates) without another model call. - // Do not expose this internal frame or reuse its tokens. - delete(a.captures, targetKey(args)) - prepared, e := nativeScrollArgs(frame, args, func() (pointerFrame, error) { - nativeArgs := observationArgs(args) - nativeArgs["max_image_dimension"] = 0 - nativeArgs["include_accessibility_tree"] = false - native, err := a.peer.tool("get_window_state", nativeArgs) - if err != nil { - return pointerFrame{}, err - } - src := source(native, args, float64(time.Now().UnixMilli())/1000) - f, ok := capturedFrame(native, src) - if !ok { - return pointerFrame{}, errors.New("native capture unavailable; get_window_state again") - } - return f, nil - }) - if e != nil { - return failure(e.Error(), "not_started", "none", "invalid_coordinates") - } - args = prepared - } else if name == "drag" { - // Upstream drag already scales using its current screenshot. - args = copyArgs(args) - delete(args, "capture_id") + if name == "scroll" || name == "drag" { + // v0.38.0 maps screenshot pixels itself. Keep x/y unchanged. + args = withoutCaptureBinding(args) } } upstream := name @@ -402,13 +380,40 @@ func (a *adapter) handle(name string, args obj) obj { } if writeTool(name) && rec["execution"] != "not_started" { // Observe once after delivery; NEVER repeat an uncertain write. - observation, origin, e := a.observe(observationArgs(args)) + focus := keyboardFocus() + next, reason := postActionTarget(args, nativeWindowInfo(num(args["window_id"])), focus, nativeShellWindows()) + structured["post_action_focus"] = focus + structured["observation_target"] = obj{"requested": obj{"pid": args["pid"], "window_id": args["window_id"]}, + "observed": obj{"pid": next["pid"], "window_id": next["window_id"]}, "reason": reason} + observation, origin, e := a.observe(next) if e == nil && origin != nil { src = origin structured["observation"] = observation["structuredContent"] r["content"] = append(items(r["content"]), items(observation["content"])...) } else { structured["observation_error"] = "Post-action evidence unavailable; inspect before another action." + if e != nil { + structured["observation_error"] = e.Error() + } else if observation != nil { + var errors []string + if observation["isError"] == true { + for _, item := range items(observation["content"]) { + block := object(item) + if block["type"] == "text" { + text := []rune(str(block["text"])) + if len(text) > 2000 { + text = text[:2000] + } + errors = append(errors, string(text)) + } + } + } + // Failed identity/geometry validation must never forward its image + // or full tree as task evidence, even inside diagnostics. + structured["observation_diagnostics"] = obj{"driver_errors": errors, + "native_target": nativeWindowInfo(num(next["window_id"])), "isError": observation["isError"], + "image_size": func() []int { w, h := imageSize(observation); return []int{w, h} }()} + } } } if name == "verify_state" { // Read-only verification may include a screenshot; it is not a task verdict. diff --git a/plugins/cua-driver/patches/cua-driver-0.38.0-untitled-shell.patch b/plugins/cua-driver/patches/cua-driver-0.38.0-untitled-shell.patch new file mode 100644 index 0000000..1ced6b9 --- /dev/null +++ b/plugins/cua-driver/patches/cua-driver-0.38.0-untitled-shell.patch @@ -0,0 +1,33 @@ +diff --git a/libs/cua-driver/rust/crates/platform-windows/src/win32/windows.rs b/libs/cua-driver/rust/crates/platform-windows/src/win32/windows.rs +index b13588d..b0bccf5 100644 +--- a/libs/cua-driver/rust/crates/platform-windows/src/win32/windows.rs ++++ b/libs/cua-driver/rust/crates/platform-windows/src/win32/windows.rs +@@ -281,14 +281,26 @@ fn window_info_by_handle(hwnd: u64) -> Option { + return None; + } + let title_len = unsafe { GetWindowTextLengthW(native) }; +- if title_len == 0 { ++ // An exact PID/HWND remains a valid identity for untitled native shell ++ // surfaces. Keep ordinary application/overlay discovery filters intact. ++ let untitled_shell = matches!( ++ window_class_name(native).as_str(), ++ "Shell_TrayWnd" ++ | "Shell_SecondaryTrayWnd" ++ | "NotifyIconOverflowWindow" ++ | "TopLevelWindowForOverflowXamlIsland" ++ | "#32768" ++ | "Progman" ++ | "WorkerW" ++ ); ++ if title_len == 0 && !untitled_shell { + return None; + } + let mut buf = vec![0u16; (title_len + 1) as usize]; + unsafe { GetWindowTextW(native, &mut buf) }; + let title_len = buf.iter().position(|&c| c == 0).unwrap_or(buf.len()); + let title = String::from_utf16_lossy(&buf[..title_len]); +- if title.trim().is_empty() { ++ if title.trim().is_empty() && !untitled_shell { + return None; + } + let minimized = unsafe { IsIconic(native) }.as_bool(); diff --git a/plugins/cua-driver/plugin.json b/plugins/cua-driver/plugin.json index 377045b..5943cd3 100644 --- a/plugins/cua-driver/plugin.json +++ b/plugins/cua-driver/plugin.json @@ -2,7 +2,7 @@ "schema_version": 2, "id": "cua-driver", "name": "电脑控制 · Cua", - "version": "0.36.0+artemis.4", + "version": "0.38.0+artemis.1", "description": "通过 Cua Driver 的窗口截图与可访问性树操作 Windows 应用及浏览器窗口;复用 ARTEMIS 的任务调度、步骤证据与录像。", "artemis_version": ">=2.0.0,<3.0.0", "platforms": [ diff --git a/plugins/cua-driver/shell.go b/plugins/cua-driver/shell.go new file mode 100644 index 0000000..5bd4bfb --- /dev/null +++ b/plugins/cua-driver/shell.go @@ -0,0 +1,106 @@ +package main + +import "strings" + +// Shell surfaces often have an empty title and are omitted by upstream's +// ordinary application-window enumeration. They still require exact HWND/PID. +func shellKind(class string) string { + switch class { + case "Shell_TrayWnd", "Shell_SecondaryTrayWnd": + return "taskbar" + case "NotifyIconOverflowWindow", "TopLevelWindowForOverflowXamlIsland": + return "tray" + case "#32768": + return "menu" + case "Progman", "WorkerW": + return "desktop" + } + return "" +} + +func shellMatches(window, query obj) bool { + if query["pid"] != nil && num(query["pid"]) != num(window["pid"]) { + return false + } + app := strings.TrimSuffix(strings.ToLower(str(query["app"])), ".exe") + name := strings.TrimSuffix(strings.ToLower(str(window["app_name"])), ".exe") + return (app == "" || strings.Contains(name, app)) && (query["on_screen_only"] != true || window["is_on_screen"] == true) +} + +func supplementShellWindows(result, query obj, native []obj) obj { + if result["isError"] == true { + return result + } + data := object(result["structuredContent"]) + windows, ok := data["windows"].([]any) + if !ok { // Do not turn an upstream observation failure into a valid list. + return result + } + known := map[int64]obj{} + for _, value := range windows { + w := object(value) + known[num(w["window_id"])] = w + } + for _, w := range native { + if shellKind(str(w["class_name"])) == "" || !shellMatches(w, query) { + continue + } + if old, exists := known[num(w["window_id"])]; exists { + old["class_name"] = w["class_name"] + old["window_kind"] = shellKind(str(w["class_name"])) + } else { + w["window_kind"] = shellKind(str(w["class_name"])) + windows = append(windows, w) + known[num(w["window_id"])] = w + } + } + data["windows"] = windows + result["structuredContent"] = data + return result +} + +// Follow only native keyboard focus or a uniquely associated active menu. +// A different foreground application never silently becomes screenshot scope. +func postActionTarget(args, requested, focus obj, candidates []obj) (obj, string) { + selected := object(focus["focus_root"]) + if focus["probe_ok"] != true { + return observationArgs(args), "keyboard_focus_unavailable" + } + if focus["menu_active"] == true { + owner := object(focus["menu_owner"]) + var menus []obj + for _, w := range candidates { + if shellKind(str(w["class_name"])) == "menu" && w["is_on_screen"] == true && + num(w["pid"]) == num(args["pid"]) && num(owner["pid"]) == num(args["pid"]) && + (num(w["owner_id"]) == num(owner["window_id"]) || + (num(w["thread_id"]) != 0 && num(w["thread_id"]) == num(owner["thread_id"]))) { + menus = append(menus, w) + } + } + if len(menus) == 1 { + selected = menus[0] + } else if len(menus) > 1 { + return observationArgs(args), "ambiguous_active_menu_observe_candidates_explicitly" + } + } + if num(selected["window_id"]) == num(args["window_id"]) { + return observationArgs(args), "keyboard_focus_remains_in_requested_window" + } + if num(selected["pid"]) != num(args["pid"]) || selected["is_on_screen"] != true { + return observationArgs(args), "unrelated_or_unverified_focus_not_followed" + } + kind := shellKind(str(selected["class_name"])) + relatedShell := shellKind(str(requested["class_name"])) != "" && kind != "" && kind != "desktop" + menuOwner := object(focus["menu_owner"]) + // Native #32768 menus can report GW_OWNER=0. GetGUIThreadInfo still + // identifies their owning window; the candidate was also matched by PID + // and UI thread above. Do not broaden this to arbitrary same-PID windows. + relatedPopup := kind == "menu" && (num(selected["owner_id"]) == num(args["window_id"]) || + (focus["menu_active"] == true && num(menuOwner["window_id"]) == num(args["window_id"]))) + if !relatedShell && !relatedPopup { + return observationArgs(args), "unrelated_focus_not_followed" + } + next := observationArgs(args) + next["pid"], next["window_id"] = selected["pid"], selected["window_id"] + return next, "associated_native_keyboard_focus" +} diff --git a/plugins/cua-driver/shell_test.go b/plugins/cua-driver/shell_test.go new file mode 100644 index 0000000..9a431c7 --- /dev/null +++ b/plugins/cua-driver/shell_test.go @@ -0,0 +1,96 @@ +package main + +import "testing" + +func shellFixtureWindow(pid, hwnd int64, class string) obj { + return obj{"pid": pid, "window_id": hwnd, "class_name": class, + "is_on_screen": true, "app_name": "explorer.exe", "title": ""} +} + +func TestUntitledShellDiscoveryKeepsAppAndPIDScope(t *testing.T) { + native := []obj{shellFixtureWindow(1, 10, "Shell_TrayWnd"), shellFixtureWindow(2, 20, "NotifyIconOverflowWindow")} + for _, query := range []obj{{"pid": 1}, {"app": "Explorer", "pid": 1}, {"app": "explorer.exe", "pid": 1}} { + r := supplementShellWindows(obj{"structuredContent": obj{"windows": []any{}}}, query, native) + windows := items(object(r["structuredContent"])["windows"]) + if len(windows) != 1 || num(object(windows[0])["window_id"]) != 10 { + t.Fatal(r) + } + } + r := supplementShellWindows(obj{"structuredContent": obj{"windows": []any{}}}, obj{"app": "Brave"}, native) + if len(items(object(r["structuredContent"])["windows"])) != 0 { + t.Fatal("widened app scope") + } + broken := obj{"isError": true} + if supplementShellWindows(broken, obj{}, native)["isError"] != true { + t.Fatal("observation error hidden") + } +} + +func TestWinBObservationFollowsActualTaskbarFocus(t *testing.T) { + args := obj{"pid": 1, "window_id": 10} + requested := shellFixtureWindow(1, 10, "Progman") + focus := obj{"probe_ok": true, "focus_root": shellFixtureWindow(1, 20, "Shell_TrayWnd"), + "foreground_window": requested} // Foreground HWND alone would give the wrong answer. + next, reason := postActionTarget(args, requested, focus, nil) + if num(next["window_id"]) != 20 || reason != "associated_native_keyboard_focus" { + t.Fatal(next, reason) + } + if num(args["window_id"]) != 10 { + t.Fatal("requested target history mutated") + } +} + +func TestPostActionNeverCapturesUnrelatedApplicationOrAmbiguousMenu(t *testing.T) { + args := obj{"pid": 1, "window_id": 10} + requested := shellFixtureWindow(1, 10, "Progman") + for _, other := range []obj{shellFixtureWindow(2, 20, "Shell_TrayWnd"), shellFixtureWindow(1, 20, "OtherApp")} { + next, _ := postActionTarget(args, requested, obj{"probe_ok": true, "focus_root": other}, nil) + if num(next["window_id"]) != 10 { + t.Fatal("unrelated focus captured", next) + } + } + menu := shellFixtureWindow(1, 30, "#32768") + menu["owner_id"] = int64(10) + menu2 := shellFixtureWindow(1, 40, "#32768") + menu2["owner_id"] = int64(10) + focus := obj{"probe_ok": true, "menu_active": true, "menu_owner": requested} + next, reason := postActionTarget(args, requested, focus, []obj{menu, menu2}) + if num(next["window_id"]) != 10 || reason != "ambiguous_active_menu_observe_candidates_explicitly" { + t.Fatal(next, reason) + } +} + +func TestOwnedNativeMenuAndOverflowHaveIndependentObservationTargets(t *testing.T) { + args := obj{"pid": 1, "window_id": 10} + owner := shellFixtureWindow(1, 10, "Shell_TrayWnd") + menu := shellFixtureWindow(1, 30, "#32768") + menu["owner_id"] = int64(10) + focus := obj{"probe_ok": true, "menu_active": true, "menu_owner": owner} + next, _ := postActionTarget(args, owner, focus, []obj{menu}) + if num(next["window_id"]) != 30 { + t.Fatal(next) + } + next, _ = postActionTarget(args, owner, obj{"probe_ok": true, + "focus_root": shellFixtureWindow(1, 40, "TopLevelWindowForOverflowXamlIsland")}, nil) + if num(next["window_id"]) != 40 { + t.Fatal(next) + } +} + +func TestNativeMenuWithoutGWOwnerRequiresVerifiedMenuOwnerAndThread(t *testing.T) { + args := obj{"pid": 1, "window_id": 10} + owner := shellFixtureWindow(1, 10, "ordinary-form") + owner["thread_id"] = int64(7) + menu := shellFixtureWindow(1, 30, "#32768") + menu["owner_id"], menu["thread_id"] = int64(0), int64(7) + focus := obj{"probe_ok": true, "menu_active": true, "menu_owner": owner} + next, _ := postActionTarget(args, owner, focus, []obj{menu}) + if num(next["window_id"]) != 30 { + t.Fatal(next) + } + menu["thread_id"] = int64(8) + next, _ = postActionTarget(args, owner, focus, []obj{menu}) + if num(next["window_id"]) != 10 { + t.Fatal("unrelated menu followed", next) + } +} diff --git a/plugins/cua-driver/shell_windows.go b/plugins/cua-driver/shell_windows.go new file mode 100644 index 0000000..a4ef896 --- /dev/null +++ b/plugins/cua-driver/shell_windows.go @@ -0,0 +1,105 @@ +package main + +import ( + "path/filepath" + "runtime" + "sync" + "syscall" + "unsafe" +) + +type windowRect struct{ left, top, right, bottom int32 } + +func windowClass(hwnd int64) string { + var value [256]uint16 + user32.NewProc("GetClassNameW").Call(uintptr(hwnd), uintptr(unsafe.Pointer(&value[0])), uintptr(len(value))) + return syscall.UTF16ToString(value[:]) +} + +func nativeWindowInfo(hwnd int64) obj { + if hwnd == 0 { + return obj{} + } + var pid uint32 + thread, _, _ := user32.NewProc("GetWindowThreadProcessId").Call(uintptr(hwnd), uintptr(unsafe.Pointer(&pid))) + if pid == 0 { + return obj{} + } + visible, _, _ := user32.NewProc("IsWindowVisible").Call(uintptr(hwnd)) + owner, _, _ := user32.NewProc("GetWindow").Call(uintptr(hwnd), 4) + root, _, _ := user32.NewProc("GetAncestor").Call(uintptr(hwnd), 2) + var title [1024]uint16 + user32.NewProc("GetWindowTextW").Call(uintptr(hwnd), uintptr(unsafe.Pointer(&title[0])), uintptr(len(title))) + app := "" + process, _, _ := kernel32.NewProc("OpenProcess").Call(0x1000, 0, uintptr(pid)) + if process != 0 { + var path [32768]uint16 + size := uint32(len(path)) + ok, _, _ := kernel32.NewProc("QueryFullProcessImageNameW").Call(process, 0, uintptr(unsafe.Pointer(&path[0])), uintptr(unsafe.Pointer(&size))) + if ok != 0 { + app = filepath.Base(syscall.UTF16ToString(path[:])) + } + kernel32.NewProc("CloseHandle").Call(process) + } + info := obj{"pid": int64(pid), "window_id": hwnd, "thread_id": int64(thread), + "root_window_id": int64(root), "owner_id": int64(owner), "class_name": windowClass(hwnd), + "title": syscall.UTF16ToString(title[:]), "app_name": app, "is_on_screen": visible != 0, + "z_index": nil, "minimized": false} + runtime.LockOSThread() + defer runtime.UnlockOSThread() + previous, _, _ := user32.NewProc("SetThreadDpiAwarenessContext").Call(^uintptr(3)) + if previous != 0 { + defer user32.NewProc("SetThreadDpiAwarenessContext").Call(previous) + var rect windowRect + ok, _, _ := user32.NewProc("GetWindowRect").Call(uintptr(hwnd), uintptr(unsafe.Pointer(&rect))) + if ok != 0 { + info["bounds"] = obj{"x": rect.left, "y": rect.top, "width": rect.right - rect.left, "height": rect.bottom - rect.top} + info["is_on_screen"] = visible != 0 && rect.right > rect.left && rect.bottom > rect.top + } + } + return info +} + +// A single static callback avoids NewCallback's non-reclaimable slot leak. +var shellEnumerationLock sync.Mutex +var shellEnumeration *[]obj +var shellEnumerator = syscall.NewCallback(func(hwnd, _ uintptr) uintptr { + if shellKind(windowClass(int64(hwnd))) != "" { + info := nativeWindowInfo(int64(hwnd)) + if info["is_on_screen"] == true { + *shellEnumeration = append(*shellEnumeration, info) + } + } + return 1 +}) + +func nativeShellWindows() []obj { + shellEnumerationLock.Lock() + defer shellEnumerationLock.Unlock() + var windows []obj + shellEnumeration = &windows + user32.NewProc("EnumWindows").Call(shellEnumerator, 0) + shellEnumeration = nil + return windows +} + +func keyboardFocus() obj { + foreground := nativeWindowInfo(foregroundWindow()) + var gui struct { + size, flags uint32 + active, focus, capture, menuOwner, moveSize, caret uintptr + rect windowRect + } + gui.size = uint32(unsafe.Sizeof(gui)) + ok, _, _ := user32.NewProc("GetGUIThreadInfo").Call(0, uintptr(unsafe.Pointer(&gui))) + result := obj{"probe_ok": ok != 0, "foreground_window": foreground} + if ok != 0 { + focused := nativeWindowInfo(int64(gui.focus)) + result["focus_window"] = focused + result["focus_root"] = nativeWindowInfo(num(focused["root_window_id"])) + result["active_window"] = nativeWindowInfo(int64(gui.active)) + result["menu_owner"] = nativeWindowInfo(int64(gui.menuOwner)) + result["menu_active"] = gui.flags&0x1c != 0 + } + return result +} diff --git a/plugins/cua-driver/window_lookup.go b/plugins/cua-driver/window_lookup.go index d639e7c..f3e5d49 100644 --- a/plugins/cua-driver/window_lookup.go +++ b/plugins/cua-driver/window_lookup.go @@ -60,6 +60,9 @@ func windowListing(result obj, query obj) obj { for _, value := range windows { window := object(value) lines = append(lines, fmt.Sprintf("- %s (pid %d), window_id %d, title %q", str(window["app_name"]), num(window["pid"]), num(window["window_id"]), str(window["title"]))) + if kind := str(window["window_kind"]); kind != "" { + lines[len(lines)-1] += fmt.Sprintf(" [%s, class %s]", kind, str(window["class_name"])) + } } if len(windows) == 0 { lookup["status"] = "no_match" diff --git a/plugins/cua-driver/windows.go b/plugins/cua-driver/windows.go index 734e8fa..c35fce2 100644 --- a/plugins/cua-driver/windows.go +++ b/plugins/cua-driver/windows.go @@ -45,8 +45,16 @@ func verifyWindow(pid, hwnd int64, b obj) (obj, bool) { } var r struct{ L, T, R, B int32 } code, _, _ := syscall.NewLazyDLL("dwmapi.dll").NewProc("DwmGetWindowAttribute").Call(uintptr(hwnd), 9, uintptr(unsafe.Pointer(&r)), unsafe.Sizeof(r)) - if code != 0 { - return nil, false + if code != 0 || r.R <= r.L || r.B <= r.T { + // Shell menus/taskbars can lack a DWM extended frame. Match their + // actual HWND rectangle; never substitute desktop or another window. + if shellKind(windowClass(hwnd)) == "" { + return nil, false + } + ok, _, _ := user32.NewProc("GetWindowRect").Call(uintptr(hwnd), uintptr(unsafe.Pointer(&r))) + if ok == 0 { + return nil, false + } } // Cua uses DWM physical bounds; refuse evidence after a window move/resize. if int64(r.L) != num(b["left"]) || int64(r.T) != num(b["top"]) || int64(r.R-r.L) != num(b["width"]) || int64(r.B-r.T) != num(b["height"]) { diff --git a/scripts/build_cua_driver.py b/scripts/build_cua_driver.py index 2593d08..85acaa3 100644 --- a/scripts/build_cua_driver.py +++ b/scripts/build_cua_driver.py @@ -12,28 +12,67 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] SOURCE = ROOT / "plugins/cua-driver" -UPSTREAM_VERSION = "0.36.0" -COMMIT = "6d6c734c127025fb39ad5d585a10382808b39dc1" -DIGEST = "f8c47024b626f4e0e70f8428d2998df39e4879110c5722a29d5fd03e367353ad" -URL = "https://github.com/trycua/cua/releases/download/cua-driver-rs-v0.36.0/cua-driver-rs-0.36.0-windows-x86_64-binary.zip" +UPSTREAM_VERSION = "0.38.0" +COMMIT = "85e2b1318bd77ccf3cfa88ccdf875a4bb7eaed19" +DIGEST = "5edaa74d0afa1b55a9668dbc4f32736507bd2bd4d75b18ca1dc8881bb12402b5" +URL = "https://github.com/trycua/cua/releases/download/cua-driver-rs-v0.38.0/cua-driver-rs-0.38.0-windows-x86_64-binary.zip" +PATCH = SOURCE / "patches/cua-driver-0.38.0-untitled-shell.patch" +PATCHED_PATH = "libs/cua-driver/rust/crates/platform-windows/src/win32/windows.rs" + + +def build_patched_runtime(source: Path, cargo: str) -> dict: + source = source.resolve() + head = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=source, text=True).strip() + if head != COMMIT: + raise ValueError("Patched Cua source must use the pinned upstream commit") + changes = subprocess.check_output(["git", "diff", "--name-only", "HEAD"], cwd=source, text=True) + if changes.splitlines() != [PATCHED_PATH]: + raise ValueError("Cua source may contain only the reviewed untitled-shell patch") + diff = subprocess.check_output( + ["git", "diff", "--binary", "HEAD", "--", PATCHED_PATH], cwd=source + ) + if diff.replace(b"\r\n", b"\n") != PATCH.read_bytes().replace(b"\r\n", b"\n"): + raise ValueError("Cua source does not match the packaged compatibility patch") + project = source / "libs/cua-driver/rust" + command = [ + cargo, + "build", + "--release", + "--locked", + "-p", + "cua-driver", + "-p", + "cua-driver-uia", + "-p", + "cua-driver-sdk", + ] + subprocess.run(command, cwd=project, check=True, timeout=1800) + binaries = { + name: project / "target/release" / name + for name in ("cua-driver.exe", "cua-driver-uia.exe", "cua_driver_sdk.dll") + } + if not all(path.is_file() for path in binaries.values()): + raise ValueError("Patched Cua runtime build is incomplete") + return binaries def main(): parser = argparse.ArgumentParser(__doc__) parser.add_argument("upstream_zip", type=Path) parser.add_argument("--go", default="go") + parser.add_argument("--cargo", default="cargo") + parser.add_argument("--upstream-source", type=Path, required=True) parser.add_argument("--output", type=Path, default=ROOT / "artifacts") args = parser.parse_args() version = json.loads((SOURCE / "plugin.json").read_text("utf-8"))["version"] if version.split("+", 1)[0] != UPSTREAM_VERSION: raise ValueError("Plugin version must match the pinned Cua Driver version") - match = re.search( - r'const adapterVersion = "([^"]+)"', (SOURCE / "main.go").read_text("utf-8") - ) + match = re.search(r'const adapterVersion = "([^"]+)"', (SOURCE / "main.go").read_text("utf-8")) if not match or match[1] != version: raise ValueError("Adapter MCP version must match plugin.json") if hashlib.sha256(args.upstream_zip.read_bytes()).hexdigest() != DIGEST: raise ValueError("Cua release archive hash mismatch") + patched_runtime = build_patched_runtime(args.upstream_source, args.cargo) output = args.output.resolve() output.mkdir(parents=True, exist_ok=True) adapter = output / "artemis-cua-adapter.exe" @@ -76,12 +115,14 @@ def main(): "LICENSE.upstream", "RELEASE.md", "VERIFICATION.md", + "BUILD.md", ): dest.write(SOURCE / name, name) for source in sorted(SOURCE.glob("*.go")): dest.write(source, f"source/{source.name}") for name in ("go.mod", "tools.json"): dest.write(SOURCE / name, f"source/{name}") + dest.write(PATCH, f"source/patches/{PATCH.name}") dest.write(adapter, "bin/artemis-cua-adapter.exe") for name in upstream.namelist(): # Only the release's executable runtime; no global installer/hooks. @@ -91,7 +132,11 @@ def main(): "cua-cursor-theme.exe", "cua_driver_sdk.dll", }: - dest.writestr(f"bin/{Path(name).name}", upstream.read(name)) + basename = Path(name).name + if basename not in patched_runtime: + dest.writestr(f"bin/{basename}", upstream.read(name)) + for name, binary in patched_runtime.items(): + dest.write(binary, f"bin/{name}") dest.writestr( "provenance.json", json.dumps( @@ -99,10 +144,20 @@ def main(): "source_commit": COMMIT, "upstream_version": UPSTREAM_VERSION, "plugin_version": version, - "adapter_revision": 5, + "adapter_revision": int(version.rsplit(".", 1)[1]), "prerelease": True, + "upstream_release_channel": "stable", "url": URL, "sha256": DIGEST, + "compatibility_patch": { + "file": f"source/patches/{PATCH.name}", + "sha256": hashlib.sha256(PATCH.read_bytes()).hexdigest(), + "rebuilt_binaries": { + name: hashlib.sha256(binary.read_bytes()).hexdigest() + for name, binary in patched_runtime.items() + }, + "cargo_locked": True, + }, }, indent=2, ), diff --git a/tests/cua_shell_smoke.py b/tests/cua_shell_smoke.py new file mode 100644 index 0000000..95a6a5c --- /dev/null +++ b/tests/cua_shell_smoke.py @@ -0,0 +1,163 @@ +"""Opt-in native-menu and Win+B regression, under the host desktop lease.""" + +import argparse +import asyncio +import base64 +import hashlib +import json +from pathlib import Path +import re +import subprocess +import tempfile + +from artemis.plugins.models import ServerConfig +from artemis.plugins.session import PluginSession +from artemis.runtime.device_lock import DeviceExecutionLock + + +async def run(executable: Path, output: Path, taskbar: bool): + output.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="artemis-shell-fixture-") as folder: + state_file = Path(folder) / "state.json" + proc = await asyncio.create_subprocess_exec( + "powershell.exe", + "-NoProfile", + "-File", + str(Path(__file__).with_name("shell-fixture.ps1")), + "-StateFile", + str(state_file), + creationflags=subprocess.CREATE_NO_WINDOW, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + ) + session = None + calls = [] + try: + async with asyncio.timeout(15): + while not state_file.exists(): + if proc.returncode is not None: + raise RuntimeError((await proc.stderr.read()).decode(errors="replace")) + await asyncio.sleep(0.1) + state = json.loads(state_file.read_text(encoding="utf-8-sig")) + target = {"pid": state["pid"], "window_id": state["hwnd"]} + session = await PluginSession( + ServerConfig(command=str(executable)), executable.parent + ).start() + + async def call(name, args): + result = await session.call(name, args) + calls.append( + { + "name": name, + "arguments": args, + "result": result.model_dump(mode="json", by_alias=True, exclude_none=True), + } + ) + assert not result.isError, "\n".join( + block.text[:1500] for block in result.content if block.type == "text" + ) + return result + + def tree(result): + return "\n".join(getattr(block, "text", "") for block in result.content) + + result = await call("get_window_state", target) + match = re.search(r'\[(\d+)\][^\n]*"Open native menu"', tree(result)) + assert match, tree(result) + opened = await call( + "click", + { + **target, + "element_token": result.structuredContent["snapshot_id"] + ":" + match[1], + "count": 1, + "delivery_mode": "foreground", + }, + ) + observation = opened.structuredContent["observation_target"] + menu = observation["observed"] + assert menu["window_id"] != target["window_id"], observation + assert observation["reason"] == "associated_native_keyboard_focus", observation + assert opened.structuredContent["artemis"]["source"]["target_id"] == str( + menu["window_id"] + ) + observed = await call("get_window_state", menu) + assert "Increment once" in tree(observed), tree(observed) + png = next(block for block in observed.content if block.type == "image") + image = base64.b64decode(png.data, validate=True) + (output / "native-menu.png").write_bytes(image) + listed = await call("list_windows", {"pid": proc.pid}) + assert any( + window["window_id"] == menu["window_id"] and window.get("window_kind") == "menu" + for window in listed.structuredContent["windows"] + ) + # Escape belongs to the owner queue; no fixture or user business action. + await call("press_key", {**target, "key": "escape", "delivery_mode": "foreground"}) + final_state = json.loads(state_file.read_text(encoding="utf-8-sig")) + assert final_state["selected_count"] == 0, final_state + assert final_state["menu_attempts"] == 1, final_state + + shell_verified = False + if taskbar: + windows = await call("list_windows", {"app": "explorer", "on_screen_only": True}) + desktop = next( + window + for window in windows.structuredContent["windows"] + if window.get("class_name") == "Progman" + ) + desktop = {key: desktop[key] for key in ("pid", "window_id")} + await call("get_window_state", desktop) + key_result = await call( + "press_key", + {**desktop, "key": "b", "modifiers": ["win"], "delivery_mode": "foreground"}, + ) + switched = key_result.structuredContent["observation_target"] + assert switched["observed"]["window_id"] != desktop["window_id"], switched + assert switched["reason"] == "associated_native_keyboard_focus", switched + assert key_result.structuredContent["post_action_focus"]["focus_root"][ + "class_name" + ] in {"Shell_TrayWnd", "Shell_SecondaryTrayWnd"} + assert key_result.structuredContent["artemis"]["source"]["target_id"] == str( + switched["observed"]["window_id"] + ) + shell_verified = True + await call("bring_to_front", target) + report = { + "native_menu_discovered_and_captured": True, + "menu_source_matches_actual_hwnd": True, + "menu_png_sha256": hashlib.sha256(image).hexdigest(), + "win_b_focus_and_capture": shell_verified, + "fixture_business_submissions": 0, + "calls": len(calls), + } + (output / "summary.json").write_text(json.dumps(report, indent=2), encoding="utf-8") + print(json.dumps(report)) + finally: + # Save only this fixture's protocol evidence; no pairing payloads. + (output / "calls.json").write_text( + json.dumps(calls, ensure_ascii=False, indent=2), encoding="utf-8" + ) + if session: + await session.aclose() + if proc.returncode is None: + proc.terminate() + await proc.wait() + if state_file.exists(): + (output / "fixture-state.json").write_text( + state_file.read_text(encoding="utf-8-sig"), encoding="utf-8" + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(__doc__) + parser.add_argument("executable", type=Path) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--taskbar", action="store_true") + args = parser.parse_args() + lock = DeviceExecutionLock( + "cua-shell-regression", description="Cua shell regression", resource_ids=["desktop:local"] + ) + lock.acquire(timeout=0) + try: + asyncio.run(run(args.executable.resolve(), args.output.resolve(), args.taskbar)) + finally: + lock.release() diff --git a/tests/shell-fixture.ps1 b/tests/shell-fixture.ps1 new file mode 100644 index 0000000..6fe4d1c --- /dev/null +++ b/tests/shell-fixture.ps1 @@ -0,0 +1,51 @@ +param([Parameter(Mandatory = $true)][string]$StateFile) +$ErrorActionPreference = 'Stop' +Add-Type -AssemblyName System.Windows.Forms +Add-Type -AssemblyName System.Drawing +Add-Type -ReferencedAssemblies System.Windows.Forms,System.Drawing,System.Web.Extensions @' +using System; +using System.IO; +using System.Collections.Generic; +using System.Diagnostics; +using System.Runtime.InteropServices; +using System.Web.Script.Serialization; +using System.Windows.Forms; +using System.Drawing; +public sealed class ArtemisMenuFixture : Form { + [DllImport("user32.dll")] static extern IntPtr CreatePopupMenu(); + [DllImport("user32.dll", CharSet=CharSet.Unicode)] static extern bool AppendMenuW(IntPtr menu, uint flags, UIntPtr id, string text); + [DllImport("user32.dll")] static extern uint TrackPopupMenu(IntPtr menu, uint flags, int x, int y, int reserved, IntPtr owner, IntPtr rect); + [DllImport("user32.dll")] static extern bool DestroyMenu(IntPtr menu); + readonly string stateFile; + int count, attempts; + long result = -1; + string error = ""; + public ArtemisMenuFixture(string path) { + stateFile = path; + Text = "ARTEMIS native menu fixture"; + Size = new Size(560, 360); + StartPosition = FormStartPosition.CenterScreen; + var button = new Button { Text="Open native menu", Location=new Point(30,30), Size=new Size(240,55) }; + button.Click += delegate { + var menu = CreatePopupMenu(); + try { + attempts++; + Save(); + if (!AppendMenuW(menu, 0, new UIntPtr(1), "Increment once")) throw new Exception("AppendMenuW failed"); + result = TrackPopupMenu(menu, 0x100, Left+45, Top+140, 0, Handle, IntPtr.Zero); + if (result == 1) count++; + } catch (Exception ex) { error = ex.Message; } + finally { DestroyMenu(menu); Save(); } + }; + Controls.Add(button); + Shown += delegate { Save(); }; + } + void Save() { + File.WriteAllText(stateFile, new JavaScriptSerializer().Serialize(new Dictionary { + {"pid",Process.GetCurrentProcess().Id}, {"hwnd",Handle.ToInt64()}, {"selected_count",count}, + {"menu_attempts",attempts}, {"menu_result",result}, {"menu_error",error} + })); + } +} +'@ +[System.Windows.Forms.Application]::Run([ArtemisMenuFixture]::new($StateFile))