From c89588a80d1f8872d64b05bb08ccc083afc6601f Mon Sep 17 00:00:00 2001 From: chenmingxuan Date: Sat, 17 Oct 2026 13:01:39 +0800 Subject: [PATCH] feat: replace marketplace OCU with Cua Driver 0.36.0 --- README.md | 43 +- catalog.json | 163 ++++++- plugins/cua-driver/LICENSE.upstream | 21 + plugins/cua-driver/RELEASE.md | 11 + plugins/cua-driver/SKILL.md | 13 + plugins/cua-driver/VERIFICATION.md | 44 ++ plugins/cua-driver/go.mod | 3 + plugins/cua-driver/main.go | 448 +++++++++++++++++++ plugins/cua-driver/main_test.go | 61 +++ plugins/cua-driver/plugin.json | 169 ++++++++ plugins/cua-driver/tools.json | 645 ++++++++++++++++++++++++++++ plugins/cua-driver/windows.go | 69 +++ scripts/build_cua_driver.py | 115 +++++ tests/benchmark-fixture.ps1 | 47 ++ tests/benchmark_plugins.py | 279 ++++++++++++ tests/capture-fixture.ps1 | 8 +- tests/cua_fixture_validation.py | 115 +++++ tests/cua_smoke.py | 141 ++++++ tests/summarize_benchmark.py | 208 +++++++++ 19 files changed, 2563 insertions(+), 40 deletions(-) create mode 100644 plugins/cua-driver/LICENSE.upstream create mode 100644 plugins/cua-driver/RELEASE.md create mode 100644 plugins/cua-driver/SKILL.md create mode 100644 plugins/cua-driver/VERIFICATION.md create mode 100644 plugins/cua-driver/go.mod create mode 100644 plugins/cua-driver/main.go create mode 100644 plugins/cua-driver/main_test.go create mode 100644 plugins/cua-driver/plugin.json create mode 100644 plugins/cua-driver/tools.json create mode 100644 plugins/cua-driver/windows.go create mode 100644 scripts/build_cua_driver.py create mode 100644 tests/benchmark-fixture.ps1 create mode 100644 tests/benchmark_plugins.py create mode 100644 tests/cua_fixture_validation.py create mode 100644 tests/cua_smoke.py create mode 100644 tests/summarize_benchmark.py diff --git a/README.md b/README.md index 714c5fa..6ca7660 100644 --- a/README.md +++ b/README.md @@ -1,38 +1,35 @@ # Artemis Plugins -ARTEMIS 2.0.0 的插件市场仓库，用于分发经过验证的插件，供用户在 ARTEMIS 插件页面安装。 +ARTEMIS 2.0.0 的插件市场。当前电脑控制插件为 **Cua 0.36.0**，支持 Windows x64，通过窗口截图与可访问性树操作电脑应用和浏览器窗口，复用 ARTEMIS 的调度、资源互斥、步骤、截图附件和录像。 -电脑控制 1.0.0 以开发预览提供市场安装，适用于 Windows x64 和 ARTEMIS 2.0.0 开发版。已验证基础窗口、浏览器表单和 Flash 手机联动；复杂控制台、多 DPI、多显示器和长程场景仍需验证。具体结果见 [验证记录](plugins/open-computer-use/VERIFICATION.md)。 +在 ARTEMIS 的「插件」页面安装「电脑控制 · Cua」，授权并启用即可。安卓任务可根据提示词按需调用电脑控制，不需要单独配置 Codex MCP 或安装 Go、Node、Python。 -## 仓库内容 +## 从 OCU 切换 -- `catalog.json`：市场索引，当前契约版本为 1，空列表表示没有已发布插件。 -- `plugins/open-computer-use`：首发插件清单、适配 Skill 与验证记录。 -- `scripts/build_computer_use.py`：校验固定的 npm 1.1.0 安装包与上游源码，在构建阶段应用 Windows 截图和输入适配，运行 Go 测试并编译，生成包含源码、许可证的 ZIP 与 SHA-256；不执行 npm 安装脚本。 -- 安装包通过本仓库的发行附件分发；`artifacts/` 下的本地测试包不进入 Git。 +Open Computer Use（OCU）已从市场索引下架，当前市场只提供 Cua。已安装 OCU 的用户请先安装并启用 Cua，再停用 OCU，避免同一任务加载两套电脑控制工具。排队、暂停或运行中的任务结束后再切换。 -首发计划仅适配 [open-codex-computer-use](https://github.com/iFurySt/open-codex-computer-use)，通过同一插件操作 Windows 应用及浏览器窗口。用户在插件页面安装、授权并启用后，安卓任务可由 AI 按需调用，无需选择组合执行目标。 +旧插件的源码、发行附件、历史任务与证据继续保留。下架不会删除用户的已安装版本，也不会替其他用户自动授权新插件。需要复核旧任务时仍可按其固定版本处理。 -## 本地构建和验证 +## 验证范围 -构建机需要 Python 和 Go 1.22+；最终用户安装包不需要 Go。准备 npm `open-computer-use@1.1.0` 原始 tarball，以及上游提交 `7d8cb3fa5618d1ed8ec71773434ac3677f1922c9` 的干净 Git 检出： +本版本沿用 Cua 0.1.2 候选的驱动与动作实现，固定上游 [Cua Driver 0.36.0](https://github.com/trycua/cua/releases/tag/cua-driver-rs-v0.36.0) 预发布版，继续标注开发预览。 + +已测试原生窗口表单、34 像素视觉目标、滚动、截图保存、ARTEMIS 窗口录像和子进程退出。对照测试独立验收 3/3 完成；ARTEMIS AI 裁定为 2 次通过、1 次未判定。浏览器复杂控制台、多 DPI、多显示器与安卓混合长程任务仍需进一步验证，详见 [验证记录](plugins/cua-driver/VERIFICATION.md)。 + +## 构建与验证 + +构建机需要 Python 和 Go；最终用户只安装 ZIP。从固定上游发行下载 `cua-driver-rs-0.36.0-windows-x86_64-binary.zip`，构建脚本校验 SHA256 后打包，不执行全局安装器。 ```sh -python scripts/build_computer_use.py open-computer-use-1.1.0.tgz --upstream-source <上游源码目录> --go -python tests/windows_capture_smoke.py artifacts/open-computer-use.exe -python tests/browser_smoke.py artifacts/open-computer-use.exe --chrome +python scripts/build_cua_driver.py <上游ZIP> --go +python tests/cua_smoke.py <安装后的artemis-cua-adapter.exe> artifacts/cua-smoke +python tests/cua_fixture_validation.py <安装后的artemis-cua-adapter.exe> ``` -第二条命令使用安装了 ARTEMIS 依赖的 Python，创建并关闭两个临时窗口，验证遮挡截图、坐标对应和按钮实际收到点击。不会操作用户已有应用。 +验证脚本使用已安装 ARTEMIS 依赖的 Python，并将 ARTEMIS 源码目录放入 `PYTHONPATH`。它们只操作自行创建的测试窗口。历史 A/B 脚本仍固定 OCU 1.1.0 与 Cua 0.1.2，便于复现已有测量；它不是安装后必须执行的步骤。 -第三条命令创建独立 Chrome 配置和本地测试表单，通过 MCP 填写中文、提交一次并读回结果，退出后关闭测试浏览器。 +`catalog.json` 是市场索引，包含清单、HTTPS 安装包地址、SHA256 和验证说明。`artifacts/` 保存本机安装包与测试资料，不进入 Git。版本文件不可原地覆盖，任务固定使用其启动时的版本。 -ARTEMIS 适配将屏幕区域截取改为目标窗口 `PrintWindow`；失败或空白画面不回退为截屏。鼠标消息根据目标窗口的子控件路由，工具声明不再向模型推荐 Windows 未实现的点击方式。适配源文件与构建后的完整源码都随包分发，未修改上游仓库。 +插件版本跟随所用 Cua Driver 上游版本：当前为 `0.36.0`，对应上游标签 `cua-driver-rs-v0.36.0`，本仓库发行标签为 `cua-driver-v0.36.0`。只修改 ARTEMIS 适配而未升级上游时使用 `+artemis.N` 后缀区分不可变安装包；包内 `provenance.json` 记录上游提交、原包校验值和适配修订，构建会检查版本对应关系。 -## 发布约定 - -每个可安装版本需要声明支持的 ARTEMIS 版本和平台，固定运行依赖，提供安装包校验值、更新说明及实际验证记录。插件配置不包含账号密钥、个人目录或设备信息。 - -安装使用最新已验证且兼容的版本；后续升级由用户在 ARTEMIS 中操作，正在执行的任务固定插件版本。 - -[ARTEMIS 主项目](https://git.playones.com/chenmingxuan/Artemis-Eapil) +[发行说明](plugins/cua-driver/RELEASE.md) · [ARTEMIS 主项目](https://git.playones.com/chenmingxuan/Artemis-Eapil) diff --git a/catalog.json b/catalog.json index 98dfdde..677c6c8 100644 --- a/catalog.json +++ b/catalog.json @@ -3,11 +3,11 @@ "plugins": [ { "manifest": { - "schema_version": 1, - "id": "open-computer-use", - "name": "电脑控制", - "version": "1.0.0", - "description": "通过 Windows 窗口截图与可访问性树观察并操作桌面应用、浏览器控制台；可与手机任务组合。", + "schema_version": 2, + "id": "cua-driver", + "name": "电脑控制 · Cua", + "version": "0.36.0", + "description": "通过 Cua Driver 的窗口截图与可访问性树操作 Windows 应用及浏览器窗口；复用 ARTEMIS 的任务调度、步骤证据与录像。", "artemis_version": ">=2.0.0,<3.0.0", "platforms": [ "win32" @@ -15,14 +15,11 @@ "architectures": [ "amd64" ], - "source": "https://github.com/iFurySt/open-codex-computer-use", + "source": "https://github.com/trycua/cua", "license": "MIT", "server": { "transport": "stdio", - "command": "${plugin_dir}/bin/open-computer-use.exe", - "args": [ - "mcp" - ] + "command": "${plugin_dir}/bin/artemis-cua-adapter.exe" }, "skill": "SKILL.md", "targets": [ @@ -33,17 +30,151 @@ "desktop:local" ], "observation": { - "tool": "list_apps", - "arguments": {}, + "tool": "list_windows", + "arguments": { + "on_screen_only": true + }, "refresh_tools": [ - "get_app_state" + "get_window_state" + ] + }, + "capabilities": { + "kind": "visual", + "recording": "windows", + "tools": { + "list_windows": { + "effect": "read", + "observation": true, + "idempotent": true, + "target_argument": "", + "cancellation": "unknown", + "evidence": [ + "text" + ] + }, + "get_window_state": { + "effect": "read", + "observation": true, + "idempotent": true, + "target_argument": "window_id", + "cancellation": "unknown", + "evidence": [ + "text", + "image" + ] + }, + "verify_state": { + "effect": "read", + "observation": false, + "idempotent": true, + "target_argument": "window_id", + "cancellation": "unknown", + "evidence": [ + "text", + "image" + ] + }, + "click": { + "effect": "write", + "observation": false, + "idempotent": false, + "target_argument": "window_id", + "cancellation": "unknown", + "evidence": [ + "text", + "image" + ] + }, + "drag": { + "effect": "write", + "observation": false, + "idempotent": false, + "target_argument": "window_id", + "cancellation": "unknown", + "evidence": [ + "text", + "image" + ] + }, + "type_text": { + "effect": "write", + "observation": false, + "idempotent": false, + "target_argument": "window_id", + "cancellation": "unknown", + "evidence": [ + "text", + "image" + ] + }, + "press_key": { + "effect": "write", + "observation": false, + "idempotent": false, + "target_argument": "window_id", + "cancellation": "unknown", + "evidence": [ + "text", + "image" + ] + }, + "set_value": { + "effect": "write", + "observation": false, + "idempotent": false, + "target_argument": "window_id", + "cancellation": "unknown", + "evidence": [ + "text", + "image" + ] + }, + "scroll": { + "effect": "write", + "observation": false, + "idempotent": false, + "target_argument": "window_id", + "cancellation": "unknown", + "evidence": [ + "text", + "image" + ] + }, + "save_screenshot": { + "effect": "write", + "observation": false, + "idempotent": false, + "target_argument": "window_id", + "cancellation": "unknown", + "evidence": [ + "text", + "image", + "file" + ] + }, + "click_at": { + "effect": "write", + "observation": false, + "idempotent": false, + "target_argument": "window_id", + "cancellation": "unknown", + "evidence": [ + "text", + "image" + ] + } + }, + "required_tools": [ + "list_windows", + "get_window_state", + "save_screenshot" ] } }, "release_stage": "preview", - "archive_url": "https://git.playones.com/attachments/3273b838-4d64-442d-84c2-1a590ce8a81d", - "sha256": "b4fcc9ca6a1f0137bba6f4db93d524fa57cfa02a0d53178ac28e7521a97785bf", - "verification": "Windows x64 / ARTEMIS 2.0.0 开发版；2026-10-14。\n已验证：窗口观察、中文输入、子控件点击、遮挡截图；Chrome 基础表单填写、提交与重新读取；Flash/Pro 纯电脑基础任务；Flash 电脑修改参数、手机验证、恢复参数和再次验证，步骤截图及 Word/Excel 导出。\n未覆盖：混合 Pro、复杂控制台、多 DPI、多显示器、长程压力和真实批次组合。部分应用不支持窗口截图，最小化窗口不可用于坐标操作。完整记录见插件仓库 VERIFICATION.md。" + "archive_url": "https://git.playones.com/chenmingxuan/Artemis-Plugins/releases/download/cua-driver-v0.36.0/cua-driver-0.36.0-windows-amd64.zip", + "sha256": "dea40dce182ac6b982c7da17d3610d5db87e2639205ca32f179c55ca1d7833ce", + "verification": "Windows x64 / ARTEMIS 2.0.0 开发版。固定 Cua Driver 0.36.0 预发布版。已验证原生窗口表单、控件与像素点击、34 像素目标、滚动、trace 截图及哈希、ARTEMIS PC 窗口录像、子进程退出；同一复合用例重复 3 次，独立验收 3/3，AI 裁定 2 次通过、1 次未判定。0.36.0 沿用已测 0.1.2 动作实现，取代 OCU 市场入口。未覆盖复杂浏览器控制台、多 DPI、多显示器和跨端长任务。完整验证与已知限制见仓库 plugins/cua-driver/VERIFICATION.md。" } ] } diff --git a/plugins/cua-driver/LICENSE.upstream b/plugins/cua-driver/LICENSE.upstream new file mode 100644 index 0000000..b8b198c --- /dev/null +++ b/plugins/cua-driver/LICENSE.upstream @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Cua AI, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/plugins/cua-driver/RELEASE.md b/plugins/cua-driver/RELEASE.md new file mode 100644 index 0000000..9bf8a2f --- /dev/null +++ b/plugins/cua-driver/RELEASE.md @@ -0,0 +1,11 @@ +# Cua 电脑控制 0.36.0 + +作为 ARTEMIS 插件市场的默认电脑控制实现，替代已下架的 Open Computer Use。此版本沿用候选 0.1.2 的驱动和动作逻辑，更新版本标识、名称和分发文档；上游仍是固定的 Cua Driver 0.36.0 预发布版，市场标注开发预览。 + +上游提交：`6d6c734c127025fb39ad5d585a10382808b39dc1`。Windows x64 原始 ZIP SHA256：`f8c47024b626f4e0e70f8428d2998df39e4879110c5722a29d5fd03e367353ad`。采用 MIT Driver，许可证随包提供；不包含 Cua Spaces、云环境或模型服务。 + +提供 11 个窗口观察、控件/像素点击、输入、按键、拖动、滚动及截图工具。操作后返回真实窗口观察，保留动作回执，截图校验 PID/HWND 与物理边界，生成文件保存到当前任务 trace。共享 ARTEMIS 的 `desktop:local` 资源、任务固定版本、步骤和录像，不另建调度器。 + +适配器与依赖随包分发，无需额外安装 Go、Python、Node 或另一份 Codex MCP。关闭、取消或宿主退出后通过 Windows Job Object 回收 Driver/UIA 子进程；仅在该子进程环境关闭遥测。 + +升级：任务结束后，在插件页面安装并授权启用 Cua，再停用 OCU。下架不删除旧包、历史报告或用户配置，不自动替其他用户授权。完整实测结果与尚未验证的范围见 [VERIFICATION.md](VERIFICATION.md)。 diff --git a/plugins/cua-driver/SKILL.md b/plugins/cua-driver/SKILL.md new file mode 100644 index 0000000..1077b66 --- /dev/null +++ b/plugins/cua-driver/SKILL.md @@ -0,0 +1,13 @@ +# Cua 电脑控制（Windows） + +通过本地 Cua Driver 操作用户授权的 Windows 窗口。调度、资源互斥、模型、步骤、截图和录像由 ARTEMIS 提供；插件本身不运行 AI，不另起后台服务。 + +1. `list_windows` 找到目标的 `pid` 和 `window_id`；只操作任务要求的窗口，不关闭其他应用。 +2. `get_window_state` 同时读取截图和 UIA 控件树。最新 `snapshot_id` 加 `:N` 即控件 `element_token`。优先用 `click` 点击最新控件；视觉目标用 `click_at`，坐标仅取本次截图上的窗口局部像素，不能用屏幕坐标，需携带本次 `capture_id`。两种点击参数互斥，不混用。 +3. 所有窗口操作明确传 `pid`、`window_id`。`set_value` 适合直接设置输入框；不支持时可明确聚焦后 `type_text`。观察控件支持的动作，不猜元素编号。`press_key` 传单键及 `modifiers` 数组。 +4. 每个动作后插件自动返回一次新截图和控件树，新 snapshot 替换旧 token。接着用新 token，无需重复获取相同状态。仅当画面确实还在变化或需要核验时再次观察。一次调用只执行一次动作，超时不自动重发。 +5. 默认后台操作。`effect=unverifiable/suspected_noop` 仅代表投递/效果未确认，必须检查返回画面和读回值；`refused` 是未执行。后台无效时，可在任务授权范围内显式 `delivery_mode=foreground` 后再次检查。不要仅凭工具成功声称测试通过。 +6. `scroll` 需要最新窗口/滚动区域，使用 up/down/left/right、amount、by=line/page；先观察是否真的滚动。`get_window_state(since="latest")` 可在同一窗口读取差异，但新 snapshot_id 仍替换旧 token。 +7. `save_screenshot` 保存最新窗口 PNG 到当前任务 trace，filename 仅是文件名，不能传路径，不覆盖已有文件。实际成功返回 SHA256 和附件；不得把工具内存截图当成已保存文件。 + +不提供 Cua 自带调度、录屏、回放、配置修改、安装器、任意脚本或自动升级接口。浏览器本轮使用窗口视觉/UIA 通路；未接入 DOM 或已有登录浏览器调试权限。此包基于上游 0.36.0 预发布版，通过已列出的本机验证，其他场景的支持范围以 VERIFICATION.md 为准。 diff --git a/plugins/cua-driver/VERIFICATION.md b/plugins/cua-driver/VERIFICATION.md new file mode 100644 index 0000000..191d754 --- /dev/null +++ b/plugins/cua-driver/VERIFICATION.md @@ -0,0 +1,44 @@ +# Cua 验证记录 + +## 候选阶段记录 + +以下记录对应市场替换前的候选测试。适配版本 0.1.2；上游 Cua Driver 0.36.0（预发布），仅 Windows x64。默认模型和默认插件保持原样。 + +端到端联调还发现 ARTEMIS SDK 配置构造遗漏 `recording`，导致任务指定窗口录制却使用默认屏幕录制。主项目已补传该配置，覆盖从任务默认值、SDK Builder 到 PluginRecordings 的回归；新工作进程实际录像索引已确认 `scope=window`。早期屏幕录像只保留在本机，测量报告披露各次实际录制范围，不混同为统一录制条件。 + +已验证： + +- 11 个工具符合 ARTEMIS schema 2 清单，命名、只读声明与窗口参数匹配。 +- HWND/PID 与物理窗口边界核对、真实窗口截图、动作后新 snapshot/token。 +- 独立窗口确认点击实际到达；表单值与勾选状态正确，蓝色小目标命中，滚动到末尾后完成。 +- 后台滚动的明确预检拒绝与前台执行区分；未自动重试写操作。 +- 新 PNG 保存到 trace，读回 SHA256 校验，附件被证据索引接收；拒绝覆盖已有文件与路径逃逸。 +- 使用 ARTEMIS 原有录像管理器录制 PC 窗口，H.264 文件确实生成；关闭会话后候选适配器、Driver 和 UIA 子进程均退出。 +- Go 回归覆盖动作回执、未知结果、拒绝、参数边界与两种点击路径互斥；Python 工具经过 Ruff 检查。 + +联调发现并修正：0.1.0 同一点击工具允许控件与像素参数，模型混填导致上游拒绝；0.1.1 拆分 `click` 与 `click_at`。0.1.2 补充上游 Windows 类级 `background_unavailable` 的未执行回执；带 actuator cause 的失败仍保留不确定状态。 + +实际 A/B 数据由 `tests/benchmark_plugins.py` 和 `tests/summarize_benchmark.py` 生成，保存于本机 `artifacts/`。测试使用独立窗口、固定任务和统一调度，比较表单、34 像素视觉目标、滚动及截图留档；不能由这些小样本外推浏览器、所有 DPI、真实路由器或长程任务的成功率。模型无工具调用、兜底模型与网关缓存情况必须随数据披露。 + +2026-10-17 完成一个复合用例各 3 次：Flash，主模型 `mimo-v2.6-flash/high`，保留 `gpt-6-luna` 兜底，480 秒任务时限。对照为当前 OCU 1.1.0 适配包。独立窗口事件判定结果如下（token 包含执行、摘要、压缩与最终核验）： + +| 指标 | OCU 1.1.0 | Cua 0.1.2 | +| --- | --- | --- | +| 全部动作与截图独立验收 | 0/3 | 3/3 | +| 蓝色目标恰好命中一次、没有误点 | 3/3 | 3/3 | +| 平均总 token | 683,748 | 504,954 | +| 缓存输入 / 总输入 | 51.0% | 49.7% | +| 平均任务结束/停止耗时 | 478.8 秒（超时） | 303.2 秒 | +| 平均插件调用耗时合计 | 32.2 秒 | 5.2 秒 | + +OCU 在滚动回退中漏传 `WindowBounds`，导致反复报坐标不可用；此为对照版本的具体缺陷，本次未中途修改。Cua 第一轮有一次旧 token 拒绝，刷新后成功。两者的小目标点击均成功，不能据此宣称 Cua 普遍点击更准。Cua 的 token 少 26.1% 是本轮观察值，包含旧插件失败重试开销，不是所有成功任务的固定节省。 + +Cua 第三轮虽然独立验收通过，ARTEMIS 最终核验却没有生成结论，保留 `inconclusive`；AI 裁定实际为 2 次通过、1 次未判定。三次截图均可解码、SHA256 与保存回执一致；最后一轮窗口录像全片解码通过，约 173 秒、5.8 MiB。测试结束后队列空闲，候选适配器、Driver、UIA helper 和测试窗口进程均已退出。 + +主项目 SDK 配置修复通过默认回归 2761 项（2 项跳过、8 项未选中），Ruff、质量基线和核心类型检查通过。候选继续仅在非默认测试配置启用；没有修改市场索引。录像范围的中途修复、模型兜底以及最终核验延迟均在本地报告中保留，不将本次开发测试包装为严格受控的性能实验。 + +未验证：真实业务跨端长程任务、Chrome/Brave 特定页面、多个 DPI/显示器、非 Windows 平台、DOM 路由。这些未测范围仍不作稳定性保证；市场替换决定见下一节。 + +## 市场替换版本 0.36.0 + +0.36.0 沿用 0.1.2 已测驱动和动作实现，仅更新发行版本、名称及文档，作为市场默认电脑控制插件取代 OCU。上游仍固定 0.36.0 预发布版，市场继续标注开发预览；上文的未测范围和 AI 裁定缺失情况仍然有效。旧 OCU 从 catalog.json 下架，既有源码、发行附件和任务证据保留。已安装 OCU 的用户应安装并启用 Cua，再停用 OCU；不会静默为其他用户授权新代码。 diff --git a/plugins/cua-driver/go.mod b/plugins/cua-driver/go.mod new file mode 100644 index 0000000..fcca543 --- /dev/null +++ b/plugins/cua-driver/go.mod @@ -0,0 +1,3 @@ +module artemis-cua-adapter + +go 1.22 diff --git a/plugins/cua-driver/main.go b/plugins/cua-driver/main.go new file mode 100644 index 0000000..8a86938 --- /dev/null +++ b/plugins/cua-driver/main.go @@ -0,0 +1,448 @@ +// ARTEMIS adapter for the pinned MIT Cua Driver. No model, scheduler, or recorder. +package main + +import ( + "bufio" + "bytes" + "crypto/sha256" + _ "embed" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" +) + +//go:embed tools.json +var toolsJSON []byte + +type obj = map[string]any +type request struct { + JSONRPC string `json:"jsonrpc"` + ID json.RawMessage `json:"id"` + Method string `json:"method"` + Params obj `json:"params"` +} +type peer struct { + cmd *exec.Cmd + input io.WriteCloser + lines chan []byte + next int + mu sync.Mutex + dead bool + job uintptr +} + +func object(v any) obj { + m, _ := v.(map[string]any) + if m == nil { + return obj{} + } + return m +} +func str(v any) string { s, _ := v.(string); return s } +func num(v any) int64 { + switch n := v.(type) { + case float64: + return int64(n) + case int64: + return n + case int: + return int64(n) + } + return 0 +} +func items(v any) []any { a, _ := v.([]any); return a } +func textBlock(s string) obj { return obj{"type": "text", "text": s} } +func receipt(execution, side, code string) obj { + return obj{"version": 1, "execution": execution, "side_effect": side, "retryable": false, "error_code": code} +} +func failure(msg, execution, side, code string) obj { + return obj{"isError": true, "content": []any{textBlock(msg)}, "structuredContent": obj{"artemis": receipt(execution, side, code)}} +} +func writeTool(name string) bool { + return name != "list_windows" && name != "get_window_state" && name != "verify_state" +} + +func startPeer(exe string) (*peer, error) { + c := exec.Command(exe, "mcp", "--direct") + c.Env = append(os.Environ(), "DO_NOT_TRACK=1", "CUA_TELEMETRY=0", "CUA_DRIVER_RS_TELEMETRY_ENABLED=false") + configureProcess(c) + in, err := c.StdinPipe() + if err != nil { + return nil, err + } + out, err := c.StdoutPipe() + if err != nil { + return nil, err + } + c.Stderr = os.Stderr + if err = c.Start(); err != nil { + return nil, err + } + job, err := ownProcess(c.Process.Pid) + if err != nil { + _ = c.Process.Kill() + _ = c.Wait() + return nil, err + } + p := &peer{cmd: c, input: in, lines: make(chan []byte, 32), job: job} + go func() { + defer close(p.lines) + s := bufio.NewScanner(out) + s.Buffer(make([]byte, 65536), 32<<20) + for s.Scan() { + p.lines <- append([]byte(nil), s.Bytes()...) + } + }() + _, err = p.call("initialize", obj{"protocolVersion": "2024-11-05", "clientInfo": obj{"name": "artemis-cua-adapter", "version": "0.36.0"}, "capabilities": obj{}}) + if err != nil { + p.stop() + return nil, err + } + err = json.NewEncoder(in).Encode(obj{"jsonrpc": "2.0", "method": "notifications/initialized"}) + if err != nil { + p.stop() + return nil, err + } + return p, nil +} +func (p *peer) stop() { + p.mu.Lock() + defer p.mu.Unlock() + if p.dead { + return + } + p.dead = true + _ = p.input.Close() + closeJob(p.job) + _ = p.cmd.Process.Kill() + _ = p.cmd.Wait() +} +func (p *peer) call(method string, params obj) (obj, error) { + p.mu.Lock() + dead := p.dead + p.mu.Unlock() + if dead { + return nil, errors.New("driver closed") + } + p.next++ + id := p.next + if err := json.NewEncoder(p.input).Encode(obj{"jsonrpc": "2.0", "id": id, "method": method, "params": params}); err != nil { + return nil, err + } + timer := time.NewTimer(45 * time.Second) + defer timer.Stop() + for { + select { + case raw, ok := <-p.lines: + if !ok { + return nil, errors.New("driver disconnected") + } + var r obj + if json.Unmarshal(raw, &r) != nil { + continue + } + if num(r["id"]) != int64(id) { + continue + } + if r["error"] != nil { + return nil, errors.New("driver protocol error") + } + return object(r["result"]), nil + case <-timer.C: + p.stop() + return nil, errors.New("driver response timed out; action not replayed") + } + } +} +func (p *peer) tool(name string, args obj) (obj, error) { + return p.call("tools/call", obj{"name": name, "arguments": args}) +} + +type adapter struct { + peer *peer + tools []any + schemas map[string]obj +} + +func newAdapter(p *peer) *adapter { + a := &adapter{peer: p, schemas: map[string]obj{}} + _ = json.Unmarshal(toolsJSON, &a.tools) + for _, v := range a.tools { + t := object(v) + a.schemas[str(t["name"])] = object(t["inputSchema"]) + } + return a +} +func (a *adapter) validate(name string, args obj) error { + schema, ok := a.schemas[name] + if !ok { + return errors.New("tool is not exposed") + } + props := object(schema["properties"]) + for key := range args { + if _, ok := props[key]; !ok { + return fmt.Errorf("unsupported argument: %s", key) + } + } + for _, key := range items(schema["required"]) { + if _, ok := args[str(key)]; !ok { + return fmt.Errorf("missing argument: %s", str(key)) + } + } + if name != "list_windows" && (num(args["pid"]) <= 0 || num(args["window_id"]) <= 0) { + return errors.New("a concrete pid and window_id are required") + } + if name == "save_screenshot" && !validFilename(str(args["filename"])) { + return errors.New("filename must be a new plain PNG filename") + } + if name == "get_window_state" && (args["include_screenshot"] == false || args["include_accessibility_tree"] == false) { + return errors.New("candidate observations require both screenshot and accessibility tree") + } + return nil +} +func validFilename(s string) bool { + return s != "" && len(s) <= 160 && strings.EqualFold(filepath.Ext(s), ".png") && !strings.ContainsAny(s, "/\\:<>\"|?*\x00") && s != "." && s != ".." && !strings.HasSuffix(s, ".") && !strings.HasSuffix(s, " ") +} +func actionReceipt(r obj, write bool) obj { + if !write { + if r["isError"] == true { + return receipt("not_started", "none", "observation_failed") + } + return receipt("completed", "none", "") + } + s := object(r["structuredContent"]) + // v0.36.0 class-level preflight refusal has no effect field. It occurs + // before input injection. Do not classify actuator failures with a cause + // as safe-to-repeat; retain their uncertain execution below. + if s["code"] == "background_unavailable" && s["cause"] == nil && s["event_kind"] != nil && r["isError"] == true { + return receipt("not_started", "none", "background_unavailable") + } + switch str(s["effect"]) { + case "confirmed": + return receipt("completed", "confirmed", "") + case "partial": + return receipt("partial", "confirmed", "partial_delivery") + case "refused": + return receipt("not_started", "none", "driver_refused") + case "unverifiable", "suspected_noop": + return receipt("completed", "possible", str(s["effect"])) + default: + return receipt("unknown", "possible", "unclassified_result") + } +} +func imageSize(r obj) (int, int) { + for _, v := range items(r["content"]) { + c := object(v) + if c["type"] == "image" { + raw, err := base64.StdEncoding.DecodeString(str(c["data"])) + if err == nil { + cfg, _, err := image.DecodeConfig(bytes.NewReader(raw)) + if err == nil { + return cfg.Width, cfg.Height + } + } + } + } + return 0, 0 +} +func source(r obj, args obj, started float64) obj { + s := object(r["structuredContent"]) + w, h := imageSize(r) + if r["isError"] == true || w <= 0 || h <= 0 || num(s["pid"]) != num(args["pid"]) || num(s["window_id"]) != num(args["window_id"]) { + return nil + } + bounds := object(s["window_bounds"]) + rect := obj{"left": num(bounds["x"]), "top": num(bounds["y"]), "width": num(bounds["width"]), "height": num(bounds["height"])} + screen, ok := verifyWindow(num(args["pid"]), num(args["window_id"]), rect) + if !ok { + return nil + } + return obj{"kind": "computer", "target_id": fmt.Sprint(num(args["window_id"])), "label": s["window_title"], "captured_at": started, "width": w, "height": h, "window": rect, "screen": screen} +} +func (a *adapter) observe(args obj) (obj, obj, error) { + started := float64(time.Now().UnixMilli()) / 1000 + r, err := a.peer.tool("get_window_state", args) + if err != nil { + return nil, nil, err + } + return r, source(r, args, started), nil +} +func observationArgs(args obj) obj { + return obj{"pid": args["pid"], "window_id": args["window_id"], "max_image_dimension": 1280, "max_elements": 250, "tree_format": "markdown"} +} +func (a *adapter) handle(name string, args obj) obj { + if err := a.validate(name, args); err != nil { + return failure(err.Error(), "not_started", "none", "invalid_input") + } + if name != "list_windows" { + if !windowOwned(num(args["pid"]), num(args["window_id"])) { + return failure("window is gone or belongs to a different process", "not_started", "none", "invalid_target") + } + } + if name == "save_screenshot" { + return a.save(args) + } + var r, src obj + var err error + if name == "get_window_state" { + for k, v := range observationArgs(args) { + if _, ok := args[k]; !ok { + args[k] = v + } + } + r, src, err = a.observe(args) + } else { + upstream := name + if name == "click_at" { + upstream = "click" + } + r, err = a.peer.tool(upstream, args) + } + if err != nil { + a.peer.stop() + if writeTool(name) { + return failure(err.Error(), "unknown", "possible", "transport_lost") + } + return failure(err.Error(), "not_started", "none", "transport_lost") + } + rec := actionReceipt(r, writeTool(name)) + structured := object(r["structuredContent"]) + if writeTool(name) && rec["execution"] != "not_started" { + // Observe once after delivery; NEVER repeat an uncertain write. + observation, origin, e := a.observe(observationArgs(args)) + if e == nil && origin != nil { + src = origin + structured["observation"] = observation["structuredContent"] + r["content"] = append(items(r["content"]), items(observation["content"])...) + } else { + structured["observation_error"] = "Post-action evidence unavailable; inspect before another action." + } + } + if name == "verify_state" { // Read-only verification may include a screenshot; it is not a task verdict. + src = source(r, args, float64(time.Now().UnixMilli())/1000) + } + if src != nil { + rec["source"] = src + } else if name == "get_window_state" && r["isError"] != true { + return failure("capture target or geometry could not be validated", "not_started", "none", "invalid_capture") + } + if structured["effect"] == "refused" { + r["isError"] = true + } + structured["artemis"] = rec + r["structuredContent"] = structured + return r +} +func (a *adapter) save(args obj) obj { + root := os.Getenv("ARTEMIS_ARTIFACTS_DIR") + if root == "" { + return failure("task artifact directory unavailable", "not_started", "none", "missing_artifacts") + } + r, src, err := a.observe(observationArgs(args)) + if err != nil || src == nil { + return failure("fresh window capture failed", "not_started", "none", "capture_failed") + } + var png []byte + for _, v := range items(r["content"]) { + b := object(v) + if b["type"] == "image" && b["mimeType"] == "image/png" { + png, _ = base64.StdEncoding.DecodeString(str(b["data"])) + break + } + } + if len(png) == 0 { + return failure("expected PNG capture", "not_started", "none", "capture_format") + } + if err = os.MkdirAll(root, 0700); err != nil { + return failure("artifact directory unavailable", "not_started", "none", "file_error") + } + path := filepath.Join(root, str(args["filename"])) + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return failure("file exists or cannot be created", "not_started", "none", "file_error") + } + n, writeErr := f.Write(png) + closeErr := f.Close() + actual, readErr := os.ReadFile(path) + if writeErr != nil || closeErr != nil || readErr != nil || n != len(png) || !bytes.Equal(actual, png) { + return failure("artifact write could not be verified", "partial", "possible", "file_error") + } + rec := receipt("completed", "confirmed", "") + rec["source"] = src + rec["attachments"] = []any{obj{"name": str(args["filename"]), "sha256": fmt.Sprintf("%x", sha256.Sum256(actual)), "media_type": "image/png"}} + object(r["structuredContent"])["artemis"] = rec + r["content"] = append(items(r["content"]), textBlock("Saved verified task attachment: "+str(args["filename"]))) + return r +} +func serve(a *adapter, in io.Reader, out io.Writer) { + input := make(chan request) + go func() { + defer close(input) + defer a.peer.stop() + s := bufio.NewScanner(in) + s.Buffer(make([]byte, 65536), 2<<20) + for s.Scan() { + var q request + if json.Unmarshal(s.Bytes(), &q) == nil { + if q.Method == "notifications/cancelled" { + a.peer.stop() + continue + } + input <- q + } + } + }() + enc := json.NewEncoder(out) + for q := range input { + if len(q.ID) == 0 { + continue + } + var result any + var rpcErr any + switch q.Method { + case "initialize": + result = obj{"protocolVersion": "2024-11-05", "capabilities": obj{"tools": obj{}}, "serverInfo": obj{"name": "artemis-cua-driver", "version": "0.36.0"}} + case "ping": + result = obj{} + case "tools/list": + result = obj{"tools": a.tools} + case "tools/call": + result = a.handle(str(q.Params["name"]), object(q.Params["arguments"])) + default: + rpcErr = obj{"code": -32601, "message": "method not found"} + } + response := obj{"jsonrpc": "2.0", "id": q.ID} + if rpcErr != nil { + response["error"] = rpcErr + } else { + response["result"] = result + } + if enc.Encode(response) != nil { + return + } + } +} +func main() { + exe, err := os.Executable() + if err != nil { + os.Exit(1) + } + p, err := startPeer(filepath.Join(filepath.Dir(exe), "cua-driver.exe")) + if err != nil { + fmt.Fprintln(os.Stderr, "Cua startup failed:", err) + os.Exit(1) + } + defer p.stop() + serve(newAdapter(p), os.Stdin, os.Stdout) +} diff --git a/plugins/cua-driver/main_test.go b/plugins/cua-driver/main_test.go new file mode 100644 index 0000000..6a8896b --- /dev/null +++ b/plugins/cua-driver/main_test.go @@ -0,0 +1,61 @@ +package main + +import "testing" + +func TestUnknownAndRefusedAreNotConfirmed(t *testing.T) { + cases := []struct{ effect, execution, side string }{{"confirmed", "completed", "confirmed"}, {"partial", "partial", "confirmed"}, {"unverifiable", "completed", "possible"}, {"suspected_noop", "completed", "possible"}, {"refused", "not_started", "none"}, {"", "unknown", "possible"}} + for _, c := range cases { + r := actionReceipt(obj{"structuredContent": obj{"effect": c.effect}}, true) + if r["execution"] != c.execution || r["side_effect"] != c.side || r["retryable"] != false { + t.Fatalf("%s: %v", c.effect, r) + } + } +} +func TestAttachmentCannotEscape(t *testing.T) { + for _, s := range []string{"../out.png", "..\\out.png", "C:out.png", "a.png:stream", "", "a.png ", "a.jpg"} { + if validFilename(s) { + t.Fatal(s) + } + } + if !validFilename("留档.png") { + t.Fatal("plain Unicode PNG rejected") + } +} +func TestUnexposedAndMissingTargetRejected(t *testing.T) { + a := newAdapter(nil) + for _, c := range []struct { + name string + args obj + }{{"run_script", obj{}}, {"click", obj{"pid": 1, "window_id": 2, "scope": "desktop"}}, {"get_window_state", obj{"pid": 1}}, {"click", obj{"pid": 0, "window_id": 2}}} { + if a.validate(c.name, c.args) == nil { + t.Fatal(c) + } + } +} +func TestSupportedArguments(t *testing.T) { + a := newAdapter(nil) + if err := a.validate("click", obj{"pid": 1, "window_id": 2, "element_token": "s00000001:0"}); err != nil { + t.Fatal(err) + } +} + +func TestBackgroundPreflightVsActuatorUncertainty(t *testing.T) { + r := obj{"isError": true, "structuredContent": obj{"code": "background_unavailable", "event_kind": "mouse_scroll"}} + if actionReceipt(r, true)["execution"] != "not_started" { + t.Fatal("preflight refusal must be explicit") + } + object(r["structuredContent"])["cause"] = "unknown actuator failure" + if actionReceipt(r, true)["execution"] != "unknown" { + t.Fatal("actuator uncertainty must not be hidden") + } +} + +func TestClickModesAreDisjoint(t *testing.T) { + a := newAdapter(nil) + if a.validate("click", obj{"pid": 1, "window_id": 2, "element_token": "s1:0", "capture_id": "capture"}) == nil { + t.Fatal("mixed click modes") + } + if a.validate("click_at", obj{"pid": 1, "window_id": 2, "x": 5, "y": 5}) == nil { + t.Fatal("pixel action lacks capture") + } +} diff --git a/plugins/cua-driver/plugin.json b/plugins/cua-driver/plugin.json new file mode 100644 index 0000000..30d9ba8 --- /dev/null +++ b/plugins/cua-driver/plugin.json @@ -0,0 +1,169 @@ +{ + "schema_version": 2, + "id": "cua-driver", + "name": "电脑控制 · Cua", + "version": "0.36.0", + "description": "通过 Cua Driver 的窗口截图与可访问性树操作 Windows 应用及浏览器窗口；复用 ARTEMIS 的任务调度、步骤证据与录像。", + "artemis_version": ">=2.0.0,<3.0.0", + "platforms": [ + "win32" + ], + "architectures": [ + "amd64" + ], + "source": "https://github.com/trycua/cua", + "license": "MIT", + "server": { + "transport": "stdio", + "command": "${plugin_dir}/bin/artemis-cua-adapter.exe" + }, + "skill": "SKILL.md", + "targets": [ + "computer", + "mixed" + ], + "resources": [ + "desktop:local" + ], + "observation": { + "tool": "list_windows", + "arguments": { + "on_screen_only": true + }, + "refresh_tools": [ + "get_window_state" + ] + }, + "capabilities": { + "kind": "visual", + "recording": "windows", + "tools": { + "list_windows": { + "effect": "read", + "observation": true, + "idempotent": true, + "target_argument": "", + "cancellation": "unknown", + "evidence": [ + "text" + ] + }, + "get_window_state": { + "effect": "read", + "observation": true, + "idempotent": true, + "target_argument": "window_id", + "cancellation": "unknown", + "evidence": [ + "text", + "image" + ] + }, + "verify_state": { + "effect": "read", + "observation": false, + "idempotent": true, + "target_argument": "window_id", + "cancellation": "unknown", + "evidence": [ + "text", + "image" + ] + }, + "click": { + "effect": "write", + "observation": false, + "idempotent": false, + "target_argument": "window_id", + "cancellation": "unknown", + "evidence": [ + "text", + "image" + ] + }, + "drag": { + "effect": "write", + "observation": false, + "idempotent": false, + "target_argument": "window_id", + "cancellation": "unknown", + "evidence": [ + "text", + "image" + ] + }, + "type_text": { + "effect": "write", + "observation": false, + "idempotent": false, + "target_argument": "window_id", + "cancellation": "unknown", + "evidence": [ + "text", + "image" + ] + }, + "press_key": { + "effect": "write", + "observation": false, + "idempotent": false, + "target_argument": "window_id", + "cancellation": "unknown", + "evidence": [ + "text", + "image" + ] + }, + "set_value": { + "effect": "write", + "observation": false, + "idempotent": false, + "target_argument": "window_id", + "cancellation": "unknown", + "evidence": [ + "text", + "image" + ] + }, + "scroll": { + "effect": "write", + "observation": false, + "idempotent": false, + "target_argument": "window_id", + "cancellation": "unknown", + "evidence": [ + "text", + "image" + ] + }, + "save_screenshot": { + "effect": "write", + "observation": false, + "idempotent": false, + "target_argument": "window_id", + "cancellation": "unknown", + "evidence": [ + "text", + "image", + "file" + ] + }, + "click_at": { + "effect": "write", + "observation": false, + "idempotent": false, + "target_argument": "window_id", + "cancellation": "unknown", + "evidence": [ + "text", + "image" + ] + } + }, + "required_tools": [ + "list_windows", + "get_window_state", + "save_screenshot" + ] + } +} diff --git a/plugins/cua-driver/tools.json b/plugins/cua-driver/tools.json new file mode 100644 index 0000000..cb6dcf5 --- /dev/null +++ b/plugins/cua-driver/tools.json @@ -0,0 +1,645 @@ +[ + { + "name": "list_windows", + "description": "List visible application windows with exact pid/window_id and physical bounds. Pick only the user-authorized target.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "app": { + "description": "Only windows of apps whose name contains this text (case-insensitive), or whose bundle id equals it. Prefer this or `pid` over an unfiltered call, which returns every window on the host.", + "type": "string" + }, + "on_screen_only": { + "description": "When true, drop windows that aren't currently on-screen. Default false.", + "type": "boolean" + }, + "pid": { + "description": "Optional pid filter. When set, only this pid's windows are returned.", + "type": "integer" + } + }, + "type": "object", + "required": [] + }, + "annotations": { + "readOnlyHint": true, + "idempotentHint": true + } + }, + { + "name": "get_window_state", + "description": "Read fresh window screenshot and compact UIA tree. Every [N] uses element_token=:N; a new snapshot invalidates old tokens. Pixel x/y are screenshot-local; use capture_id. since=latest returns changes. Keep the new snapshot_id.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "max_elements": { + "description": "Cap on the total number of UIA nodes walked. Truncates depth-first; markdown and structured elements truncate together. Default 250 (5 000 with full_output:true); the response states when the tree was cut. Raise it to read further into a la", + "minimum": 1, + "type": "integer" + }, + "max_image_dimension": { + "description": "Per-call long-edge limit for the returned screenshot, in pixels (aspect ratio preserved). Explicit values override configured behavior; 0 returns native resolution. Omit to preserve the configured default.", + "minimum": 0, + "type": "integer" + }, + "pid": { + "description": "Process ID from `list_apps`.", + "type": "integer" + }, + "query": { + "description": "Optional case-insensitive substring. Projects both tree_markdown and structured elements to matches plus ancestors while preserving original indices. Compare total_element_count with returned_element_count.", + "type": "string" + }, + "since": { + "description": "A `snapshot_id` from an earlier get_window_state of the same window, or \"latest\" for this window's most recent read with the same query/max_elements/max_depth. Use it on every re-read after acting. Returns only what changed against that sna", + "type": "string" + }, + "window_id": { + "description": "HWND of the target window. Must belong to `pid`. Enumerate via `list_windows` or read from `launch_app`'s `windows` array.", + "type": "integer" + } + }, + "required": [ + "pid", + "window_id" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": true, + "idempotentHint": true + } + }, + { + "name": "verify_state", + "description": "Read-only assertion for exact window. satisfied/unsatisfied/unknown describes the observation, not the whole task verdict.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "expect": { + "description": "One to eight predicates, combined with logical AND.", + "items": { + "additionalProperties": false, + "properties": { + "element": { + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "exists": { + "description": "Assert that at least one trusted element matches the selector.\n\nElement walks are not yet exhaustive on every platform, so absence\ncannot be proven. `false` is rejected instead of returning an\nindefinitely-unknown predicate.", + "type": "boolean" + }, + "selected": { + "type": "boolean" + }, + "selector": { + "additionalProperties": false, + "properties": { + "label_contains": { + "minLength": 1, + "type": "string" + }, + "role": { + "minLength": 1, + "type": "string" + } + }, + "required": [], + "type": "object" + }, + "value_equals": { + "type": "string" + } + }, + "required": [ + "selector" + ], + "type": "object" + }, + "window": { + "additionalProperties": false, + "properties": { + "bounds": { + "additionalProperties": false, + "properties": { + "height": { + "type": "number" + }, + "tolerance_px": { + "maximum": 100, + "minimum": 0, + "type": "number" + }, + "width": { + "type": "number" + }, + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + }, + "required": [ + "x", + "y", + "width", + "height" + ], + "type": "object" + }, + "exists": { + "type": "boolean" + } + }, + "required": [], + "type": "object" + } + }, + "required": [], + "type": "object" + }, + "maxItems": 8, + "minItems": 1, + "type": "array" + }, + "include_screenshot": { + "description": "Return the final window screenshot as image content for a multimodal\ncaller. The driver does not interpret that image.", + "type": "boolean" + }, + "pid": { + "description": "Exact process whose window may be observed.", + "minimum": 1, + "type": "integer" + }, + "stable_samples": { + "default": 2, + "description": "Consecutive satisfied samples required before returning success.", + "maximum": 5, + "minimum": 1, + "type": "integer" + }, + "timeout_ms": { + "default": 5000, + "description": "Bounded wait. Zero performs one sample.", + "maximum": 10000, + "minimum": 0, + "type": "integer" + }, + "window_id": { + "description": "Exact native window identifier.", + "type": "integer" + } + }, + "required": [ + "pid", + "window_id", + "expect" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": true, + "idempotentHint": true + } + }, + { + "name": "click", + "description": "Click the latest UIA element_token. This tool has no pixel coordinates. For visual targets use click_at. Returns effect and fresh observation; old tokens become stale.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "button": { + "description": "Mouse button. Default \"left\".", + "enum": [ + "left", + "right", + "middle" + ], + "type": "string" + }, + "count": { + "description": "Click count — 1 (single), 2 (double), 3 (triple). Default 1.", + "maximum": 3, + "minimum": 1, + "type": "integer" + }, + "delivery_mode": { + "description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "element_token": { + "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current sna", + "pattern": "^s[0-9a-f]{8}:[0-9]+$", + "type": "string" + }, + "pid": { + "description": "Target process ID for window scope. Optional with element_token (the token carries the pid). Omit with scope=desktop for screen-absolute coordinates from get_desktop_state.", + "type": "integer" + }, + "window_id": { + "description": "HWND of the target window. Omit when element_token is supplied (the token carries it).", + "type": "integer" + } + }, + "type": "object", + "required": [ + "pid", + "window_id", + "element_token" + ] + }, + "annotations": { + "readOnlyHint": false, + "idempotentHint": false + } + }, + { + "name": "drag", + "description": "Drag between screenshot-local coordinates. Returns actual delivery effect and fresh post-action observation.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "delivery_mode": { + "description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "duration_ms": { + "description": "Wall-clock duration of drag path. Default: 500.", + "maximum": 10000, + "minimum": 0, + "type": "integer" + }, + "from_x": { + "description": "Drag-start X in window-local screenshot pixels.", + "type": "number" + }, + "from_y": { + "description": "Drag-start Y in window-local screenshot pixels.", + "type": "number" + }, + "pid": { + "description": "Target process ID.", + "type": "integer" + }, + "to_x": { + "description": "Drag-end X in window-local screenshot pixels.", + "type": "number" + }, + "to_y": { + "description": "Drag-end Y in window-local screenshot pixels.", + "type": "number" + }, + "window_id": { + "description": "Target window handle (HWND). Optional — driver picks frontmost window of pid when omitted.", + "type": "integer" + } + }, + "required": [ + "pid", + "window_id", + "from_x", + "from_y", + "to_x", + "to_y" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "idempotentHint": false + } + }, + { + "name": "type_text", + "description": "Type text into the focused target window or specified latest element. Fresh observation follows. If background is unsupported, explicitly choose foreground within task scope.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "delivery_mode": { + "description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "element_token": { + "description": "Opaque element handle from get_window_state. When supplied, type_text writes through UIA ValuePattern and reads that exact element back by handle. The shared ActionResult is confirmed only when the complete expected value is synchronously v", + "pattern": "^s[0-9a-f]{8}:[0-9]+$", + "type": "string" + }, + "pid": { + "description": "Target process ID.", + "type": "integer" + }, + "text": { + "description": "Text to insert at the focused element's cursor.", + "type": "string" + }, + "window_id": { + "description": "HWND of the target window. Omit when element_token is supplied (the token carries it).", + "type": "integer" + }, + "x": { + "description": "Window-local screenshot-pixel X of the field to type into — the element px action form. Pass x,y (no element_token) and the tool pixel-clicks there to establish real renderer focus, then types. Use for Chromium/Electron inputs the UIA/WM_CH", + "type": "number" + }, + "y": { + "description": "Window-local screenshot-pixel Y of the field (see x).", + "type": "number" + } + }, + "required": [ + "pid", + "window_id", + "text" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "idempotentHint": false + } + }, + { + "name": "press_key", + "description": "Send key and modifiers to the exact window, optionally focus element_token or x/y first. Fresh observation follows.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "delivery_mode": { + "description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "element_token": { + "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current sna", + "pattern": "^s[0-9a-f]{8}:[0-9]+$", + "type": "string" + }, + "key": { + "description": "Key name (return, tab, escape, up, down, left, right, space, delete, home, end, pageup, pagedown, f1-f12, letter, digit).", + "type": "string" + }, + "modifiers": { + "description": "Optional modifier names held while the key is pressed (ctrl/shift/alt/win).", + "items": { + "type": "string" + }, + "type": "array" + }, + "pid": { + "description": "Target process ID.", + "type": "integer" + }, + "window_id": { + "description": "HWND for the target window. Omit when element_token is supplied; otherwise auto-resolves the pid's first visible window.", + "type": "integer" + }, + "x": { + "description": "Window-local screenshot-pixel X — the element px action form: pixel-click there to focus, then send the key. Use when the key must go to a Chromium/Electron surface the UIA path can't focus. Pass with y, no element_token.", + "type": "number" + }, + "y": { + "description": "Window-local screenshot-pixel Y (see x).", + "type": "number" + } + }, + "required": [ + "pid", + "window_id", + "key" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "idempotentHint": false + } + }, + { + "name": "set_value", + "description": "Set the latest UIA element_token value. Returns delivery effect/readback and fresh screenshot/tree.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "delivery_mode": { + "description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "element_token": { + "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current sna", + "pattern": "^s[0-9a-f]{8}:[0-9]+$", + "type": "string" + }, + "pid": { + "description": "Target process ID. Optional with element_token (the token carries the pid).", + "type": "integer" + }, + "value": { + "description": "New value. UIA will coerce to the element's native type.", + "type": "string" + }, + "window_id": { + "description": "HWND of the window. Omit when element_token is supplied (the token carries it).", + "type": "integer" + } + }, + "required": [ + "pid", + "window_id", + "value" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "idempotentHint": false + } + }, + { + "name": "scroll", + "description": "Scroll the target window/element at x,y. amount=1..50, by=line/page. Verify content changed in fresh returned observation.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "amount": { + "description": "Number of scroll ticks. Default 3.", + "maximum": 50, + "minimum": 1, + "type": "integer" + }, + "by": { + "description": "Scroll granularity. Default: line.", + "enum": [ + "line", + "page" + ], + "type": "string" + }, + "delivery_mode": { + "description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "direction": { + "description": "Scroll direction.", + "enum": [ + "up", + "down", + "left", + "right" + ], + "type": "string" + }, + "element_token": { + "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current sna", + "pattern": "^s[0-9a-f]{8}:[0-9]+$", + "type": "string" + }, + "pid": { + "description": "Target process ID for window scope. Optional with element_token (the token carries the pid). Omit with scope=desktop for screen-absolute coordinates from get_desktop_state.", + "type": "integer" + }, + "window_id": { + "description": "HWND of the target window. Omit when element_token is supplied; otherwise auto-resolves the pid's first visible window.", + "type": "integer" + }, + "x": { + "description": "With pid/window_id: window-local screenshot X used to target a nested scroll surface in foreground mode. Without pid/window_id: screen-absolute X for desktop scope. Must be paired with y.", + "type": "number" + }, + "y": { + "description": "With pid/window_id: window-local screenshot Y used to target a nested scroll surface in foreground mode. Without pid/window_id: screen-absolute Y for desktop scope. Must be paired with x.", + "type": "number" + } + }, + "required": [ + "pid", + "window_id", + "direction" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "idempotentHint": false + } + }, + { + "name": "save_screenshot", + "description": "Save fresh window screenshot to this task trace. Plain .png filename only; no overwrite. Returns verified attachment hash, screenshot and latest tree.", + "inputSchema": { + "type": "object", + "additionalProperties": false, + "properties": { + "pid": { + "type": "integer", + "minimum": 1 + }, + "window_id": { + "type": "integer", + "minimum": 1 + }, + "filename": { + "type": "string" + } + }, + "required": [ + "pid", + "window_id", + "filename" + ] + }, + "annotations": { + "readOnlyHint": false, + "idempotentHint": false + } + }, + { + "name": "click_at", + "description": "Click a visual target at x,y in the latest window screenshot, with its exact capture_id. Never pass an element token. Returns effect and fresh observation.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "button": { + "description": "Mouse button. Default \"left\".", + "enum": [ + "left", + "right", + "middle" + ], + "type": "string" + }, + "capture_id": { + "description": "Optional ID from get_window_state or get_desktop_state. When present, x and y are admitted against that exact capture and the ID is consumed before native dispatch.", + "minLength": 1, + "type": "string" + }, + "count": { + "description": "Click count — 1 (single), 2 (double), 3 (triple). Default 1.", + "maximum": 3, + "minimum": 1, + "type": "integer" + }, + "delivery_mode": { + "description": "Input delivery mode. 'background' (default) never swaps foreground: it routes through UIA Invoke / PostMessage and the window is never raised. For targets whose input stack silently drops posted events (Chromium/Electron content, GTK button", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "pid": { + "description": "Target process ID for window scope. Optional with element_token (the token carries the pid). Omit with scope=desktop for screen-absolute coordinates from get_desktop_state.", + "type": "integer" + }, + "window_id": { + "description": "HWND of the target window. Omit when element_token is supplied (the token carries it).", + "type": "integer" + }, + "x": { + "description": "X in window-local screenshot pixels — same space as the PNG get_window_state returns. Must be provided together with y.", + "type": "number" + }, + "y": { + "description": "Y in window-local screenshot pixels. Must be provided together with x.", + "type": "number" + } + }, + "type": "object", + "required": [ + "pid", + "window_id", + "x", + "y", + "capture_id" + ] + }, + "annotations": { + "readOnlyHint": false, + "idempotentHint": false + } + } +] \ No newline at end of file diff --git a/plugins/cua-driver/windows.go b/plugins/cua-driver/windows.go new file mode 100644 index 0000000..ed2d5c8 --- /dev/null +++ b/plugins/cua-driver/windows.go @@ -0,0 +1,69 @@ +package main + +import ( + "fmt" + "os/exec" + "syscall" + "unsafe" +) + +var user32 = syscall.NewLazyDLL("user32.dll") +var kernel32 = syscall.NewLazyDLL("kernel32.dll") + +func configureProcess(c *exec.Cmd) { + c.SysProcAttr = &syscall.SysProcAttr{HideWindow: true} + user32.NewProc("SetThreadDpiAwarenessContext").Call(^uintptr(3)) +} +func windowOwned(pid, hwnd int64) bool { + var actual uint32 + r, _, _ := user32.NewProc("IsWindow").Call(uintptr(hwnd)) + if r == 0 { + return false + } + user32.NewProc("GetWindowThreadProcessId").Call(uintptr(hwnd), uintptr(unsafe.Pointer(&actual))) + return int64(actual) == pid +} +func verifyWindow(pid, hwnd int64, b obj) (obj, bool) { + if !windowOwned(pid, hwnd) { + return nil, false + } + var r struct{ L, T, R, B int32 } + code, _, _ := syscall.NewLazyDLL("dwmapi.dll").NewProc("DwmGetWindowAttribute").Call(uintptr(hwnd), 9, uintptr(unsafe.Pointer(&r)), unsafe.Sizeof(r)) + if code != 0 { + return nil, false + } + // Cua uses DWM physical bounds; refuse evidence after a window move/resize. + if int64(r.L) != num(b["left"]) || int64(r.T) != num(b["top"]) || int64(r.R-r.L) != num(b["width"]) || int64(r.B-r.T) != num(b["height"]) { + return nil, false + } + metric := func(n uintptr) int64 { x, _, _ := user32.NewProc("GetSystemMetrics").Call(n); return int64(int32(x)) } + return obj{"left": metric(76), "top": metric(77), "width": metric(78), "height": metric(79)}, true +} +func ownProcess(pid int) (uintptr, error) { + job, _, e := kernel32.NewProc("CreateJobObjectW").Call(0, 0) + if job == 0 { + return 0, e + } + // JOBOBJECT_EXTENDED_LIMIT_INFORMATION, Windows x64. Kill the entire owned + // child tree when the adapter dies, including Cua's UIA helper. + data := make([]byte, 144) + *(*uint32)(unsafe.Pointer(&data[16])) = 0x2000 + ok, _, _ := kernel32.NewProc("SetInformationJobObject").Call(job, 9, uintptr(unsafe.Pointer(&data[0])), uintptr(len(data))) + h, _, _ := kernel32.NewProc("OpenProcess").Call(0x0101, 0, uintptr(pid)) + if h == 0 || ok == 0 { + closeJob(job) + return 0, fmt.Errorf("cannot own driver process") + } + defer kernel32.NewProc("CloseHandle").Call(h) + ok, _, e = kernel32.NewProc("AssignProcessToJobObject").Call(job, h) + if ok == 0 { + closeJob(job) + return 0, e + } + return job, nil +} +func closeJob(h uintptr) { + if h != 0 { + kernel32.NewProc("CloseHandle").Call(h) + } +} diff --git a/scripts/build_cua_driver.py b/scripts/build_cua_driver.py new file mode 100644 index 0000000..59840bd --- /dev/null +++ b/scripts/build_cua_driver.py @@ -0,0 +1,115 @@ +"""Build a self-contained Windows plugin from a hash-pinned Cua release.""" + +import argparse +import hashlib +import json +import os +import subprocess +import urllib.request +import zipfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SOURCE = ROOT / "plugins/cua-driver" +UPSTREAM_VERSION = "0.36.0" +COMMIT = "6d6c734c127025fb39ad5d585a10382808b39dc1" +DIGEST = "f8c47024b626f4e0e70f8428d2998df39e4879110c5722a29d5fd03e367353ad" +URL = "https://github.com/trycua/cua/releases/download/cua-driver-rs-v0.36.0/cua-driver-rs-0.36.0-windows-x86_64-binary.zip" + + +def main(): + parser = argparse.ArgumentParser(__doc__) + parser.add_argument("upstream_zip", type=Path) + parser.add_argument("--go", default="go") + parser.add_argument("--output", type=Path, default=ROOT / "artifacts") + args = parser.parse_args() + version = json.loads((SOURCE / "plugin.json").read_text("utf-8"))["version"] + if version.split("+", 1)[0] != UPSTREAM_VERSION: + raise ValueError("Plugin version must match the pinned Cua Driver version") + if hashlib.sha256(args.upstream_zip.read_bytes()).hexdigest() != DIGEST: + raise ValueError("Cua release archive hash mismatch") + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=True) + adapter = output / "artemis-cua-adapter.exe" + env = { + **os.environ, + "CGO_ENABLED": "0", + "GOOS": "windows", + "GOARCH": "amd64", + "GOTOOLCHAIN": "local", + } + for command in ( + [args.go, "test", "./..."], + [ + args.go, + "build", + "-trimpath", + "-buildvcs=false", + "-ldflags=-s -w", + "-o", + str(adapter), + ".", + ], + ): + subprocess.run(command, cwd=SOURCE, env=env, check=True, timeout=180) + license_file = SOURCE / "LICENSE.upstream" + if not license_file.exists(): + with urllib.request.urlopen( + f"https://raw.githubusercontent.com/trycua/cua/{COMMIT}/LICENSE.md", + timeout=30, + ) as response: + license_file.write_bytes(response.read()) + target = output / f"cua-driver-{version}-windows-amd64.zip" + with ( + zipfile.ZipFile(target, "w", zipfile.ZIP_DEFLATED) as dest, + zipfile.ZipFile(args.upstream_zip) as upstream, + ): + for name in ( + "plugin.json", + "SKILL.md", + "LICENSE.upstream", + "RELEASE.md", + "VERIFICATION.md", + ): + dest.write(SOURCE / name, name) + for source in sorted(SOURCE.glob("*.go")): + dest.write(source, f"source/{source.name}") + for name in ("go.mod", "tools.json"): + dest.write(SOURCE / name, f"source/{name}") + dest.write(adapter, "bin/artemis-cua-adapter.exe") + for name in upstream.namelist(): + # Only the release's executable runtime; no global installer/hooks. + if Path(name).name in { + "cua-driver.exe", + "cua-driver-uia.exe", + "cua-cursor-theme.exe", + "cua_driver_sdk.dll", + }: + dest.writestr(f"bin/{Path(name).name}", upstream.read(name)) + dest.writestr( + "provenance.json", + json.dumps( + { + "source_commit": COMMIT, + "upstream_version": UPSTREAM_VERSION, + "plugin_version": version, + "adapter_revision": 1, + "prerelease": True, + "url": URL, + "sha256": DIGEST, + }, + indent=2, + ), + ) + print( + json.dumps( + { + "archive": str(target), + "sha256": hashlib.sha256(target.read_bytes()).hexdigest(), + } + ) + ) + + +if __name__ == "__main__": + main() diff --git a/tests/benchmark-fixture.ps1 b/tests/benchmark-fixture.ps1 new file mode 100644 index 0000000..4e5146b --- /dev/null +++ b/tests/benchmark-fixture.ps1 @@ -0,0 +1,47 @@ +param([Parameter(Mandatory=$true)][string]$StateFile, [string]$Title='ARTEMIS Plugin Benchmark') +Add-Type -TypeDefinition 'using System; using System.Runtime.InteropServices; public static class BenchmarkDpi { [DllImport("user32.dll")] public static extern IntPtr SetThreadDpiAwarenessContext(IntPtr c); }' +[void][BenchmarkDpi]::SetThreadDpiAwarenessContext([IntPtr](-4)) +Add-Type -AssemblyName System.Windows.Forms +Add-Type -AssemblyName System.Drawing +$form = New-Object System.Windows.Forms.Form +$form.Text=$Title +$form.Size=New-Object System.Drawing.Size(880,660) +$form.StartPosition='Manual' +$form.Location=New-Object System.Drawing.Point(80,80) +$form.TopMost=$true +$form.Font=New-Object System.Drawing.Font('Segoe UI',11) +$state=@{pid=$PID;title=$Title;applied=$false;ssid='';night=$false;apply_count=0;invalid_apply=0;target_clicks=0;wrong_clicks=0;finished=$false;events=@()} +function Save-State { + $state | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath $StateFile -Encoding UTF8 +} +function Log-Event([string]$name) {$state.events += @{event=$name;at=[DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds()};Save-State} +$heading=New-Object System.Windows.Forms.Label +$heading.Text='Router test console - owned fixture';$heading.AutoSize=$true;$heading.Location=New-Object System.Drawing.Point(20,18) +$form.Controls.Add($heading) +$label=New-Object System.Windows.Forms.Label +$label.Text='Network name';$label.Location=New-Object System.Drawing.Point(20,60);$label.AutoSize=$true;$form.Controls.Add($label) +$networkInput=New-Object System.Windows.Forms.TextBox +$networkInput.AccessibleName='Network name';$networkInput.Location=New-Object System.Drawing.Point(170,56);$networkInput.Size=New-Object System.Drawing.Size(280,32);$form.Controls.Add($networkInput) +$check=New-Object System.Windows.Forms.CheckBox +$check.Text='Night mode';$check.AccessibleName='Night mode';$check.Location=New-Object System.Drawing.Point(470,56);$check.Size=New-Object System.Drawing.Size(140,32);$form.Controls.Add($check) +$apply=New-Object System.Windows.Forms.Button +$apply.Text='Apply';$apply.AccessibleName='Apply';$apply.Location=New-Object System.Drawing.Point(630,54);$apply.Size=New-Object System.Drawing.Size(130,36);$form.Controls.Add($apply) +$status=New-Object System.Windows.Forms.Label +$status.Text='Not applied';$status.AutoSize=$true;$status.Location=New-Object System.Drawing.Point(20,104);$form.Controls.Add($status) +$apply.Add_Click({$state.apply_count++;$state.ssid=$networkInput.Text;$state.night=$check.Checked;$state.applied=($networkInput.Text -eq 'ARTEMIS-LAB' -and $check.Checked);if(!$state.applied){$state.invalid_apply++};$status.Text="Saved: $($networkInput.Text); Night mode: $($check.Checked)";Log-Event 'apply'}) +$hint=New-Object System.Windows.Forms.Label +$hint.Text='Visual task: click the BLUE circle only, then scroll the panel to Finish.';$hint.AutoSize=$true;$hint.Location=New-Object System.Drawing.Point(20,150);$form.Controls.Add($hint) +$canvas=New-Object System.Windows.Forms.Panel +$canvas.Location=New-Object System.Drawing.Point(20,185);$canvas.Size=New-Object System.Drawing.Size(760,100);$canvas.BackColor=[System.Drawing.Color]::White;$form.Controls.Add($canvas) +$canvas.Add_Paint({param($sender,$e) for($i=0;$i -lt 8;$i++){$color=if($i -eq 5){[System.Drawing.Brushes]::RoyalBlue}else{[System.Drawing.Brushes]::IndianRed};$e.Graphics.FillEllipse($color,25+90*$i,24,34,34)}}) +$canvas.Add_MouseClick({param($sender,$e) $hit=$false;for($i=0;$i -lt 8;$i++){if([Math]::Pow($e.X-(42+90*$i),2)+[Math]::Pow($e.Y-41,2) -le 289){$hit=$true;if($i -eq 5){$state.target_clicks++;$hint.Text='Blue target selected. Scroll panel to Finish.';Log-Event 'target'}else{$state.wrong_clicks++;Log-Event 'wrong-target'}}};if(!$hit){$state.wrong_clicks++;Log-Event 'miss'}}) +$panel=New-Object System.Windows.Forms.Panel +$panel.AccessibleName='Scrollable settings';$panel.Location=New-Object System.Drawing.Point(20,310);$panel.Size=New-Object System.Drawing.Size(760,260);$panel.AutoScroll=$true;$panel.BorderStyle='FixedSingle';$form.Controls.Add($panel) +for($i=0;$i -lt 12;$i++){$row=New-Object System.Windows.Forms.Label;$row.Text="Diagnostic row $($i+1) - scroll down for Finish";$row.AutoSize=$true;$row.Location=New-Object System.Drawing.Point(15,(15+60*$i));$panel.Controls.Add($row)} +$finish=New-Object System.Windows.Forms.Button +$finish.Text='Finish';$finish.AccessibleName='Finish';$finish.Location=New-Object System.Drawing.Point(20,760);$finish.Size=New-Object System.Drawing.Size(160,40);$panel.Controls.Add($finish) +$finish.Add_Click({$state.scroll_value=$panel.VerticalScroll.Value;$state.finished=($state.target_clicks -eq 1 -and $state.wrong_clicks -eq 0 -and $state.scroll_value -gt 0);$finish.Text=if($state.finished){'Finished successfully'}else{'Target not selected'};Log-Event 'finish'}) +$timer=New-Object System.Windows.Forms.Timer +$timer.Interval=900000;$timer.Add_Tick({$form.Close()}) +$form.Add_Shown({Save-State;$timer.Start()}) +try{[System.Windows.Forms.Application]::Run($form)}finally{$timer.Dispose();$form.Dispose()} diff --git a/tests/benchmark_plugins.py b/tests/benchmark_plugins.py new file mode 100644 index 0000000..14a5465 --- /dev/null +++ b/tests/benchmark_plugins.py @@ -0,0 +1,279 @@ +"""Opt-in sequential A/B through ARTEMIS's existing queue, never a second agent. + +Usage: python tests/benchmark_plugins.py --traces --output +The user-started service must already run. Only this script's owned fixture is closed. +""" + +import argparse +import json +import sqlite3 +import subprocess +import time +import urllib.request +import uuid +from pathlib import Path + +PROMPT = """只操作测试窗口 ARTEMIS Plugin Benchmark（进程 PID {pid}），不要观察或操作其他应用。先按此 PID 找到窗口。 +1. 将 Network name 填为 ARTEMIS-LAB，勾选 Night mode，点击 Apply，确认显示 Saved: ARTEMIS-LAB; Night mode: True。 +2. 只点击唯一的蓝色圆形一次，不要点击红色圆形。在下方 Scrollable settings 面板向下滚动，找到并点击 Finish，确认 Finished successfully。 +3. 保存这个窗口的截图到任务 trace，文件名 proof.png，然后结束并如实说明实际结果。不要关闭窗口。 +""" + + +def api(base, route, body=None): + request = urllib.request.Request( + base + route, + data=json.dumps(body).encode() if body is not None else None, + headers={"Content-Type": "application/json"}, + ) + with urllib.request.urlopen(request, timeout=30) as response: + return json.load(response) + + +def collect(traces, sid): + with sqlite3.connect( + f"file:{(traces / 'data_engine.db').as_posix()}?mode=ro", uri=True + ) as db: + db.row_factory = sqlite3.Row + session = dict( + db.execute("SELECT * FROM sessions WHERE session_id=?", (sid,)).fetchone() + ) + events = [ + dict(row) + for row in db.execute( + "SELECT name,type,status,duration,payload FROM traces WHERE session_id=? ORDER BY timestamp", + (sid,), + ) + ] + steps = db.execute( + "SELECT count(*) FROM steps WHERE session_id=?", (sid,) + ).fetchone()[0] + usage, calls, requests = [], [], [] + for event in events: + payload = json.loads(event["payload"] or "{}") + if event["name"] == "llm_usage": + usage.append(payload) + if "plugin_id" in payload and "tool_name" in payload: + # Tool trace contains the real call receipt, not the separate log line. + calls.append( + { + "name": payload["tool_name"], + "status": event["status"], + "duration": event["duration"], + "args": payload.get("args"), + "result": payload.get("result"), + } + ) + if event["name"] == "llm_request": + requests.append(payload) + return { + "session": session, + "steps": steps, + "llm_usage": usage, + "plugin_calls": calls, + "requests": requests, + } + + +def main(): + parser = argparse.ArgumentParser(__doc__) + parser.add_argument("--base", default="http://127.0.0.1:8000") + parser.add_argument("--traces", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--model", default="mimo-v2.6-flash") + parser.add_argument("--repetitions", type=int, default=3) + parser.add_argument("--resume", action="store_true") + args = parser.parse_args() + args.output.mkdir(parents=True, exist_ok=args.resume) + config = { + "model": args.model, + "reasoning_effort": "high", + "execution_target": "computer", + "record_video": True, + "recording": {"codec": "h264", "computer_scope": "window"}, + "analyze_video": False, + "timeout_seconds": 480, + "request_timeout_seconds": 120, + "max_retries": 1, + } + protocol = { + "prompt": PROMPT, + "config": config, + "repetitions": args.repetitions, + "acceptance": "applied && finished && exactly one target click && no wrong clicks && verified proof.png", + "order": "OCU,Cua / Cua,OCU / OCU,Cua", + "fixture": "benchmark-fixture.ps1", + } + protocol_path = args.output / "protocol.json" + if args.resume and protocol_path.exists(): + if json.loads(protocol_path.read_text("utf-8")) != protocol: + raise ValueError( + "Resume requires the original prompt, model and task settings" + ) + else: + protocol_path.write_text( + json.dumps(protocol, ensure_ascii=False, indent=2), "utf-8" + ) + for repetition in range(args.repetitions): + order = ( + ["open-computer-use", "cua-driver"] + if repetition % 2 == 0 + else ["cua-driver", "open-computer-use"] + ) + for plugin in order: + folder = args.output / f"{repetition + 1}-{plugin}" + if args.resume and (folder / "result.json").is_file(): + continue + queue = api(args.base, "/api/status") + if queue.get("active_tasks") or queue.get("queue"): + raise RuntimeError( + "Other tasks are active/queued; benchmark will not interfere" + ) + folder.mkdir() + state_file = folder / "fixture.json" + stderr = (folder / "fixture-stderr.log").open("w") + fixture = subprocess.Popen( + [ + "powershell.exe", + "-NoProfile", + "-File", + str(Path(__file__).with_name("benchmark-fixture.ps1").resolve()), + "-StateFile", + str(state_file.resolve()), + ], + creationflags=subprocess.CREATE_NO_WINDOW, + stdout=subprocess.DEVNULL, + stderr=stderr, + ) + sid = str(uuid.uuid4()) + done = False + try: + for _ in range(100): + if state_file.exists(): + break + time.sleep(0.1) + if not state_file.exists(): + raise RuntimeError("fixture did not start") + body = { + "goal": PROMPT.format(pid=fixture.pid), + "session_id": sid, + "profile": "flash", + "ingress": "client", + "runtime_overrides": { + **config, + "plugins": [ + { + "id": plugin, + "profile": "benchmark" + if plugin == "cua-driver" + else "default", + "version": "0.1.2" + if plugin == "cua-driver" + else "1.1.0", + } + ], + }, + } + (folder / "request.json").write_text( + json.dumps(body, ensure_ascii=False, indent=2), "utf-8" + ) + started = time.monotonic() + response = api(args.base, "/api/run", body) + (folder / "submission.json").write_text(json.dumps(response), "utf-8") + print( + json.dumps( + { + "event": "submitted", + "plugin": plugin, + "round": repetition + 1, + "trace_id": sid, + "response": response, + } + ), + flush=True, + ) + if response.get("status") in {"rejected", "error"}: + raise RuntimeError("task was rejected") + while time.monotonic() - started < 600: + time.sleep(15) + status = api(args.base, f"/api/sessions/{sid}") + state = json.loads(state_file.read_text("utf-8-sig")) + print( + json.dumps( + { + "event": "progress", + "trace_id": sid, + "status": status["status"], + "elapsed": round(time.monotonic() - started), + "applied": state["applied"], + "finished": state["finished"], + } + ), + flush=True, + ) + if status["status"] not in { + "running", + "pending", + "queued", + "stopping", + "starting", + }: + done = True + break + if not done: + api(args.base, "/api/stop", {"session_id": sid}) + time.sleep(3) + elapsed = time.monotonic() - started + result = collect(args.traces.resolve(), sid) + state = json.loads(state_file.read_text("utf-8-sig")) + attachment = ( + args.traces / sid / "artifacts/plugins" / plugin / "proof.png" + ) + result.update( + trace_id=sid, + plugin=plugin, + repetition=repetition + 1, + fixture=state, + observed_elapsed=elapsed, + watchdog_timeout=not done, + actual_pass=bool( + state["applied"] + and state["finished"] + and state["target_clicks"] == 1 + and state["wrong_clicks"] == 0 + and attachment.exists() + ), + attachment_exists=attachment.exists(), + ) + (folder / "result.json").write_text( + json.dumps(result, ensure_ascii=False, indent=2), "utf-8" + ) + print( + json.dumps( + { + "event": "done", + "trace_id": sid, + "actual_pass": result["actual_pass"], + } + ), + flush=True, + ) + # Release/recording cleanup belongs to ARTEMIS, wait before next fixture. + for _ in range(15): + if not api(args.base, "/api/status").get("active_tasks"): + break + time.sleep(2) + finally: + if not done: + try: + api(args.base, "/api/stop", {"session_id": sid}) + except (OSError, ValueError) as error: + print(f"Cleanup request failed for {sid}: {error}", flush=True) + if fixture.poll() is None: + fixture.terminate() + fixture.wait(10) + stderr.close() + + +if __name__ == "__main__": + main() diff --git a/tests/capture-fixture.ps1 b/tests/capture-fixture.ps1 index 36597e0..8608984 100644 --- a/tests/capture-fixture.ps1 +++ b/tests/capture-fixture.ps1 @@ -1,4 +1,8 @@ -param([string]$Title, [string]$ReadyFile, [switch]$Cover, [int]$Width=500, [int]$Height=300) +param([string]$Title, [string]$ReadyFile, [switch]$Cover, [int]$Width=500, [int]$Height=300, [switch]$PerMonitorDpi) +if ($PerMonitorDpi) { + Add-Type -TypeDefinition 'using System; using System.Runtime.InteropServices; public static class FixtureDpi { [DllImport("user32.dll")] public static extern IntPtr SetThreadDpiAwarenessContext(IntPtr c); }' + [void][FixtureDpi]::SetThreadDpiAwarenessContext([IntPtr](-4)) +} Add-Type -AssemblyName System.Windows.Forms Add-Type -AssemblyName System.Drawing $form = New-Object System.Windows.Forms.Form @@ -20,6 +24,8 @@ $button.Location = New-Object System.Drawing.Point(100, 100) $button.Size = New-Object System.Drawing.Size(180, 50) $button.BackColor = [System.Drawing.Color]::Red $button.Add_Click({ [System.IO.File]::WriteAllText(($ReadyFile + '.clicked'), 'clicked') }) +$button.Add_MouseDown({ param($sender, $event) [System.IO.File]::AppendAllText(($ReadyFile + '.mouse'), "down:$($event.X),$($event.Y)`n") }) +$button.Add_MouseUp({ param($sender, $event) [System.IO.File]::AppendAllText(($ReadyFile + '.mouse'), "up:$($event.X),$($event.Y)`n") }) $form.Controls.Add($button) $timer = New-Object System.Windows.Forms.Timer $timer.Interval = 120000 diff --git a/tests/cua_fixture_validation.py b/tests/cua_fixture_validation.py new file mode 100644 index 0000000..86898b8 --- /dev/null +++ b/tests/cua_fixture_validation.py @@ -0,0 +1,115 @@ +"""Validate the benchmark's independent oracle and Cua foreground escalation.""" + +import asyncio +import json +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +from artemis.plugins.models import ServerConfig +from artemis.plugins.session import PluginSession + + +async def main(executable): + with tempfile.TemporaryDirectory( + prefix="artemis-benchmark-validation-" + ) as temporary: + state_file = Path(temporary) / "state.json" + proc = await asyncio.create_subprocess_exec( + "powershell.exe", + "-NoProfile", + "-File", + str(Path(__file__).with_name("benchmark-fixture.ps1")), + "-StateFile", + str(state_file), + creationflags=subprocess.CREATE_NO_WINDOW, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + session = None + try: + for _ in range(100): + if state_file.exists(): + break + await asyncio.sleep(0.1) + assert state_file.exists() + session = await PluginSession( + ServerConfig(command=str(executable)), executable.parent + ).start() + listing = await session.call("list_windows", {"pid": proc.pid}) + target = { + "pid": proc.pid, + "window_id": listing.structuredContent["windows"][0]["window_id"], + } + + async def element(label, tool, **extra): + state = await session.call("get_window_state", target) + text = "\n".join(getattr(c, "text", "") for c in state.content) + match = re.search(r'\[(\d+)\][^\n]*"' + re.escape(label) + '"', text) + assert match, text + token = state.structuredContent["snapshot_id"] + ":" + match[1] + result = await session.call( + tool, {**target, "element_token": token, **extra} + ) + assert not result.isError, result + + await element("Network name", "set_value", value="ARTEMIS-LAB") + await element("Night mode", "click") + await element("Apply", "click") + assert json.loads(state_file.read_text("utf-8-sig"))["applied"] + state = await session.call("get_window_state", target) + pixel = await session.call( + "click_at", + { + **target, + "x": 512, + "y": 256, + "capture_id": state.structuredContent["capture_id"], + }, + ) + assert not pixel.isError, pixel + scrolled = await session.call( + "scroll", + { + **target, + "direction": "down", + "amount": 4, + "by": "page", + "x": 400, + "y": 470, + }, + ) + if scrolled.isError: + assert ( + scrolled.structuredContent["artemis"]["execution"] == "not_started" + ), scrolled + scrolled = await session.call( + "scroll", + { + **target, + "direction": "down", + "amount": 4, + "by": "page", + "x": 400, + "y": 470, + "delivery_mode": "foreground", + }, + ) + assert not scrolled.isError, scrolled + await element("Finish", "click") + result = json.loads(state_file.read_text("utf-8-sig")) + print(json.dumps(result)) + assert ( + result["applied"] and result["finished"] and result["wrong_clicks"] == 0 + ) + finally: + if session: + await session.aclose() + proc.terminate() + await asyncio.wait_for(proc.wait(), 10) + + +if __name__ == "__main__": + asyncio.run(main(Path(sys.argv[1]).resolve())) diff --git a/tests/cua_smoke.py b/tests/cua_smoke.py new file mode 100644 index 0000000..a50ad1a --- /dev/null +++ b/tests/cua_smoke.py @@ -0,0 +1,141 @@ +"""Opt-in actual HWND capture, action, artifact, recording and shutdown proof.""" + +import asyncio +import json +import subprocess +import tempfile +import time +from pathlib import Path +from types import SimpleNamespace + +from artemis.config.agent import RecordingConfig +from artemis.plugins.evidence import ( + decode_receipt, + persist_attachments, + source_identity, + validate_tools, +) +from artemis.plugins.models import PluginCallResult, PluginManifest, ServerConfig +from artemis.plugins.recording import PluginRecordings +from artemis.plugins.session import PluginSession + + +async def run(executable, output): + output.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="artemis-cua-smoke-") as temporary: + ready = Path(temporary) / "ready" + proc = await asyncio.create_subprocess_exec( + "powershell.exe", + "-NoProfile", + "-File", + str(Path(__file__).with_name("capture-fixture.ps1")), + "-Title", + "ARTEMIS Cua Smoke", + "-ReadyFile", + str(ready), + "-PerMonitorDpi", + creationflags=subprocess.CREATE_NO_WINDOW, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + session = recorder = None + try: + for _ in range(100): + if ready.exists(): + break + await asyncio.sleep(0.1) + assert ready.exists() + artifacts = output / "artifacts/plugins/cua-driver" + session = await PluginSession( + ServerConfig( + command=str(executable), + env={"ARTEMIS_ARTIFACTS_DIR": str(artifacts)}, + ), + executable.parent, + ).start() + manifest = PluginManifest.model_validate_json( + ( + Path(__file__).parents[1] / "plugins/cua-driver/plugin.json" + ).read_text("utf-8") + ) + validate_tools(manifest, session.tools) + windows = await session.call("list_windows", {"pid": proc.pid}) + target = { + "pid": proc.pid, + "window_id": windows.structuredContent["windows"][0]["window_id"], + } + raw = await session.call("get_window_state", target) + assert not raw.isError, raw + outcome = PluginCallResult( + status="success", structured_content=raw.structuredContent + ) + decode_receipt(outcome) + assert outcome.source and outcome.source.width > 0 + snapshot = raw.structuredContent["snapshot_id"] + clicked = await session.call( + "click", {**target, "element_token": f"{snapshot}:0"} + ) + assert not clicked.isError, clicked + assert Path(str(ready) + ".clicked").exists() + assert clicked.structuredContent["artemis"]["source"] + assert clicked.structuredContent["observation"]["snapshot_id"] != snapshot + saved = await session.call( + "save_screenshot", {**target, "filename": "cua-proof.png"} + ) + assert not saved.isError, saved + receipt = saved.structuredContent["artemis"] + files = persist_attachments(output, "cua-driver", receipt["attachments"]) + duplicate = await session.call( + "save_screenshot", {**target, "filename": "cua-proof.png"} + ) + assert ( + duplicate.isError + and duplicate.structuredContent["artemis"]["execution"] == "not_started" + ) + context = SimpleNamespace( + data_engine=SimpleNamespace( + base_dir=output, session_start_time=time.time() + ), + agent_config=SimpleNamespace( + recording=RecordingConfig(codec="h264", computer_scope="window") + ), + execution_setup=SimpleNamespace( + record_video_enabled=True, video_recording_tools_enabled=False + ), + ) + recorder = PluginRecordings(context) + await recorder.observe( + source_identity("cua-driver", outcome.source), + "cua-driver", + outcome.source, + ) + assert recorder.active + await asyncio.sleep(5) + await recorder.stop() + clips = [ + s for track in recorder.sources.values() for s in track["segments"] + ] + assert clips and clips[0]["duration"] >= 3 + result = { + "actual_click": True, + "source": outcome.source.model_dump(), + "attachments": files, + "recordings": clips, + } + (output / "result.json").write_text( + json.dumps(result, ensure_ascii=False, indent=2), "utf-8" + ) + print(json.dumps(result, ensure_ascii=False)) + finally: + if recorder: + await recorder.stop() + if session: + await session.aclose() + proc.terminate() + await asyncio.wait_for(proc.wait(), 10) + + +if __name__ == "__main__": + import sys + + asyncio.run(run(Path(sys.argv[1]).resolve(), Path(sys.argv[2]).resolve())) diff --git a/tests/summarize_benchmark.py b/tests/summarize_benchmark.py new file mode 100644 index 0000000..4e6abb7 --- /dev/null +++ b/tests/summarize_benchmark.py @@ -0,0 +1,208 @@ +"""Summarize measured trace usage; never infer token counts from text length.""" + +import argparse +import hashlib +import json +import sqlite3 +from collections import Counter, defaultdict +from pathlib import Path +from statistics import mean + + +def enrich(row, data, traces): + """Read actual evidence separately; the fixture never feeds the agent.""" + session_root = traces / row["trace_id"] + outcome = session_root / "run_outcome.json" + if outcome.exists(): + value = json.loads(outcome.read_text("utf-8")) + row["ai_verdict"] = value.get("test_result", {}).get("status") + row["ai_verdict_reason"] = value.get("test_result", {}).get("reason") + row["outcome_code"] = value.get("code") + recordings = session_root / "evidence/recordings/recordings.json" + row["recordings"] = [] + if recordings.exists(): + for source in json.loads(recordings.read_text("utf-8")).get("sources", []): + row["recordings"].append( + { + "scope": source.get("scope"), + "status": source.get("status"), + "area": source.get("area"), + "segments": [ + { + **segment, + "exists": (session_root / segment["file"]).is_file(), + } + for segment in source.get("segments", []) + ], + } + ) + proof = session_root / "artifacts/plugins" / row["plugin"] / "proof.png" + row["proof_sha256"] = ( + hashlib.sha256(proof.read_bytes()).hexdigest() if proof.is_file() else None + ) + start = data["session"]["start_time"] + finish = next( + (e["at"] / 1000 for e in data["fixture"]["events"] if e["event"] == "finish"), + None, + ) + row["fixture_finish_seconds"] = ( + round(finish - start, 3) if finish is not None else None + ) + row["evidence_ready_seconds"] = ( + round(max(finish, proof.stat().st_mtime) - start, 3) + if finish is not None and proof.is_file() and row["actual_pass"] + else None + ) + row["post_evidence_seconds"] = ( + round(row["seconds"] - row["evidence_ready_seconds"], 3) + if row["evidence_ready_seconds"] is not None + else None + ) + with sqlite3.connect( + f"file:{(traces / 'data_engine.db').as_posix()}?mode=ro", uri=True + ) as db: + errors = [] + observed_plugins = set() + for status, payload in db.execute( + "SELECT status,payload FROM traces WHERE session_id=? AND type='tool' ORDER BY timestamp", + (row["trace_id"],), + ): + value = json.loads(payload or "{}") + if value.get("plugin_id"): + observed_plugins.add(value["plugin_id"]) + if value.get("plugin_id") and status != "success": + # Keep the diagnostic first line, not private window inventory. + message = str(value.get("error", "")).splitlines() + errors.append( + { + "plugin": value["plugin_id"], + "tool": value.get("tool_name"), + "message": message[0] if message else "", + } + ) + row["tool_errors"] = errors + row["observed_plugins"] = sorted(observed_plugins) + + +def summarize(root, traces=None): + rows = [] + for path in sorted(root.glob("*-*/result.json")): + data = json.loads(path.read_text("utf-8")) + usage = data["llm_usage"] + calls = data["plugin_calls"] + session = data["session"] + requests = data.get("requests", []) + usage_by_node = defaultdict( + lambda: {"calls": 0, "input": 0, "cached": 0, "output": 0} + ) + models_from_usage = Counter() + for item in usage: + source_parts = item.get("source", "").split(":") + model_index = 2 if source_parts[0] == "lens" else 1 + model = ( + source_parts[model_index] + if len(source_parts) > model_index + else "unknown" + ) + models_from_usage[model] += 1 + node = usage_by_node[item.get("node", "unknown")] + node["calls"] += 1 + node["input"] += item.get("prompt_tokens", 0) + node["cached"] += item.get("cached_tokens", 0) + node["output"] += item.get("completion_tokens", 0) + models = Counter( + p.get("model", "unknown") for p in requests if p.get("phase") == "completed" + ) + input_tokens = sum(u.get("prompt_tokens", 0) for u in usage) + cached = sum(u.get("cached_tokens", 0) for u in usage) + output_tokens = sum(u.get("completion_tokens", 0) for u in usage) + duration = (session.get("end_time") or session["start_time"]) - session[ + "start_time" + ] + row = { + "plugin": data["plugin"], + "round": data["repetition"], + "trace_id": data["trace_id"], + "actual_pass": data["actual_pass"], + "execution_status": session["status"], + "input_tokens": input_tokens, + "cached_tokens": cached, + "uncached_input": input_tokens - cached, + "output_tokens": output_tokens, + "total_tokens": input_tokens + output_tokens, + "cache_rate": cached / input_tokens if input_tokens else None, + "llm_calls": len(usage), + "models": dict(models), + "models_from_usage": dict(models_from_usage), + "usage_by_node": dict(usage_by_node), + "request_phases": dict(Counter(p.get("phase") for p in requests)), + "request_seconds_by_role": { + role: round( + sum( + p.get("elapsed_seconds", 0) + for p in requests + if p.get("role") == role + ), + 3, + ) + for role in sorted({p.get("role", "unknown") for p in requests}) + }, + "fallback_requests": sum(bool(p.get("is_fallback")) for p in requests), + "retried_requests": sum((p.get("attempt") or 1) > 1 for p in requests), + "executor_text_only_responses": sum( + p.get("role") == "FlashRunner" + and p.get("phase") == "completed" + and not p.get("tools") + for p in requests + ), + "seconds": round(duration, 3), + "plugin_calls": len(calls), + "plugin_seconds": round(sum(c.get("duration") or 0 for c in calls), 3), + "plugin_errors": sum(c["status"] != "success" for c in calls), + "scroll_calls": sum(c["name"] == "scroll" for c in calls), + "fixture": data["fixture"], + } + if traces is not None: + enrich(row, data, traces) + rows.append(row) + aggregates = {} + for plugin in sorted({r["plugin"] for r in rows}): + group = [r for r in rows if r["plugin"] == plugin] + aggregate = {"runs": len(group), "passed": sum(r["actual_pass"] for r in group)} + for key in ( + "input_tokens", + "cached_tokens", + "uncached_input", + "output_tokens", + "total_tokens", + "seconds", + "llm_calls", + "plugin_calls", + "plugin_seconds", + "plugin_errors", + "scroll_calls", + ): + aggregate[f"mean_{key}"] = round(mean(r[key] for r in group), 3) + aggregate["cache_rate"] = sum(r["cached_tokens"] for r in group) / max( + 1, sum(r["input_tokens"] for r in group) + ) + aggregates[plugin] = aggregate + report = {"runs": rows, "aggregate": aggregates} + (root / "summary.json").write_text( + json.dumps(report, ensure_ascii=False, indent=2), "utf-8" + ) + return report + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(__doc__) + parser.add_argument("directory", type=Path) + parser.add_argument("--traces", type=Path) + options = parser.parse_args() + print( + json.dumps( + summarize(options.directory, options.traces)["aggregate"], + ensure_ascii=False, + indent=2, + ) + )