fix: validate Cua window discovery filters and guide recovery
This commit is contained in:
1 parent
c89588a80d
commit
1c1d700e9b
12 files changed
+336
-30
No files matched your search
@@ -1,6 +1,6 @@
|
|||||||
# Artemis Plugins
|
# Artemis Plugins
|
||||||
|
|
||||||
ARTEMIS 2.0.0 的插件市场。当前电脑控制插件为 **Cua 0.36.0**,支持 Windows x64,通过窗口截图与可访问性树操作电脑应用和浏览器窗口,复用 ARTEMIS 的调度、资源互斥、步骤、截图附件和录像。
|
ARTEMIS 2.0.0 的插件市场。当前电脑控制插件为 **Cua 0.36.0+artemis.1**,支持 Windows x64,通过窗口截图与可访问性树操作电脑应用和浏览器窗口,复用 ARTEMIS 的调度、资源互斥、步骤、截图附件和录像。
|
||||||
|
|
||||||
在 ARTEMIS 的「插件」页面安装「电脑控制 · Cua」,授权并启用即可。安卓任务可根据提示词按需调用电脑控制,不需要单独配置 Codex MCP 或安装 Go、Node、Python。
|
在 ARTEMIS 的「插件」页面安装「电脑控制 · Cua」,授权并启用即可。安卓任务可根据提示词按需调用电脑控制,不需要单独配置 Codex MCP 或安装 Go、Node、Python。
|
||||||
|
|
||||||
@@ -24,12 +24,13 @@ Open Computer Use(OCU)已从市场索引下架,当前市场只提供 Cua
|
|||||||
python scripts/build_cua_driver.py <上游ZIP> --go <Go可执行文件>
|
python scripts/build_cua_driver.py <上游ZIP> --go <Go可执行文件>
|
||||||
python tests/cua_smoke.py <安装后的artemis-cua-adapter.exe> artifacts/cua-smoke
|
python tests/cua_smoke.py <安装后的artemis-cua-adapter.exe> artifacts/cua-smoke
|
||||||
python tests/cua_fixture_validation.py <安装后的artemis-cua-adapter.exe>
|
python tests/cua_fixture_validation.py <安装后的artemis-cua-adapter.exe>
|
||||||
|
python tests/cua_discovery_smoke.py <安装后的artemis-cua-adapter.exe> --app Brave --output artifacts/cua-discovery.json
|
||||||
```
|
```
|
||||||
|
|
||||||
验证脚本使用已安装 ARTEMIS 依赖的 Python,并将 ARTEMIS 源码目录放入 `PYTHONPATH`。它们只操作自行创建的测试窗口。历史 A/B 脚本仍固定 OCU 1.1.0 与 Cua 0.1.2,便于复现已有测量;它不是安装后必须执行的步骤。
|
验证脚本使用已安装 ARTEMIS 依赖的 Python,并将 ARTEMIS 源码目录放入 `PYTHONPATH`。前两项实测只操作自行创建的测试窗口;窗口发现实测只读取指定的已打开应用,不改变其页面。历史 A/B 脚本仍固定 OCU 1.1.0 与 Cua 0.1.2,便于复现已有测量;它不是安装后必须执行的步骤。
|
||||||
|
|
||||||
`catalog.json` 是市场索引,包含清单、HTTPS 安装包地址、SHA256 和验证说明。`artifacts/` 保存本机安装包与测试资料,不进入 Git。版本文件不可原地覆盖,任务固定使用其启动时的版本。
|
`catalog.json` 是市场索引,包含清单、HTTPS 安装包地址、SHA256 和验证说明。`artifacts/` 保存本机安装包与测试资料,不进入 Git。版本文件不可原地覆盖,任务固定使用其启动时的版本。
|
||||||
|
|
||||||
插件版本跟随所用 Cua Driver 上游版本:当前为 `0.36.0`,对应上游标签 `cua-driver-rs-v0.36.0`,本仓库发行标签为 `cua-driver-v0.36.0`。只修改 ARTEMIS 适配而未升级上游时使用 `+artemis.N` 后缀区分不可变安装包;包内 `provenance.json` 记录上游提交、原包校验值和适配修订,构建会检查版本对应关系。
|
插件版本跟随所用 Cua Driver 上游版本:当前为 `0.36.0`,对应上游标签 `cua-driver-rs-v0.36.0`,当前适配修订为 `0.36.0+artemis.1`,本仓库发行标签为 `cua-driver-v0.36.0+artemis.1`。只修改 ARTEMIS 适配而未升级上游时使用 `+artemis.N` 后缀区分不可变安装包;包内 `provenance.json` 记录上游提交、原包校验值和适配修订,构建会检查版本对应关系。
|
||||||
|
|
||||||
[发行说明](plugins/cua-driver/RELEASE.md) · [ARTEMIS 主项目](https://git.playones.com/chenmingxuan/Artemis-Eapil)
|
[发行说明](plugins/cua-driver/RELEASE.md) · [ARTEMIS 主项目](https://git.playones.com/chenmingxuan/Artemis-Eapil)
|
||||||
+4
-4
@@ -6,7 +6,7 @@
|
|||||||
"schema_version": 2,
|
"schema_version": 2,
|
||||||
"id": "cua-driver",
|
"id": "cua-driver",
|
||||||
"name": "电脑控制 · Cua",
|
"name": "电脑控制 · Cua",
|
||||||
"version": "0.36.0",
|
"version": "0.36.0+artemis.1",
|
||||||
"description": "通过 Cua Driver 的窗口截图与可访问性树操作 Windows 应用及浏览器窗口;复用 ARTEMIS 的任务调度、步骤证据与录像。",
|
"description": "通过 Cua Driver 的窗口截图与可访问性树操作 Windows 应用及浏览器窗口;复用 ARTEMIS 的任务调度、步骤证据与录像。",
|
||||||
"artemis_version": ">=2.0.0,<3.0.0",
|
"artemis_version": ">=2.0.0,<3.0.0",
|
||||||
"platforms": [
|
"platforms": [
|
||||||
@@ -172,9 +172,9 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"release_stage": "preview",
|
"release_stage": "preview",
|
||||||
"archive_url": "https://git.playones.com/chenmingxuan/Artemis-Plugins/releases/download/cua-driver-v0.36.0/cua-driver-0.36.0-windows-amd64.zip",
|
"archive_url": "https://git.playones.com/chenmingxuan/Artemis-Plugins/releases/download/cua-driver-v0.36.0+artemis.1/cua-driver-0.36.0+artemis.1-windows-amd64.zip",
|
||||||
"sha256": "dea40dce182ac6b982c7da17d3610d5db87e2639205ca32f179c55ca1d7833ce",
|
"sha256": "43a90d514e0d148e4476ca8508e22a2bd1f60d8af97cb5393a7a2034e2af6aa6",
|
||||||
"verification": "Windows x64 / ARTEMIS 2.0.0 开发版。固定 Cua Driver 0.36.0 预发布版。已验证原生窗口表单、控件与像素点击、34 像素目标、滚动、trace 截图及哈希、ARTEMIS PC 窗口录像、子进程退出;同一复合用例重复 3 次,独立验收 3/3,AI 裁定 2 次通过、1 次未判定。0.36.0 沿用已测 0.1.2 动作实现,取代 OCU 市场入口。未覆盖复杂浏览器控制台、多 DPI、多显示器和跨端长任务。完整验证与已知限制见仓库 plugins/cua-driver/VERIFICATION.md。"
|
"verification": "Windows x64 / ARTEMIS 2.0.0 开发版。固定 Cua Driver 0.36.0 预发布版。已验证原生窗口表单、控件与像素点击、34 像素目标、滚动、trace 截图及哈希、ARTEMIS PC 窗口录像、子进程退出;同一复合用例重复 3 次,独立验收 3/3,AI 裁定 2 次通过、1 次未判定。0.36.0 沿用已测 0.1.2 动作实现,取代 OCU 市场入口。未覆盖复杂浏览器控制台、多 DPI、多显示器和跨端长任务。完整验证与已知限制见仓库 plugins/cua-driver/VERIFICATION.md。 本修订实测补充:Brave 按应用名发现、无效/null PID、冲突 PID 纠正、实际截图及 UIA 树通过;未重跑完整跨端业务任务。"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
@@ -1,6 +1,8 @@
|
|||||||
# Cua 电脑控制 0.36.0
|
# Cua 电脑控制 0.36.0+artemis.1
|
||||||
|
|
||||||
作为 ARTEMIS 插件市场的默认电脑控制实现,替代已下架的 Open Computer Use。此版本沿用候选 0.1.2 的驱动和动作逻辑,更新版本标识、名称和分发文档;上游仍是固定的 Cua Driver 0.36.0 预发布版,市场标注开发预览。
|
作为 ARTEMIS 插件市场的默认电脑控制实现,替代已下架的 Open Computer Use。此修订修复按应用名查找窗口时,被无效或冲突 PID 筛选为零窗口并误判浏览器不可用的问题;上游仍是固定的 Cua Driver 0.36.0 预发布版,市场标注开发预览。
|
||||||
|
|
||||||
|
修复内容:未知 PID 可省略或填 `null`;`0`、负数、非整数等无效 PID 明确拒绝。零窗口结果保留实际筛选条件,并给出同应用重新查找或检查最小化窗口的建议;去掉无关前台应用 PID 提示,不自动扩大查询范围或替用户选择其他应用。
|
||||||
|
|
||||||
上游提交:`6d6c734c127025fb39ad5d585a10382808b39dc1`。Windows x64 原始 ZIP SHA256:`f8c47024b626f4e0e70f8428d2998df39e4879110c5722a29d5fd03e367353ad`。采用 MIT Driver,许可证随包提供;不包含 Cua Spaces、云环境或模型服务。
|
上游提交:`6d6c734c127025fb39ad5d585a10382808b39dc1`。Windows x64 原始 ZIP SHA256:`f8c47024b626f4e0e70f8428d2998df39e4879110c5722a29d5fd03e367353ad`。采用 MIT Driver,许可证随包提供;不包含 Cua Spaces、云环境或模型服务。
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
通过本地 Cua Driver 操作用户授权的 Windows 窗口。调度、资源互斥、模型、步骤、截图和录像由 ARTEMIS 提供;插件本身不运行 AI,不另起后台服务。
|
通过本地 Cua Driver 操作用户授权的 Windows 窗口。调度、资源互斥、模型、步骤、截图和录像由 ARTEMIS 提供;插件本身不运行 AI,不另起后台服务。
|
||||||
|
|
||||||
1. `list_windows` 找到目标的 `pid` 和 `window_id`;只操作任务要求的窗口,不关闭其他应用。
|
1. 先按应用名发现窗口,例如 `list_windows({"app":"Brave","on_screen_only":true})`;未知 `pid` 必须省略或填 `null`,不能填 `0`。`app` 与非空 `pid` 同时提供时会取交集,只有该应用实际返回的 PID 才能用于后续调用。空结果时先按 `lookup.suggested_arguments` 纠正筛选(去掉错误 PID 或检查最小化窗口),不能直接宣称浏览器不可用,也不能把无关前台进程号当成目标 PID。取得目标的 `pid`、`window_id` 后读取真实画面;只操作任务要求的窗口,不关闭其他应用。
|
||||||
2. `get_window_state` 同时读取截图和 UIA 控件树。最新 `snapshot_id` 加 `:N` 即控件 `element_token`。优先用 `click` 点击最新控件;视觉目标用 `click_at`,坐标仅取本次截图上的窗口局部像素,不能用屏幕坐标,需携带本次 `capture_id`。两种点击参数互斥,不混用。
|
2. `get_window_state` 同时读取截图和 UIA 控件树。最新 `snapshot_id` 加 `:N` 即控件 `element_token`。优先用 `click` 点击最新控件;视觉目标用 `click_at`,坐标仅取本次截图上的窗口局部像素,不能用屏幕坐标,需携带本次 `capture_id`。两种点击参数互斥,不混用。
|
||||||
3. 所有窗口操作明确传 `pid`、`window_id`。`set_value` 适合直接设置输入框;不支持时可明确聚焦后 `type_text`。观察控件支持的动作,不猜元素编号。`press_key` 传单键及 `modifiers` 数组。
|
3. 所有窗口操作明确传 `pid`、`window_id`。`set_value` 适合直接设置输入框;不支持时可明确聚焦后 `type_text`。观察控件支持的动作,不猜元素编号。`press_key` 传单键及 `modifiers` 数组。
|
||||||
4. 每个动作后插件自动返回一次新截图和控件树,新 snapshot 替换旧 token。接着用新 token,无需重复获取相同状态。仅当画面确实还在变化或需要核验时再次观察。一次调用只执行一次动作,超时不自动重发。
|
4. 每个动作后插件自动返回一次新截图和控件树,新 snapshot 替换旧 token。接着用新 token,无需重复获取相同状态。仅当画面确实还在变化或需要核验时再次观察。一次调用只执行一次动作,超时不自动重发。
|
||||||
|
|||||||
@@ -42,3 +42,11 @@ Cua 第三轮虽然独立验收通过,ARTEMIS 最终核验却没有生成结
|
|||||||
## 市场替换版本 0.36.0
|
## 市场替换版本 0.36.0
|
||||||
|
|
||||||
0.36.0 沿用 0.1.2 已测驱动和动作实现,仅更新发行版本、名称及文档,作为市场默认电脑控制插件取代 OCU。上游仍固定 0.36.0 预发布版,市场继续标注开发预览;上文的未测范围和 AI 裁定缺失情况仍然有效。旧 OCU 从 catalog.json 下架,既有源码、发行附件和任务证据保留。已安装 OCU 的用户应安装并启用 Cua,再停用 OCU;不会静默为其他用户授权新代码。
|
0.36.0 沿用 0.1.2 已测驱动和动作实现,仅更新发行版本、名称及文档,作为市场默认电脑控制插件取代 OCU。上游仍固定 0.36.0 预发布版,市场继续标注开发预览;上文的未测范围和 AI 裁定缺失情况仍然有效。旧 OCU 从 catalog.json 下架,既有源码、发行附件和任务证据保留。已安装 OCU 的用户应安装并启用 Cua,再停用 OCU;不会静默为其他用户授权新代码。
|
||||||
|
|
||||||
|
## 窗口发现修订 0.36.0+artemis.1
|
||||||
|
|
||||||
|
修复复跑任务在按应用名找 Brave 时填入 PID 0,随后误用无关前台应用 PID 的失败链路。原生窗口基准没有覆盖这一入口,原有 A/B 数据不作为浏览器发现能力的验证。
|
||||||
|
|
||||||
|
新增 Go 回归覆盖无效 PID 在调用驱动前拒绝、null 省略、冲突筛选保持原范围并提示同应用重查、可见窗口为空时保留目标检查最小化窗口,以及驱动错误不得伪装成空列表。`tests/cua_discovery_smoke.py` 对已打开的指定应用只读验证主机 schema、适配器、应用名发现、null PID、冲突 PID 恢复,以及实际截图和 UIA 树;不点击、不导航、不修改用户页面。
|
||||||
|
|
||||||
|
本修订实测通过:真实 Brave 应用名枚举、null PID、省略 PID、冲突 PID 后按建议恢复,并取得真实截图及 UIA 树。原生窗口表单、34 像素目标、滚动、PNG 附件校验和窗口录像回归通过;Go 测试、vet 与 Python Ruff 检查通过。未重跑原业务的整段移动端/PC布局验收,不将窗口发现修复等同于原任务通过。
|
||||||
@@ -25,6 +25,8 @@ import (
|
|||||||
//go:embed tools.json
|
//go:embed tools.json
|
||||||
var toolsJSON []byte
|
var toolsJSON []byte
|
||||||
|
|
||||||
|
const adapterVersion = "0.36.0+artemis.1"
|
||||||
|
|
||||||
type obj = map[string]any
|
type obj = map[string]any
|
||||||
type request struct {
|
type request struct {
|
||||||
JSONRPC string `json:"jsonrpc"`
|
JSONRPC string `json:"jsonrpc"`
|
||||||
@@ -104,7 +106,7 @@ func startPeer(exe string) (*peer, error) {
|
|||||||
p.lines <- append([]byte(nil), s.Bytes()...)
|
p.lines <- append([]byte(nil), s.Bytes()...)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
_, err = p.call("initialize", obj{"protocolVersion": "2024-11-05", "clientInfo": obj{"name": "artemis-cua-adapter", "version": "0.36.0"}, "capabilities": obj{}})
|
_, err = p.call("initialize", obj{"protocolVersion": "2024-11-05", "clientInfo": obj{"name": "artemis-cua-adapter", "version": adapterVersion}, "capabilities": obj{}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
p.stop()
|
p.stop()
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -200,7 +202,11 @@ func (a *adapter) validate(name string, args obj) error {
|
|||||||
return fmt.Errorf("missing argument: %s", str(key))
|
return fmt.Errorf("missing argument: %s", str(key))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if name != "list_windows" && (num(args["pid"]) <= 0 || num(args["window_id"]) <= 0) {
|
if name == "list_windows" {
|
||||||
|
_, err := windowQuery(args)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if num(args["pid"]) <= 0 || num(args["window_id"]) <= 0 {
|
||||||
return errors.New("a concrete pid and window_id are required")
|
return errors.New("a concrete pid and window_id are required")
|
||||||
}
|
}
|
||||||
if name == "save_screenshot" && !validFilename(str(args["filename"])) {
|
if name == "save_screenshot" && !validFilename(str(args["filename"])) {
|
||||||
@@ -302,6 +308,12 @@ func (a *adapter) handle(name string, args obj) obj {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
r, src, err = a.observe(args)
|
r, src, err = a.observe(args)
|
||||||
|
} else if name == "list_windows" {
|
||||||
|
query, _ := windowQuery(args) // Already validated above.
|
||||||
|
r, err = a.peer.tool(name, query)
|
||||||
|
if err == nil {
|
||||||
|
r = windowListing(r, query)
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
upstream := name
|
upstream := name
|
||||||
if name == "click_at" {
|
if name == "click_at" {
|
||||||
@@ -412,7 +424,7 @@ func serve(a *adapter, in io.Reader, out io.Writer) {
|
|||||||
var rpcErr any
|
var rpcErr any
|
||||||
switch q.Method {
|
switch q.Method {
|
||||||
case "initialize":
|
case "initialize":
|
||||||
result = obj{"protocolVersion": "2024-11-05", "capabilities": obj{"tools": obj{}}, "serverInfo": obj{"name": "artemis-cua-driver", "version": "0.36.0"}}
|
result = obj{"protocolVersion": "2024-11-05", "capabilities": obj{"tools": obj{}}, "serverInfo": obj{"name": "artemis-cua-driver", "version": adapterVersion}}
|
||||||
case "ping":
|
case "ping":
|
||||||
result = obj{}
|
result = obj{}
|
||||||
case "tools/list":
|
case "tools/list":
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
"schema_version": 2,
|
"schema_version": 2,
|
||||||
"id": "cua-driver",
|
"id": "cua-driver",
|
||||||
"name": "电脑控制 · Cua",
|
"name": "电脑控制 · Cua",
|
||||||
"version": "0.36.0",
|
"version": "0.36.0+artemis.1",
|
||||||
"description": "通过 Cua Driver 的窗口截图与可访问性树操作 Windows 应用及浏览器窗口;复用 ARTEMIS 的任务调度、步骤证据与录像。",
|
"description": "通过 Cua Driver 的窗口截图与可访问性树操作 Windows 应用及浏览器窗口;复用 ARTEMIS 的任务调度、步骤证据与录像。",
|
||||||
"artemis_version": ">=2.0.0,<3.0.0",
|
"artemis_version": ">=2.0.0,<3.0.0",
|
||||||
"platforms": [
|
"platforms": [
|
||||||
|
|||||||
@@ -1,21 +1,29 @@
|
|||||||
[
|
[
|
||||||
{
|
{
|
||||||
"name": "list_windows",
|
"name": "list_windows",
|
||||||
"description": "List visible application windows with exact pid/window_id and physical bounds. Pick only the user-authorized target.",
|
"description": "Find the authorized application window. If its PID is unknown, call with app only, e.g. {\"app\":\"Brave\",\"on_screen_only\":true}; omit pid or use null, never 0. Use the returned matching pid/window_id for later actions. Empty filtered results are not proof that the app is unavailable; follow lookup.suggested_arguments when present.",
|
||||||
"inputSchema": {
|
"inputSchema": {
|
||||||
"additionalProperties": false,
|
"additionalProperties": false,
|
||||||
"properties": {
|
"properties": {
|
||||||
"app": {
|
"app": {
|
||||||
"description": "Only windows of apps whose name contains this text (case-insensitive), or whose bundle id equals it. Prefer this or `pid` over an unfiltered call, which returns every window on the host.",
|
"description": "Application name substring (case-insensitive). Start with this when the target PID is unknown. app and a non-null pid are combined filters; do not copy the PID of an unrelated foreground app.",
|
||||||
"type": "string"
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
},
|
},
|
||||||
"on_screen_only": {
|
"on_screen_only": {
|
||||||
"description": "When true, drop windows that aren't currently on-screen. Default false.",
|
"description": "When true, drop windows that aren't currently on-screen. Default false.",
|
||||||
"type": "boolean"
|
"type": "boolean"
|
||||||
},
|
},
|
||||||
"pid": {
|
"pid": {
|
||||||
"description": "Optional pid filter. When set, only this pid's windows are returned.",
|
"description": "Known positive Windows process ID from a matching list_windows result. Unknown: omit or null; never 0. With app, both filters must match the same window.",
|
||||||
"type": "integer"
|
"type": [
|
||||||
|
"integer",
|
||||||
|
"null"
|
||||||
|
],
|
||||||
|
"minimum": 1,
|
||||||
|
"maximum": 4294967295
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"type": "object",
|
"type": "object",
|
||||||
@@ -43,7 +51,7 @@
|
|||||||
"type": "integer"
|
"type": "integer"
|
||||||
},
|
},
|
||||||
"pid": {
|
"pid": {
|
||||||
"description": "Process ID from `list_apps`.",
|
"description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.",
|
||||||
"type": "integer"
|
"type": "integer"
|
||||||
},
|
},
|
||||||
"query": {
|
"query": {
|
||||||
@@ -170,7 +178,7 @@
|
|||||||
"type": "boolean"
|
"type": "boolean"
|
||||||
},
|
},
|
||||||
"pid": {
|
"pid": {
|
||||||
"description": "Exact process whose window may be observed.",
|
"description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.",
|
||||||
"minimum": 1,
|
"minimum": 1,
|
||||||
"type": "integer"
|
"type": "integer"
|
||||||
},
|
},
|
||||||
@@ -240,7 +248,7 @@
|
|||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
"pid": {
|
"pid": {
|
||||||
"description": "Target process ID for window scope. Optional with element_token (the token carries the pid). Omit with scope=desktop for screen-absolute coordinates from get_desktop_state.",
|
"description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.",
|
||||||
"type": "integer"
|
"type": "integer"
|
||||||
},
|
},
|
||||||
"window_id": {
|
"window_id": {
|
||||||
@@ -289,7 +297,7 @@
|
|||||||
"type": "number"
|
"type": "number"
|
||||||
},
|
},
|
||||||
"pid": {
|
"pid": {
|
||||||
"description": "Target process ID.",
|
"description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.",
|
||||||
"type": "integer"
|
"type": "integer"
|
||||||
},
|
},
|
||||||
"to_x": {
|
"to_x": {
|
||||||
@@ -340,7 +348,7 @@
|
|||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
"pid": {
|
"pid": {
|
||||||
"description": "Target process ID.",
|
"description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.",
|
||||||
"type": "integer"
|
"type": "integer"
|
||||||
},
|
},
|
||||||
"text": {
|
"text": {
|
||||||
@@ -403,7 +411,7 @@
|
|||||||
"type": "array"
|
"type": "array"
|
||||||
},
|
},
|
||||||
"pid": {
|
"pid": {
|
||||||
"description": "Target process ID.",
|
"description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.",
|
||||||
"type": "integer"
|
"type": "integer"
|
||||||
},
|
},
|
||||||
"window_id": {
|
"window_id": {
|
||||||
@@ -451,7 +459,7 @@
|
|||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
"pid": {
|
"pid": {
|
||||||
"description": "Target process ID. Optional with element_token (the token carries the pid).",
|
"description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.",
|
||||||
"type": "integer"
|
"type": "integer"
|
||||||
},
|
},
|
||||||
"value": {
|
"value": {
|
||||||
@@ -519,7 +527,7 @@
|
|||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
"pid": {
|
"pid": {
|
||||||
"description": "Target process ID for window scope. Optional with element_token (the token carries the pid). Omit with scope=desktop for screen-absolute coordinates from get_desktop_state.",
|
"description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.",
|
||||||
"type": "integer"
|
"type": "integer"
|
||||||
},
|
},
|
||||||
"window_id": {
|
"window_id": {
|
||||||
@@ -556,7 +564,8 @@
|
|||||||
"properties": {
|
"properties": {
|
||||||
"pid": {
|
"pid": {
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
"minimum": 1
|
"minimum": 1,
|
||||||
|
"description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID."
|
||||||
},
|
},
|
||||||
"window_id": {
|
"window_id": {
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
@@ -612,7 +621,7 @@
|
|||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
"pid": {
|
"pid": {
|
||||||
"description": "Target process ID for window scope. Optional with element_token (the token carries the pid). Omit with scope=desktop for screen-absolute coordinates from get_desktop_state.",
|
"description": "Positive process ID returned for the target app by list_windows; never guess or use another app foreground PID.",
|
||||||
"type": "integer"
|
"type": "integer"
|
||||||
},
|
},
|
||||||
"window_id": {
|
"window_id": {
|
||||||
|
|||||||
@@ -0,0 +1,92 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"math"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Optional filters must remain optional across JSON-schema clients. In
|
||||||
|
// particular, null means unspecified; zero is not a wildcard process ID.
|
||||||
|
func windowQuery(args obj) (obj, error) {
|
||||||
|
query := obj{}
|
||||||
|
for key, value := range args {
|
||||||
|
if value != nil {
|
||||||
|
query[key] = value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if value, ok := query["app"]; ok {
|
||||||
|
app, valid := value.(string)
|
||||||
|
if !valid {
|
||||||
|
return nil, errors.New("app must be an application name or null")
|
||||||
|
}
|
||||||
|
if app = strings.TrimSpace(app); app == "" {
|
||||||
|
delete(query, "app")
|
||||||
|
} else {
|
||||||
|
query["app"] = app
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if value, ok := query["pid"]; ok {
|
||||||
|
valid := num(value) > 0 && num(value) <= math.MaxUint32
|
||||||
|
switch n := value.(type) {
|
||||||
|
case float64:
|
||||||
|
valid = valid && n == math.Trunc(n)
|
||||||
|
case int, int64:
|
||||||
|
default:
|
||||||
|
valid = false
|
||||||
|
}
|
||||||
|
if !valid {
|
||||||
|
return nil, errors.New("pid must be a positive Windows process ID from a matching list_windows result. If unknown, omit pid or use null and search by app, e.g. {\"app\":\"Brave\",\"on_screen_only\":true}. Never use 0 or a different app's foreground PID")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return query, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Upstream's empty-result hint names the unrelated foreground process. Do not
|
||||||
|
// present that PID as a recovery candidate. Keep the exact filters and provide
|
||||||
|
// a separate suggested query; never silently widen the caller's PID scope.
|
||||||
|
func windowListing(result obj, query obj) obj {
|
||||||
|
if result["isError"] == true {
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
data := object(result["structuredContent"])
|
||||||
|
windows, ok := data["windows"].([]any)
|
||||||
|
if !ok {
|
||||||
|
return failure("Driver returned no window list; this is an observation error, not evidence that the application is absent", "not_started", "none", "invalid_window_list")
|
||||||
|
}
|
||||||
|
lookup := obj{"status": "matched", "filters": query, "count": len(windows)}
|
||||||
|
lines := []string{fmt.Sprintf("Found %d matching window(s).", len(windows))}
|
||||||
|
for _, value := range windows {
|
||||||
|
window := object(value)
|
||||||
|
lines = append(lines, fmt.Sprintf("- %s (pid %d), window_id %d, title %q", str(window["app_name"]), num(window["pid"]), num(window["window_id"]), str(window["title"])))
|
||||||
|
}
|
||||||
|
if len(windows) == 0 {
|
||||||
|
lookup["status"] = "no_match"
|
||||||
|
lines = append(lines, "No windows match these filters. This alone does not prove the application is unavailable or screenshot capture is broken. Do not substitute another application's foreground PID.")
|
||||||
|
if str(query["app"]) != "" && query["pid"] != nil {
|
||||||
|
next := obj{}
|
||||||
|
for key, value := range query {
|
||||||
|
if key != "pid" {
|
||||||
|
next[key] = value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
lookup["suggested_arguments"] = next
|
||||||
|
lines = append(lines, "app and pid are combined filters. Retry list_windows for the same app with pid omitted; use only a PID returned for that app.")
|
||||||
|
} else if query["on_screen_only"] == true {
|
||||||
|
next := obj{}
|
||||||
|
for key, value := range query {
|
||||||
|
next[key] = value
|
||||||
|
}
|
||||||
|
next["on_screen_only"] = false
|
||||||
|
lookup["suggested_arguments"] = next
|
||||||
|
lines = append(lines, "Retry the same filters with on_screen_only=false to check for minimized/off-screen windows. A listed window still needs a fresh get_window_state before interaction.")
|
||||||
|
} else {
|
||||||
|
lines = append(lines, "Verify the requested application name or PID. When the PID is unknown, search by the known app name with pid omitted. Do not guess a PID or operate an unrelated window.")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
data["lookup"] = lookup
|
||||||
|
result["structuredContent"] = data
|
||||||
|
result["content"] = []any{textBlock(strings.Join(lines, "\n"))}
|
||||||
|
return result
|
||||||
|
}
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
type lookupInput struct{ bytes.Buffer }
|
||||||
|
|
||||||
|
func (*lookupInput) Close() error { return nil }
|
||||||
|
|
||||||
|
func TestInvalidDiscoveryPIDStopsBeforeDriver(t *testing.T) {
|
||||||
|
for _, pid := range []any{0, -1, 1.5, "123", true, float64(4294967296)} {
|
||||||
|
result := newAdapter(nil).handle("list_windows", obj{"app": "Brave", "pid": pid})
|
||||||
|
if result["isError"] != true || object(object(result["structuredContent"])["artemis"])["execution"] != "not_started" {
|
||||||
|
t.Fatalf("invalid PID %v reached driver or was reported as no windows: %v", pid, result)
|
||||||
|
}
|
||||||
|
if !strings.Contains(str(object(items(result["content"])[0])["text"]), "omit pid") {
|
||||||
|
t.Fatalf("missing recovery instruction for %v", pid)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAppDiscoveryOmitsNullPIDAtDriverBoundary(t *testing.T) {
|
||||||
|
input := &lookupInput{}
|
||||||
|
lines := make(chan []byte, 1)
|
||||||
|
lines <- []byte(`{"id":1,"result":{"content":[],"structuredContent":{"windows":[{"app_name":"brave.exe","pid":72,"window_id":81,"title":"Browser"}]}}}`)
|
||||||
|
a := newAdapter(&peer{input: input, lines: lines})
|
||||||
|
args := obj{"app": " Brave ", "pid": nil, "on_screen_only": true}
|
||||||
|
result := a.handle("list_windows", args)
|
||||||
|
if result["isError"] == true || len(items(object(result["structuredContent"])["windows"])) != 1 {
|
||||||
|
t.Fatalf("failed app-only discovery: %v", result)
|
||||||
|
}
|
||||||
|
var sent obj
|
||||||
|
if err := json.Unmarshal(input.Bytes(), &sent); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
forwarded := object(object(sent["params"])["arguments"])
|
||||||
|
if _, present := forwarded["pid"]; present || forwarded["app"] != "Brave" || forwarded["on_screen_only"] != true {
|
||||||
|
t.Fatalf("incorrect upstream query: %v", forwarded)
|
||||||
|
}
|
||||||
|
if args["app"] != " Brave " || len(args) != 3 {
|
||||||
|
t.Fatal("changed original arguments used in trace")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConflictingFiltersPreserveScopeAndRemoveForegroundHint(t *testing.T) {
|
||||||
|
query := obj{"app": "Brave", "pid": 14492, "on_screen_only": true}
|
||||||
|
result := windowListing(obj{
|
||||||
|
"content": []any{textBlock(`No windows. Current frontmost app is "studio64.exe" (pid 14492).`)},
|
||||||
|
"structuredContent": obj{"windows": []any{}},
|
||||||
|
}, query)
|
||||||
|
data := object(result["structuredContent"])
|
||||||
|
lookup := object(data["lookup"])
|
||||||
|
next := object(lookup["suggested_arguments"])
|
||||||
|
if len(items(data["windows"])) != 0 || lookup["status"] != "no_match" || object(lookup["filters"])["pid"] != 14492 {
|
||||||
|
t.Fatal("original filter or no-match result was hidden")
|
||||||
|
}
|
||||||
|
if _, exists := next["pid"]; exists || next["app"] != "Brave" || next["on_screen_only"] != true {
|
||||||
|
t.Fatalf("incorrect app-only recovery: %v", next)
|
||||||
|
}
|
||||||
|
text := str(object(items(result["content"])[0])["text"])
|
||||||
|
if strings.Contains(text, "studio64.exe") || !strings.Contains(text, "pid omitted") {
|
||||||
|
t.Fatalf("misleading recovery: %s", text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEmptyVisibleQuerySuggestsSameTargetOffScreen(t *testing.T) {
|
||||||
|
query := obj{"pid": 12, "on_screen_only": true}
|
||||||
|
result := windowListing(obj{"structuredContent": obj{"windows": []any{}}}, query)
|
||||||
|
next := object(object(object(result["structuredContent"])["lookup"])["suggested_arguments"])
|
||||||
|
if next["pid"] != 12 || next["on_screen_only"] != false || query["on_screen_only"] != true {
|
||||||
|
t.Fatalf("changed target or original query: %v", next)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLookupErrorsNeverBecomeEmptySuccess(t *testing.T) {
|
||||||
|
for _, result := range []obj{
|
||||||
|
{"isError": true, "content": []any{textBlock("driver unavailable")}},
|
||||||
|
{"structuredContent": obj{}},
|
||||||
|
} {
|
||||||
|
if windowListing(result, obj{"app": "Brave"})["isError"] != true {
|
||||||
|
t.Fatal("observation failure became a successful empty list")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -4,6 +4,7 @@ import argparse
|
|||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
import subprocess
|
import subprocess
|
||||||
import urllib.request
|
import urllib.request
|
||||||
import zipfile
|
import zipfile
|
||||||
@@ -26,6 +27,11 @@ def main():
|
|||||||
version = json.loads((SOURCE / "plugin.json").read_text("utf-8"))["version"]
|
version = json.loads((SOURCE / "plugin.json").read_text("utf-8"))["version"]
|
||||||
if version.split("+", 1)[0] != UPSTREAM_VERSION:
|
if version.split("+", 1)[0] != UPSTREAM_VERSION:
|
||||||
raise ValueError("Plugin version must match the pinned Cua Driver version")
|
raise ValueError("Plugin version must match the pinned Cua Driver version")
|
||||||
|
match = re.search(
|
||||||
|
r'const adapterVersion = "([^"]+)"', (SOURCE / "main.go").read_text("utf-8")
|
||||||
|
)
|
||||||
|
if not match or match[1] != version:
|
||||||
|
raise ValueError("Adapter MCP version must match plugin.json")
|
||||||
if hashlib.sha256(args.upstream_zip.read_bytes()).hexdigest() != DIGEST:
|
if hashlib.sha256(args.upstream_zip.read_bytes()).hexdigest() != DIGEST:
|
||||||
raise ValueError("Cua release archive hash mismatch")
|
raise ValueError("Cua release archive hash mismatch")
|
||||||
output = args.output.resolve()
|
output = args.output.resolve()
|
||||||
@@ -93,7 +99,7 @@ def main():
|
|||||||
"source_commit": COMMIT,
|
"source_commit": COMMIT,
|
||||||
"upstream_version": UPSTREAM_VERSION,
|
"upstream_version": UPSTREAM_VERSION,
|
||||||
"plugin_version": version,
|
"plugin_version": version,
|
||||||
"adapter_revision": 1,
|
"adapter_revision": 2,
|
||||||
"prerelease": True,
|
"prerelease": True,
|
||||||
"url": URL,
|
"url": URL,
|
||||||
"sha256": DIGEST,
|
"sha256": DIGEST,
|
||||||
|
|||||||
@@ -0,0 +1,88 @@
|
|||||||
|
"""Opt-in read-only app discovery against an already open, named application."""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import jsonschema
|
||||||
|
from artemis.plugins.models import ServerConfig
|
||||||
|
from artemis.plugins.session import PluginSession
|
||||||
|
|
||||||
|
|
||||||
|
async def run(executable: Path, app: str, output: Path):
|
||||||
|
session = await PluginSession(
|
||||||
|
ServerConfig(command=str(executable)), executable.parent
|
||||||
|
).start()
|
||||||
|
try:
|
||||||
|
schema = next(t.inputSchema for t in session.tools if t.name == "list_windows")
|
||||||
|
query = {"app": app, "on_screen_only": True}
|
||||||
|
bad_query = {**query, "pid": 0}
|
||||||
|
try:
|
||||||
|
jsonschema.validate(bad_query, schema)
|
||||||
|
except jsonschema.ValidationError:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
raise AssertionError("Host schema still accepts PID 0")
|
||||||
|
bad = await session.call("list_windows", bad_query)
|
||||||
|
assert bad.isError
|
||||||
|
assert bad.structuredContent["artemis"]["execution"] == "not_started"
|
||||||
|
assert "omit pid" in "\n".join(b.text for b in bad.content if b.type == "text")
|
||||||
|
|
||||||
|
jsonschema.validate(query, schema)
|
||||||
|
discovered = await session.call("list_windows", query)
|
||||||
|
assert not discovered.isError
|
||||||
|
windows = discovered.structuredContent["windows"]
|
||||||
|
assert windows, "Open the intended app before running this opt-in probe"
|
||||||
|
|
||||||
|
nullable = {**query, "pid": None}
|
||||||
|
jsonschema.validate(nullable, schema)
|
||||||
|
found = await session.call("list_windows", nullable)
|
||||||
|
assert not found.isError and found.structuredContent["windows"]
|
||||||
|
assert "pid" not in found.structuredContent["lookup"]["filters"]
|
||||||
|
|
||||||
|
# This Python test process is not the requested application. A conflict
|
||||||
|
# must stay empty and suggest a new app-only query, without a guessed PID.
|
||||||
|
conflict = await session.call("list_windows", {**query, "pid": os.getpid()})
|
||||||
|
assert not conflict.isError and conflict.structuredContent["windows"] == []
|
||||||
|
next_query = conflict.structuredContent["lookup"]["suggested_arguments"]
|
||||||
|
assert next_query == query
|
||||||
|
assert "frontmost" not in "\n".join(
|
||||||
|
b.text for b in conflict.content if b.type == "text"
|
||||||
|
)
|
||||||
|
recovered = await session.call("list_windows", next_query)
|
||||||
|
assert not recovered.isError and recovered.structuredContent["windows"]
|
||||||
|
|
||||||
|
window = recovered.structuredContent["windows"][0]
|
||||||
|
target = {k: window[k] for k in ("pid", "window_id")}
|
||||||
|
state = await session.call("get_window_state", {**target, "max_elements": 100})
|
||||||
|
assert not state.isError, state
|
||||||
|
assert any(b.type == "image" for b in state.content)
|
||||||
|
assert state.structuredContent["snapshot_id"]
|
||||||
|
assert state.structuredContent["tree_markdown"]
|
||||||
|
assert state.structuredContent["artemis"]["source"]["target_id"] == str(
|
||||||
|
target["window_id"]
|
||||||
|
)
|
||||||
|
report = {
|
||||||
|
"app": app,
|
||||||
|
"zero_pid_rejected_by_host_and_adapter": True,
|
||||||
|
"app_only_window_count": len(windows),
|
||||||
|
"null_pid_omitted": True,
|
||||||
|
"conflicting_pid_recovered_without_substitution": True,
|
||||||
|
"screenshot_and_tree": True,
|
||||||
|
}
|
||||||
|
output.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
output.write_text(json.dumps(report, ensure_ascii=False, indent=2), "utf-8")
|
||||||
|
print(json.dumps(report, ensure_ascii=False))
|
||||||
|
finally:
|
||||||
|
await session.aclose()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
parser = argparse.ArgumentParser(__doc__)
|
||||||
|
parser.add_argument("executable", type=Path)
|
||||||
|
parser.add_argument("--app", required=True)
|
||||||
|
parser.add_argument("--output", type=Path, required=True)
|
||||||
|
args = parser.parse_args()
|
||||||
|
asyncio.run(run(args.executable.resolve(), args.app, args.output))
|
||||||
Reference in new issue
Block a user