diff --git a/artemis/agents/flash/flash_runner.md b/artemis/agents/flash/flash_runner.md index f157aea..e91077f 100644 --- a/artemis/agents/flash/flash_runner.md +++ b/artemis/agents/flash/flash_runner.md @@ -7,10 +7,12 @@ # ROLE & OBJECTIVE -You are an autonomous UI execution agent. Accomplish the user's objective using the actual available device and plugin tools. Android actions use normalized 0–1000 coordinates unless drag/action_sequence explicitly selects pixel; plugin tools use their own documented coordinates and element indices. +You are an autonomous UI execution agent. Accomplish the user's objective using the actual available device and plugin tools. Android actions use normalized 0–1000 coordinates unless drag/action_sequence explicitly selects pixel; native selector.bounds always uses original Android screen pixels. Plugin tools use their own documented coordinates and element indices. **Objective: {{ goal }}** +{% if "read_control" in available_tools %}- **原生选择器**:直接复制当前节点的 `node.native_selector` 对象,用于读取、等待、状态确保和数值反馈。列表 `Bounds` 是 0–1000 归一化坐标,不能填入需要 Android 原始像素的 `selector.bounds`,也不能反算取整后的 Bounds。不要把正在变化的时间文本或描述加进身份筛选;要等待的文本放 expected_text。NOT_FOUND 时重新观察并核对选择器,不能据此认定 APP 不提供原生读数;重复 ID 的消歧边界以 native_selector 内的原始像素为准。{% endif %} + {% if "drag" in available_tools %}- **数值目标的闭环**:先观察真实手势区域,为 drag/pinch/touch_gesture 填 `drag.region`,不要把日期栏、导航栏当作时间轴。若原生可读取持久数值,为 drag 或 action_sequence 绑定 `action_sequence.feedback`(唯一稳定 selector、value_format、target_value、tolerance);时分秒用 clock,单位为当天秒数。组合“拖动→暂停”时只在整个序列上绑定 feedback,末尾读稳定值,不能在拖动中途的浮层上核验。已验证的同一持久读数在允许误差内就是此数值条件的证据;日期、暂停等其它要求仍分别核对,不能无依据地改称误差无法判断。连续无效反馈会拒绝第三次同策略输入;放大并重新读刻度、改为足够距离或原生设值,不能靠微改坐标绕过。calibration.tolerance 给出几何误差预算;min_movement_px 只填写实际测得的起效距离,不猜固定阈值。没有可读的持久数值时保留视觉证据与不确定项,不捏造 selector 或读数。{% endif %} {% if "control_action" in available_tools %}- **先状态后动作**:对于暂停、勾选等目标状态,先 read_control 确认其真实原生语义,再用 control_action.ensure_state;已符合时跳过,否则只操作一次并有界核验。可按观察到的 checked/selected、text 或 content_desc 判断;不能假设 selected 就表示暂停。selector 不包含动作会改变的文本/描述。拖动可能恢复播放,把确保暂停放在拖动后同一动作序列中;状态核验失败时重新观察,不重复盲点切换。{% endif %} diff --git a/artemis/mcp/action_specs.py b/artemis/mcp/action_specs.py index fab5244..93248b6 100644 --- a/artemis/mcp/action_specs.py +++ b/artemis/mcp/action_specs.py @@ -878,9 +878,13 @@ _NATIVE_DESCRIPTIONS = { ), "read_control": ( "[OBSERVE] Read ONE live native control by exact AND selector copied from the current UI. " - "Use resource_id/text/content_desc/class_name and optional package_name/bounds to disambiguate. " + "Copy its native_selector object unchanged; it uses stable identity and original pixel bounds " + "only when needed to disambiguate. UI Bounds are normalized 0..1000 and MUST NOT be used " + "as selector.bounds (exact Android pixels); rounded UI bounds cannot be converted back exactly. " + "Do not add the changing text/content_desc you want to read or wait for as an identity filter. " "Returns current range/value, supported native actions, editable selection (UTF-16 offsets), " "identity token and visible subtree content_signature. Missing or ambiguous controls fail explicitly. " + "On NOT_FOUND refresh the UI and selector; it does not prove that native reading is unavailable. " "Custom canvas controls may expose no range/actions; then use calibrated gestures and visual verification." ), "control_action": ( diff --git a/artemis/mcp/native_actions.py b/artemis/mcp/native_actions.py index 3815550..1dc1d97 100644 --- a/artemis/mcp/native_actions.py +++ b/artemis/mcp/native_actions.py @@ -26,11 +26,17 @@ class ControlSelector(GestureModel): model_config = ConfigDict(str_strip_whitespace=False) resource_id: ShortText | None = None - text: Annotated[str, StringConstraints(strip_whitespace=False, max_length=4096)] | None = None + text: Annotated[str, StringConstraints(strip_whitespace=False, max_length=4096)] | None = Field( + default=None, + description="Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.", + ) content_desc: ShortText | None = None class_name: ShortText | None = None package_name: ShortText | None = None - bounds: Bounds | None = None + bounds: Bounds | None = Field( + default=None, + description="Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.", + ) @model_validator(mode="after") def meaningful_selector(self): @@ -44,7 +50,9 @@ class ControlSelector(GestureModel): class ReadControlRequest(GestureModel): - selector: ControlSelector + selector: ControlSelector = Field( + description="Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.", + ) ControlAction = Literal[ diff --git a/artemis/utils/ui_filter.py b/artemis/utils/ui_filter.py index b5dc898..cc96fec 100644 --- a/artemis/utils/ui_filter.py +++ b/artemis/utils/ui_filter.py @@ -389,6 +389,12 @@ def _filter_and_prune( bounds_data = _parse_bounds(node.get("bounds")) if bounds_data: + # Hit-test bounds may be clipped or merged later; native identity must + # retain the exact rectangle originally reported by Android. + node.setdefault( + "native_bounds_px", + [bounds_data[key] for key in ("left", "top", "right", "bottom")], + ) # We no longer check _intersects_screen to return [] immediately, # to ensure zero info loss. Out-of-bounds elements will be clipped to a point. diff --git a/artemis/utils/visualization.py b/artemis/utils/visualization.py index 30f2630..4872331 100644 --- a/artemis/utils/visualization.py +++ b/artemis/utils/visualization.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from collections import Counter import io import json import os @@ -351,6 +352,12 @@ def format_minimal_list_with_elements( idx = 1 added_items = [] # list of tuples: (text, cx, cy) items_bounds = [] # list of tuples: (idx, (left, top, right, bottom)) + resource_counts = Counter( + resource_id + for node in fused_xml + if isinstance(resource_id := node.get("resource-id") or node.get("resource_id"), str) + and resource_id + ) def normalize_bounds(left: int, top: int, right: int, bottom: int) -> str: n_left = int(max(0, min(1000, left * 1000 / width))) @@ -450,6 +457,31 @@ def format_minimal_list_with_elements( line += f" | range: {json.dumps(native_state['range'], ensure_ascii=False)}" if native_state.get("actions"): line += f" | actions: {','.join(native_state['actions'])}" + # Displayed Bounds are normalized and lossy. Native matching needs + # exact pixels; changing readouts are values, not identity filters. + native_selector: dict[str, Any] = { + key: value + for key, value in ( + ("resource_id", node.get("resource-id") or node.get("resource_id")), + ("class_name", node.get("class") or node.get("class_name")), + ("package_name", node.get("package") or node.get("package_name")), + ) + if isinstance(value, str) and 0 < len(value) <= 512 + } + if ( + native_selector.get("resource_id") or native_selector.get("class_name") + ) and "[OCR]" not in str(node.get("class", "")): + resource_id = native_selector.get("resource_id") + if not resource_id or resource_counts[resource_id] > 1: + native_selector["bounds"] = node.get("native_bounds_px") or [ + left, + top, + right, + bottom, + ] + line += f" | native_selector: {json.dumps(native_selector, ensure_ascii=False)}" + else: + native_selector = {} if error: line += f" | Error: '{error}'" if value_evidence: @@ -474,6 +506,8 @@ def format_minimal_list_with_elements( element["error"] = error element.update(value_evidence) element.update(native_state) + if native_selector: + element["native_selector"] = native_selector if native_control: element["content_desc"] = node.get("content-desc") or "" element["native_text"] = node.get("text") or "" diff --git a/docs/android-native-actions.md b/docs/android-native-actions.md index f16e34d..4251d7f 100644 --- a/docs/android-native-actions.md +++ b/docs/android-native-actions.md @@ -19,6 +19,8 @@ Flash、普通 Pro 和探索共用同一套动作参数、助手执行入口与 先 `read_control`,再按实际 `actions` 操作。范围设值使用 `action="set_progress", value=...`,严格检查实际 min/max;自绘时间轴没有 range 时不能直接设置秒数。`expected_token` 可绑定之前的读数;目标文本、位置或身份变化时返回 `STALE`,由模型重新观察,避免使用旧目标。 +观察列表中的 `native_selector` 是可直接复制到读取、等待、状态确保及数值反馈中的原生选择器。它优先使用资源 ID、类名和包名,不把当前读数或描述当作身份;重复 ID 或没有 ID 时才附加原始像素边界。列表 `Bounds` 是用于手势的 0–1000 归一化坐标,而 `selector.bounds` 必须是 Android 原始屏幕像素,不能混用,也不能将有损取整的列表坐标反算回去。可点击区域经过裁剪或合并时仍保留原生节点的原始边界。选择器仍逐项精确 AND 匹配;布局改变、原生窗口中有其它同名节点时可能失效或歧义,需要重新观察。`NOT_FOUND` 只表示本次选择器未匹配,不能直接推断控件没有原生语义。等待的目标文本放 `expected_text`,不要塞入身份筛选。 + 文本动作包括 `set_text`、`set_selection`、`replace_selection`、`ime_enter`。选择区使用 Android UTF-16 索引,不能切开代理对;例如 `A😀B中文` 中 emoji 的范围是 `[1,3]`。`replace_selection` 保留其它文字并将光标移至插入末尾;空白与空字符串按原样传入。密码字段不公开原文,不支持基于隐藏原文替换选区。`ime_enter` 仅提交控件支持的 IME 动作,不代表业务表单已成功提交。 ## 手势、组合与验证 diff --git a/skills/artemis-use/references/native-actions.md b/skills/artemis-use/references/native-actions.md index 5bd39b7..ca86ade 100644 --- a/skills/artemis-use/references/native-actions.md +++ b/skills/artemis-use/references/native-actions.md @@ -19,6 +19,8 @@ Flash、普通 Pro 和探索共用同一套动作参数、助手执行入口与 先 `read_control`,再按实际 `actions` 操作。范围设值使用 `action="set_progress", value=...`,严格检查实际 min/max;自绘时间轴没有 range 时不能直接设置秒数。`expected_token` 可绑定之前的读数;目标文本、位置或身份变化时返回 `STALE`,由模型重新观察,避免使用旧目标。 +观察列表中的 `native_selector` 是可直接复制到读取、等待、状态确保及数值反馈中的原生选择器。它优先使用资源 ID、类名和包名,不把当前读数或描述当作身份;重复 ID 或没有 ID 时才附加原始像素边界。列表 `Bounds` 是用于手势的 0–1000 归一化坐标,而 `selector.bounds` 必须是 Android 原始屏幕像素,不能混用,也不能将有损取整的列表坐标反算回去。可点击区域经过裁剪或合并时仍保留原生节点的原始边界。选择器仍逐项精确 AND 匹配;布局改变、原生窗口中有其它同名节点时可能失效或歧义,需要重新观察。`NOT_FOUND` 只表示本次选择器未匹配,不能直接推断控件没有原生语义。等待的目标文本放 `expected_text`,不要塞入身份筛选。 + 文本动作包括 `set_text`、`set_selection`、`replace_selection`、`ime_enter`。选择区使用 Android UTF-16 索引,不能切开代理对;例如 `A😀B中文` 中 emoji 的范围是 `[1,3]`。`replace_selection` 保留其它文字并将光标移至插入末尾;空白与空字符串按原样传入。密码字段不公开原文,不支持基于隐藏原文替换选区。`ime_enter` 仅提交控件支持的 IME 动作,不代表业务表单已成功提交。 ## 手势、组合与验证 diff --git a/tests/fixtures/action_surfaces/action_server_manifest.json b/tests/fixtures/action_surfaces/action_server_manifest.json index 084d18b..e8dc9e1 100644 --- a/tests/fixtures/action_surfaces/action_server_manifest.json +++ b/tests/fixtures/action_surfaces/action_server_manifest.json @@ -186,6 +186,7 @@ } ], "default": null, + "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.", "title": "Text" }, "content_desc": { @@ -245,6 +246,7 @@ } ], "default": null, + "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.", "title": "Bounds" } }, @@ -623,7 +625,8 @@ "additionalProperties": false, "properties": { "selector": { - "$ref": "#/$defs/ControlSelector" + "$ref": "#/$defs/ControlSelector", + "description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds." }, "action": { "enum": [ @@ -917,7 +920,8 @@ "additionalProperties": false, "properties": { "selector": { - "$ref": "#/$defs/ControlSelector" + "$ref": "#/$defs/ControlSelector", + "description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds." }, "state": { "enum": [ @@ -1177,6 +1181,7 @@ } ], "default": null, + "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.", "title": "Text" }, "content_desc": { @@ -1236,6 +1241,7 @@ } ], "default": null, + "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.", "title": "Bounds" } }, @@ -3306,7 +3312,7 @@ } }, "read_control": { - "description": "[OBSERVE] Read ONE live native control by exact AND selector copied from the current UI. Use resource_id/text/content_desc/class_name and optional package_name/bounds to disambiguate. Returns current range/value, supported native actions, editable selection (UTF-16 offsets), identity token and visible subtree content_signature. Missing or ambiguous controls fail explicitly. Custom canvas controls may expose no range/actions; then use calibrated gestures and visual verification.", + "description": "[OBSERVE] Read ONE live native control by exact AND selector copied from the current UI. Copy its native_selector object unchanged; it uses stable identity and original pixel bounds only when needed to disambiguate. UI Bounds are normalized 0..1000 and MUST NOT be used as selector.bounds (exact Android pixels); rounded UI bounds cannot be converted back exactly. Do not add the changing text/content_desc you want to read or wait for as an identity filter. Returns current range/value, supported native actions, editable selection (UTF-16 offsets), identity token and visible subtree content_signature. Missing or ambiguous controls fail explicitly. On NOT_FOUND refresh the UI and selector; it does not prove that native reading is unavailable. Custom canvas controls may expose no range/actions; then use calibrated gestures and visual verification.", "inputSchema": { "$defs": { "ControlSelector": { @@ -3338,6 +3344,7 @@ } ], "default": null, + "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.", "title": "Text" }, "content_desc": { @@ -3397,6 +3404,7 @@ } ], "default": null, + "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.", "title": "Bounds" } }, @@ -3591,6 +3599,7 @@ } ], "default": null, + "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.", "title": "Text" }, "content_desc": { @@ -3650,6 +3659,7 @@ } ], "default": null, + "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.", "title": "Bounds" } }, @@ -3894,6 +3904,7 @@ } ], "default": null, + "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.", "title": "Text" }, "content_desc": { @@ -3953,6 +3964,7 @@ } ], "default": null, + "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.", "title": "Bounds" } }, @@ -4164,6 +4176,7 @@ } ], "default": null, + "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.", "title": "Text" }, "content_desc": { @@ -4223,6 +4236,7 @@ } ], "default": null, + "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.", "title": "Bounds" } }, diff --git a/tests/fixtures/action_surfaces/operator_shells.json b/tests/fixtures/action_surfaces/operator_shells.json index 35b9db6..cfd2b5e 100644 --- a/tests/fixtures/action_surfaces/operator_shells.json +++ b/tests/fixtures/action_surfaces/operator_shells.json @@ -640,7 +640,7 @@ "properties": { "selector": { "additionalProperties": false, - "description": "Every supplied attribute is an exact AND match against a fresh native node.", + "description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.", "properties": { "resource_id": { "anyOf": [ @@ -667,6 +667,7 @@ } ], "default": null, + "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.", "title": "Text" }, "content_desc": { @@ -726,6 +727,7 @@ } ], "default": null, + "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.", "title": "Bounds" } }, @@ -908,7 +910,7 @@ "properties": { "selector": { "additionalProperties": false, - "description": "Every supplied attribute is an exact AND match against a fresh native node.", + "description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.", "properties": { "resource_id": { "anyOf": [ @@ -935,6 +937,7 @@ } ], "default": null, + "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.", "title": "Text" }, "content_desc": { @@ -994,6 +997,7 @@ } ], "default": null, + "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.", "title": "Bounds" } }, @@ -1124,6 +1128,7 @@ } ], "default": null, + "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.", "title": "Text" }, "content_desc": { @@ -1183,6 +1188,7 @@ } ], "default": null, + "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.", "title": "Bounds" } }, @@ -1343,6 +1349,7 @@ } ], "default": null, + "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.", "title": "Text" }, "content_desc": { @@ -1402,6 +1409,7 @@ } ], "default": null, + "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.", "title": "Bounds" } }, @@ -1783,12 +1791,12 @@ "type": "function", "function": { "name": "read_control", - "description": "[OBSERVE] Read ONE live native control by exact AND selector copied from the current UI. Use resource_id/text/content_desc/class_name and optional package_name/bounds to disambiguate. Returns current range/value, supported native actions, editable selection (UTF-16 offsets), identity token and visible subtree content_signature. Missing or ambiguous controls fail explicitly. Custom canvas controls may expose no range/actions; then use calibrated gestures and visual verification.", + "description": "[OBSERVE] Read ONE live native control by exact AND selector copied from the current UI. Copy its native_selector object unchanged; it uses stable identity and original pixel bounds only when needed to disambiguate. UI Bounds are normalized 0..1000 and MUST NOT be used as selector.bounds (exact Android pixels); rounded UI bounds cannot be converted back exactly. Do not add the changing text/content_desc you want to read or wait for as an identity filter. Returns current range/value, supported native actions, editable selection (UTF-16 offsets), identity token and visible subtree content_signature. Missing or ambiguous controls fail explicitly. On NOT_FOUND refresh the UI and selector; it does not prove that native reading is unavailable. Custom canvas controls may expose no range/actions; then use calibrated gestures and visual verification.", "parameters": { "properties": { "selector": { "additionalProperties": false, - "description": "Every supplied attribute is an exact AND match against a fresh native node.", + "description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.", "properties": { "resource_id": { "anyOf": [ @@ -1813,7 +1821,8 @@ "type": "null" } ], - "default": null + "default": null, + "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead." }, "content_desc": { "anyOf": [ @@ -1868,7 +1877,8 @@ "type": "null" } ], - "default": null + "default": null, + "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector." } }, "type": "object" @@ -1890,7 +1900,7 @@ "properties": { "selector": { "additionalProperties": false, - "description": "Every supplied attribute is an exact AND match against a fresh native node.", + "description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.", "properties": { "resource_id": { "anyOf": [ @@ -1915,7 +1925,8 @@ "type": "null" } ], - "default": null + "default": null, + "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead." }, "content_desc": { "anyOf": [ @@ -1970,7 +1981,8 @@ "type": "null" } ], - "default": null + "default": null, + "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector." } }, "type": "object" @@ -2141,7 +2153,7 @@ "properties": { "selector": { "additionalProperties": false, - "description": "Every supplied attribute is an exact AND match against a fresh native node.", + "description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.", "properties": { "resource_id": { "anyOf": [ @@ -2166,7 +2178,8 @@ "type": "null" } ], - "default": null + "default": null, + "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead." }, "content_desc": { "anyOf": [ @@ -2221,7 +2234,8 @@ "type": "null" } ], - "default": null + "default": null, + "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector." } }, "type": "object" @@ -2332,7 +2346,8 @@ "type": "null" } ], - "default": null + "default": null, + "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead." }, "content_desc": { "anyOf": [ @@ -2387,7 +2402,8 @@ "type": "null" } ], - "default": null + "default": null, + "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector." } }, "type": "object" @@ -2419,7 +2435,8 @@ "type": "null" } ], - "default": null + "default": null, + "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead." }, "content_desc": { "anyOf": [ @@ -2474,7 +2491,8 @@ "type": "null" } ], - "default": null + "default": null, + "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector." } }, "type": "object" diff --git a/tests/fixtures/action_surfaces/tool_declarations.json b/tests/fixtures/action_surfaces/tool_declarations.json index 1918f88..6aa6bc1 100644 --- a/tests/fixtures/action_surfaces/tool_declarations.json +++ b/tests/fixtures/action_surfaces/tool_declarations.json @@ -335,6 +335,7 @@ } ], "default": null, + "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.", "title": "Text" }, "content_desc": { @@ -394,6 +395,7 @@ } ], "default": null, + "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.", "title": "Bounds" } }, @@ -1172,7 +1174,7 @@ "properties": { "selector": { "additionalProperties": false, - "description": "Every supplied attribute is an exact AND match against a fresh native node.", + "description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.", "properties": { "resource_id": { "anyOf": [ @@ -1199,6 +1201,7 @@ } ], "default": null, + "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.", "title": "Text" }, "content_desc": { @@ -1258,6 +1261,7 @@ } ], "default": null, + "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.", "title": "Bounds" } }, @@ -1440,7 +1444,7 @@ "properties": { "selector": { "additionalProperties": false, - "description": "Every supplied attribute is an exact AND match against a fresh native node.", + "description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.", "properties": { "resource_id": { "anyOf": [ @@ -1467,6 +1471,7 @@ } ], "default": null, + "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.", "title": "Text" }, "content_desc": { @@ -1526,6 +1531,7 @@ } ], "default": null, + "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.", "title": "Bounds" } }, @@ -1656,6 +1662,7 @@ } ], "default": null, + "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.", "title": "Text" }, "content_desc": { @@ -1715,6 +1722,7 @@ } ], "default": null, + "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.", "title": "Bounds" } }, @@ -2044,12 +2052,12 @@ "type": "function", "function": { "name": "read_control", - "description": "[OBSERVE] Read ONE live native control by exact AND selector copied from the current UI. Use resource_id/text/content_desc/class_name and optional package_name/bounds to disambiguate. Returns current range/value, supported native actions, editable selection (UTF-16 offsets), identity token and visible subtree content_signature. Missing or ambiguous controls fail explicitly. Custom canvas controls may expose no range/actions; then use calibrated gestures and visual verification.", + "description": "[OBSERVE] Read ONE live native control by exact AND selector copied from the current UI. Copy its native_selector object unchanged; it uses stable identity and original pixel bounds only when needed to disambiguate. UI Bounds are normalized 0..1000 and MUST NOT be used as selector.bounds (exact Android pixels); rounded UI bounds cannot be converted back exactly. Do not add the changing text/content_desc you want to read or wait for as an identity filter. Returns current range/value, supported native actions, editable selection (UTF-16 offsets), identity token and visible subtree content_signature. Missing or ambiguous controls fail explicitly. On NOT_FOUND refresh the UI and selector; it does not prove that native reading is unavailable. Custom canvas controls may expose no range/actions; then use calibrated gestures and visual verification.", "parameters": { "properties": { "selector": { "additionalProperties": false, - "description": "Every supplied attribute is an exact AND match against a fresh native node.", + "description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.", "properties": { "resource_id": { "anyOf": [ @@ -2074,7 +2082,8 @@ "type": "null" } ], - "default": null + "default": null, + "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead." }, "content_desc": { "anyOf": [ @@ -2129,7 +2138,8 @@ "type": "null" } ], - "default": null + "default": null, + "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector." } }, "type": "object" @@ -2151,7 +2161,7 @@ "properties": { "selector": { "additionalProperties": false, - "description": "Every supplied attribute is an exact AND match against a fresh native node.", + "description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.", "properties": { "resource_id": { "anyOf": [ @@ -2176,7 +2186,8 @@ "type": "null" } ], - "default": null + "default": null, + "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead." }, "content_desc": { "anyOf": [ @@ -2231,7 +2242,8 @@ "type": "null" } ], - "default": null + "default": null, + "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector." } }, "type": "object" @@ -2402,7 +2414,7 @@ "properties": { "selector": { "additionalProperties": false, - "description": "Every supplied attribute is an exact AND match against a fresh native node.", + "description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.", "properties": { "resource_id": { "anyOf": [ @@ -2427,7 +2439,8 @@ "type": "null" } ], - "default": null + "default": null, + "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead." }, "content_desc": { "anyOf": [ @@ -2482,7 +2495,8 @@ "type": "null" } ], - "default": null + "default": null, + "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector." } }, "type": "object" @@ -2593,7 +2607,8 @@ "type": "null" } ], - "default": null + "default": null, + "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead." }, "content_desc": { "anyOf": [ @@ -2648,7 +2663,8 @@ "type": "null" } ], - "default": null + "default": null, + "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector." } }, "type": "object" @@ -2680,7 +2696,8 @@ "type": "null" } ], - "default": null + "default": null, + "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead." }, "content_desc": { "anyOf": [ @@ -2735,7 +2752,8 @@ "type": "null" } ], - "default": null + "default": null, + "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector." } }, "type": "object" diff --git a/tests/unit/agents/test_flash_runner.py b/tests/unit/agents/test_flash_runner.py index 721d397..482a2ad 100644 --- a/tests/unit/agents/test_flash_runner.py +++ b/tests/unit/agents/test_flash_runner.py @@ -136,11 +136,16 @@ async def test_fallback_observation_renders_the_indexed_ui_list(mock_context, tm assert isinstance(exec_result.ui_elements_text, str) assert exec_result.ui_elements_text == ( "[1] Text: 'Settings' | Bounds: [37,50][962,112] | resource_id: 'android:id/title'" + ' | native_selector: {"resource_id": "android:id/title", "class_name": "android.widget.TextView"}' ) assert "{'text'" not in exec_result.ui_elements_text # State refreshed exactly as the executor's success path does. assert state.indexed_elements[0]["index"] == 1 assert state.indexed_elements[0]["text"] == "Settings" + assert state.indexed_elements[0]["native_selector"] == { + "resource_id": "android:id/title", + "class_name": "android.widget.TextView", + } assert state.indexed_points == [state.indexed_elements[0]["center"]] assert state.latest_screenshot != "old.jpg" assert state.latest_screenshot.startswith(str(tmp_path)) diff --git a/tests/unit/mcp/test_native_observation_selectors.py b/tests/unit/mcp/test_native_observation_selectors.py new file mode 100644 index 0000000..575614d --- /dev/null +++ b/tests/unit/mcp/test_native_observation_selectors.py @@ -0,0 +1,152 @@ +"""Model-visible selectors must round-trip to exact native matching safely.""" + +from copy import deepcopy +import json +from unittest.mock import AsyncMock + +import pytest + +from artemis.mcp.native_actions import ControlSelector +from artemis.mcp.native_executor import execute_native +from artemis.utils.ui_filter import filter_ui_hierarchy +from artemis.utils.visualization import format_minimal_list_with_elements +from tests.unit.utils.test_native_control_state import encode_node, hierarchy_node + + +def native_node(raw): + return { + "resource_id": raw.get("resource-id", ""), + "class_name": raw["class"], + "package_name": raw["package"], + "bounds": [30, 549, 110, 629], + "text": "17:42:20", + "content_desc": "current 17:42:20", + "visible": True, + "actions": ["set_selection"], + } + + +def exact_native(nodes): + async def inspect(command, args, *, timeout=None): + assert command == "inspect_control", "A failed read must never dispatch input" + matches = [ + node + for node in nodes + if all(node.get(key) == value for key, value in args["selector"].items()) + ] + if len(matches) == 1: + return {"success": True, "execution": "observed", "code": "OK", "node": matches[0]} + return { + "success": False, + "execution": "not_started", + "code": "NOT_FOUND" if not matches else "AMBIGUOUS", + "error": "Exact native selector did not resolve uniquely", + } + + return AsyncMock(side_effect=inspect) + + +@pytest.mark.parametrize("encoding", ["xml", "json"]) +@pytest.mark.parametrize("screen", [(720, 1600), (1080, 2400), (1600, 720)]) +@pytest.mark.asyncio +async def test_copy_visible_selector_reads_a_changing_value_without_lossy_ui_bounds( + encoding, screen +): + raw = hierarchy_node( + text="17:42:17", + **{ + "class": "android.widget.TextView", + "content-desc": "current 17:42:17", + "bounds": "[30,549][110,629]", + "range": None, + "actions": ["set_selection"], + }, + ) + source = encode_node(raw, encoding) + original = deepcopy(source) + text, elements, _ = format_minimal_list_with_elements([source], *screen) + selector = json.loads(text.split(" | native_selector: ", 1)[1]) + assert ( + selector + == elements[0]["native_selector"] + == { + "resource_id": raw["resource-id"], + "class_name": raw["class"], + "package_name": raw["package"], + } + ) + assert source == original + ControlSelector.model_validate(selector) + native = exact_native([native_node(raw)]) + result = await execute_native("read_control", {"selector": selector}, *screen, native) + assert result["success"] and result["node"]["text"] == "17:42:20" + native.assert_awaited_once() + + +@pytest.mark.parametrize("resource_id", ["", "fixture:id/shared"]) +@pytest.mark.asyncio +async def test_no_id_and_repeated_ids_use_original_pixels_to_resolve_each_node(resource_id): + rows = [ + hierarchy_node( + text=f"row {i}", + **{ + "resource-id": resource_id, + "bounds": f"[30,{549 + i * 100}][110,{629 + i * 100}]", + }, + ) + for i in range(2) + ] + _, elements, _ = format_minimal_list_with_elements(rows, 720, 1600) + nodes = [ + native_node(row) | {"bounds": [30, 549 + i * 100, 110, 629 + i * 100]} + for i, row in enumerate(rows) + ] + native = exact_native(nodes) + for element, node in zip(elements, nodes, strict=True): + selector = element["native_selector"] + assert selector["bounds"] == node["bounds"] + result = await execute_native("read_control", {"selector": selector}, 720, 1600, native) + assert result["success"] and result["node"]["bounds"] == node["bounds"] + + +def test_clipped_hit_area_does_not_replace_the_original_native_identity_bounds(): + raw = hierarchy_node( + text="partly visible", + **{ + "resource-id": "", + "bounds": "[-20,-40][300,60]", + }, + ) + filtered = filter_ui_hierarchy([raw], screen_width=720, screen_height=1600) + _, elements, _ = format_minimal_list_with_elements(filtered, 720, 1600) + assert elements[0]["bounds"] == [0, 0, 300, 60] + assert elements[0]["native_selector"]["bounds"] == [-20, -40, 300, 60] + ControlSelector.model_validate(elements[0]["native_selector"]) + + +def test_ocr_only_geometry_is_not_presented_as_a_native_selector(): + text, elements, _ = format_minimal_list_with_elements( + [{"text": "17:42:17", "class": "[OCR]", "bounds": "[30,549][110,629]"}], + 720, + 1600, + ) + assert "native_selector" not in text + assert "native_selector" not in elements[0] + + +@pytest.mark.parametrize("action", ["read_control", "control_action"]) +@pytest.mark.parametrize("extra", [{"text": "17:42:17"}, {"bounds": [41, 343, 152, 393]}]) +@pytest.mark.asyncio +async def test_explicit_stale_text_or_wrong_bounds_are_not_silently_dropped_or_retried( + action, extra +): + raw = hierarchy_node() + native = exact_native([native_node(raw)]) + request = {"selector": {"resource_id": raw["resource-id"], **extra}} + if action == "control_action": + request["action"] = "click" + result = await execute_native(action, request, 720, 1600, native) + assert not result["success"] and result["code"] == "NOT_FOUND" + assert result["execution"] == "not_started" + assert native.await_args.args[1]["selector"] == request["selector"] + native.assert_awaited_once() diff --git a/tests/unit/test_hierarchy_fidelity.py b/tests/unit/test_hierarchy_fidelity.py index 58c10b0..ed2eddc 100644 --- a/tests/unit/test_hierarchy_fidelity.py +++ b/tests/unit/test_hierarchy_fidelity.py @@ -78,10 +78,15 @@ def test_minimal_list_shows_hint_for_empty_inputs_and_errors(): text, items, labels = format_minimal_list_with_elements(elements[1:], 1000, 1000) lines = text.splitlines() assert ( - lines[0] - == "[1] Hint: 'Search settings' | Bounds: [0,0][100,100] | Editable: true | Value: empty" + lines[0] == "[1] Hint: 'Search settings' | Bounds: [0,0][100,100]" + ' | native_selector: {"class_name": "android.widget.EditText", "package_name": "app", "bounds": [0, 0, 100, 100]}' + " | Editable: true | Value: empty" + ) + assert lines[1] == ( + "[2] Text: 'abc' | Bounds: [0,200][100,300]" + ' | native_selector: {"class_name": "android.widget.TextView", "package_name": "app", "bounds": [0, 200, 100, 300]}' + " | Error: 'Password too short'" ) - assert lines[1] == "[2] Text: 'abc' | Bounds: [0,200][100,300] | Error: 'Password too short'" assert len(lines) == 2 and labels == ["1", "2"] assert items[0]["is_hint"] is True and items[0]["text"] == "Search settings" assert items[1]["error"] == "Password too short"