fix(android): expose stable native selectors with exact pixel bounds

artemis/agents/flash/flash_runner.md

@@ -7,10 +7,12 @@
 
 
 # ROLE & OBJECTIVE
-You are an autonomous UI execution agent. Accomplish the user's objective using the actual available device and plugin tools. Android actions use normalized 0–1000 coordinates unless drag/action_sequence explicitly selects pixel; plugin tools use their own documented coordinates and element indices.
+You are an autonomous UI execution agent. Accomplish the user's objective using the actual available device and plugin tools. Android actions use normalized 0–1000 coordinates unless drag/action_sequence explicitly selects pixel; native selector.bounds always uses original Android screen pixels. Plugin tools use their own documented coordinates and element indices.
 
 **Objective: {{ goal }}**
 
+{% if "read_control" in available_tools %}- **原生选择器**：直接复制当前节点的 `node.native_selector` 对象，用于读取、等待、状态确保和数值反馈。列表 `Bounds` 是 0–1000 归一化坐标，不能填入需要 Android 原始像素的 `selector.bounds`，也不能反算取整后的 Bounds。不要把正在变化的时间文本或描述加进身份筛选；要等待的文本放 expected_text。NOT_FOUND 时重新观察并核对选择器，不能据此认定 APP 不提供原生读数；重复 ID 的消歧边界以 native_selector 内的原始像素为准。{% endif %}
+
 {% if "drag" in available_tools %}- **数值目标的闭环**：先观察真实手势区域，为 drag/pinch/touch_gesture 填 `drag.region`，不要把日期栏、导航栏当作时间轴。若原生可读取持久数值，为 drag 或 action_sequence 绑定 `action_sequence.feedback`（唯一稳定 selector、value_format、target_value、tolerance）；时分秒用 clock，单位为当天秒数。组合“拖动→暂停”时只在整个序列上绑定 feedback，末尾读稳定值，不能在拖动中途的浮层上核验。已验证的同一持久读数在允许误差内就是此数值条件的证据；日期、暂停等其它要求仍分别核对，不能无依据地改称误差无法判断。连续无效反馈会拒绝第三次同策略输入；放大并重新读刻度、改为足够距离或原生设值，不能靠微改坐标绕过。calibration.tolerance 给出几何误差预算；min_movement_px 只填写实际测得的起效距离，不猜固定阈值。没有可读的持久数值时保留视觉证据与不确定项，不捏造 selector 或读数。{% endif %}
 {% if "control_action" in available_tools %}- **先状态后动作**：对于暂停、勾选等目标状态，先 read_control 确认其真实原生语义，再用 control_action.ensure_state；已符合时跳过，否则只操作一次并有界核验。可按观察到的 checked/selected、text 或 content_desc 判断；不能假设 selected 就表示暂停。selector 不包含动作会改变的文本/描述。拖动可能恢复播放，把确保暂停放在拖动后同一动作序列中；状态核验失败时重新观察，不重复盲点切换。{% endif %}
 

artemis/mcp/action_specs.py

@@ -878,9 +878,13 @@ _NATIVE_DESCRIPTIONS = {
     ),
     "read_control": (
         "[OBSERVE] Read ONE live native control by exact AND selector copied from the current UI. "
-        "Use resource_id/text/content_desc/class_name and optional package_name/bounds to disambiguate. "
+        "Copy its native_selector object unchanged; it uses stable identity and original pixel bounds "
+        "only when needed to disambiguate. UI Bounds are normalized 0..1000 and MUST NOT be used "
+        "as selector.bounds (exact Android pixels); rounded UI bounds cannot be converted back exactly. "
+        "Do not add the changing text/content_desc you want to read or wait for as an identity filter. "
         "Returns current range/value, supported native actions, editable selection (UTF-16 offsets), "
         "identity token and visible subtree content_signature. Missing or ambiguous controls fail explicitly. "
+        "On NOT_FOUND refresh the UI and selector; it does not prove that native reading is unavailable. "
         "Custom canvas controls may expose no range/actions; then use calibrated gestures and visual verification."
     ),
     "control_action": (

artemis/mcp/native_actions.py

@@ -26,11 +26,17 @@ class ControlSelector(GestureModel):
     model_config = ConfigDict(str_strip_whitespace=False)
 
     resource_id: ShortText | None = None
-    text: Annotated[str, StringConstraints(strip_whitespace=False, max_length=4096)] | None = None
+    text: Annotated[str, StringConstraints(strip_whitespace=False, max_length=4096)] | None = Field(
+        default=None,
+        description="Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
+    )
     content_desc: ShortText | None = None
     class_name: ShortText | None = None
     package_name: ShortText | None = None
-    bounds: Bounds | None = None
+    bounds: Bounds | None = Field(
+        default=None,
+        description="Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
+    )
 
     @model_validator(mode="after")
     def meaningful_selector(self):
@@ -44,7 +50,9 @@ class ControlSelector(GestureModel):
 
 
 class ReadControlRequest(GestureModel):
-    selector: ControlSelector
+    selector: ControlSelector = Field(
+        description="Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.",
+    )
 
 
 ControlAction = Literal[

artemis/utils/ui_filter.py

@@ -389,6 +389,12 @@ def _filter_and_prune(
     bounds_data = _parse_bounds(node.get("bounds"))
 
     if bounds_data:
+        # Hit-test bounds may be clipped or merged later; native identity must
+        # retain the exact rectangle originally reported by Android.
+        node.setdefault(
+            "native_bounds_px",
+            [bounds_data[key] for key in ("left", "top", "right", "bottom")],
+        )
         # We no longer check _intersects_screen to return [] immediately,
         # to ensure zero info loss. Out-of-bounds elements will be clipped to a point.
 

artemis/utils/visualization.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from collections import Counter
 import io
 import json
 import os
@@ -351,6 +352,12 @@ def format_minimal_list_with_elements(
     idx = 1
     added_items = []  # list of tuples: (text, cx, cy)
     items_bounds = []  # list of tuples: (idx, (left, top, right, bottom))
+    resource_counts = Counter(
+        resource_id
+        for node in fused_xml
+        if isinstance(resource_id := node.get("resource-id") or node.get("resource_id"), str)
+        and resource_id
+    )
 
     def normalize_bounds(left: int, top: int, right: int, bottom: int) -> str:
         n_left = int(max(0, min(1000, left * 1000 / width)))
@@ -450,6 +457,31 @@ def format_minimal_list_with_elements(
                     line += f" | range: {json.dumps(native_state['range'], ensure_ascii=False)}"
                 if native_state.get("actions"):
                     line += f" | actions: {','.join(native_state['actions'])}"
+            # Displayed Bounds are normalized and lossy. Native matching needs
+            # exact pixels; changing readouts are values, not identity filters.
+            native_selector: dict[str, Any] = {
+                key: value
+                for key, value in (
+                    ("resource_id", node.get("resource-id") or node.get("resource_id")),
+                    ("class_name", node.get("class") or node.get("class_name")),
+                    ("package_name", node.get("package") or node.get("package_name")),
+                )
+                if isinstance(value, str) and 0 < len(value) <= 512
+            }
+            if (
+                native_selector.get("resource_id") or native_selector.get("class_name")
+            ) and "[OCR]" not in str(node.get("class", "")):
+                resource_id = native_selector.get("resource_id")
+                if not resource_id or resource_counts[resource_id] > 1:
+                    native_selector["bounds"] = node.get("native_bounds_px") or [
+                        left,
+                        top,
+                        right,
+                        bottom,
+                    ]
+                line += f" | native_selector: {json.dumps(native_selector, ensure_ascii=False)}"
+            else:
+                native_selector = {}
             if error:
                 line += f" | Error: '{error}'"
             if value_evidence:
@@ -474,6 +506,8 @@ def format_minimal_list_with_elements(
                 element["error"] = error
             element.update(value_evidence)
             element.update(native_state)
+            if native_selector:
+                element["native_selector"] = native_selector
             if native_control:
                 element["content_desc"] = node.get("content-desc") or ""
                 element["native_text"] = node.get("text") or ""

docs/android-native-actions.md

@@ -19,6 +19,8 @@ Flash、普通 Pro 和探索共用同一套动作参数、助手执行入口与
 
 先 `read_control`，再按实际 `actions` 操作。范围设值使用 `action="set_progress", value=...`，严格检查实际 min/max；自绘时间轴没有 range 时不能直接设置秒数。`expected_token` 可绑定之前的读数；目标文本、位置或身份变化时返回 `STALE`，由模型重新观察，避免使用旧目标。
 
+观察列表中的 `native_selector` 是可直接复制到读取、等待、状态确保及数值反馈中的原生选择器。它优先使用资源 ID、类名和包名，不把当前读数或描述当作身份；重复 ID 或没有 ID 时才附加原始像素边界。列表 `Bounds` 是用于手势的 0–1000 归一化坐标，而 `selector.bounds` 必须是 Android 原始屏幕像素，不能混用，也不能将有损取整的列表坐标反算回去。可点击区域经过裁剪或合并时仍保留原生节点的原始边界。选择器仍逐项精确 AND 匹配；布局改变、原生窗口中有其它同名节点时可能失效或歧义，需要重新观察。`NOT_FOUND` 只表示本次选择器未匹配，不能直接推断控件没有原生语义。等待的目标文本放 `expected_text`，不要塞入身份筛选。
+
 文本动作包括 `set_text`、`set_selection`、`replace_selection`、`ime_enter`。选择区使用 Android UTF-16 索引，不能切开代理对；例如 `A😀B中文` 中 emoji 的范围是 `[1,3]`。`replace_selection` 保留其它文字并将光标移至插入末尾；空白与空字符串按原样传入。密码字段不公开原文，不支持基于隐藏原文替换选区。`ime_enter` 仅提交控件支持的 IME 动作，不代表业务表单已成功提交。
 
 ## 手势、组合与验证

skills/artemis-use/references/native-actions.md

@@ -19,6 +19,8 @@ Flash、普通 Pro 和探索共用同一套动作参数、助手执行入口与
 
 先 `read_control`，再按实际 `actions` 操作。范围设值使用 `action="set_progress", value=...`，严格检查实际 min/max；自绘时间轴没有 range 时不能直接设置秒数。`expected_token` 可绑定之前的读数；目标文本、位置或身份变化时返回 `STALE`，由模型重新观察，避免使用旧目标。
 
+观察列表中的 `native_selector` 是可直接复制到读取、等待、状态确保及数值反馈中的原生选择器。它优先使用资源 ID、类名和包名，不把当前读数或描述当作身份；重复 ID 或没有 ID 时才附加原始像素边界。列表 `Bounds` 是用于手势的 0–1000 归一化坐标，而 `selector.bounds` 必须是 Android 原始屏幕像素，不能混用，也不能将有损取整的列表坐标反算回去。可点击区域经过裁剪或合并时仍保留原生节点的原始边界。选择器仍逐项精确 AND 匹配；布局改变、原生窗口中有其它同名节点时可能失效或歧义，需要重新观察。`NOT_FOUND` 只表示本次选择器未匹配，不能直接推断控件没有原生语义。等待的目标文本放 `expected_text`，不要塞入身份筛选。
+
 文本动作包括 `set_text`、`set_selection`、`replace_selection`、`ime_enter`。选择区使用 Android UTF-16 索引，不能切开代理对；例如 `A😀B中文` 中 emoji 的范围是 `[1,3]`。`replace_selection` 保留其它文字并将光标移至插入末尾；空白与空字符串按原样传入。密码字段不公开原文，不支持基于隐藏原文替换选区。`ime_enter` 仅提交控件支持的 IME 动作，不代表业务表单已成功提交。
 
 ## 手势、组合与验证

tests/fixtures/action_surfaces/action_server_manifest.json

@@ -186,6 +186,7 @@
                 }
               ],
               "default": null,
+              "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
               "title": "Text"
             },
             "content_desc": {
@@ -245,6 +246,7 @@
                 }
               ],
               "default": null,
+              "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
               "title": "Bounds"
             }
           },
@@ -623,7 +625,8 @@
           "additionalProperties": false,
           "properties": {
             "selector": {
-              "$ref": "#/$defs/ControlSelector"
+              "$ref": "#/$defs/ControlSelector",
+              "description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds."
             },
             "action": {
               "enum": [
@@ -917,7 +920,8 @@
           "additionalProperties": false,
           "properties": {
             "selector": {
-              "$ref": "#/$defs/ControlSelector"
+              "$ref": "#/$defs/ControlSelector",
+              "description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds."
             },
             "state": {
               "enum": [
@@ -1177,6 +1181,7 @@
                 }
               ],
               "default": null,
+              "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
               "title": "Text"
             },
             "content_desc": {
@@ -1236,6 +1241,7 @@
                 }
               ],
               "default": null,
+              "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
               "title": "Bounds"
             }
           },
@@ -3306,7 +3312,7 @@
     }
   },
   "read_control": {
-    "description": "[OBSERVE] Read ONE live native control by exact AND selector copied from the current UI. Use resource_id/text/content_desc/class_name and optional package_name/bounds to disambiguate. Returns current range/value, supported native actions, editable selection (UTF-16 offsets), identity token and visible subtree content_signature. Missing or ambiguous controls fail explicitly. Custom canvas controls may expose no range/actions; then use calibrated gestures and visual verification.",
+    "description": "[OBSERVE] Read ONE live native control by exact AND selector copied from the current UI. Copy its native_selector object unchanged; it uses stable identity and original pixel bounds only when needed to disambiguate. UI Bounds are normalized 0..1000 and MUST NOT be used as selector.bounds (exact Android pixels); rounded UI bounds cannot be converted back exactly. Do not add the changing text/content_desc you want to read or wait for as an identity filter. Returns current range/value, supported native actions, editable selection (UTF-16 offsets), identity token and visible subtree content_signature. Missing or ambiguous controls fail explicitly. On NOT_FOUND refresh the UI and selector; it does not prove that native reading is unavailable. Custom canvas controls may expose no range/actions; then use calibrated gestures and visual verification.",
     "inputSchema": {
       "$defs": {
         "ControlSelector": {
@@ -3338,6 +3344,7 @@
                 }
               ],
               "default": null,
+              "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
               "title": "Text"
             },
             "content_desc": {
@@ -3397,6 +3404,7 @@
                 }
               ],
               "default": null,
+              "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
               "title": "Bounds"
             }
           },
@@ -3591,6 +3599,7 @@
                 }
               ],
               "default": null,
+              "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
               "title": "Text"
             },
             "content_desc": {
@@ -3650,6 +3659,7 @@
                 }
               ],
               "default": null,
+              "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
               "title": "Bounds"
             }
           },
@@ -3894,6 +3904,7 @@
                 }
               ],
               "default": null,
+              "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
               "title": "Text"
             },
             "content_desc": {
@@ -3953,6 +3964,7 @@
                 }
               ],
               "default": null,
+              "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
               "title": "Bounds"
             }
           },
@@ -4164,6 +4176,7 @@
                 }
               ],
               "default": null,
+              "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
               "title": "Text"
             },
             "content_desc": {
@@ -4223,6 +4236,7 @@
                 }
               ],
               "default": null,
+              "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
               "title": "Bounds"
             }
           },

tests/fixtures/action_surfaces/operator_shells.json

@@ -640,7 +640,7 @@
                       "properties": {
                         "selector": {
                           "additionalProperties": false,
-                          "description": "Every supplied attribute is an exact AND match against a fresh native node.",
+                          "description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.",
                           "properties": {
                             "resource_id": {
                               "anyOf": [
@@ -667,6 +667,7 @@
                                 }
                               ],
                               "default": null,
+                              "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
                               "title": "Text"
                             },
                             "content_desc": {
@@ -726,6 +727,7 @@
                                 }
                               ],
                               "default": null,
+                              "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
                               "title": "Bounds"
                             }
                           },
@@ -908,7 +910,7 @@
                   "properties": {
                     "selector": {
                       "additionalProperties": false,
-                      "description": "Every supplied attribute is an exact AND match against a fresh native node.",
+                      "description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.",
                       "properties": {
                         "resource_id": {
                           "anyOf": [
@@ -935,6 +937,7 @@
                             }
                           ],
                           "default": null,
+                          "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
                           "title": "Text"
                         },
                         "content_desc": {
@@ -994,6 +997,7 @@
                             }
                           ],
                           "default": null,
+                          "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
                           "title": "Bounds"
                         }
                       },
@@ -1124,6 +1128,7 @@
                           }
                         ],
                         "default": null,
+                        "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
                         "title": "Text"
                       },
                       "content_desc": {
@@ -1183,6 +1188,7 @@
                           }
                         ],
                         "default": null,
+                        "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
                         "title": "Bounds"
                       }
                     },
@@ -1343,6 +1349,7 @@
                           }
                         ],
                         "default": null,
+                        "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
                         "title": "Text"
                       },
                       "content_desc": {
@@ -1402,6 +1409,7 @@
                           }
                         ],
                         "default": null,
+                        "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
                         "title": "Bounds"
                       }
                     },
@@ -1783,12 +1791,12 @@
     "type": "function",
     "function": {
       "name": "read_control",
-      "description": "[OBSERVE] Read ONE live native control by exact AND selector copied from the current UI. Use resource_id/text/content_desc/class_name and optional package_name/bounds to disambiguate. Returns current range/value, supported native actions, editable selection (UTF-16 offsets), identity token and visible subtree content_signature. Missing or ambiguous controls fail explicitly. Custom canvas controls may expose no range/actions; then use calibrated gestures and visual verification.",
+      "description": "[OBSERVE] Read ONE live native control by exact AND selector copied from the current UI. Copy its native_selector object unchanged; it uses stable identity and original pixel bounds only when needed to disambiguate. UI Bounds are normalized 0..1000 and MUST NOT be used as selector.bounds (exact Android pixels); rounded UI bounds cannot be converted back exactly. Do not add the changing text/content_desc you want to read or wait for as an identity filter. Returns current range/value, supported native actions, editable selection (UTF-16 offsets), identity token and visible subtree content_signature. Missing or ambiguous controls fail explicitly. On NOT_FOUND refresh the UI and selector; it does not prove that native reading is unavailable. Custom canvas controls may expose no range/actions; then use calibrated gestures and visual verification.",
       "parameters": {
         "properties": {
           "selector": {
             "additionalProperties": false,
-            "description": "Every supplied attribute is an exact AND match against a fresh native node.",
+            "description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.",
             "properties": {
               "resource_id": {
                 "anyOf": [
@@ -1813,7 +1821,8 @@
                     "type": "null"
                   }
                 ],
-                "default": null
+                "default": null,
+                "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead."
               },
               "content_desc": {
                 "anyOf": [
@@ -1868,7 +1877,8 @@
                     "type": "null"
                   }
                 ],
-                "default": null
+                "default": null,
+                "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector."
               }
             },
             "type": "object"
@@ -1890,7 +1900,7 @@
         "properties": {
           "selector": {
             "additionalProperties": false,
-            "description": "Every supplied attribute is an exact AND match against a fresh native node.",
+            "description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.",
             "properties": {
               "resource_id": {
                 "anyOf": [
@@ -1915,7 +1925,8 @@
                     "type": "null"
                   }
                 ],
-                "default": null
+                "default": null,
+                "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead."
               },
               "content_desc": {
                 "anyOf": [
@@ -1970,7 +1981,8 @@
                     "type": "null"
                   }
                 ],
-                "default": null
+                "default": null,
+                "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector."
               }
             },
             "type": "object"
@@ -2141,7 +2153,7 @@
         "properties": {
           "selector": {
             "additionalProperties": false,
-            "description": "Every supplied attribute is an exact AND match against a fresh native node.",
+            "description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.",
             "properties": {
               "resource_id": {
                 "anyOf": [
@@ -2166,7 +2178,8 @@
                     "type": "null"
                   }
                 ],
-                "default": null
+                "default": null,
+                "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead."
               },
               "content_desc": {
                 "anyOf": [
@@ -2221,7 +2234,8 @@
                     "type": "null"
                   }
                 ],
-                "default": null
+                "default": null,
+                "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector."
               }
             },
             "type": "object"
@@ -2332,7 +2346,8 @@
                     "type": "null"
                   }
                 ],
-                "default": null
+                "default": null,
+                "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead."
               },
               "content_desc": {
                 "anyOf": [
@@ -2387,7 +2402,8 @@
                     "type": "null"
                   }
                 ],
-                "default": null
+                "default": null,
+                "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector."
               }
             },
             "type": "object"
@@ -2419,7 +2435,8 @@
                     "type": "null"
                   }
                 ],
-                "default": null
+                "default": null,
+                "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead."
               },
               "content_desc": {
                 "anyOf": [
@@ -2474,7 +2491,8 @@
                     "type": "null"
                   }
                 ],
-                "default": null
+                "default": null,
+                "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector."
               }
             },
             "type": "object"

tests/fixtures/action_surfaces/tool_declarations.json

@@ -335,6 +335,7 @@
                           }
                         ],
                         "default": null,
+                        "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
                         "title": "Text"
                       },
                       "content_desc": {
@@ -394,6 +395,7 @@
                           }
                         ],
                         "default": null,
+                        "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
                         "title": "Bounds"
                       }
                     },
@@ -1172,7 +1174,7 @@
                       "properties": {
                         "selector": {
                           "additionalProperties": false,
-                          "description": "Every supplied attribute is an exact AND match against a fresh native node.",
+                          "description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.",
                           "properties": {
                             "resource_id": {
                               "anyOf": [
@@ -1199,6 +1201,7 @@
                                 }
                               ],
                               "default": null,
+                              "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
                               "title": "Text"
                             },
                             "content_desc": {
@@ -1258,6 +1261,7 @@
                                 }
                               ],
                               "default": null,
+                              "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
                               "title": "Bounds"
                             }
                           },
@@ -1440,7 +1444,7 @@
                   "properties": {
                     "selector": {
                       "additionalProperties": false,
-                      "description": "Every supplied attribute is an exact AND match against a fresh native node.",
+                      "description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.",
                       "properties": {
                         "resource_id": {
                           "anyOf": [
@@ -1467,6 +1471,7 @@
                             }
                           ],
                           "default": null,
+                          "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
                           "title": "Text"
                         },
                         "content_desc": {
@@ -1526,6 +1531,7 @@
                             }
                           ],
                           "default": null,
+                          "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
                           "title": "Bounds"
                         }
                       },
@@ -1656,6 +1662,7 @@
                           }
                         ],
                         "default": null,
+                        "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
                         "title": "Text"
                       },
                       "content_desc": {
@@ -1715,6 +1722,7 @@
                           }
                         ],
                         "default": null,
+                        "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
                         "title": "Bounds"
                       }
                     },
@@ -2044,12 +2052,12 @@
     "type": "function",
     "function": {
       "name": "read_control",
-      "description": "[OBSERVE] Read ONE live native control by exact AND selector copied from the current UI. Use resource_id/text/content_desc/class_name and optional package_name/bounds to disambiguate. Returns current range/value, supported native actions, editable selection (UTF-16 offsets), identity token and visible subtree content_signature. Missing or ambiguous controls fail explicitly. Custom canvas controls may expose no range/actions; then use calibrated gestures and visual verification.",
+      "description": "[OBSERVE] Read ONE live native control by exact AND selector copied from the current UI. Copy its native_selector object unchanged; it uses stable identity and original pixel bounds only when needed to disambiguate. UI Bounds are normalized 0..1000 and MUST NOT be used as selector.bounds (exact Android pixels); rounded UI bounds cannot be converted back exactly. Do not add the changing text/content_desc you want to read or wait for as an identity filter. Returns current range/value, supported native actions, editable selection (UTF-16 offsets), identity token and visible subtree content_signature. Missing or ambiguous controls fail explicitly. On NOT_FOUND refresh the UI and selector; it does not prove that native reading is unavailable. Custom canvas controls may expose no range/actions; then use calibrated gestures and visual verification.",
       "parameters": {
         "properties": {
           "selector": {
             "additionalProperties": false,
-            "description": "Every supplied attribute is an exact AND match against a fresh native node.",
+            "description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.",
             "properties": {
               "resource_id": {
                 "anyOf": [
@@ -2074,7 +2082,8 @@
                     "type": "null"
                   }
                 ],
-                "default": null
+                "default": null,
+                "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead."
               },
               "content_desc": {
                 "anyOf": [
@@ -2129,7 +2138,8 @@
                     "type": "null"
                   }
                 ],
-                "default": null
+                "default": null,
+                "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector."
               }
             },
             "type": "object"
@@ -2151,7 +2161,7 @@
         "properties": {
           "selector": {
             "additionalProperties": false,
-            "description": "Every supplied attribute is an exact AND match against a fresh native node.",
+            "description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.",
             "properties": {
               "resource_id": {
                 "anyOf": [
@@ -2176,7 +2186,8 @@
                     "type": "null"
                   }
                 ],
-                "default": null
+                "default": null,
+                "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead."
               },
               "content_desc": {
                 "anyOf": [
@@ -2231,7 +2242,8 @@
                     "type": "null"
                   }
                 ],
-                "default": null
+                "default": null,
+                "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector."
               }
             },
             "type": "object"
@@ -2402,7 +2414,7 @@
         "properties": {
           "selector": {
             "additionalProperties": false,
-            "description": "Every supplied attribute is an exact AND match against a fresh native node.",
+            "description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.",
             "properties": {
               "resource_id": {
                 "anyOf": [
@@ -2427,7 +2439,8 @@
                     "type": "null"
                   }
                 ],
-                "default": null
+                "default": null,
+                "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead."
               },
               "content_desc": {
                 "anyOf": [
@@ -2482,7 +2495,8 @@
                     "type": "null"
                   }
                 ],
-                "default": null
+                "default": null,
+                "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector."
               }
             },
             "type": "object"
@@ -2593,7 +2607,8 @@
                     "type": "null"
                   }
                 ],
-                "default": null
+                "default": null,
+                "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead."
               },
               "content_desc": {
                 "anyOf": [
@@ -2648,7 +2663,8 @@
                     "type": "null"
                   }
                 ],
-                "default": null
+                "default": null,
+                "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector."
               }
             },
             "type": "object"
@@ -2680,7 +2696,8 @@
                     "type": "null"
                   }
                 ],
-                "default": null
+                "default": null,
+                "description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead."
               },
               "content_desc": {
                 "anyOf": [
@@ -2735,7 +2752,8 @@
                     "type": "null"
                   }
                 ],
-                "default": null
+                "default": null,
+                "description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector."
               }
             },
             "type": "object"

tests/unit/agents/test_flash_runner.py

@@ -136,11 +136,16 @@ async def test_fallback_observation_renders_the_indexed_ui_list(mock_context, tm
     assert isinstance(exec_result.ui_elements_text, str)
     assert exec_result.ui_elements_text == (
         "[1] Text: 'Settings' | Bounds: [37,50][962,112] | resource_id: 'android:id/title'"
+        ' | native_selector: {"resource_id": "android:id/title", "class_name": "android.widget.TextView"}'
     )
     assert "{'text'" not in exec_result.ui_elements_text
     # State refreshed exactly as the executor's success path does.
     assert state.indexed_elements[0]["index"] == 1
     assert state.indexed_elements[0]["text"] == "Settings"
+    assert state.indexed_elements[0]["native_selector"] == {
+        "resource_id": "android:id/title",
+        "class_name": "android.widget.TextView",
+    }
     assert state.indexed_points == [state.indexed_elements[0]["center"]]
     assert state.latest_screenshot != "old.jpg"
     assert state.latest_screenshot.startswith(str(tmp_path))

tests/unit/mcp/test_native_observation_selectors.py

@@ -0,0 +1,152 @@
+"""Model-visible selectors must round-trip to exact native matching safely."""
+
+from copy import deepcopy
+import json
+from unittest.mock import AsyncMock
+
+import pytest
+
+from artemis.mcp.native_actions import ControlSelector
+from artemis.mcp.native_executor import execute_native
+from artemis.utils.ui_filter import filter_ui_hierarchy
+from artemis.utils.visualization import format_minimal_list_with_elements
+from tests.unit.utils.test_native_control_state import encode_node, hierarchy_node
+
+
+def native_node(raw):
+    return {
+        "resource_id": raw.get("resource-id", ""),
+        "class_name": raw["class"],
+        "package_name": raw["package"],
+        "bounds": [30, 549, 110, 629],
+        "text": "17:42:20",
+        "content_desc": "current 17:42:20",
+        "visible": True,
+        "actions": ["set_selection"],
+    }
+
+
+def exact_native(nodes):
+    async def inspect(command, args, *, timeout=None):
+        assert command == "inspect_control", "A failed read must never dispatch input"
+        matches = [
+            node
+            for node in nodes
+            if all(node.get(key) == value for key, value in args["selector"].items())
+        ]
+        if len(matches) == 1:
+            return {"success": True, "execution": "observed", "code": "OK", "node": matches[0]}
+        return {
+            "success": False,
+            "execution": "not_started",
+            "code": "NOT_FOUND" if not matches else "AMBIGUOUS",
+            "error": "Exact native selector did not resolve uniquely",
+        }
+
+    return AsyncMock(side_effect=inspect)
+
+
+@pytest.mark.parametrize("encoding", ["xml", "json"])
+@pytest.mark.parametrize("screen", [(720, 1600), (1080, 2400), (1600, 720)])
+@pytest.mark.asyncio
+async def test_copy_visible_selector_reads_a_changing_value_without_lossy_ui_bounds(
+    encoding, screen
+):
+    raw = hierarchy_node(
+        text="17:42:17",
+        **{
+            "class": "android.widget.TextView",
+            "content-desc": "current 17:42:17",
+            "bounds": "[30,549][110,629]",
+            "range": None,
+            "actions": ["set_selection"],
+        },
+    )
+    source = encode_node(raw, encoding)
+    original = deepcopy(source)
+    text, elements, _ = format_minimal_list_with_elements([source], *screen)
+    selector = json.loads(text.split(" | native_selector: ", 1)[1])
+    assert (
+        selector
+        == elements[0]["native_selector"]
+        == {
+            "resource_id": raw["resource-id"],
+            "class_name": raw["class"],
+            "package_name": raw["package"],
+        }
+    )
+    assert source == original
+    ControlSelector.model_validate(selector)
+    native = exact_native([native_node(raw)])
+    result = await execute_native("read_control", {"selector": selector}, *screen, native)
+    assert result["success"] and result["node"]["text"] == "17:42:20"
+    native.assert_awaited_once()
+
+
+@pytest.mark.parametrize("resource_id", ["", "fixture:id/shared"])
+@pytest.mark.asyncio
+async def test_no_id_and_repeated_ids_use_original_pixels_to_resolve_each_node(resource_id):
+    rows = [
+        hierarchy_node(
+            text=f"row {i}",
+            **{
+                "resource-id": resource_id,
+                "bounds": f"[30,{549 + i * 100}][110,{629 + i * 100}]",
+            },
+        )
+        for i in range(2)
+    ]
+    _, elements, _ = format_minimal_list_with_elements(rows, 720, 1600)
+    nodes = [
+        native_node(row) | {"bounds": [30, 549 + i * 100, 110, 629 + i * 100]}
+        for i, row in enumerate(rows)
+    ]
+    native = exact_native(nodes)
+    for element, node in zip(elements, nodes, strict=True):
+        selector = element["native_selector"]
+        assert selector["bounds"] == node["bounds"]
+        result = await execute_native("read_control", {"selector": selector}, 720, 1600, native)
+        assert result["success"] and result["node"]["bounds"] == node["bounds"]
+
+
+def test_clipped_hit_area_does_not_replace_the_original_native_identity_bounds():
+    raw = hierarchy_node(
+        text="partly visible",
+        **{
+            "resource-id": "",
+            "bounds": "[-20,-40][300,60]",
+        },
+    )
+    filtered = filter_ui_hierarchy([raw], screen_width=720, screen_height=1600)
+    _, elements, _ = format_minimal_list_with_elements(filtered, 720, 1600)
+    assert elements[0]["bounds"] == [0, 0, 300, 60]
+    assert elements[0]["native_selector"]["bounds"] == [-20, -40, 300, 60]
+    ControlSelector.model_validate(elements[0]["native_selector"])
+
+
+def test_ocr_only_geometry_is_not_presented_as_a_native_selector():
+    text, elements, _ = format_minimal_list_with_elements(
+        [{"text": "17:42:17", "class": "[OCR]", "bounds": "[30,549][110,629]"}],
+        720,
+        1600,
+    )
+    assert "native_selector" not in text
+    assert "native_selector" not in elements[0]
+
+
+@pytest.mark.parametrize("action", ["read_control", "control_action"])
+@pytest.mark.parametrize("extra", [{"text": "17:42:17"}, {"bounds": [41, 343, 152, 393]}])
+@pytest.mark.asyncio
+async def test_explicit_stale_text_or_wrong_bounds_are_not_silently_dropped_or_retried(
+    action, extra
+):
+    raw = hierarchy_node()
+    native = exact_native([native_node(raw)])
+    request = {"selector": {"resource_id": raw["resource-id"], **extra}}
+    if action == "control_action":
+        request["action"] = "click"
+    result = await execute_native(action, request, 720, 1600, native)
+    assert not result["success"] and result["code"] == "NOT_FOUND"
+    assert result["execution"] == "not_started"
+    assert native.await_args.args[1]["selector"] == request["selector"]
+    native.assert_awaited_once()

tests/unit/test_hierarchy_fidelity.py

@@ -78,10 +78,15 @@ def test_minimal_list_shows_hint_for_empty_inputs_and_errors():
     text, items, labels = format_minimal_list_with_elements(elements[1:], 1000, 1000)
     lines = text.splitlines()
     assert (
-        lines[0]
-        == "[1] Hint: 'Search settings' | Bounds: [0,0][100,100] | Editable: true | Value: empty"
+        lines[0] == "[1] Hint: 'Search settings' | Bounds: [0,0][100,100]"
+        ' | native_selector: {"class_name": "android.widget.EditText", "package_name": "app", "bounds": [0, 0, 100, 100]}'
+        " | Editable: true | Value: empty"
+    )
+    assert lines[1] == (
+        "[2] Text: 'abc' | Bounds: [0,200][100,300]"
+        ' | native_selector: {"class_name": "android.widget.TextView", "package_name": "app", "bounds": [0, 200, 100, 300]}'
+        " | Error: 'Password too short'"
     )
-    assert lines[1] == "[2] Text: 'abc' | Bounds: [0,200][100,300] | Error: 'Password too short'"
     assert len(lines) == 2 and labels == ["1", "2"]
     assert items[0]["is_hint"] is True and items[0]["text"] == "Search settings"
     assert items[1]["error"] == "Password too short"