chenmingxuan/Artemis-Eapil
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 32 Wiki Activity

fix(android): expose stable native selectors with exact pixel bounds

Browse Source
This commit is contained in:
chenmingxuan committed 2026-11-03 00:02:35 +08:00
1 parent 4ea7939d9c
commit dc3a48f6b2
13 files changed
+313 -43
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/fixtures/action_surfaces/action_server_manifest.json
Vendored
+17 -3
View File
@@ -186,6 +186,7 @@
}
],
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
"title": "Text"
},
"content_desc": {
@@ -245,6 +246,7 @@
}
],
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
"title": "Bounds"
}
},
@@ -623,7 +625,8 @@
"additionalProperties": false,
"properties": {
"selector": {
"$ref": "#/$defs/ControlSelector"
"$ref": "#/$defs/ControlSelector",
"description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds."
},
"action": {
"enum": [
@@ -917,7 +920,8 @@
"additionalProperties": false,
"properties": {
"selector": {
"$ref": "#/$defs/ControlSelector"
"$ref": "#/$defs/ControlSelector",
"description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds."
},
"state": {
"enum": [
@@ -1177,6 +1181,7 @@
}
],
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
"title": "Text"
},
"content_desc": {
@@ -1236,6 +1241,7 @@
}
],
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
"title": "Bounds"
}
},
@@ -3306,7 +3312,7 @@
}
},
"read_control": {
"description": "[OBSERVE] Read ONE live native control by exact AND selector copied from the current UI. Use resource_id/text/content_desc/class_name and optional package_name/bounds to disambiguate. Returns current range/value, supported native actions, editable selection (UTF-16 offsets), identity token and visible subtree content_signature. Missing or ambiguous controls fail explicitly. Custom canvas controls may expose no range/actions; then use calibrated gestures and visual verification.",
"description": "[OBSERVE] Read ONE live native control by exact AND selector copied from the current UI. Copy its native_selector object unchanged; it uses stable identity and original pixel bounds only when needed to disambiguate. UI Bounds are normalized 0..1000 and MUST NOT be used as selector.bounds (exact Android pixels); rounded UI bounds cannot be converted back exactly. Do not add the changing text/content_desc you want to read or wait for as an identity filter. Returns current range/value, supported native actions, editable selection (UTF-16 offsets), identity token and visible subtree content_signature. Missing or ambiguous controls fail explicitly. On NOT_FOUND refresh the UI and selector; it does not prove that native reading is unavailable. Custom canvas controls may expose no range/actions; then use calibrated gestures and visual verification.",
"inputSchema": {
"$defs": {
"ControlSelector": {
@@ -3338,6 +3344,7 @@
}
],
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
"title": "Text"
},
"content_desc": {
@@ -3397,6 +3404,7 @@
}
],
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
"title": "Bounds"
}
},
@@ -3591,6 +3599,7 @@
}
],
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
"title": "Text"
},
"content_desc": {
@@ -3650,6 +3659,7 @@
}
],
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
"title": "Bounds"
}
},
@@ -3894,6 +3904,7 @@
}
],
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
"title": "Text"
},
"content_desc": {
@@ -3953,6 +3964,7 @@
}
],
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
"title": "Bounds"
}
},
@@ -4164,6 +4176,7 @@
}
],
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
"title": "Text"
},
"content_desc": {
@@ -4223,6 +4236,7 @@
}
],
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
"title": "Bounds"
}
},
tests/fixtures/action_surfaces/operator_shells.json
Vendored
+34 -16
View File
@@ -640,7 +640,7 @@
"properties": {
"selector": {
"additionalProperties": false,
"description": "Every supplied attribute is an exact AND match against a fresh native node.",
"description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.",
"properties": {
"resource_id": {
"anyOf": [
@@ -667,6 +667,7 @@
}
],
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
"title": "Text"
},
"content_desc": {
@@ -726,6 +727,7 @@
}
],
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
"title": "Bounds"
}
},
@@ -908,7 +910,7 @@
"properties": {
"selector": {
"additionalProperties": false,
"description": "Every supplied attribute is an exact AND match against a fresh native node.",
"description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.",
"properties": {
"resource_id": {
"anyOf": [
@@ -935,6 +937,7 @@
}
],
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
"title": "Text"
},
"content_desc": {
@@ -994,6 +997,7 @@
}
],
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
"title": "Bounds"
}
},
@@ -1124,6 +1128,7 @@
}
],
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
"title": "Text"
},
"content_desc": {
@@ -1183,6 +1188,7 @@
}
],
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
"title": "Bounds"
}
},
@@ -1343,6 +1349,7 @@
}
],
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
"title": "Text"
},
"content_desc": {
@@ -1402,6 +1409,7 @@
}
],
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
"title": "Bounds"
}
},
@@ -1783,12 +1791,12 @@
"type": "function",
"function": {
"name": "read_control",
"description": "[OBSERVE] Read ONE live native control by exact AND selector copied from the current UI. Use resource_id/text/content_desc/class_name and optional package_name/bounds to disambiguate. Returns current range/value, supported native actions, editable selection (UTF-16 offsets), identity token and visible subtree content_signature. Missing or ambiguous controls fail explicitly. Custom canvas controls may expose no range/actions; then use calibrated gestures and visual verification.",
"description": "[OBSERVE] Read ONE live native control by exact AND selector copied from the current UI. Copy its native_selector object unchanged; it uses stable identity and original pixel bounds only when needed to disambiguate. UI Bounds are normalized 0..1000 and MUST NOT be used as selector.bounds (exact Android pixels); rounded UI bounds cannot be converted back exactly. Do not add the changing text/content_desc you want to read or wait for as an identity filter. Returns current range/value, supported native actions, editable selection (UTF-16 offsets), identity token and visible subtree content_signature. Missing or ambiguous controls fail explicitly. On NOT_FOUND refresh the UI and selector; it does not prove that native reading is unavailable. Custom canvas controls may expose no range/actions; then use calibrated gestures and visual verification.",
"parameters": {
"properties": {
"selector": {
"additionalProperties": false,
"description": "Every supplied attribute is an exact AND match against a fresh native node.",
"description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.",
"properties": {
"resource_id": {
"anyOf": [
@@ -1813,7 +1821,8 @@
"type": "null"
}
],
"default": null
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead."
},
"content_desc": {
"anyOf": [
@@ -1868,7 +1877,8 @@
"type": "null"
}
],
"default": null
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector."
}
},
"type": "object"
@@ -1890,7 +1900,7 @@
"properties": {
"selector": {
"additionalProperties": false,
"description": "Every supplied attribute is an exact AND match against a fresh native node.",
"description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.",
"properties": {
"resource_id": {
"anyOf": [
@@ -1915,7 +1925,8 @@
"type": "null"
}
],
"default": null
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead."
},
"content_desc": {
"anyOf": [
@@ -1970,7 +1981,8 @@
"type": "null"
}
],
"default": null
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector."
}
},
"type": "object"
@@ -2141,7 +2153,7 @@
"properties": {
"selector": {
"additionalProperties": false,
"description": "Every supplied attribute is an exact AND match against a fresh native node.",
"description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.",
"properties": {
"resource_id": {
"anyOf": [
@@ -2166,7 +2178,8 @@
"type": "null"
}
],
"default": null
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead."
},
"content_desc": {
"anyOf": [
@@ -2221,7 +2234,8 @@
"type": "null"
}
],
"default": null
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector."
}
},
"type": "object"
@@ -2332,7 +2346,8 @@
"type": "null"
}
],
"default": null
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead."
},
"content_desc": {
"anyOf": [
@@ -2387,7 +2402,8 @@
"type": "null"
}
],
"default": null
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector."
}
},
"type": "object"
@@ -2419,7 +2435,8 @@
"type": "null"
}
],
"default": null
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead."
},
"content_desc": {
"anyOf": [
@@ -2474,7 +2491,8 @@
"type": "null"
}
],
"default": null
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector."
}
},
"type": "object"
tests/fixtures/action_surfaces/tool_declarations.json
Vendored
+34 -16
View File
@@ -335,6 +335,7 @@
}
],
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
"title": "Text"
},
"content_desc": {
@@ -394,6 +395,7 @@
}
],
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
"title": "Bounds"
}
},
@@ -1172,7 +1174,7 @@
"properties": {
"selector": {
"additionalProperties": false,
"description": "Every supplied attribute is an exact AND match against a fresh native node.",
"description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.",
"properties": {
"resource_id": {
"anyOf": [
@@ -1199,6 +1201,7 @@
}
],
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
"title": "Text"
},
"content_desc": {
@@ -1258,6 +1261,7 @@
}
],
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
"title": "Bounds"
}
},
@@ -1440,7 +1444,7 @@
"properties": {
"selector": {
"additionalProperties": false,
"description": "Every supplied attribute is an exact AND match against a fresh native node.",
"description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.",
"properties": {
"resource_id": {
"anyOf": [
@@ -1467,6 +1471,7 @@
}
],
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
"title": "Text"
},
"content_desc": {
@@ -1526,6 +1531,7 @@
}
],
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
"title": "Bounds"
}
},
@@ -1656,6 +1662,7 @@
}
],
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
"title": "Text"
},
"content_desc": {
@@ -1715,6 +1722,7 @@
}
],
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
"title": "Bounds"
}
},
@@ -2044,12 +2052,12 @@
"type": "function",
"function": {
"name": "read_control",
"description": "[OBSERVE] Read ONE live native control by exact AND selector copied from the current UI. Use resource_id/text/content_desc/class_name and optional package_name/bounds to disambiguate. Returns current range/value, supported native actions, editable selection (UTF-16 offsets), identity token and visible subtree content_signature. Missing or ambiguous controls fail explicitly. Custom canvas controls may expose no range/actions; then use calibrated gestures and visual verification.",
"description": "[OBSERVE] Read ONE live native control by exact AND selector copied from the current UI. Copy its native_selector object unchanged; it uses stable identity and original pixel bounds only when needed to disambiguate. UI Bounds are normalized 0..1000 and MUST NOT be used as selector.bounds (exact Android pixels); rounded UI bounds cannot be converted back exactly. Do not add the changing text/content_desc you want to read or wait for as an identity filter. Returns current range/value, supported native actions, editable selection (UTF-16 offsets), identity token and visible subtree content_signature. Missing or ambiguous controls fail explicitly. On NOT_FOUND refresh the UI and selector; it does not prove that native reading is unavailable. Custom canvas controls may expose no range/actions; then use calibrated gestures and visual verification.",
"parameters": {
"properties": {
"selector": {
"additionalProperties": false,
"description": "Every supplied attribute is an exact AND match against a fresh native node.",
"description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.",
"properties": {
"resource_id": {
"anyOf": [
@@ -2074,7 +2082,8 @@
"type": "null"
}
],
"default": null
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead."
},
"content_desc": {
"anyOf": [
@@ -2129,7 +2138,8 @@
"type": "null"
}
],
"default": null
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector."
}
},
"type": "object"
@@ -2151,7 +2161,7 @@
"properties": {
"selector": {
"additionalProperties": false,
"description": "Every supplied attribute is an exact AND match against a fresh native node.",
"description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.",
"properties": {
"resource_id": {
"anyOf": [
@@ -2176,7 +2186,8 @@
"type": "null"
}
],
"default": null
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead."
},
"content_desc": {
"anyOf": [
@@ -2231,7 +2242,8 @@
"type": "null"
}
],
"default": null
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector."
}
},
"type": "object"
@@ -2402,7 +2414,7 @@
"properties": {
"selector": {
"additionalProperties": false,
"description": "Every supplied attribute is an exact AND match against a fresh native node.",
"description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.",
"properties": {
"resource_id": {
"anyOf": [
@@ -2427,7 +2439,8 @@
"type": "null"
}
],
"default": null
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead."
},
"content_desc": {
"anyOf": [
@@ -2482,7 +2495,8 @@
"type": "null"
}
],
"default": null
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector."
}
},
"type": "object"
@@ -2593,7 +2607,8 @@
"type": "null"
}
],
"default": null
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead."
},
"content_desc": {
"anyOf": [
@@ -2648,7 +2663,8 @@
"type": "null"
}
],
"default": null
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector."
}
},
"type": "object"
@@ -2680,7 +2696,8 @@
"type": "null"
}
],
"default": null
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead."
},
"content_desc": {
"anyOf": [
@@ -2735,7 +2752,8 @@
"type": "null"
}
],
"default": null
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector."
}
},
"type": "object"
tests/unit/agents/test_flash_runner.py
+5
View File
@@ -136,11 +136,16 @@ async def test_fallback_observation_renders_the_indexed_ui_list(mock_context, tm
assert isinstance(exec_result.ui_elements_text, str)
assert exec_result.ui_elements_text == (
"[1] Text: 'Settings' | Bounds: [37,50][962,112] | resource_id: 'android:id/title'"
' | native_selector: {"resource_id": "android:id/title", "class_name": "android.widget.TextView"}'
)
assert "{'text'" not in exec_result.ui_elements_text
# State refreshed exactly as the executor's success path does.
assert state.indexed_elements[0]["index"] == 1
assert state.indexed_elements[0]["text"] == "Settings"
assert state.indexed_elements[0]["native_selector"] == {
"resource_id": "android:id/title",
"class_name": "android.widget.TextView",
}
assert state.indexed_points == [state.indexed_elements[0]["center"]]
assert state.latest_screenshot != "old.jpg"
assert state.latest_screenshot.startswith(str(tmp_path))
tests/unit/mcp/test_native_observation_selectors.py
+152
View File
@@ -0,0 +1,152 @@
"""Model-visible selectors must round-trip to exact native matching safely."""
from copy import deepcopy
import json
from unittest.mock import AsyncMock
import pytest
from artemis.mcp.native_actions import ControlSelector
from artemis.mcp.native_executor import execute_native
from artemis.utils.ui_filter import filter_ui_hierarchy
from artemis.utils.visualization import format_minimal_list_with_elements
from tests.unit.utils.test_native_control_state import encode_node, hierarchy_node
def native_node(raw):
return {
"resource_id": raw.get("resource-id", ""),
"class_name": raw["class"],
"package_name": raw["package"],
"bounds": [30, 549, 110, 629],
"text": "17:42:20",
"content_desc": "current 17:42:20",
"visible": True,
"actions": ["set_selection"],
}
def exact_native(nodes):
async def inspect(command, args, *, timeout=None):
assert command == "inspect_control", "A failed read must never dispatch input"
matches = [
node
for node in nodes
if all(node.get(key) == value for key, value in args["selector"].items())
]
if len(matches) == 1:
return {"success": True, "execution": "observed", "code": "OK", "node": matches[0]}
return {
"success": False,
"execution": "not_started",
"code": "NOT_FOUND" if not matches else "AMBIGUOUS",
"error": "Exact native selector did not resolve uniquely",
}
return AsyncMock(side_effect=inspect)
@pytest.mark.parametrize("encoding", ["xml", "json"])
@pytest.mark.parametrize("screen", [(720, 1600), (1080, 2400), (1600, 720)])
@pytest.mark.asyncio
async def test_copy_visible_selector_reads_a_changing_value_without_lossy_ui_bounds(
encoding, screen
):
raw = hierarchy_node(
text="17:42:17",
**{
"class": "android.widget.TextView",
"content-desc": "current 17:42:17",
"bounds": "[30,549][110,629]",
"range": None,
"actions": ["set_selection"],
},
)
source = encode_node(raw, encoding)
original = deepcopy(source)
text, elements, _ = format_minimal_list_with_elements([source], *screen)
selector = json.loads(text.split(" | native_selector: ", 1)[1])
assert (
selector
== elements[0]["native_selector"]
== {
"resource_id": raw["resource-id"],
"class_name": raw["class"],
"package_name": raw["package"],
}
)
assert source == original
ControlSelector.model_validate(selector)
native = exact_native([native_node(raw)])
result = await execute_native("read_control", {"selector": selector}, *screen, native)
assert result["success"] and result["node"]["text"] == "17:42:20"
native.assert_awaited_once()
@pytest.mark.parametrize("resource_id", ["", "fixture:id/shared"])
@pytest.mark.asyncio
async def test_no_id_and_repeated_ids_use_original_pixels_to_resolve_each_node(resource_id):
rows = [
hierarchy_node(
text=f"row {i}",
**{
"resource-id": resource_id,
"bounds": f"[30,{549 + i * 100}][110,{629 + i * 100}]",
},
)
for i in range(2)
]
_, elements, _ = format_minimal_list_with_elements(rows, 720, 1600)
nodes = [
native_node(row) | {"bounds": [30, 549 + i * 100, 110, 629 + i * 100]}
for i, row in enumerate(rows)
]
native = exact_native(nodes)
for element, node in zip(elements, nodes, strict=True):
selector = element["native_selector"]
assert selector["bounds"] == node["bounds"]
result = await execute_native("read_control", {"selector": selector}, 720, 1600, native)
assert result["success"] and result["node"]["bounds"] == node["bounds"]
def test_clipped_hit_area_does_not_replace_the_original_native_identity_bounds():
raw = hierarchy_node(
text="partly visible",
**{
"resource-id": "",
"bounds": "[-20,-40][300,60]",
},
)
filtered = filter_ui_hierarchy([raw], screen_width=720, screen_height=1600)
_, elements, _ = format_minimal_list_with_elements(filtered, 720, 1600)
assert elements[0]["bounds"] == [0, 0, 300, 60]
assert elements[0]["native_selector"]["bounds"] == [-20, -40, 300, 60]
ControlSelector.model_validate(elements[0]["native_selector"])
def test_ocr_only_geometry_is_not_presented_as_a_native_selector():
text, elements, _ = format_minimal_list_with_elements(
[{"text": "17:42:17", "class": "[OCR]", "bounds": "[30,549][110,629]"}],
720,
1600,
)
assert "native_selector" not in text
assert "native_selector" not in elements[0]
@pytest.mark.parametrize("action", ["read_control", "control_action"])
@pytest.mark.parametrize("extra", [{"text": "17:42:17"}, {"bounds": [41, 343, 152, 393]}])
@pytest.mark.asyncio
async def test_explicit_stale_text_or_wrong_bounds_are_not_silently_dropped_or_retried(
action, extra
):
raw = hierarchy_node()
native = exact_native([native_node(raw)])
request = {"selector": {"resource_id": raw["resource-id"], **extra}}
if action == "control_action":
request["action"] = "click"
result = await execute_native(action, request, 720, 1600, native)
assert not result["success"] and result["code"] == "NOT_FOUND"
assert result["execution"] == "not_started"
assert native.await_args.args[1]["selector"] == request["selector"]
native.assert_awaited_once()
tests/unit/test_hierarchy_fidelity.py
+8 -3
View File
@@ -78,10 +78,15 @@ def test_minimal_list_shows_hint_for_empty_inputs_and_errors():
text, items, labels = format_minimal_list_with_elements(elements[1:], 1000, 1000)
lines = text.splitlines()
assert (
lines[0]
== "[1] Hint: 'Search settings' | Bounds: [0,0][100,100] | Editable: true | Value: empty"
lines[0] == "[1] Hint: 'Search settings' | Bounds: [0,0][100,100]"
' | native_selector: {"class_name": "android.widget.EditText", "package_name": "app", "bounds": [0, 0, 100, 100]}'
" | Editable: true | Value: empty"
)
assert lines[1] == (
"[2] Text: 'abc' | Bounds: [0,200][100,300]"
' | native_selector: {"class_name": "android.widget.TextView", "package_name": "app", "bounds": [0, 200, 100, 300]}'
" | Error: 'Password too short'"
)
assert lines[1] == "[2] Text: 'abc' | Bounds: [0,200][100,300] | Error: 'Password too short'"
assert len(lines) == 2 and labels == ["1", "2"]
assert items[0]["is_hint"] is True and items[0]["text"] == "Search settings"
assert items[1]["error"] == "Password too short"