chenmingxuan/Artemis-Eapil
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 32 Wiki Activity

fix(android): expose stable native selectors with exact pixel bounds

Browse Source
This commit is contained in:
chenmingxuan committed 2026-11-03 00:02:35 +08:00
1 parent 4ea7939d9c
commit dc3a48f6b2
13 files changed
+313 -43
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/fixtures/action_surfaces/operator_shells.json
Vendored
+34 -16
View File
@@ -640,7 +640,7 @@
"properties": {
"selector": {
"additionalProperties": false,
"description": "Every supplied attribute is an exact AND match against a fresh native node.",
"description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.",
"properties": {
"resource_id": {
"anyOf": [
@@ -667,6 +667,7 @@
}
],
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
"title": "Text"
},
"content_desc": {
@@ -726,6 +727,7 @@
}
],
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
"title": "Bounds"
}
},
@@ -908,7 +910,7 @@
"properties": {
"selector": {
"additionalProperties": false,
"description": "Every supplied attribute is an exact AND match against a fresh native node.",
"description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.",
"properties": {
"resource_id": {
"anyOf": [
@@ -935,6 +937,7 @@
}
],
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
"title": "Text"
},
"content_desc": {
@@ -994,6 +997,7 @@
}
],
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
"title": "Bounds"
}
},
@@ -1124,6 +1128,7 @@
}
],
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
"title": "Text"
},
"content_desc": {
@@ -1183,6 +1188,7 @@
}
],
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
"title": "Bounds"
}
},
@@ -1343,6 +1349,7 @@
}
],
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead.",
"title": "Text"
},
"content_desc": {
@@ -1402,6 +1409,7 @@
}
],
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector.",
"title": "Bounds"
}
},
@@ -1783,12 +1791,12 @@
"type": "function",
"function": {
"name": "read_control",
"description": "[OBSERVE] Read ONE live native control by exact AND selector copied from the current UI. Use resource_id/text/content_desc/class_name and optional package_name/bounds to disambiguate. Returns current range/value, supported native actions, editable selection (UTF-16 offsets), identity token and visible subtree content_signature. Missing or ambiguous controls fail explicitly. Custom canvas controls may expose no range/actions; then use calibrated gestures and visual verification.",
"description": "[OBSERVE] Read ONE live native control by exact AND selector copied from the current UI. Copy its native_selector object unchanged; it uses stable identity and original pixel bounds only when needed to disambiguate. UI Bounds are normalized 0..1000 and MUST NOT be used as selector.bounds (exact Android pixels); rounded UI bounds cannot be converted back exactly. Do not add the changing text/content_desc you want to read or wait for as an identity filter. Returns current range/value, supported native actions, editable selection (UTF-16 offsets), identity token and visible subtree content_signature. Missing or ambiguous controls fail explicitly. On NOT_FOUND refresh the UI and selector; it does not prove that native reading is unavailable. Custom canvas controls may expose no range/actions; then use calibrated gestures and visual verification.",
"parameters": {
"properties": {
"selector": {
"additionalProperties": false,
"description": "Every supplied attribute is an exact AND match against a fresh native node.",
"description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.",
"properties": {
"resource_id": {
"anyOf": [
@@ -1813,7 +1821,8 @@
"type": "null"
}
],
"default": null
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead."
},
"content_desc": {
"anyOf": [
@@ -1868,7 +1877,8 @@
"type": "null"
}
],
"default": null
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector."
}
},
"type": "object"
@@ -1890,7 +1900,7 @@
"properties": {
"selector": {
"additionalProperties": false,
"description": "Every supplied attribute is an exact AND match against a fresh native node.",
"description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.",
"properties": {
"resource_id": {
"anyOf": [
@@ -1915,7 +1925,8 @@
"type": "null"
}
],
"default": null
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead."
},
"content_desc": {
"anyOf": [
@@ -1970,7 +1981,8 @@
"type": "null"
}
],
"default": null
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector."
}
},
"type": "object"
@@ -2141,7 +2153,7 @@
"properties": {
"selector": {
"additionalProperties": false,
"description": "Every supplied attribute is an exact AND match against a fresh native node.",
"description": "Copy the current observation's native_selector object unchanged. Every supplied attribute is an exact AND filter. Do not add changing text/content_desc or normalized UI Bounds.",
"properties": {
"resource_id": {
"anyOf": [
@@ -2166,7 +2178,8 @@
"type": "null"
}
],
"default": null
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead."
},
"content_desc": {
"anyOf": [
@@ -2221,7 +2234,8 @@
"type": "null"
}
],
"default": null
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector."
}
},
"type": "object"
@@ -2332,7 +2346,8 @@
"type": "null"
}
],
"default": null
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead."
},
"content_desc": {
"anyOf": [
@@ -2387,7 +2402,8 @@
"type": "null"
}
],
"default": null
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector."
}
},
"type": "object"
@@ -2419,7 +2435,8 @@
"type": "null"
}
],
"default": null
"default": null,
"description": "Exact identity filter, NOT the text to read or wait for. Omit changing readouts; use the observed native_selector instead."
},
"content_desc": {
"anyOf": [
@@ -2474,7 +2491,8 @@
"type": "null"
}
],
"default": null
"default": null,
"description": "Exact original Android screen PIXELS [left,top,right,bottom], only when needed to disambiguate. Never copy normalized 0..1000 UI Bounds or convert their rounded values back to pixels. Prefer the observed native_selector."
}
},
"type": "object"