chenmingxuan
fac4ccb760
Add dark theme, richer Markdown and readable plugin results. Improve Android input evidence, response integrity and diagnostics. Unify lifecycle, guidance receipts and evidence sources across clients. Keep direct action dispatch without an added AI quota approval layer. Align version metadata at 2.2.0 and bundle Helper 1.2.2. Validation: 3063 backend tests before the version bump; 44 after. Frontend: 317 tests and production build passed.
160 lines
6.2 KiB
Python
160 lines
6.2 KiB
Python
"""What the agent sees must be what is on screen, whichever backend dumped it.
|
|
|
|
Covers the host-side half of the UIAutomator parity work: negative bounds no
|
|
longer lose an element's coordinates, hints and errors reach the element list
|
|
the model reads, an empty helper answer degrades to UIAutomator2 in ``auto``
|
|
mode, and the parity comparison used by ``artemis helper parity`` flags the
|
|
regressions it exists for.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import MagicMock
|
|
|
|
import pytest
|
|
|
|
from artemis.clients.accessibility_client import AccessibilityClient, HelperEmptyHierarchy
|
|
from artemis.clients.screen_client_factory import FallbackScreenClient
|
|
from artemis.clients.ui_automator_client import UIAutomatorClient, _parse_hierarchy_xml_to_elements
|
|
from artemis.core.diagnostics.hierarchy_parity import compare_dumps
|
|
from artemis.utils import ui_filter
|
|
from artemis.utils.visualization import format_minimal_list_with_elements
|
|
|
|
|
|
def _node(**attrs: str) -> str:
|
|
base = {
|
|
"index": "0",
|
|
"text": "",
|
|
"resource-id": "",
|
|
"class": "android.widget.TextView",
|
|
"package": "app",
|
|
"content-desc": "",
|
|
"clickable": "false",
|
|
"enabled": "true",
|
|
"bounds": "[0,0][100,100]",
|
|
}
|
|
base.update(attrs)
|
|
return "<node " + " ".join(f'{k}="{v}"' for k, v in base.items()) + " />"
|
|
|
|
|
|
def _xml(*nodes: str) -> str:
|
|
return '<?xml version="1.0"?><hierarchy rotation="0">' + "".join(nodes) + "</hierarchy>"
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Negative bounds
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_negative_bounds_keep_their_coordinates_and_get_clipped():
|
|
elements = _parse_hierarchy_xml_to_elements(_xml(_node(text="Row", bounds="[-20,-40][300,60]")))
|
|
row = elements[1]
|
|
assert row["parsed_bounds"] == {"left": -20, "top": -40, "right": 300, "bottom": 60}
|
|
|
|
assert ui_filter._parse_bounds("[-20,-40][300,60]") == {
|
|
"left": -20,
|
|
"top": -40,
|
|
"right": 300,
|
|
"bottom": 60,
|
|
}
|
|
filtered = ui_filter.filter_ui_hierarchy([row], screen_width=1080, screen_height=2400)
|
|
assert len(filtered) == 1
|
|
assert filtered[0]["parsed_bounds"] == {"left": 0, "top": 0, "right": 300, "bottom": 60}
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Hints and errors reach the model
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_minimal_list_shows_hint_for_empty_inputs_and_errors():
|
|
elements = _parse_hierarchy_xml_to_elements(
|
|
_xml(
|
|
_node(text="", hint="Search settings", **{"class": "android.widget.EditText"}),
|
|
_node(text="abc", error="Password too short", bounds="[0,200][100,300]"),
|
|
_node(text="", bounds="[0,400][100,500]"), # nothing to say: stays out
|
|
)
|
|
)
|
|
text, items, labels = format_minimal_list_with_elements(elements[1:], 1000, 1000)
|
|
lines = text.splitlines()
|
|
assert (
|
|
lines[0]
|
|
== "[1] Hint: 'Search settings' | Bounds: [0,0][100,100] | Editable: true | Value: empty"
|
|
)
|
|
assert lines[1] == "[2] Text: 'abc' | Bounds: [0,200][100,300] | Error: 'Password too short'"
|
|
assert len(lines) == 2 and labels == ["1", "2"]
|
|
assert items[0]["is_hint"] is True and items[0]["text"] == "Search settings"
|
|
assert items[1]["error"] == "Password too short"
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Empty helper answer degrades instead of showing a blank screen
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_empty_helper_hierarchy_falls_back_to_uiautomator():
|
|
helper = MagicMock(spec=AccessibilityClient)
|
|
u2 = MagicMock(spec=UIAutomatorClient)
|
|
helper.get_screen_data.side_effect = HelperEmptyHierarchy("no window")
|
|
u2.get_screen_data.return_value = "from-u2"
|
|
client = FallbackScreenClient(
|
|
"dev", helper=helper, uiautomator_factory=lambda: u2, state_probe=lambda _s: "device"
|
|
)
|
|
assert client.get_screen_data() == "from-u2"
|
|
assert client.active_backend == "uiautomator"
|
|
assert "HelperEmptyHierarchy" in client.backend_history[-1]["reason"]
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Parity comparison
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_parity_passes_when_both_backends_describe_the_same_screen():
|
|
u2 = _xml(
|
|
_node(text="Wi-Fi", bounds="[0,100][1080,200]"),
|
|
_node(**{"content-desc": "Back", "bounds": "[0,0][120,120]"}),
|
|
)
|
|
helper = _xml(
|
|
_node(text="Wi-Fi", bounds="[2,101][1078,199]"),
|
|
_node(**{"content-desc": "Back", "bounds": "[0,0][120,120]"}),
|
|
)
|
|
report = compare_dumps(helper, u2, 1080, 2400)
|
|
assert report["ok"], report["problems"]
|
|
assert report["match"]["recall"] == 1.0 and report["match"]["precision"] == 1.0
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"helper_extra, expected",
|
|
[
|
|
(_node(text="Hidden drawer item", bounds="[-1080,300][0,400]"), "negative bounds"),
|
|
(_node(text="Next page", bounds="[1080,300][2160,400]"), "outside the screen"),
|
|
(
|
|
"".join(
|
|
_node(text=f"Ghost {i}", bounds=f"[0,{i * 50}][100,{i * 50 + 40}]")
|
|
for i in range(8)
|
|
),
|
|
"not on screen",
|
|
),
|
|
],
|
|
)
|
|
def test_parity_flags_elements_uiautomator_would_not_show(helper_extra, expected):
|
|
u2 = _xml(_node(text="Wi-Fi", bounds="[0,100][1080,200]"))
|
|
helper = _xml(_node(text="Wi-Fi", bounds="[0,100][1080,200]"), helper_extra)
|
|
report = compare_dumps(helper, u2, 1080, 2400)
|
|
assert not report["ok"]
|
|
assert any(expected in problem for problem in report["problems"]), report["problems"]
|
|
|
|
|
|
def test_parity_flags_missing_elements():
|
|
u2 = _xml(
|
|
_node(text="Wi-Fi", bounds="[0,100][1080,200]"),
|
|
_node(text="Bluetooth", bounds="[0,200][1080,300]"),
|
|
)
|
|
helper = _xml(_node(text="Wi-Fi", bounds="[0,100][1080,200]"))
|
|
report = compare_dumps(helper, u2, 1080, 2400)
|
|
assert not report["ok"]
|
|
assert report["match"]["missing_in_helper"] == [
|
|
"'Bluetooth' at {'left': 0, 'top': 200, 'right': 1080, 'bottom': 300}"
|
|
]
|