532 lines
20 KiB
Python
532 lines
20 KiB
Python
# Copyright 2026 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""HTTP client for the Artemis Accessibility Helper running on a device.
|
|
|
|
A lightweight, conflict-free alternative to UIAutomator2: it never takes the
|
|
singleton ``UiAutomationConnection``, so it runs alongside Mobly, Appium and
|
|
Espresso. This class only speaks the helper's HTTP API. Installing, enabling,
|
|
port forwarding, the session token and hot-plug recovery live in
|
|
:mod:`artemis.runtime.helper_manager`; this client asks the manager for a
|
|
session lazily and hands transport failures back to it once.
|
|
|
|
The screen-data contract is byte-for-byte the one :class:`UIAutomatorClient`
|
|
exposes (``UIAutomatorScreenData`` with the element dictionaries produced by
|
|
``_parse_hierarchy_xml_to_elements``), so every consumer downstream
|
|
(``filter_ui_hierarchy``, the Explorer index, the MCP actuators that compare
|
|
``clickable == "true"``) sees identical shapes whichever backend is active.
|
|
|
|
One observation is one request: ``/snapshot?fields=xml`` returns the XML and,
|
|
on Android 11+, the screenshot taken at the same instant with the bitmap's own
|
|
dimensions. On older Android the same answer carries the XML and the client
|
|
only adds an adb screencap; the hierarchy is never dumped twice.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from io import BytesIO
|
|
import json
|
|
import subprocess
|
|
import time
|
|
from typing import Any
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
from PIL import Image
|
|
|
|
from artemis.runtime.native_protocol import input_completion_seconds
|
|
|
|
from artemis.clients.ui_automator_client import (
|
|
UIAutomatorScreenData,
|
|
_parse_hierarchy_xml_to_elements,
|
|
_pil_to_base64,
|
|
)
|
|
from artemis.runtime.adb_endpoint import adb_command
|
|
from artemis.runtime.awake_service import ensure_device_awake
|
|
from artemis.runtime.helper_manager import (
|
|
DEVICE_PORT,
|
|
PACKAGE_NAME,
|
|
SERVICE_NAME,
|
|
AccessibilityHelperManager,
|
|
HelperSession,
|
|
HelperUnavailable,
|
|
ProvisionEvent,
|
|
helper_manager,
|
|
)
|
|
from artemis.utils.logger import get_logger
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
# Kept for callers that imported these from here.
|
|
DEFAULT_PORT = DEVICE_PORT
|
|
TOKEN_HEADER = "X-Artemis-Token"
|
|
__all__ = [
|
|
"AccessibilityClient",
|
|
"DEFAULT_PORT",
|
|
"HelperEmptyHierarchy",
|
|
"HelperRequestError",
|
|
"HelperUnavailable",
|
|
"PACKAGE_NAME",
|
|
"SERVICE_NAME",
|
|
"TOKEN_HEADER",
|
|
"normalize_helper_elements",
|
|
]
|
|
|
|
_BOOL_ATTRIBUTES = (
|
|
"checkable",
|
|
"checked",
|
|
"clickable",
|
|
"enabled",
|
|
"focusable",
|
|
"focused",
|
|
"scrollable",
|
|
"long-clickable",
|
|
"password",
|
|
"selected",
|
|
"visible-to-user",
|
|
)
|
|
_GLOBAL_KEYS = ("back", "home", "recents", "notifications", "quick_settings")
|
|
|
|
|
|
class HelperEmptyHierarchy(RuntimeError):
|
|
"""The helper answered but found no window to dump.
|
|
|
|
Raised instead of returning a screen with zero elements, so ``auto`` mode
|
|
can try UIAutomator2 and ``helper`` mode reports the real cause instead of
|
|
showing the agent an empty screen.
|
|
"""
|
|
|
|
|
|
class HelperRequestError(RuntimeError):
|
|
"""The helper answered with an HTTP error that a new tunnel cannot fix."""
|
|
|
|
def __init__(self, status: int, path: str, body: str):
|
|
self.status = status
|
|
self.path = path
|
|
self.body = body
|
|
super().__init__(f"accessibility helper {path} answered HTTP {status}: {body[:200]}")
|
|
|
|
|
|
def normalize_helper_elements(elements: Any) -> list[dict[str, Any]]:
|
|
"""Coerce the helper's JSON element list into the UIAutomator element shape.
|
|
|
|
Used only when a dump carries elements but no XML. The JSON encodes flags
|
|
as booleans and omits ``accessibilityText``; the XML parser emits string
|
|
flags and mirrors ``content-desc`` into ``accessibilityText``, and several
|
|
consumers compare against the string ``"true"``.
|
|
"""
|
|
if isinstance(elements, dict):
|
|
elements = [elements]
|
|
if not isinstance(elements, list):
|
|
return []
|
|
normalized: list[dict[str, Any]] = []
|
|
for raw in elements:
|
|
if not isinstance(raw, dict):
|
|
continue
|
|
element = dict(raw)
|
|
for key in _BOOL_ATTRIBUTES:
|
|
if key in element and isinstance(element[key], bool):
|
|
element[key] = "true" if element[key] else "false"
|
|
desc = element.get("content-desc")
|
|
if desc is not None and "accessibilityText" not in element:
|
|
element["accessibilityText"] = desc
|
|
bounds = element.get("parsed_bounds")
|
|
if isinstance(bounds, dict):
|
|
element["parsed_bounds"] = {
|
|
side: int(bounds.get(side, 0)) for side in ("left", "top", "right", "bottom")
|
|
}
|
|
element.pop("children", None)
|
|
normalized.append(element)
|
|
return normalized
|
|
|
|
|
|
class AccessibilityClient:
|
|
"""Screen-data client over the helper's loopback HTTP API."""
|
|
|
|
backend_name = "helper"
|
|
|
|
def __init__(
|
|
self,
|
|
device_id: str,
|
|
manager: AccessibilityHelperManager | None = None,
|
|
*,
|
|
provision_on_connect: bool = True,
|
|
request_timeout: float = 6.0,
|
|
) -> None:
|
|
self._device_id = device_id
|
|
self._manager = manager or helper_manager
|
|
self._provision_on_connect = provision_on_connect
|
|
self._request_timeout = request_timeout
|
|
self._session: HelperSession | None = None
|
|
self._awake_strategy: str | None = None
|
|
|
|
# ------------------------------------------------------------------ #
|
|
# Session lifecycle
|
|
# ------------------------------------------------------------------ #
|
|
|
|
@property
|
|
def device_id(self) -> str:
|
|
return self._device_id
|
|
|
|
@property
|
|
def session(self) -> HelperSession | None:
|
|
return self._session
|
|
|
|
@property
|
|
def active_backend(self) -> str | None:
|
|
return self.backend_name if self._session is not None else None
|
|
|
|
def connect(self, on_event: ProvisionEvent | None = None) -> None:
|
|
"""Task-path entry: provision (install / upgrade / enable) and attach.
|
|
|
|
``on_event`` receives ``installing`` / ``upgrading`` before the slow step.
|
|
"""
|
|
self._session = self._manager.attach(
|
|
self._device_id, provision=self._provision_on_connect, on_event=on_event
|
|
)
|
|
if self._awake_strategy is None:
|
|
self._awake_strategy = ensure_device_awake(self._device_id)
|
|
|
|
def _ensure_session(self) -> HelperSession:
|
|
"""Observer-path entry: attach to a helper that is already running."""
|
|
if self._session is None:
|
|
self._session = self._manager.attach(self._device_id, provision=False)
|
|
if self._awake_strategy is None:
|
|
self._awake_strategy = ensure_device_awake(self._device_id)
|
|
return self._session
|
|
|
|
def disconnect(self) -> None:
|
|
self._manager.detach(self._device_id)
|
|
self._session = None
|
|
self._awake_strategy = None
|
|
|
|
def ping(self) -> bool:
|
|
try:
|
|
session = self._ensure_session()
|
|
except HelperUnavailable:
|
|
return False
|
|
return self._manager.ping(session.local_port) is not None
|
|
|
|
@property
|
|
def native_capabilities(self) -> frozenset[str]:
|
|
"""Advertised at attach time; capability queries never block on network IO."""
|
|
return frozenset(self._session.capabilities) if self._session is not None else frozenset()
|
|
|
|
def native_request(
|
|
self, command: str, args: dict[str, Any], *, timeout: float | None = None
|
|
) -> dict[str, Any]:
|
|
"""Preserve native receipts. A failed POST is never sent to another backend."""
|
|
read_only = command == "inspect_control"
|
|
if command == "inspect_control" and self._session is None:
|
|
try:
|
|
self._ensure_session()
|
|
except (HelperUnavailable, OSError, RuntimeError) as exc:
|
|
return {
|
|
"success": False,
|
|
"code": "UNAVAILABLE",
|
|
"execution": "not_started",
|
|
"error": str(exc),
|
|
}
|
|
session = self._session
|
|
if session is None:
|
|
return {
|
|
"success": False,
|
|
"code": "UNAVAILABLE",
|
|
"execution": "not_started",
|
|
"error": "Connect the Android helper before issuing native operations.",
|
|
}
|
|
if (
|
|
command not in {"inspect_control", "control_action", "touch_gesture", "cancel_gesture"}
|
|
or command not in session.capabilities
|
|
):
|
|
return {
|
|
"success": False,
|
|
"code": "UNSUPPORTED",
|
|
"execution": "not_started",
|
|
"error": "The attached Android helper does not advertise this native action. Reconnect after upgrading the bundled helper.",
|
|
}
|
|
completion_window = input_completion_seconds(command, args)
|
|
started = time.monotonic()
|
|
request_timeout = max(timeout or self._request_timeout, completion_window)
|
|
|
|
def drain_uncertain_input() -> None:
|
|
remaining = completion_window - (time.monotonic() - started)
|
|
if remaining > 0:
|
|
time.sleep(remaining)
|
|
|
|
try:
|
|
result = json.loads(
|
|
self._http_once(
|
|
session, "/action", {**args, "cmd": command}, request_timeout
|
|
).decode("utf-8")
|
|
)
|
|
if not isinstance(result, dict) or not isinstance(result.get("success"), bool):
|
|
raise ValueError("Malformed native action receipt.")
|
|
if result.get("execution") not in {"not_started", "observed", "completed"}:
|
|
drain_uncertain_input()
|
|
return result
|
|
except (
|
|
HelperUnavailable,
|
|
HelperRequestError,
|
|
urllib.error.URLError,
|
|
OSError,
|
|
ValueError,
|
|
TimeoutError,
|
|
) as exc:
|
|
status = (
|
|
exc.status if isinstance(exc, HelperRequestError) else getattr(exc, "code", None)
|
|
)
|
|
rejected = status in {400, 401, 403, 404, 405, 413, 422, 429}
|
|
if not read_only and not rejected:
|
|
drain_uncertain_input()
|
|
return {
|
|
"success": False,
|
|
"code": "UNAVAILABLE" if read_only else "TRANSPORT_ERROR",
|
|
"execution": "not_started" if read_only or rejected else "unknown",
|
|
"error": f"{exc}. Observe current state; do not replay an unacknowledged write.",
|
|
}
|
|
|
|
# ------------------------------------------------------------------ #
|
|
# Transport
|
|
# ------------------------------------------------------------------ #
|
|
|
|
def _http(
|
|
self, path: str, payload: dict[str, Any] | None = None, timeout: float | None = None
|
|
) -> bytes:
|
|
"""One request with two single-shot repairs.
|
|
|
|
* HTTP 401: the helper lost its token (re-bound service, reboot): push it
|
|
again and retry once.
|
|
* Transport failure on a read: rebuild the tunnel once and retry.
|
|
Actions are not replayed because they may already have executed.
|
|
|
|
Any other HTTP status is a :class:`HelperRequestError`; a new tunnel
|
|
would not change the answer.
|
|
"""
|
|
session = self._ensure_session()
|
|
try:
|
|
return self._http_once(session, path, payload, timeout)
|
|
except urllib.error.HTTPError as exc:
|
|
body = _read_error_body(exc)
|
|
if exc.code != 401:
|
|
raise HelperRequestError(exc.code, path, body) from exc
|
|
logger.warning(
|
|
f"Accessibility helper on {self._device_id} rejected the session token "
|
|
f"({body[:120]}); pushing it again."
|
|
)
|
|
self._manager.push_token(self._device_id)
|
|
try:
|
|
return self._http_once(session, path, payload, timeout)
|
|
except urllib.error.HTTPError as again:
|
|
raise HelperRequestError(again.code, path, _read_error_body(again)) from again
|
|
except (urllib.error.URLError, OSError, TimeoutError) as exc:
|
|
if payload is not None:
|
|
raise
|
|
logger.warning(
|
|
f"Accessibility helper request {path} on {self._device_id} failed ({exc}); "
|
|
"rebuilding the tunnel once."
|
|
)
|
|
self._session = self._manager.reattach(self._device_id)
|
|
try:
|
|
return self._http_once(self._session, path, payload, timeout)
|
|
except urllib.error.HTTPError as again:
|
|
raise HelperRequestError(again.code, path, _read_error_body(again)) from again
|
|
|
|
def _http_once(
|
|
self,
|
|
session: HelperSession,
|
|
path: str,
|
|
payload: dict[str, Any] | None,
|
|
timeout: float | None,
|
|
) -> bytes:
|
|
data = (
|
|
json.dumps(payload, ensure_ascii=False).encode("utf-8") if payload is not None else None
|
|
)
|
|
headers: dict[str, str] = {}
|
|
token = session.token or self._manager.host_token()
|
|
if token:
|
|
headers[TOKEN_HEADER] = token
|
|
if data is not None:
|
|
headers["Content-Type"] = "application/json; charset=utf-8"
|
|
request = urllib.request.Request(f"{session.base_url}{path}", data=data, headers=headers)
|
|
with urllib.request.urlopen(request, timeout=timeout or self._request_timeout) as resp:
|
|
return resp.read()
|
|
|
|
def _rpc(self, cmd: str, params: dict[str, Any] | None = None) -> bool:
|
|
payload = {"cmd": cmd, **(params or {})}
|
|
try:
|
|
data = json.loads(self._http("/action", payload).decode("utf-8"))
|
|
except (
|
|
HelperUnavailable,
|
|
HelperRequestError,
|
|
urllib.error.URLError,
|
|
OSError,
|
|
ValueError,
|
|
TimeoutError,
|
|
) as exc:
|
|
logger.warning(f"Accessibility helper command '{cmd}' failed: {exc}")
|
|
return False
|
|
return bool(data.get("success", False))
|
|
|
|
# ------------------------------------------------------------------ #
|
|
# Screen data
|
|
# ------------------------------------------------------------------ #
|
|
|
|
def get_hierarchy(self) -> str:
|
|
"""Raw UIAutomator-format XML, the same contract as ``UIAutomatorClient``."""
|
|
return self._http("/dump_xml").decode("utf-8")
|
|
|
|
def get_hierarchy_json(self, *, include_invisible: bool = False) -> dict[str, Any]:
|
|
"""The helper's JSON dump (XML plus its own element list); diagnostics only."""
|
|
query = "fields=xml,elements" + ("&include_invisible=1" if include_invisible else "")
|
|
return json.loads(self._http(f"/dump?{query}").decode("utf-8"))
|
|
|
|
def _snapshot(self) -> dict[str, Any]:
|
|
"""``/snapshot?fields=xml``: hierarchy plus, on Android 11+, the screenshot."""
|
|
data = json.loads(self._http("/snapshot?fields=xml", timeout=8.0).decode("utf-8"))
|
|
if not isinstance(data, dict):
|
|
raise HelperRequestError(200, "/snapshot", "non-object answer")
|
|
if not data.get("success"):
|
|
raise HelperEmptyHierarchy(
|
|
f"accessibility helper on {self._device_id} returned no hierarchy: "
|
|
f"{data.get('error') or 'unknown reason'}"
|
|
)
|
|
return data
|
|
|
|
def get_atomic_snapshot(self) -> dict[str, Any] | None:
|
|
"""Screenshot + hierarchy captured in one call on the device (Android 11+), else None."""
|
|
data = self._snapshot()
|
|
if data.get("has_screenshot") and data.get("screenshot_base64"):
|
|
return data
|
|
return None
|
|
|
|
def get_screenshot(self) -> Image.Image | None:
|
|
try:
|
|
result = subprocess.run(
|
|
adb_command(["-s", self._device_id, "exec-out", "screencap", "-p"]),
|
|
stdin=subprocess.DEVNULL,
|
|
capture_output=True,
|
|
timeout=10,
|
|
check=True,
|
|
)
|
|
return Image.open(BytesIO(result.stdout))
|
|
except (OSError, subprocess.SubprocessError) as exc:
|
|
logger.error(f"Failed to capture screenshot via adb: {exc}")
|
|
return None
|
|
|
|
def get_screenshot_base64(self) -> str | None:
|
|
screenshot = self.get_screenshot()
|
|
return None if screenshot is None else _pil_to_base64(screenshot, format="JPEG")
|
|
|
|
@staticmethod
|
|
def _elements_from_dump(dump: dict[str, Any]) -> tuple[str, list[dict[str, Any]]]:
|
|
xml = dump.get("xml") or ""
|
|
if isinstance(xml, str) and xml.strip():
|
|
return xml, _parse_hierarchy_xml_to_elements(xml)
|
|
return "", normalize_helper_elements(dump.get("elements"))
|
|
|
|
def get_screen_data(self) -> UIAutomatorScreenData:
|
|
snapshot = self._snapshot()
|
|
xml, elements = self._elements_from_dump(snapshot)
|
|
if snapshot.get("has_screenshot") and snapshot.get("screenshot_base64"):
|
|
width, height = _dimensions(snapshot)
|
|
return UIAutomatorScreenData(
|
|
base64=str(snapshot["screenshot_base64"]),
|
|
hierarchy_xml=xml,
|
|
elements=elements,
|
|
width=width,
|
|
height=height,
|
|
)
|
|
|
|
# Android 10 and below (or a rate-limited capture that did not recover):
|
|
# keep the hierarchy we already have and only add an adb screencap.
|
|
reason = snapshot.get("screenshot_error")
|
|
if reason:
|
|
logger.debug(f"Helper snapshot without screenshot ({reason}); using adb screencap")
|
|
screenshot = self.get_screenshot()
|
|
if screenshot is None:
|
|
raise RuntimeError("Failed to capture screenshot")
|
|
return UIAutomatorScreenData(
|
|
base64=_pil_to_base64(screenshot, format="JPEG"),
|
|
hierarchy_xml=xml,
|
|
elements=elements,
|
|
width=screenshot.width,
|
|
height=screenshot.height,
|
|
)
|
|
|
|
# ------------------------------------------------------------------ #
|
|
# Input
|
|
# ------------------------------------------------------------------ #
|
|
|
|
def set_clipboard(self, text: str) -> bool:
|
|
"""Put ``text`` on the device clipboard so the driver can paste it."""
|
|
return self._rpc("clipboard", {"text": text})
|
|
|
|
def send_text(self, text: str) -> bool:
|
|
"""Append ``text`` to the focused field, the way typing through an IME does."""
|
|
return self._rpc("type", {"text": text, "append": True})
|
|
|
|
def clear_text(self) -> bool:
|
|
return self._rpc("clear", {})
|
|
|
|
def press_key(self, key: str) -> bool:
|
|
key_lower = str(key).lower()
|
|
if key_lower in _GLOBAL_KEYS:
|
|
return self._rpc("global", {"action": key_lower})
|
|
result = subprocess.run(
|
|
adb_command(
|
|
[
|
|
"-s",
|
|
self._device_id,
|
|
"shell",
|
|
"input",
|
|
"keyevent",
|
|
f"KEYCODE_{key_lower.upper()}",
|
|
]
|
|
),
|
|
stdin=subprocess.DEVNULL,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=15,
|
|
check=False,
|
|
)
|
|
return result.returncode == 0
|
|
|
|
def tap(self, x: float, y: float) -> bool:
|
|
return self._rpc("tap", {"x": x, "y": y})
|
|
|
|
def swipe(self, x1: float, y1: float, x2: float, y2: float, duration_ms: int = 300) -> bool:
|
|
return self._rpc("swipe", {"x1": x1, "y1": y1, "x2": x2, "y2": y2, "duration": duration_ms})
|
|
|
|
|
|
def _read_error_body(exc: urllib.error.HTTPError) -> str:
|
|
try:
|
|
return exc.read().decode("utf-8", "replace")
|
|
except OSError:
|
|
return str(exc)
|
|
|
|
|
|
def _dimensions(snapshot: dict[str, Any]) -> tuple[int, int]:
|
|
"""The screenshot's own size; a snapshot without it is a broken answer, not a 1080x2400 one."""
|
|
try:
|
|
width = int(snapshot.get("width") or 0)
|
|
height = int(snapshot.get("height") or 0)
|
|
except (TypeError, ValueError):
|
|
width = height = 0
|
|
if width <= 0 or height <= 0:
|
|
raise HelperRequestError(
|
|
200, "/snapshot", f"screenshot without dimensions (width={width}, height={height})"
|
|
)
|
|
return width, height
|