diff --git a/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.ts b/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.ts
index 5601311..4372372 100644
--- a/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.ts
+++ b/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.ts
@@ -61,6 +61,10 @@ interface StartupWorkStage {
started: string;
completed: string;
completedMessage: string;
+ /** Sub-steps that replace the live message while the stage is still running. */
+ liveDetailStages?: string[];
+ /** A later event whose message is a better completion line than the stage's own. */
+ completedDetailStage?: string;
}
const STARTUP_WORK_STAGES: StartupWorkStage[] = [
@@ -72,7 +76,12 @@ const STARTUP_WORK_STAGES: StartupWorkStage[] = [
{
started: 'uiautomator',
completed: 'uiautomator_ready',
- completedMessage: 'UI Automator is ready'
+ completedMessage: 'UI hierarchy service is ready',
+ // First task on a device installs / upgrades the accessibility helper
+ // (a few seconds): say so instead of a generic "connecting".
+ liveDetailStages: ['helper_install', 'helper_upgrade'],
+ // "UI hierarchy source: Artemis accessibility helper v1.1.3" (or UIAutomator2).
+ completedDetailStage: 'hierarchy_backend'
},
{
started: 'environment',
@@ -119,10 +128,21 @@ export function buildStartupWorkItems(
const endTimestamp = completed?.timestamp
|| (isActive ? nowSeconds : events[events.length - 1]?.timestamp || startTimestamp);
+ const liveDetail = (stage.liveDetailStages || [])
+ .map((detailStage) => byStage.get(detailStage))
+ .filter((event): event is StartupProgressEvent => Boolean(event))
+ .filter((event) => event.timestamp >= startTimestamp)
+ .sort((a, b) => b.timestamp - a.timestamp)[0];
+ const completedDetail = stage.completedDetailStage
+ ? byStage.get(stage.completedDetailStage)
+ : undefined;
+ const message = completed
+ ? (completedDetail?.message || explicitlyCompleted?.message || stage.completedMessage)
+ : (liveDetail?.message || started!.message);
+
return [{
...(explicitlyCompleted || started!),
- message: explicitlyCompleted?.message
- || (completed ? stage.completedMessage : started!.message),
+ message,
isActive,
elapsed: formatStartupElapsed(endTimestamp - startTimestamp)
}];
@@ -198,6 +218,7 @@ import { drawActionCoordinatesOnOverlay } from '../../utils/image-overlay.util';
import {
consolidateLogsToBlocks,
+ isBackendSwitchNote,
groupBlocksToPhases,
extractBlockTokens,
formatTokenCount,
@@ -1795,6 +1816,7 @@ export class AgentStreamComponent implements AfterViewInit {
|| this.isDisplayableLLMFailure(t)
|| this.isLLMRetry(t)
|| this.isReportStatusAction(t)
+ || this.isBackendSwitchNote(t)
);
const hasAndroidActions = Boolean(block.data?.action_taken) && (this.isAndroidAction(block.data.action_taken) || this.isReportStatusAction(block.data.action_taken));
return hasNative || hasRaw || hasReset || hasVisibleTools || hasAndroidActions;
@@ -1826,6 +1848,15 @@ export class AgentStreamComponent implements AfterViewInit {
&& (tool?.name === 'llm_pause' || tool?.payload?.pause === true);
}
+ /** Mid-run change of the UI-hierarchy source (helper <-> UIAutomator2). */
+ public isBackendSwitchNote(tool: any): boolean {
+ return isBackendSwitchNote(tool);
+ }
+
+ public getBackendSwitchMessage(tool: any): string {
+ return String(tool?.payload?.message || 'UI hierarchy source changed');
+ }
+
public resumePausedTask(event: Event): void {
event.stopPropagation();
this.agentService.resumeTask();
diff --git a/apps/showcase_ui/src/app/utils/stream-aggregator.util.spec.ts b/apps/showcase_ui/src/app/utils/stream-aggregator.util.spec.ts
index ee8dc8f..d6f1212 100644
--- a/apps/showcase_ui/src/app/utils/stream-aggregator.util.spec.ts
+++ b/apps/showcase_ui/src/app/utils/stream-aggregator.util.spec.ts
@@ -1,4 +1,4 @@
-import { consolidateLogsToBlocks, getSortedStepEvents, persistedStreamToSegments } from './stream-aggregator.util';
+import { consolidateLogsToBlocks, getSortedStepEvents, isBackendSwitchNote, persistedStreamToSegments } from './stream-aggregator.util';
describe('stream aggregator timeline ordering', () => {
it('interleaves text, tools, and actions by timestamp', () => {
@@ -413,6 +413,51 @@ describe('stream aggregator step ownership', () => {
expect(blocks.length).toBe(1);
expect(blocks[0].data.generic_tools.map((t: any) => t.trace_id)).toEqual(['trace-note']);
});
+
+ it('keeps a mid-run hierarchy source switch in the step where it happened, as a note', () => {
+ const initial = {
+ type: 'trace_recorded',
+ timestamp: '2026-09-02T03:57:20.000Z',
+ data: {
+ trace_id: 'trace-backend-initial',
+ type: 'log',
+ name: 'hierarchy_backend',
+ payload: { backend: 'helper', previous_backend: null, message: 'UI hierarchy source: Artemis accessibility helper v1.1.3' }
+ }
+ };
+ const switched = {
+ type: 'trace_recorded',
+ timestamp: '2026-09-02T03:57:26.000Z',
+ data: {
+ trace_id: 'trace-backend-switch',
+ type: 'log',
+ name: 'hierarchy_backend',
+ timestamp: '2026-09-02T03:57:26.000Z',
+ payload: {
+ backend: 'uiautomator',
+ previous_backend: 'helper',
+ reason: 'HelperUnavailable: tunnel gone',
+ message: 'UI hierarchy source switched from Artemis accessibility helper to UIAutomator2 because HelperUnavailable: tunnel gone'
+ }
+ }
+ };
+ const ordinaryLog = {
+ type: 'trace_recorded',
+ timestamp: '2026-09-02T03:57:27.000Z',
+ data: { trace_id: 'trace-log', type: 'log', name: 'artemis.runtime', payload: { message: 'noise' } }
+ };
+
+ const blocks = consolidateLogsToBlocks([initial, plannerStream, switched, ordinaryLog]);
+
+ // The initial line belongs to the startup block; ordinary logs stay out.
+ expect(blocks.length).toBe(1);
+ expect(blocks[0].data.generic_tools.map((t: any) => t.trace_id)).toEqual(['trace-backend-switch']);
+ expect(isBackendSwitchNote(blocks[0].data.generic_tools[0])).toBeTrue();
+ expect(isBackendSwitchNote(initial.data)).toBeFalse();
+
+ const events = getSortedStepEvents(blocks[0].data);
+ expect(events.map((e) => e.type)).toEqual(['text', 'tool']);
+ });
});
describe('stream aggregator checker history relocation', () => {
diff --git a/apps/showcase_ui/src/app/utils/stream-aggregator.util.ts b/apps/showcase_ui/src/app/utils/stream-aggregator.util.ts
index d204a69..8b57e47 100644
--- a/apps/showcase_ui/src/app/utils/stream-aggregator.util.ts
+++ b/apps/showcase_ui/src/app/utils/stream-aggregator.util.ts
@@ -247,8 +247,17 @@ function relocateCheckerTools(blocks: StepBlock[]): void {
}
/**
- * Consolidate raw SSE logs into deduplicated and ordered StepBlocks
+ * A `hierarchy_backend` log trace that records a *switch* (helper -> UIAutomator2
+ * or back). The initial "source: ..." line belongs to the startup block; only
+ * the mid-run change is worth a row in the timeline.
*/
+export function isBackendSwitchNote(tool: any): boolean {
+ return tool?.type === 'log'
+ && tool?.name === 'hierarchy_backend'
+ && Boolean(tool?.payload?.previous_backend);
+}
+
+/** Consolidate raw SSE logs into deduplicated and ordered StepBlocks. */
export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
if (!rawLogs || rawLogs.length === 0) return [];
@@ -487,7 +496,7 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
const isTool = log.data.type === 'tool' || isAction;
const isVisibleLLMEvent = log.data.type === 'llm_call'
&& (log.data.status === 'failed' || log.data.status === 'retrying');
- if (!isTool && !isVisibleLLMEvent) return;
+ if (!isTool && !isVisibleLLMEvent && !isBackendSwitchNote(log.data)) return;
let stepId = log.data.step_id;
let existingIndex = -1;
diff --git a/apps/showcase_ui/src/app/utils/tool-formatter.util.ts b/apps/showcase_ui/src/app/utils/tool-formatter.util.ts
index d0181c2..225c1f9 100644
--- a/apps/showcase_ui/src/app/utils/tool-formatter.util.ts
+++ b/apps/showcase_ui/src/app/utils/tool-formatter.util.ts
@@ -279,6 +279,7 @@ export function shouldShowTool(tool: any, _stepData?: any): boolean {
if (!tool || !tool.name) return false;
if (tool.type === 'llm_call') return false;
if (tool.type === 'agent') return false;
+ if (tool.type === 'log') return false; // system notes render as quiet rows, not tool cards
if (isInternalPlumbingTool(tool)) return false;
return true;
diff --git a/artemis/agents/outputter/outputter.py b/artemis/agents/outputter/outputter.py
index 925760a..b84f429 100644
--- a/artemis/agents/outputter/outputter.py
+++ b/artemis/agents/outputter/outputter.py
@@ -185,6 +185,27 @@ async def outputter(
except Exception as e:
logger.error(f"Failed to attach verdict ledger to outputter: {e}")
+ # Run environment facts the model must not guess: which UI-hierarchy source
+ # served the run and whether it switched mid-way (a fallback to UIAutomator2
+ # explains slower steps and possible conflicts with other automation tools).
+ from artemis.clients.screen_client_factory import hierarchy_backend_sentence
+
+ relative = ctx.data_engine.get_relative_time if ctx.data_engine else None
+ environment_line = hierarchy_backend_sentence(
+ getattr(ctx, "ui_adb_client", None), relative_time=relative
+ )
+ if environment_line:
+ human_message_content.append(
+ {
+ "type": "text",
+ "text": (
+ "## Run Environment (facts, include verbatim under a short "
+ "'Environment' note only if the source changed mid-run or the "
+ "user asked about it)" + chr(10) + environment_line
+ ),
+ }
+ )
+
if screenshot_b64:
human_message_content.append(
{
diff --git a/artemis/clients/accessibility_client.py b/artemis/clients/accessibility_client.py
index e55505c..a86384f 100644
--- a/artemis/clients/accessibility_client.py
+++ b/artemis/clients/accessibility_client.py
@@ -12,96 +12,324 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-"""Artemis Accessibility Helper client for Android device automation.
+"""HTTP client for the Artemis Accessibility Helper running on a device.
-Provides a lightweight, conflict-free alternative to UIAutomator2.
-Runs concurrently with Mobly, Appium, and Espresso without monopolizing
-the singleton UiAutomationConnection.
+A lightweight, conflict-free alternative to UIAutomator2: it never takes the
+singleton ``UiAutomationConnection``, so it runs alongside Mobly, Appium and
+Espresso. This class only speaks the helper's HTTP API. Installing, enabling,
+port forwarding, the session token and hot-plug recovery live in
+:mod:`artemis.runtime.helper_manager`; this client asks the manager for a
+session lazily and hands transport failures back to it once.
+
+The screen-data contract is byte-for-byte the one :class:`UIAutomatorClient`
+exposes (``UIAutomatorScreenData`` with the element dictionaries produced by
+``_parse_hierarchy_xml_to_elements``), so every consumer downstream
+(``filter_ui_hierarchy``, the Explorer index, the MCP actuators that compare
+``clickable == "true"``) sees identical shapes whichever backend is active.
+
+One observation is one request: ``/snapshot?fields=xml`` returns the XML and,
+on Android 11+, the screenshot taken at the same instant with the bitmap's own
+dimensions. On older Android the same answer carries the XML and the client
+only adds an adb screencap; the hierarchy is never dumped twice.
"""
+from __future__ import annotations
+
from io import BytesIO
import json
-import os
import subprocess
-import time
+from typing import Any
import urllib.error
import urllib.request
from PIL import Image
-from artemis.clients.ui_automator_client import UIAutomatorScreenData, _pil_to_base64
+from artemis.clients.ui_automator_client import (
+ UIAutomatorScreenData,
+ _parse_hierarchy_xml_to_elements,
+ _pil_to_base64,
+)
from artemis.runtime.adb_endpoint import adb_command
+from artemis.runtime.awake_service import ensure_device_awake
+from artemis.runtime.helper_manager import (
+ DEVICE_PORT,
+ PACKAGE_NAME,
+ SERVICE_NAME,
+ AccessibilityHelperManager,
+ HelperSession,
+ HelperUnavailable,
+ ProvisionEvent,
+ helper_manager,
+)
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
-PACKAGE_NAME = "com.artemis.helper"
-SERVICE_NAME = f"{PACKAGE_NAME}/.ArtemisAccessibilityService"
-DEFAULT_PORT = 18888
+# Kept for callers that imported these from here.
+DEFAULT_PORT = DEVICE_PORT
+TOKEN_HEADER = "X-Artemis-Token"
+__all__ = [
+ "AccessibilityClient",
+ "DEFAULT_PORT",
+ "HelperEmptyHierarchy",
+ "HelperRequestError",
+ "HelperUnavailable",
+ "PACKAGE_NAME",
+ "SERVICE_NAME",
+ "TOKEN_HEADER",
+ "normalize_helper_elements",
+]
+
+_BOOL_ATTRIBUTES = (
+ "checkable",
+ "checked",
+ "clickable",
+ "enabled",
+ "focusable",
+ "focused",
+ "scrollable",
+ "long-clickable",
+ "password",
+ "selected",
+ "visible-to-user",
+)
+_GLOBAL_KEYS = ("back", "home", "recents", "notifications", "quick_settings")
+
+
+class HelperEmptyHierarchy(RuntimeError):
+ """The helper answered but found no window to dump.
+
+ Raised instead of returning a screen with zero elements, so ``auto`` mode
+ can try UIAutomator2 and ``helper`` mode reports the real cause instead of
+ showing the agent an empty screen.
+ """
+
+
+class HelperRequestError(RuntimeError):
+ """The helper answered with an HTTP error that a new tunnel cannot fix."""
+
+ def __init__(self, status: int, path: str, body: str):
+ self.status = status
+ self.path = path
+ self.body = body
+ super().__init__(f"accessibility helper {path} answered HTTP {status}: {body[:200]}")
+
+
+def normalize_helper_elements(elements: Any) -> list[dict[str, Any]]:
+ """Coerce the helper's JSON element list into the UIAutomator element shape.
+
+ Used only when a dump carries elements but no XML. The JSON encodes flags
+ as booleans and omits ``accessibilityText``; the XML parser emits string
+ flags and mirrors ``content-desc`` into ``accessibilityText``, and several
+ consumers compare against the string ``"true"``.
+ """
+ if isinstance(elements, dict):
+ elements = [elements]
+ if not isinstance(elements, list):
+ return []
+ normalized: list[dict[str, Any]] = []
+ for raw in elements:
+ if not isinstance(raw, dict):
+ continue
+ element = dict(raw)
+ for key in _BOOL_ATTRIBUTES:
+ if key in element and isinstance(element[key], bool):
+ element[key] = "true" if element[key] else "false"
+ desc = element.get("content-desc")
+ if desc is not None and "accessibilityText" not in element:
+ element["accessibilityText"] = desc
+ bounds = element.get("parsed_bounds")
+ if isinstance(bounds, dict):
+ element["parsed_bounds"] = {
+ side: int(bounds.get(side, 0)) for side in ("left", "top", "right", "bottom")
+ }
+ element.pop("children", None)
+ normalized.append(element)
+ return normalized
class AccessibilityClient:
- """Non-exclusive accessibility client for Artemis UI automation."""
+ """Screen-data client over the helper's loopback HTTP API."""
- def __init__(self, device_id: str, local_port: int = DEFAULT_PORT):
+ backend_name = "helper"
+
+ def __init__(
+ self,
+ device_id: str,
+ manager: AccessibilityHelperManager | None = None,
+ *,
+ provision_on_connect: bool = True,
+ request_timeout: float = 6.0,
+ ) -> None:
self._device_id = device_id
- self._local_port = local_port
- self._base_url = f"http://127.0.0.1:{local_port}"
+ self._manager = manager or helper_manager
+ self._provision_on_connect = provision_on_connect
+ self._request_timeout = request_timeout
+ self._session: HelperSession | None = None
+ self._awake_strategy: str | None = None
- def ensure_service_ready(self, apk_path: str | None = None) -> bool:
- """Ensure ArtemisAccessibilityHelper is installed, enabled, and port-forwarded."""
- # 1. Check if package is installed
- if not self._is_installed():
- if apk_path is None:
- default_apk = os.path.abspath(
- os.path.join(
- os.path.dirname(__file__),
- "../../packages/artemis-accessibility-helper/ArtemisAccessibilityHelper.apk",
- )
- )
- if os.path.exists(default_apk):
- apk_path = default_apk
+ # ------------------------------------------------------------------ #
+ # Session lifecycle
+ # ------------------------------------------------------------------ #
- if apk_path and os.path.exists(apk_path):
- logger.info(f"Installing ArtemisAccessibilityHelper on {self._device_id}...")
- self._run_adb(["install", "-r", "-g", apk_path])
- else:
- logger.warning(
- f"ArtemisAccessibilityHelper not installed on {self._device_id} and no APK provided."
- )
+ @property
+ def device_id(self) -> str:
+ return self._device_id
- # 2. Ensure accessibility service is enabled via secure settings
- self._enable_accessibility_service()
+ @property
+ def session(self) -> HelperSession | None:
+ return self._session
- # 3. Setup port forward
- self._setup_port_forward()
+ @property
+ def active_backend(self) -> str | None:
+ return self.backend_name if self._session is not None else None
- # 4. Verify connection
- for attempt in range(5):
- if self.ping():
- logger.info(
- f"ArtemisAccessibilityHelper connected successfully on {self._device_id}"
- )
- return True
- time.sleep(0.5)
+ def connect(self, on_event: ProvisionEvent | None = None) -> None:
+ """Task-path entry: provision (install / upgrade / enable) and attach.
- logger.warning(
- f"Failed to connect to ArtemisAccessibilityHelper on port {self._local_port}"
+ ``on_event`` receives ``installing`` / ``upgrading`` before the slow step.
+ """
+ self._session = self._manager.attach(
+ self._device_id, provision=self._provision_on_connect, on_event=on_event
)
- return False
+ if self._awake_strategy is None:
+ self._awake_strategy = ensure_device_awake(self._device_id)
+
+ def _ensure_session(self) -> HelperSession:
+ """Observer-path entry: attach to a helper that is already running."""
+ if self._session is None:
+ self._session = self._manager.attach(self._device_id, provision=False)
+ if self._awake_strategy is None:
+ self._awake_strategy = ensure_device_awake(self._device_id)
+ return self._session
+
+ def disconnect(self) -> None:
+ self._manager.detach(self._device_id)
+ self._session = None
+ self._awake_strategy = None
def ping(self) -> bool:
- """Health check probe."""
try:
- req = urllib.request.Request(f"{self._base_url}/ping")
- with urllib.request.urlopen(req, timeout=2.0) as resp:
- data = json.loads(resp.read().decode())
- return data.get("success", False)
- except Exception:
+ session = self._ensure_session()
+ except HelperUnavailable:
return False
+ return self._manager.ping(session.local_port) is not None
+
+ # ------------------------------------------------------------------ #
+ # Transport
+ # ------------------------------------------------------------------ #
+
+ def _http(
+ self, path: str, payload: dict[str, Any] | None = None, timeout: float | None = None
+ ) -> bytes:
+ """One request with two single-shot repairs.
+
+ * HTTP 401: the helper lost its token (re-bound service, reboot): push it
+ again and retry once.
+ * Transport failure on a read: rebuild the tunnel once and retry.
+ Actions are not replayed because they may already have executed.
+
+ Any other HTTP status is a :class:`HelperRequestError`; a new tunnel
+ would not change the answer.
+ """
+ session = self._ensure_session()
+ try:
+ return self._http_once(session, path, payload, timeout)
+ except urllib.error.HTTPError as exc:
+ body = _read_error_body(exc)
+ if exc.code != 401:
+ raise HelperRequestError(exc.code, path, body) from exc
+ logger.warning(
+ f"Accessibility helper on {self._device_id} rejected the session token "
+ f"({body[:120]}); pushing it again."
+ )
+ self._manager.push_token(self._device_id)
+ try:
+ return self._http_once(session, path, payload, timeout)
+ except urllib.error.HTTPError as again:
+ raise HelperRequestError(again.code, path, _read_error_body(again)) from again
+ except (urllib.error.URLError, OSError, TimeoutError) as exc:
+ if payload is not None:
+ raise
+ logger.warning(
+ f"Accessibility helper request {path} on {self._device_id} failed ({exc}); "
+ "rebuilding the tunnel once."
+ )
+ self._session = self._manager.reattach(self._device_id)
+ try:
+ return self._http_once(self._session, path, payload, timeout)
+ except urllib.error.HTTPError as again:
+ raise HelperRequestError(again.code, path, _read_error_body(again)) from again
+
+ def _http_once(
+ self,
+ session: HelperSession,
+ path: str,
+ payload: dict[str, Any] | None,
+ timeout: float | None,
+ ) -> bytes:
+ data = (
+ json.dumps(payload, ensure_ascii=False).encode("utf-8") if payload is not None else None
+ )
+ headers: dict[str, str] = {}
+ token = session.token or self._manager.host_token()
+ if token:
+ headers[TOKEN_HEADER] = token
+ if data is not None:
+ headers["Content-Type"] = "application/json; charset=utf-8"
+ request = urllib.request.Request(f"{session.base_url}{path}", data=data, headers=headers)
+ with urllib.request.urlopen(request, timeout=timeout or self._request_timeout) as resp:
+ return resp.read()
+
+ def _rpc(self, cmd: str, params: dict[str, Any] | None = None) -> bool:
+ payload = {"cmd": cmd, **(params or {})}
+ try:
+ data = json.loads(self._http("/action", payload).decode("utf-8"))
+ except (
+ HelperUnavailable,
+ HelperRequestError,
+ urllib.error.URLError,
+ OSError,
+ ValueError,
+ TimeoutError,
+ ) as exc:
+ logger.warning(f"Accessibility helper command '{cmd}' failed: {exc}")
+ return False
+ return bool(data.get("success", False))
+
+ # ------------------------------------------------------------------ #
+ # Screen data
+ # ------------------------------------------------------------------ #
+
+ def get_hierarchy(self) -> str:
+ """Raw UIAutomator-format XML, the same contract as ``UIAutomatorClient``."""
+ return self._http("/dump_xml").decode("utf-8")
+
+ def get_hierarchy_json(self, *, include_invisible: bool = False) -> dict[str, Any]:
+ """The helper's JSON dump (XML plus its own element list); diagnostics only."""
+ query = "fields=xml,elements" + ("&include_invisible=1" if include_invisible else "")
+ return json.loads(self._http(f"/dump?{query}").decode("utf-8"))
+
+ def _snapshot(self) -> dict[str, Any]:
+ """``/snapshot?fields=xml``: hierarchy plus, on Android 11+, the screenshot."""
+ data = json.loads(self._http("/snapshot?fields=xml", timeout=8.0).decode("utf-8"))
+ if not isinstance(data, dict):
+ raise HelperRequestError(200, "/snapshot", "non-object answer")
+ if not data.get("success"):
+ raise HelperEmptyHierarchy(
+ f"accessibility helper on {self._device_id} returned no hierarchy: "
+ f"{data.get('error') or 'unknown reason'}"
+ )
+ return data
+
+ def get_atomic_snapshot(self) -> dict[str, Any] | None:
+ """Screenshot + hierarchy captured in one call on the device (Android 11+), else None."""
+ data = self._snapshot()
+ if data.get("has_screenshot") and data.get("screenshot_base64"):
+ return data
+ return None
def get_screenshot(self) -> Image.Image | None:
- """Capture screenshot via high-speed ADB exec-out."""
try:
result = subprocess.run(
adb_command(["-s", self._device_id, "exec-out", "screencap", "-p"]),
@@ -111,160 +339,111 @@ class AccessibilityClient:
check=True,
)
return Image.open(BytesIO(result.stdout))
- except Exception as e:
- logger.error(f"Failed to capture screenshot via adb: {e}")
+ except (OSError, subprocess.SubprocessError) as exc:
+ logger.error(f"Failed to capture screenshot via adb: {exc}")
return None
- def get_hierarchy(self) -> dict:
- """Fetch hierarchy JSON directly from ArtemisAccessibilityHelper."""
- req = urllib.request.Request(f"{self._base_url}/dump")
- with urllib.request.urlopen(req, timeout=5.0) as resp:
- return json.loads(resp.read().decode())
+ def get_screenshot_base64(self) -> str | None:
+ screenshot = self.get_screenshot()
+ return None if screenshot is None else _pil_to_base64(screenshot, format="JPEG")
- def get_hierarchy_xml(self) -> str:
- """Fetch raw standard UIAutomator XML string directly from helper."""
- req = urllib.request.Request(f"{self._base_url}/dump_xml")
- with urllib.request.urlopen(req, timeout=5.0) as resp:
- return resp.read().decode("utf-8")
-
- def get_atomic_snapshot(self) -> dict | None:
- """Fetch atomic snapshot (screenshot Base64 + hierarchy) directly in one call."""
- try:
- req = urllib.request.Request(f"{self._base_url}/snapshot")
- with urllib.request.urlopen(req, timeout=6.0) as resp:
- data = json.loads(resp.read().decode())
- if (
- data.get("success")
- and data.get("has_screenshot")
- and data.get("screenshot_base64")
- ):
- return data
- except Exception as e:
- logger.debug(f"Atomic snapshot request failed (falling back to dual path): {e}")
- return None
+ @staticmethod
+ def _elements_from_dump(dump: dict[str, Any]) -> tuple[str, list[dict[str, Any]]]:
+ xml = dump.get("xml") or ""
+ if isinstance(xml, str) and xml.strip():
+ return xml, _parse_hierarchy_xml_to_elements(xml)
+ return "", normalize_helper_elements(dump.get("elements"))
def get_screen_data(self) -> UIAutomatorScreenData:
- """Retrieve complete screen data (screenshot + parsed elements + standard XML).
-
- Prioritizes the atomic hardware snapshot API (Android 11+) to eliminate
- any temporal phase mismatch between visual frame and layout DOM tree.
- Falls back to separate ADB screencap + HTTP dump on older platforms.
- """
- # 1. Try atomic snapshot
- atomic = self.get_atomic_snapshot()
- if atomic is not None:
+ snapshot = self._snapshot()
+ xml, elements = self._elements_from_dump(snapshot)
+ if snapshot.get("has_screenshot") and snapshot.get("screenshot_base64"):
+ width, height = _dimensions(snapshot)
return UIAutomatorScreenData(
- base64=atomic.get("screenshot_base64", ""),
- hierarchy_xml=atomic.get("xml", ""),
- elements=atomic.get("elements", []),
- width=atomic.get("width", 1080),
- height=atomic.get("height", 2400),
+ base64=str(snapshot["screenshot_base64"]),
+ hierarchy_xml=xml,
+ elements=elements,
+ width=width,
+ height=height,
)
- # 2. Fallback: separate screenshot and hierarchy dump
+ # Android 10 and below (or a rate-limited capture that did not recover):
+ # keep the hierarchy we already have and only add an adb screencap.
+ reason = snapshot.get("screenshot_error")
+ if reason:
+ logger.debug(f"Helper snapshot without screenshot ({reason}); using adb screencap")
screenshot = self.get_screenshot()
if screenshot is None:
raise RuntimeError("Failed to capture screenshot")
-
- hierarchy_data = self.get_hierarchy()
- elements = hierarchy_data.get("elements", [])
- hierarchy_xml = hierarchy_data.get("xml", "")
-
return UIAutomatorScreenData(
base64=_pil_to_base64(screenshot, format="JPEG"),
- hierarchy_xml=hierarchy_xml,
+ hierarchy_xml=xml,
elements=elements,
width=screenshot.width,
height=screenshot.height,
)
- def tap(self, x: float, y: float) -> bool:
- """Perform tap gesture via AccessibilityService."""
- return self._send_rpc("tap", {"x": x, "y": y})
+ # ------------------------------------------------------------------ #
+ # Input
+ # ------------------------------------------------------------------ #
- def swipe(self, x1: float, y1: float, x2: float, y2: float, duration_ms: int = 300) -> bool:
- """Perform swipe gesture via AccessibilityService."""
- return self._send_rpc(
- "swipe",
- {"x1": x1, "y1": y1, "x2": x2, "y2": y2, "duration": duration_ms},
- )
+ def set_clipboard(self, text: str) -> bool:
+ """Put ``text`` on the device clipboard so the driver can paste it."""
+ return self._rpc("clipboard", {"text": text})
def send_text(self, text: str) -> bool:
- """Send text input via AccessibilityNodeInfo.ACTION_SET_TEXT."""
- success = self._send_rpc("type", {"text": text})
- if not success:
- # Fallback to adb shell input text
- safe_text = text.replace(" ", "%s")
- self._run_adb(["shell", "input", "text", safe_text])
- return True
- return success
+ """Append ``text`` to the focused field, the way typing through an IME does."""
+ return self._rpc("type", {"text": text, "append": True})
def clear_text(self) -> bool:
- """Clear text of focused input."""
- return self._send_rpc("clear", {})
+ return self._rpc("clear", {})
def press_key(self, key: str) -> bool:
- """Perform global navigation action (back, home, recents)."""
- key_lower = key.lower()
- if key_lower in ("back", "home", "recents", "notifications", "quick_settings"):
- return self._send_rpc("global", {"action": key_lower})
- # Fallback to adb keyevent
- self._run_adb(["shell", "input", "keyevent", f"KEYCODE_{key.upper()}"])
- return True
-
- def _send_rpc(self, cmd: str, params: dict) -> bool:
- """Send JSON-RPC payload to helper server."""
- payload = {"cmd": cmd, **params}
- try:
- req = urllib.request.Request(
- f"{self._base_url}/action",
- data=json.dumps(payload).encode("utf-8"),
- headers={"Content-Type": "application/json"},
- )
- with urllib.request.urlopen(req, timeout=5.0) as resp:
- data = json.loads(resp.read().decode())
- return data.get("success", False)
- except Exception as e:
- logger.warning(f"RPC command '{cmd}' failed: {e}")
- return False
-
- def _is_installed(self) -> bool:
- res = self._run_adb(["shell", "pm", "list", "packages", PACKAGE_NAME])
- return f"package:{PACKAGE_NAME}" in res.stdout
-
- def _enable_accessibility_service(self) -> None:
- """Silently enable the accessibility service via adb settings without UI prompts."""
- res = self._run_adb(
- ["shell", "settings", "get", "secure", "enabled_accessibility_services"]
- )
- current_services = res.stdout.strip()
- if SERVICE_NAME not in current_services:
- new_services = (
- f"{current_services}:{SERVICE_NAME}"
- if current_services and current_services != "null"
- else SERVICE_NAME
- )
- self._run_adb(
+ key_lower = str(key).lower()
+ if key_lower in _GLOBAL_KEYS:
+ return self._rpc("global", {"action": key_lower})
+ result = subprocess.run(
+ adb_command(
[
+ "-s",
+ self._device_id,
"shell",
- "settings",
- "put",
- "secure",
- "enabled_accessibility_services",
- new_services,
+ "input",
+ "keyevent",
+ f"KEYCODE_{key_lower.upper()}",
]
- )
- self._run_adb(["shell", "settings", "put", "secure", "accessibility_enabled", "1"])
-
- def _setup_port_forward(self) -> None:
- self._run_adb(["forward", f"tcp:{self._local_port}", f"tcp:{DEFAULT_PORT}"])
-
- def _run_adb(self, args: list[str]) -> subprocess.CompletedProcess:
- return subprocess.run(
- adb_command(["-s", self._device_id] + args),
+ ),
stdin=subprocess.DEVNULL,
capture_output=True,
text=True,
timeout=15,
check=False,
)
+ return result.returncode == 0
+
+ def tap(self, x: float, y: float) -> bool:
+ return self._rpc("tap", {"x": x, "y": y})
+
+ def swipe(self, x1: float, y1: float, x2: float, y2: float, duration_ms: int = 300) -> bool:
+ return self._rpc("swipe", {"x1": x1, "y1": y1, "x2": x2, "y2": y2, "duration": duration_ms})
+
+
+def _read_error_body(exc: urllib.error.HTTPError) -> str:
+ try:
+ return exc.read().decode("utf-8", "replace")
+ except OSError:
+ return str(exc)
+
+
+def _dimensions(snapshot: dict[str, Any]) -> tuple[int, int]:
+ """The screenshot's own size; a snapshot without it is a broken answer, not a 1080x2400 one."""
+ try:
+ width = int(snapshot.get("width") or 0)
+ height = int(snapshot.get("height") or 0)
+ except (TypeError, ValueError):
+ width = height = 0
+ if width <= 0 or height <= 0:
+ raise HelperRequestError(
+ 200, "/snapshot", f"screenshot without dimensions (width={width}, height={height})"
+ )
+ return width, height
diff --git a/artemis/clients/screen_client_factory.py b/artemis/clients/screen_client_factory.py
new file mode 100644
index 0000000..311f262
--- /dev/null
+++ b/artemis/clients/screen_client_factory.py
@@ -0,0 +1,400 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Select the UI-hierarchy backend for a device and build its screen client.
+
+Three backends, chosen by ``ARTEMIS_HIERARCHY_BACKEND`` (environment or
+``.env``):
+
+* ``auto`` (default): the Accessibility Helper, with UIAutomator2 as a
+ per-call fallback. UIAutomator2 is only constructed when the helper fails,
+ so a healthy helper never triggers ``u2.connect`` (which pushes its own APKs
+ and uninstalls Maestro).
+* ``helper``: the helper only. Failures surface as errors; use it in CI to
+ prove the helper is really what serves the hierarchy.
+* ``uiautomator``: the pre-helper path, unchanged.
+
+Every entrypoint that used to build ``UIAutomatorClient`` directly (the SDK
+agent, the MCP device controller, and through it the device smoke test) goes
+through :func:`create_screen_client`, so the switch applies everywhere.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from enum import Enum
+import os
+import subprocess
+import time
+from typing import Any
+
+from PIL import Image
+
+from artemis.clients.accessibility_client import AccessibilityClient, HelperUnavailable
+from artemis.clients.ui_automator_client import UIAutomatorClient, UIAutomatorScreenData
+from artemis.config.constants import ENV_ARTEMIS_HIERARCHY_BACKEND
+from artemis.runtime.adb_endpoint import adb_command
+from artemis.runtime.helper_manager import ProvisionEvent
+from artemis.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+#: ``(previous_backend, new_backend, reason)``; ``previous_backend`` is None on first use.
+BackendChangeListener = Callable[[str | None, str, str | None], None]
+
+
+class DeviceOfflineError(RuntimeError):
+ """The device is not attached / authorized; no backend can serve it.
+
+ Raised instead of trying UIAutomator2 when the helper fails because the
+ phone itself is gone, so the failure names the real cause and does not
+ burn UIAutomator2's own multi-attempt connect timeout.
+ """
+
+ def __init__(self, device_id: str, state: str | None):
+ self.device_id = device_id
+ self.state = state
+ detail = f"adb reports state '{state}'" if state else "adb does not list it"
+ super().__init__(
+ f"Device {device_id} is not available ({detail}). Reconnect the device or "
+ "accept the USB debugging prompt, then retry."
+ )
+
+
+def device_state(device_id: str, timeout: float = 5.0) -> str | None:
+ """``adb get-state`` for one serial: 'device', 'offline', 'unauthorized', or None."""
+ try:
+ result = subprocess.run(
+ adb_command(["-s", device_id, "get-state"]),
+ stdin=subprocess.DEVNULL,
+ capture_output=True,
+ text=True,
+ timeout=timeout,
+ check=False,
+ )
+ except (OSError, subprocess.SubprocessError):
+ return None
+ text = (result.stdout or "").strip()
+ if result.returncode != 0 or not text:
+ # "error: device unauthorized" / "device offline" arrive on stderr.
+ err = (result.stderr or "").lower()
+ for known in ("unauthorized", "offline"):
+ if known in err:
+ return known
+ return None
+ return text
+
+
+class HierarchyBackend(str, Enum):
+ AUTO = "auto"
+ HELPER = "helper"
+ UIAUTOMATOR = "uiautomator"
+
+
+def resolve_backend(value: str | HierarchyBackend | None = None) -> HierarchyBackend:
+ """Backend from an explicit value, else the environment, else settings, else ``auto``."""
+ if isinstance(value, HierarchyBackend):
+ return value
+ text = value
+ if text is None:
+ text = os.environ.get(ENV_ARTEMIS_HIERARCHY_BACKEND)
+ if text is None:
+ try:
+ from artemis.config import settings
+
+ text = getattr(settings, "ARTEMIS_HIERARCHY_BACKEND", None)
+ except (ImportError, ValueError):
+ text = None
+ normalized = (text or "auto").strip().lower()
+ try:
+ return HierarchyBackend(normalized)
+ except ValueError:
+ logger.warning(
+ f"Unknown hierarchy backend {text!r}; valid values are "
+ f"{', '.join(b.value for b in HierarchyBackend)}. Using 'auto'."
+ )
+ return HierarchyBackend.AUTO
+
+
+class FallbackScreenClient:
+ """Helper first, UIAutomator2 on failure, with a retry window for the helper.
+
+ After a helper failure the client serves from UIAutomator2 for
+ ``retry_after`` seconds before trying the helper again, so one dead tunnel
+ does not cost a 5-second timeout on every step.
+ """
+
+ def __init__(
+ self,
+ device_id: str,
+ helper: AccessibilityClient | None = None,
+ uiautomator_factory: Callable[[], UIAutomatorClient] | None = None,
+ *,
+ retry_after: float = 30.0,
+ clock: Callable[[], float] = time.monotonic,
+ state_probe: Callable[[str], str | None] = device_state,
+ ) -> None:
+ self._device_id = device_id
+ self._helper = helper or AccessibilityClient(device_id)
+ self._uiautomator_factory = uiautomator_factory or (lambda: UIAutomatorClient(device_id))
+ self._uiautomator: UIAutomatorClient | None = None
+ self._retry_after = retry_after
+ self._clock = clock
+ self._state_probe = state_probe
+ self._helper_down_until: float | None = None
+ self._active_backend: str | None = None
+ self._last_failure: str | None = None
+ #: Every backend switch, oldest first: {time, backend, reason}.
+ self.backend_history: list[dict[str, Any]] = []
+ self._listeners: list[BackendChangeListener] = []
+
+ # ------------------------------------------------------------------ #
+
+ @property
+ def device_id(self) -> str:
+ return self._device_id
+
+ @property
+ def helper(self) -> AccessibilityClient:
+ return self._helper
+
+ @property
+ def active_backend(self) -> str | None:
+ """Backend that served the last call: ``"helper"``, ``"uiautomator"`` or ``None``."""
+ return self._active_backend
+
+ @property
+ def uiautomator(self) -> UIAutomatorClient:
+ if self._uiautomator is None:
+ self._uiautomator = self._uiautomator_factory()
+ return self._uiautomator
+
+ def add_backend_listener(self, listener: BackendChangeListener) -> None:
+ """Be told whenever the serving backend changes (first use included)."""
+ self._listeners.append(listener)
+
+ def remove_backend_listener(self, listener: BackendChangeListener) -> None:
+ if listener in self._listeners:
+ self._listeners.remove(listener)
+
+ def _set_active(self, backend: str, reason: str | None = None) -> None:
+ previous = self._active_backend
+ if previous == backend:
+ return
+ self._active_backend = backend
+ self.backend_history.append({"time": time.time(), "backend": backend, "reason": reason})
+ for listener in list(self._listeners):
+ try:
+ listener(previous, backend, reason)
+ except Exception as exc: # pylint: disable=broad-exception-caught
+ logger.debug(f"Backend listener failed: {exc}")
+
+ def _ensure_device_online(self) -> None:
+ """Fail fast with the real cause when the phone itself is gone."""
+ state = self._state_probe(self._device_id)
+ if state != "device":
+ raise DeviceOfflineError(self._device_id, state)
+
+ def _helper_allowed(self) -> bool:
+ if self._helper_down_until is None:
+ return True
+ if self._clock() >= self._helper_down_until:
+ self._helper_down_until = None
+ return True
+ return False
+
+ def _degrade(self, reason: BaseException) -> None:
+ # Never fall back to UIAutomator2 for a phone that is simply not there.
+ self._ensure_device_online()
+ self._last_failure = f"{reason.__class__.__name__}: {reason}"
+ if self._helper_down_until is None:
+ logger.warning(
+ f"Accessibility helper unavailable on {self._device_id} ({reason}); "
+ f"serving from UIAutomator2 and retrying the helper in {self._retry_after:g}s."
+ )
+ self._helper_down_until = self._clock() + self._retry_after
+
+ def _release_uiautomation(self) -> None:
+ """Stop our UIAutomator2 server before giving the helper another chance.
+
+ Its UiAutomation connection keeps every accessibility service unbound,
+ so while it lives the helper cannot answer and the retry would be a
+ guaranteed failure. UIAutomator2 restarts its server by itself on the
+ next call if the helper turns out to be still dead.
+ """
+ if self._uiautomator is not None and self._active_backend == "uiautomator":
+ self._uiautomator.stop_server()
+
+ def _call(self, name: str, *args: Any, **kwargs: Any) -> Any:
+ if self._helper_allowed():
+ self._release_uiautomation()
+ try:
+ result = getattr(self._helper, name)(*args, **kwargs)
+ except (RuntimeError, OSError, ValueError, subprocess.SubprocessError) as exc:
+ self._degrade(exc)
+ else:
+ self._set_active("helper", "helper recovered" if self._last_failure else None)
+ self._last_failure = None
+ return result
+ result = getattr(self.uiautomator, name)(*args, **kwargs)
+ self._set_active("uiautomator", self._last_failure)
+ return result
+
+ # ------------------------------------------------------------------ #
+ # Lifecycle
+ # ------------------------------------------------------------------ #
+
+ def connect(self, on_event: ProvisionEvent | None = None) -> None:
+ self._release_uiautomation()
+ self.backend_history.clear()
+ self._active_backend = None
+ if self._helper_allowed():
+ self._release_uiautomation()
+ try:
+ self._helper.connect(on_event=on_event)
+ except (RuntimeError, OSError, ValueError, subprocess.SubprocessError) as exc:
+ self._degrade(exc)
+ else:
+ self._set_active("helper")
+ return
+ self.uiautomator.connect()
+ self._set_active("uiautomator", self._last_failure)
+
+ def disconnect(self) -> None:
+ try:
+ self._helper.disconnect()
+ finally:
+ if self._uiautomator is not None:
+ # Release UiAutomation so the helper is back for the next task
+ # on this device (possibly served by another process).
+ self._uiautomator.disconnect(stop_server=True)
+ self._active_backend = None
+
+ # ------------------------------------------------------------------ #
+ # Screen data and input, same surface as UIAutomatorClient
+ # ------------------------------------------------------------------ #
+
+ def get_screen_data(self) -> UIAutomatorScreenData:
+ return self._call("get_screen_data")
+
+ def get_hierarchy(self) -> str:
+ return self._call("get_hierarchy")
+
+ def get_screenshot(self) -> Image.Image | None:
+ return self._call("get_screenshot")
+
+ def get_screenshot_base64(self) -> str | None:
+ return self._call("get_screenshot_base64")
+
+ def set_clipboard(self, text: str) -> bool:
+ return self._call("set_clipboard", text)
+
+ def send_text(self, text: str) -> Any:
+ return self._call("send_text", text)
+
+ def clear_text(self) -> bool:
+ return self._call("clear_text")
+
+ def press_key(self, key: str) -> Any:
+ return self._call("press_key", key)
+
+
+ScreenClient = AccessibilityClient | UIAutomatorClient | FallbackScreenClient
+
+
+def create_screen_client(
+ device_id: str, backend: str | HierarchyBackend | None = None
+) -> ScreenClient:
+ """Build the screen client for ``device_id`` according to the configured backend."""
+ resolved = resolve_backend(backend)
+ if resolved is HierarchyBackend.UIAUTOMATOR:
+ return UIAutomatorClient(device_id=device_id)
+ if resolved is HierarchyBackend.HELPER:
+ return AccessibilityClient(device_id)
+ return FallbackScreenClient(device_id)
+
+
+def describe_backend(client: Any) -> str | None:
+ """Best-effort name of the backend a client is serving from, for diagnostics."""
+ active = getattr(client, "active_backend", None)
+ if active:
+ return str(active)
+ if isinstance(client, UIAutomatorClient):
+ return "uiautomator"
+ if isinstance(client, AccessibilityClient):
+ return "helper"
+ return None
+
+
+def helper_version(client: Any) -> str | None:
+ """Version name of the helper a client is attached to, if any."""
+ helper = getattr(client, "helper", None) or client
+ session = getattr(helper, "session", None)
+ name = getattr(session, "version_name", None)
+ return str(name) if name else None
+
+
+def hierarchy_backend_summary(client: Any) -> dict[str, Any] | None:
+ """Machine-readable account of which backend served a run, for reports."""
+ backend = describe_backend(client)
+ if backend is None:
+ return None
+ history = list(getattr(client, "backend_history", None) or [])
+ return {
+ "backend": backend,
+ "helper_version": helper_version(client),
+ "switches": history[1:] if len(history) > 1 else [],
+ }
+
+
+_BACKEND_LABEL = {"helper": "the Artemis accessibility helper", "uiautomator": "UIAutomator2"}
+
+
+def hierarchy_backend_sentence(
+ client: Any, relative_time: Callable[[float], str] | None = None
+) -> str | None:
+ """One plain sentence for reports: what served the hierarchy and whether it changed."""
+ summary = hierarchy_backend_summary(client)
+ if summary is None:
+ return None
+ history = getattr(client, "backend_history", None) or []
+ first = (history[0].get("backend") if history else None) or summary["backend"]
+ text = f"UI hierarchy source: {_BACKEND_LABEL.get(first, first)}"
+ if first == "helper" and summary.get("helper_version"):
+ text += f" v{summary['helper_version']}"
+ for switch in summary["switches"]:
+ when = relative_time(switch["time"]) if relative_time else None
+ text += f"; switched to {_BACKEND_LABEL.get(switch['backend'], switch['backend'])}"
+ if when:
+ text += f" at {when}"
+ if switch.get("reason"):
+ text += f" because {str(switch['reason']).rstrip('. ')}"
+ return text + "."
+
+
+__all__ = [
+ "BackendChangeListener",
+ "DeviceOfflineError",
+ "FallbackScreenClient",
+ "HelperUnavailable",
+ "HierarchyBackend",
+ "ScreenClient",
+ "create_screen_client",
+ "describe_backend",
+ "device_state",
+ "helper_version",
+ "hierarchy_backend_sentence",
+ "hierarchy_backend_summary",
+ "resolve_backend",
+]
diff --git a/artemis/clients/ui_automator_client.py b/artemis/clients/ui_automator_client.py
index b5d3b7a..34c1d21 100644
--- a/artemis/clients/ui_automator_client.py
+++ b/artemis/clients/ui_automator_client.py
@@ -117,7 +117,7 @@ def _parse_hierarchy_xml_to_elements(hierarchy_xml: str) -> list[dict]:
elif attr_name == "bounds":
element["bounds"] = attr_value
- match = _re.match(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", attr_value)
+ match = _re.match(r"\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]", attr_value)
if match:
x1, y1, x2, y2 = map(int, match.groups())
element["parsed_bounds"] = {
@@ -190,7 +190,7 @@ def _is_package_installed(device_id: str, pkg: str) -> bool:
except subprocess.TimeoutExpired:
logger.warning("Timeout checking installed packages")
return False
- except Exception as e:
+ except (OSError, subprocess.SubprocessError) as e:
logger.warning(f"Error checking installed packages: {e}")
return False
@@ -400,7 +400,7 @@ class UIAutomatorClient:
check=True,
)
return Image.open(BytesIO(result.stdout))
- except Exception as e:
+ except (OSError, subprocess.SubprocessError) as e:
logger.warning(
f"Failed to capture screenshot via adb: {e}, falling back to uiautomator2"
)
@@ -446,12 +446,39 @@ class UIAutomatorClient:
height=screenshot.height,
)
- def disconnect(self) -> None:
- """Disconnect this client without ending the host's awake lifetime."""
+ def disconnect(self, stop_server: bool = False) -> None:
+ """Disconnect this client without ending the host's awake lifetime.
+
+ ``stop_server`` also kills uiautomator2's device server. Its UiAutomation
+ connection suppresses every accessibility service (the Artemis helper
+ included) for as long as it lives, so a caller that prefers the helper
+ must release it; the pure UIAutomator2 backend keeps it for fast reconnects.
+ """
+ device = self._device
self._awake_strategy = None
self._device = None
+ if stop_server and device is not None:
+ self.stop_server(device)
logger.info("UIAutomator2 client disconnected")
+ def stop_server(self, device: "Device | None" = None) -> bool:
+ """Kill uiautomator2's on-device server (see :meth:`disconnect`)."""
+ device = device or self._device
+ if device is None:
+ return False
+ try:
+ device.stop_uiautomator()
+ logger.info(f"UIAutomator2 device server stopped on {self._device_id}")
+ return True
+ except (
+ OSError,
+ RuntimeError,
+ subprocess.SubprocessError,
+ u2.exceptions.DeviceError,
+ ) as exc:
+ logger.debug(f"Stopping the UIAutomator2 server on {self._device_id} failed: {exc}")
+ return False
+
def get_client(device_id: str) -> UIAutomatorClient:
"""Factory function to create a UIAutomatorClient.
diff --git a/artemis/config/constants.py b/artemis/config/constants.py
index 40d535c..972bc6d 100644
--- a/artemis/config/constants.py
+++ b/artemis/config/constants.py
@@ -46,6 +46,11 @@ ENV_ARTEMIS_DEVICE_ID = "ARTEMIS_DEVICE_ID"
ENV_ADB_HOST = "ADB_HOST"
ENV_ADB_PORT = "ADB_PORT"
ENV_ADB_SERVER_SOCKET = "ADB_SERVER_SOCKET"
+# UI hierarchy backend: "auto" (helper, UIAutomator2 fallback), "helper", "uiautomator"
+ENV_ARTEMIS_HIERARCHY_BACKEND = "ARTEMIS_HIERARCHY_BACKEND"
+# "true" (default) lets a task install / upgrade the Accessibility Helper APK on a
+# device it holds; "false" only attaches to a helper someone installed by hand.
+ENV_ARTEMIS_HELPER_AUTO_INSTALL = "ARTEMIS_HELPER_AUTO_INSTALL"
# LLM & Vision OCR API Keys
ENV_GOOGLE_API_KEY = "GOOGLE_API_KEY"
diff --git a/artemis/config/settings.py b/artemis/config/settings.py
index a85b353..7129dd8 100644
--- a/artemis/config/settings.py
+++ b/artemis/config/settings.py
@@ -113,6 +113,12 @@ class Settings(BaseSettings):
ADB_HOST: str | None = Field(default=DEFAULT_ADB_HOST)
ADB_PORT: int | None = Field(default=DEFAULT_ADB_PORT)
ADB_DEVICE_SERIAL: str | None = None
+ # UI hierarchy backend: "auto" = Accessibility Helper with UIAutomator2
+ # fallback, "helper" = helper only, "uiautomator" = UIAutomator2 only.
+ ARTEMIS_HIERARCHY_BACKEND: str = Field(default="auto")
+ # Whether a task may install / upgrade the Accessibility Helper APK on a device
+ # it holds. False = only attach to a helper installed by `artemis helper install`.
+ ARTEMIS_HELPER_AUTO_INSTALL: bool = Field(default=True)
# Execution Defaults
PROJECT_NAME: str | None = None
diff --git a/artemis/core/diagnostics/device_smoke.py b/artemis/core/diagnostics/device_smoke.py
index 25c6cba..2fcc4ac 100644
--- a/artemis/core/diagnostics/device_smoke.py
+++ b/artemis/core/diagnostics/device_smoke.py
@@ -268,6 +268,7 @@ async def smoke_test_device(
{"ok": bool, "serial": str | None, "elapsed_seconds": float,
"screenshot_bytes": int | None, "element_count": int | None,
+ "hierarchy_backend": "helper" | "uiautomator" | None,
"error": str | None, "fix": list[str]}
"""
started = time.monotonic()
@@ -278,6 +279,7 @@ async def smoke_test_device(
"elapsed_seconds": 0.0,
"screenshot_bytes": None,
"element_count": None,
+ "hierarchy_backend": None,
"error": None,
"fix": [],
}
@@ -342,6 +344,9 @@ async def smoke_test_device(
result["screenshot_bytes"] = _decoded_length(getattr(device_data, "base64", None))
result["element_count"] = _count_elements(getattr(device_data, "elements", None))
+ from artemis.clients.screen_client_factory import describe_backend
+
+ result["hierarchy_backend"] = describe_backend(getattr(controller_ctx, "ui_adb_client", None))
if result["screenshot_bytes"] is None:
return _finish("Screen capture returned no screenshot data")
diff --git a/artemis/core/diagnostics/hierarchy_parity.py b/artemis/core/diagnostics/hierarchy_parity.py
new file mode 100644
index 0000000..e1b905a
--- /dev/null
+++ b/artemis/core/diagnostics/hierarchy_parity.py
@@ -0,0 +1,216 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Real-device parity check between the Accessibility Helper and UIAutomator2.
+
+The unit tests prove the two backends produce the same element *shapes*; only
+a real screen can prove they describe the same *content*. ``compare_backends``
+dumps the current screen with both, then checks what the agent depends on:
+
+* every labelled UIAutomator element (text or content-desc) has a helper
+ element with the same label at the same place (bounds IoU >= 0.5), and vice
+ versa, so neither backend invents or loses what is on screen;
+* the helper emits no negative and no off-screen bounds (its bounds are
+ clipped to the display, like UIAutomator's);
+* dump latency of both, for the record.
+
+``artemis helper parity`` is the CLI front end.
+"""
+
+from __future__ import annotations
+
+from statistics import median
+import time
+from typing import Any
+
+from artemis.clients.ui_automator_client import _parse_hierarchy_xml_to_elements
+
+MIN_RECALL = 0.9
+MIN_PRECISION = 0.8
+
+
+def _label(element: dict[str, Any]) -> str:
+ text = str(element.get("text") or "").strip()
+ if text:
+ return text
+ return str(element.get("content-desc") or "").strip()
+
+
+def _iou(a: dict[str, int], b: dict[str, int]) -> float:
+ left = max(a["left"], b["left"])
+ top = max(a["top"], b["top"])
+ right = min(a["right"], b["right"])
+ bottom = min(a["bottom"], b["bottom"])
+ if right <= left or bottom <= top:
+ return 0.0
+ inter = (right - left) * (bottom - top)
+ area_a = max(1, (a["right"] - a["left"]) * (a["bottom"] - a["top"]))
+ area_b = max(1, (b["right"] - b["left"]) * (b["bottom"] - b["top"]))
+ return inter / (area_a + area_b - inter)
+
+
+def analyze_xml(xml: str, width: int, height: int) -> dict[str, Any]:
+ """Counts and labelled elements of one dump, plus the bounds problems the helper must not have."""
+ elements = _parse_hierarchy_xml_to_elements(xml)
+ labelled: list[tuple[str, dict[str, int]]] = []
+ negative = offscreen = 0
+ package = None
+ for element in elements:
+ bounds = element.get("parsed_bounds")
+ if package is None and element.get("package"):
+ package = element["package"]
+ if not isinstance(bounds, dict):
+ continue
+ if min(bounds.values()) < 0:
+ negative += 1
+ if (
+ bounds["left"] >= width
+ or bounds["top"] >= height
+ or bounds["right"] > width
+ or bounds["bottom"] > height
+ ):
+ offscreen += 1
+ label = _label(element)
+ if label:
+ labelled.append((label, bounds))
+ return {
+ "nodes": len(elements),
+ "labelled": len(labelled),
+ "labels": labelled,
+ "negative_bounds": negative,
+ "offscreen_bounds": offscreen,
+ "package": package,
+ }
+
+
+def match_labels(
+ source: list[tuple[str, dict[str, int]]], target: list[tuple[str, dict[str, int]]]
+) -> tuple[int, list[str]]:
+ """How many labelled ``source`` elements have a same-label, same-place element in ``target``."""
+ used = [False] * len(target)
+ found = 0
+ missing: list[str] = []
+ for label, bounds in source:
+ hit = False
+ for i, (other_label, other_bounds) in enumerate(target):
+ if used[i] or other_label != label:
+ continue
+ if _iou(bounds, other_bounds) >= 0.5:
+ used[i] = True
+ hit = True
+ break
+ if hit:
+ found += 1
+ else:
+ missing.append(f"{label!r} at {bounds}")
+ return found, missing
+
+
+def compare_dumps(helper_xml: str, uiautomator_xml: str, width: int, height: int) -> dict[str, Any]:
+ """Pure comparison of two dumps of the same screen."""
+ helper = analyze_xml(helper_xml, width, height)
+ u2 = analyze_xml(uiautomator_xml, width, height)
+ u2_found, missing_in_helper = match_labels(u2["labels"], helper["labels"])
+ helper_found, extra_in_helper = match_labels(helper["labels"], u2["labels"])
+ recall = u2_found / u2["labelled"] if u2["labelled"] else 1.0
+ precision = helper_found / helper["labelled"] if helper["labelled"] else 1.0
+ problems: list[str] = []
+ if helper["negative_bounds"]:
+ problems.append(
+ f"helper emitted {helper['negative_bounds']} element(s) with negative bounds"
+ )
+ if helper["offscreen_bounds"]:
+ problems.append(
+ f"helper emitted {helper['offscreen_bounds']} element(s) outside the screen"
+ )
+ if recall < MIN_RECALL:
+ problems.append(
+ f"only {recall:.0%} of UIAutomator's labelled elements appear in the helper dump "
+ f"(need {MIN_RECALL:.0%})"
+ )
+ if precision < MIN_PRECISION:
+ problems.append(
+ f"only {precision:.0%} of the helper's labelled elements appear in the UIAutomator dump "
+ f"(need {MIN_PRECISION:.0%}); the helper is showing things that are not on screen"
+ )
+ for side in (helper, u2):
+ side.pop("labels", None)
+ return {
+ "ok": not problems,
+ "problems": problems,
+ "helper": helper,
+ "uiautomator": u2,
+ "match": {
+ "uiautomator_labelled": u2["labelled"],
+ "uiautomator_in_helper": u2_found,
+ "recall": recall,
+ "helper_labelled": helper["labelled"],
+ "helper_in_uiautomator": helper_found,
+ "precision": precision,
+ "missing_in_helper": missing_in_helper,
+ "extra_in_helper": extra_in_helper,
+ },
+ }
+
+
+def compare_backends(serial: str, rounds: int = 3) -> dict[str, Any]:
+ """Dump the current screen of ``serial`` with both backends and compare (device required)."""
+ from artemis.clients.accessibility_client import AccessibilityClient
+ from artemis.clients.ui_automator_client import UIAutomatorClient
+
+ helper = AccessibilityClient(serial)
+ u2 = UIAutomatorClient(serial)
+
+ def timed(dump) -> tuple[str, float]:
+ times: list[float] = []
+ xml = ""
+ for _ in range(rounds):
+ started = time.perf_counter()
+ xml = dump()
+ times.append((time.perf_counter() - started) * 1000.0)
+ return xml, median(times)
+
+ # The two cannot run at once: UIAutomator2's UiAutomation connection unbinds
+ # the helper. Dump with the helper first, then with UIAutomator2, then stop
+ # its server so the helper is back for whoever comes next.
+ helper_version = None
+ try:
+ helper.connect()
+ helper_version = helper.session.version_name if helper.session else None
+ info = helper.get_hierarchy_json()
+ width, height = int(info.get("width") or 0), int(info.get("height") or 0)
+ if width <= 0 or height <= 0:
+ screen = helper.get_screen_data()
+ width, height = screen.width, screen.height
+ helper_xml, helper_ms = timed(helper.get_hierarchy)
+ finally:
+ helper.disconnect()
+
+ try:
+ u2.connect()
+ u2_xml, u2_ms = timed(u2.get_hierarchy)
+ finally:
+ u2.disconnect(stop_server=True)
+
+ report = compare_dumps(helper_xml, u2_xml, width, height)
+ report["serial"] = serial
+ report["screen"] = {"width": width, "height": height}
+ report["helper"]["median_ms"] = helper_ms
+ report["helper"]["version"] = helper_version
+ report["uiautomator"]["median_ms"] = u2_ms
+ report["xml"] = {"helper": helper_xml, "uiautomator": u2_xml}
+ return report
+
+
+__all__ = ["analyze_xml", "compare_backends", "compare_dumps", "match_labels"]
diff --git a/artemis/data_engine/engine.py b/artemis/data_engine/engine.py
index a1af9ba..e880d3a 100644
--- a/artemis/data_engine/engine.py
+++ b/artemis/data_engine/engine.py
@@ -21,6 +21,7 @@ import os
from pathlib import Path
import re
import socket
+import sqlite3
import threading
import time
from typing import Any
@@ -785,6 +786,19 @@ class DataEngine:
except Exception as e:
logger.error(f"Failed to record video failure in DataEngine: {e}")
+ def update_session_device_info(self, **fields: Any) -> None:
+ """Merge ``fields`` into the running session's ``device_info`` record."""
+ if not self.storage or not self.current_session_id:
+ return
+ try:
+ session = self.storage.get_session(self.current_session_id)
+ if session is None:
+ return
+ session.device_info = {**(session.device_info or {}), **fields}
+ self.storage.update_session(session)
+ except (OSError, ValueError, sqlite3.Error) as e:
+ logger.warning(f"Failed to update session device_info in DataEngine: {e}")
+
def update_video_path(self, local_video_path: str | Path):
"""Update the video path across tables when traces or videos are moved."""
if not self.storage or not self.current_session_id:
diff --git a/artemis/drivers/android/hierarchy.py b/artemis/drivers/android/hierarchy.py
index bc754ed..c5762e0 100644
--- a/artemis/drivers/android/hierarchy.py
+++ b/artemis/drivers/android/hierarchy.py
@@ -33,7 +33,7 @@ def parse_ui_hierarchy(xml_string: str) -> list[dict[str, Any]]:
for node in root.iter():
attribs = node.attrib
bounds_str = attribs.get("bounds", "")
- match = re.match(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", bounds_str)
+ match = re.match(r"\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]", bounds_str)
if not match:
continue
diff --git a/artemis/drivers/factory.py b/artemis/drivers/factory.py
index b17e10a..ba1bcbf 100644
--- a/artemis/drivers/factory.py
+++ b/artemis/drivers/factory.py
@@ -33,7 +33,9 @@ logger = get_logger(__name__)
def create_driver(ctx: "ArtemisContext") -> BaseDeviceDriver:
"""Instantiates the appropriate BaseDeviceDriver based on the runtime context."""
- # 1. Cloud mode check
+ # 1. Cloud mode check. Cloud devices are reached through the gateway's
+ # RemoteUIAutomatorClient; ARTEMIS_HIERARCHY_BACKEND does not apply there
+ # because the Accessibility Helper needs a local adb forward.
if os.environ.get("ARTEMIS_CLOUD_MODE") == "1":
if ctx.adb_client is None:
from cloud_service.virtualization import RemoteAdbClient
diff --git a/artemis/graph/graph.py b/artemis/graph/graph.py
index 32c138c..ab854a1 100644
--- a/artemis/graph/graph.py
+++ b/artemis/graph/graph.py
@@ -447,8 +447,15 @@ async def exit_settlement_node(state: State, ctx: ArtemisContext):
if snapshot.all_check_items or records or verify_blocked:
update["run_outcome"] = outcome.model_dump()
# BLOCKED/partial wrap-ups carry the last findings in the metadata file.
- extra = {"last_findings": blocked_findings} if blocked_findings else None
- write_run_outcome(base_dir, outcome, extra)
+ extra: dict = {"last_findings": blocked_findings} if blocked_findings else {}
+ # Which UI-hierarchy source served the run, so a report reader can tell a
+ # helper run from a UIAutomator2 fallback without reading logs.
+ from artemis.clients.screen_client_factory import hierarchy_backend_summary
+
+ environment = hierarchy_backend_summary(getattr(ctx, "ui_adb_client", None))
+ if environment:
+ extra["environment"] = environment
+ write_run_outcome(base_dir, outcome, extra or None)
publish_checker_event(
ctx,
{
diff --git a/artemis/interfaces/cli/commands/doctor.py b/artemis/interfaces/cli/commands/doctor.py
index 49c9584..1c141ec 100644
--- a/artemis/interfaces/cli/commands/doctor.py
+++ b/artemis/interfaces/cli/commands/doctor.py
@@ -27,6 +27,7 @@ from dataclasses import dataclass
import json
import platform as sys_platform
import shutil
+import subprocess
from typing import Any
from rich.console import Console
@@ -194,6 +195,66 @@ def _showcase_row() -> ExtraRow:
)
+def _helper_row(results: list[ProbeResult]) -> ExtraRow | None:
+ """State of the Artemis accessibility helper on the one ready, idle device.
+
+ Tasks install the helper themselves when they take a device; this row lets a
+ person pre-install it so the first task does not pay the install delay, and
+ tells them the helper is a thing they can remove.
+ """
+ from artemis.clients.screen_client_factory import resolve_backend
+ from artemis.runtime.helper_manager import helper_manager
+
+ if resolve_backend().value == "uiautomator":
+ return None
+ adb = next((r for r in results if r.id == "android_adb"), None)
+ devices = (adb.metadata.get("devices") if adb else None) or []
+ ready = [str(d.get("serial")) for d in devices if d.get("state") == "device"]
+ if len(ready) != 1:
+ return None
+ serial = ready[0]
+ try:
+ status = helper_manager.status(serial)
+ except (OSError, ValueError, subprocess.SubprocessError) as exc:
+ return ExtraRow(
+ key="accessibility_helper",
+ title="Artemis Accessibility Helper",
+ status="missing",
+ status_markup="[dim]⚪ Unknown[/dim]",
+ summary="Could not read",
+ detail=f"{serial}: {exc}",
+ )
+ healthy = status["installed"] and status["enabled"] and not status["outdated"]
+ if healthy and status["reachable"]:
+ version = status.get("installed_version")
+ note = " (newer than the bundled build)" if status.get("newer_than_bundled") else ""
+ return ExtraRow(
+ key="accessibility_helper",
+ title="Artemis Accessibility Helper",
+ status="pass",
+ status_markup="[bold green]✔ Ready[/bold green]",
+ summary=f"v{version} on {serial}{note}",
+ detail=f"Remove any time with: artemis helper uninstall --serial {serial}",
+ )
+ if not status["installed"]:
+ summary, why = "Not installed", "the first task on this device will install it (about 3 s)"
+ elif status["outdated"]:
+ summary = f"Outdated (v{status['installed_version']} < v{status['bundled_version']})"
+ why = "the next task will upgrade it"
+ elif not status["enabled"]:
+ summary, why = "Service disabled", "the next task will try to enable it"
+ else:
+ summary, why = "Not answering", "unlock the phone or reinstall with --force"
+ return ExtraRow(
+ key="accessibility_helper",
+ title="Artemis Accessibility Helper",
+ status="missing",
+ status_markup="[bold yellow]○ Pending[/bold yellow]",
+ summary=summary,
+ detail=f"{why}; pre-install now: artemis helper install --serial {serial}",
+ )
+
+
# ---------------------------------------------------------------------------
# Rendering
# ---------------------------------------------------------------------------
@@ -373,6 +434,9 @@ def doctor_command(
"""Run diagnostics to inspect system dependencies, device connectivity, and configuration."""
results, fixes = asyncio.run(_diagnose(fix))
extras = [_npm_row(), _showcase_row()]
+ helper_row = _helper_row(results)
+ if helper_row is not None:
+ extras.append(helper_row)
verdict = base_verdict(results)
if json_output:
diff --git a/artemis/interfaces/cli/commands/helper.py b/artemis/interfaces/cli/commands/helper.py
new file mode 100644
index 0000000..9bc696a
--- /dev/null
+++ b/artemis/interfaces/cli/commands/helper.py
@@ -0,0 +1,226 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""``artemis helper``: manage the Accessibility Helper APK on attached devices.
+
+Tasks provision the helper themselves when they take a device, so these
+commands exist for the cases the automatic path deliberately leaves alone:
+inspecting a device, forcing a reinstall, and removing the helper (which is
+never done automatically).
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Annotated
+
+from rich.console import Console
+from rich.table import Table
+import typer
+
+from artemis.runtime.device_pool import device_pool
+from artemis.runtime.helper_manager import helper_manager
+from artemis.utils.logger import get_logger
+
+logger = get_logger(__name__)
+helper_app = typer.Typer(help="Install, inspect, or remove the Artemis Accessibility Helper APK.")
+
+SerialOption = Annotated[
+ str | None,
+ typer.Option("--serial", "-s", help="Device serial. Defaults to the only ready device."),
+]
+
+
+def _resolve_serial(serial: str | None) -> str:
+ if serial:
+ return serial
+ ready = [d.serial for d in device_pool.get_ready_devices()]
+ if len(ready) == 1:
+ return ready[0]
+ if not ready:
+ typer.secho("No authorized device is attached.", fg=typer.colors.RED)
+ else:
+ typer.secho(
+ "Several devices are ready; pass --serial, or --all to target every idle one: "
+ + ", ".join(ready),
+ fg=typer.colors.RED,
+ )
+ raise typer.Exit(code=1)
+
+
+def _idle_ready_serials() -> list[str]:
+ """Every authorized device that no Artemis task currently holds."""
+ return [d.serial for d in device_pool.get_ready_devices() if not d.is_busy]
+
+
+def _bool(value: object) -> str:
+ return "[green]yes[/green]" if value else "[red]no[/red]"
+
+
+@helper_app.command("status")
+def helper_status(
+ serial: SerialOption = None,
+ as_json: Annotated[bool, typer.Option("--json", help="Print raw JSON.")] = False,
+) -> None:
+ """Show install, version, service and tunnel state of the helper on a device."""
+ target = _resolve_serial(serial)
+ status = helper_manager.status(target)
+ if as_json:
+ typer.echo(json.dumps(status, indent=2))
+ return
+ table = Table(title=f"Accessibility helper on {target}")
+ table.add_column("Field", style="cyan")
+ table.add_column("Value")
+ table.add_row("Installed", _bool(status["installed"]))
+ version = str(status["installed_version"])
+ if status.get("newer_than_bundled"):
+ version += f" [yellow](newer than the bundled v{status['bundled_version']}; not downgraded)[/yellow]"
+ elif status.get("outdated"):
+ version += f" [yellow](bundled is v{status['bundled_version']}; the next task upgrades it)[/yellow]"
+ table.add_row("Installed version", version)
+ table.add_row("Bundled version", str(status["bundled_version"]))
+ table.add_row("Service enabled", _bool(status["enabled"]))
+ tunnel = {
+ "session": f"this process, host port {status['forward_port']}",
+ "shared": f"another Artemis process, host port {status['forward_port']}",
+ "probe": "none (probed through a temporary forward)",
+ None: "none",
+ }[status.get("tunnel")]
+ table.add_row("Tunnel", tunnel)
+ table.add_row("Service answering", _bool(status["reachable"]))
+ if status.get("reachable"):
+ protocol = status.get("protocol_version")
+ proto_text = str(protocol)
+ if not status.get("protocol_supported"):
+ proto_text += " [red](too old for this host; reinstall with --force)[/red]"
+ table.add_row("Protocol", proto_text)
+ table.add_row("Session token pushed", _bool(status.get("token_set")))
+ table.add_row("Auto-install by tasks", _bool(status.get("auto_install", True)))
+ table.add_row("Transport id", str(status["transport_id"]))
+ Console().print(table)
+
+
+def _install_one(target: str, force: bool) -> bool:
+ def on_event(event: str, details: dict) -> None:
+ if event == "installing":
+ typer.echo(f"{target}: installing v{details.get('version_name')}...")
+ elif event == "upgrading":
+ typer.echo(
+ f"{target}: upgrading v{details.get('from_version')} -> "
+ f"v{details.get('to_version')}..."
+ )
+
+ result = helper_manager.provision(target, force=force, on_event=on_event)
+ if result.ok:
+ typer.secho(
+ f"{target}: {result.action} (version {result.installed_version}, service enabled).",
+ fg=typer.colors.GREEN,
+ )
+ return True
+ typer.secho(f"{target}: {result.action}: {result.error}", fg=typer.colors.RED)
+ return False
+
+
+@helper_app.command("install")
+def helper_install(
+ serial: SerialOption = None,
+ force: Annotated[bool, typer.Option("--force", help="Reinstall even when up to date.")] = False,
+ all_devices: Annotated[
+ bool, typer.Option("--all", help="Every authorized device that no task is using.")
+ ] = False,
+) -> None:
+ """Install or upgrade the helper to the bundled version and enable its service."""
+ if all_devices:
+ targets = _idle_ready_serials()
+ if not targets:
+ typer.secho("No idle, authorized device is attached.", fg=typer.colors.RED)
+ raise typer.Exit(code=1)
+ else:
+ targets = [_resolve_serial(serial)]
+ outcomes = [_install_one(target, force) for target in targets]
+ if not all(outcomes):
+ raise typer.Exit(code=1)
+
+
+@helper_app.command("uninstall")
+def helper_uninstall(
+ serial: SerialOption = None,
+ yes: Annotated[bool, typer.Option("--yes", "-y", help="Do not ask for confirmation.")] = False,
+) -> None:
+ """Disable the service and remove the helper package from a device."""
+ target = _resolve_serial(serial)
+ if not yes and not typer.confirm(f"Remove the Artemis accessibility helper from {target}?"):
+ raise typer.Exit(code=0)
+ if helper_manager.uninstall(target):
+ typer.secho(f"{target}: helper removed.", fg=typer.colors.GREEN)
+ return
+ typer.secho(f"{target}: uninstall failed (is the package installed?).", fg=typer.colors.RED)
+ raise typer.Exit(code=1)
+
+
+@helper_app.command("parity")
+def helper_parity(
+ serial: SerialOption = None,
+ rounds: Annotated[int, typer.Option("--rounds", help="Timed dumps per backend.")] = 3,
+ as_json: Annotated[bool, typer.Option("--json", help="Print raw JSON.")] = False,
+) -> None:
+ """Dump the current screen with the helper and with UIAutomator2 and compare them.
+
+ The two backends must describe the same visible screen: every labelled
+ UIAutomator element should have a helper element with the same label at
+ the same place, the helper must emit no off-screen or negative bounds, and
+ its dump should not be slower. Run it on a busy screen (a scrolled list, a
+ dialog over an app) to catch regressions the unit tests cannot see.
+ """
+ from artemis.core.diagnostics.hierarchy_parity import compare_backends
+
+ target = _resolve_serial(serial)
+ report = compare_backends(target, rounds=max(1, rounds))
+ if as_json:
+ typer.echo(json.dumps(report, indent=2, ensure_ascii=False))
+ raise typer.Exit(code=0 if report["ok"] else 1)
+
+ table = Table(title=f"Hierarchy parity on {target}")
+ table.add_column("Metric", style="cyan")
+ table.add_column("Helper")
+ table.add_column("UIAutomator2")
+ h, u = report["helper"], report["uiautomator"]
+ table.add_row("Dump time, median", f"{h['median_ms']:.0f} ms", f"{u['median_ms']:.0f} ms")
+ table.add_row("Nodes", str(h["nodes"]), str(u["nodes"]))
+ table.add_row("Labelled nodes (text / desc)", str(h["labelled"]), str(u["labelled"]))
+ table.add_row("Negative bounds", str(h["negative_bounds"]), str(u["negative_bounds"]))
+ table.add_row("Off-screen bounds", str(h["offscreen_bounds"]), str(u["offscreen_bounds"]))
+ table.add_row("Foreground package", str(h["package"]), str(u["package"]))
+ Console().print(table)
+
+ m = report["match"]
+ typer.echo(
+ f"UIAutomator labels found in helper dump: {m['uiautomator_in_helper']}/{m['uiautomator_labelled']}"
+ f" ({m['recall']:.0%}); helper labels found in UIAutomator dump: "
+ f"{m['helper_in_uiautomator']}/{m['helper_labelled']} ({m['precision']:.0%})"
+ )
+ for label in m["missing_in_helper"][:10]:
+ typer.echo(f" helper lacks: {label}")
+ for label in m["extra_in_helper"][:10]:
+ typer.echo(f" helper adds: {label}")
+ for problem in report["problems"]:
+ typer.secho(f" ! {problem}", fg=typer.colors.RED)
+ typer.secho(
+ "PARITY OK" if report["ok"] else "PARITY FAILED",
+ fg=typer.colors.GREEN if report["ok"] else typer.colors.RED,
+ )
+ raise typer.Exit(code=0 if report["ok"] else 1)
+
+
+__all__ = ["helper_app"]
diff --git a/artemis/interfaces/cli/main.py b/artemis/interfaces/cli/main.py
index fe42218..d11cbed 100644
--- a/artemis/interfaces/cli/main.py
+++ b/artemis/interfaces/cli/main.py
@@ -19,6 +19,7 @@ from typing import Annotated
from artemis._version import __version__
from artemis.interfaces.cli.commands.batch import batch_command
from artemis.interfaces.cli.commands.doctor import doctor_command
+from artemis.interfaces.cli.commands.helper import helper_app
from artemis.interfaces.cli.commands.init import init_command
from artemis.interfaces.cli.commands.mcp import mcp_command
from artemis.interfaces.cli.commands.run import run_command
@@ -63,6 +64,9 @@ app.command(name="batch", help="Execute a batch sequence of automation tasks.")(
app.command(name="mcp", help="Start the Artemis Model Context Protocol (MCP) server.")(mcp_command)
app.add_typer(server_app, name="server", help="Cloud Run proxy and web dashboard server.")
app.add_typer(trace_app, name="trace", help="Inspect and query execution traces.")
+app.add_typer(
+ helper_app, name="helper", help="Manage the Accessibility Helper APK on attached devices."
+)
def version_callback(value: bool):
diff --git a/artemis/mcp/adb_server.py b/artemis/mcp/adb_server.py
index 507b103..be2b71a 100644
--- a/artemis/mcp/adb_server.py
+++ b/artemis/mcp/adb_server.py
@@ -39,7 +39,7 @@ except Exception as exc: # pylint: disable=broad-exception-caught
# Best-effort compatibility shim for FastMCP/pydantic version drift.
logger.debug("FastMCP Settings model_rebuild skipped: %s", exc, exc_info=True)
-from artemis.clients.ui_automator_client import UIAutomatorClient
+from artemis.clients.screen_client_factory import create_screen_client
from artemis.context import ArtemisContext, DeviceContext, DevicePlatform
from artemis.controllers.unified_controller import UnifiedMobileController
from artemis.platform import platform
@@ -146,7 +146,10 @@ def _get_controller(device_serial: str | None = None):
device = devices[0]
device_id = device.serial
- ui_client = UIAutomatorClient(device_id=device_id)
+ # Observer path: the helper is used only when it is already installed and
+ # running; nothing is installed from here (that happens inside a task's
+ # device-lock boundary or via `artemis helper install`).
+ ui_client = create_screen_client(device_id)
try:
ui_data = ui_client.get_screen_data()
width, height = ui_data.width, ui_data.height
diff --git a/artemis/runtime/helper_manager.py b/artemis/runtime/helper_manager.py
new file mode 100644
index 0000000..3196d5e
--- /dev/null
+++ b/artemis/runtime/helper_manager.py
@@ -0,0 +1,1030 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Lifecycle of the Artemis Accessibility Helper APK on each Android device.
+
+The helper (``packages/artemis-accessibility-helper``) is an accessibility
+service that serves the UI hierarchy over a loopback HTTP port on the phone.
+This module owns everything *around* that HTTP API, split by lifetime:
+
+* **Provision** (persistent on the device, survives reboots): the APK is
+ installed at the bundled version and the service is enabled in secure
+ settings. Only callers that hold the device (a task inside its
+ ``DeviceExecutionLock`` boundary, ``artemis helper install``,
+ ``mobile_diagnose(attempt_fix=True)``) provision; read-only observers never
+ install anything. A cross-process file mutex serialises concurrent installs.
+* **Attach** (one process, one device, one session): an ``adb forward`` from a
+ host port that adb allocates (``tcp:0``) to the fixed device port. Existing
+ forwards for the same serial are reused so two Artemis processes share one
+ tunnel; the local port is the identity of the tunnel, never a fixed number,
+ which is what keeps several devices on one host apart.
+* **Serve** (per request): handled by :class:`AccessibilityClient`; on a
+ transport error it calls :meth:`AccessibilityHelperManager.reattach` once.
+* **Detach / evict**: forwards this process created are removed on detach.
+ A physical unplug needs no host work at all: adb drops the forward with the
+ transport and the on-device service keeps running. The ``transport_id`` that
+ ``adb devices -l`` reports changes on every replug, so a session records it
+ and :meth:`attach` treats a changed id as "stale, rebuild the tunnel".
+* **Token**: the loopback port is reachable by every app on the phone, so the
+ helper serves nothing (except ``/ping``) without the session token this host
+ pushed. The token is one random secret per host, kept in a file under the
+ Artemis temp directory so every Artemis process on the machine presents the
+ same one, and delivered with an ``am broadcast`` that only the adb shell user
+ can send (the receiver is guarded by ``WRITE_SECURE_SETTINGS``). It is pushed
+ on every attach and again whenever the helper answers 401 (a re-bound service
+ starts without one).
+
+Locking is per device: provisioning one phone (up to a minute when the install
+is slow) never blocks attaching to another.
+
+**UiAutomation suppresses the helper.** Android unbinds every accessibility
+service while a ``UiAutomation`` connection created without
+``FLAG_DONT_SUPPRESS_ACCESSIBILITY_SERVICES`` is alive. uiautomator2's device
+server (``app_process ... com.wetest.uia2.Main``) and Appium's UiAutomator2
+server both do that by default, so the helper is silent for as long as one of
+them runs. When the helper does not answer, :meth:`attach` looks for such a
+holder: uiautomator2's server (Artemis's own fallback, or one left behind by a
+previous run) is stopped so the system re-binds the helper within a second;
+Appium's is never touched, the error names it instead.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from contextlib import contextmanager
+from dataclasses import asdict, dataclass, field
+import hashlib
+import json
+import os
+from pathlib import Path
+import re
+import secrets
+import subprocess
+import threading
+import time
+from typing import Any
+import urllib.error
+import urllib.request
+
+from artemis.config.constants import ENV_ARTEMIS_HELPER_AUTO_INSTALL
+from artemis.config.paths import get_temp_dir
+from artemis.runtime.adb_endpoint import adb_command
+from artemis.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+PACKAGE_NAME = "com.artemis.helper"
+SERVICE_NAME = f"{PACKAGE_NAME}/.ArtemisAccessibilityService"
+TOKEN_RECEIVER = f"{PACKAGE_NAME}/.TokenReceiver"
+TOKEN_ACTION = f"{PACKAGE_NAME}.SET_TOKEN"
+#: Loopback port the service binds on the device. Never used as a host port.
+DEVICE_PORT = 18888
+#: Oldest helper HTTP contract this host can talk to (see PROTOCOL_VERSION in the APK).
+MIN_PROTOCOL_VERSION = 2
+TOKEN_FILE_NAME = "session.token"
+
+HELPER_PACKAGE_DIR = (
+ Path(__file__).resolve().parent.parent.parent / "packages" / "artemis-accessibility-helper"
+)
+BUNDLED_APK_PATH = HELPER_PACKAGE_DIR / "ArtemisAccessibilityHelper.apk"
+BUNDLED_MANIFEST_PATH = HELPER_PACKAGE_DIR / "helper_manifest.json"
+
+_PING_ATTEMPTS_AFTER_PROVISION = 12
+_PING_ATTEMPTS_LAZY = 3
+_PING_INTERVAL_SECONDS = 0.5
+_UIAUTOMATION_RELEASE_SETTLE_SECONDS = 1.0
+
+#: Process command-line markers of UiAutomation holders. The first group is
+#: uiautomator2 (openatx), which Artemis itself uses as the fallback backend and
+#: may stop; the second is Appium's server, which belongs to someone else.
+_UIAUTOMATOR2_MARKERS = ("com.wetest.uia2.Main", "com.github.uiautomator")
+_APPIUM_MARKERS = ("io.appium.uiautomator2",)
+_PROVISION_MUTEX_TIMEOUT_SECONDS = 90.0
+_PROVISION_MUTEX_STALE_SECONDS = 180.0
+_ENABLE_ATTEMPTS = 5
+_ENABLE_SETTLE_SECONDS = 0.4
+_REVIVE_SETTLE_SECONDS = 0.3
+
+AdbRunner = Callable[[list[str]], subprocess.CompletedProcess]
+Pinger = Callable[[int], dict[str, Any] | None]
+#: Progress callback: ``(event, details)`` with events ``installing`` / ``upgrading`` /
+#: ``enabling`` fired *before* the slow step so a UI can show what is happening.
+ProvisionEvent = Callable[[str, dict[str, Any]], None]
+
+#: Where a person enables the service by hand when secure settings are rejected
+#: (some OEM ROMs and Android 13+ restricted settings for sideloaded apps).
+MANUAL_ENABLE_PATH = (
+ "Settings > Accessibility > Installed apps (or Downloaded apps) > "
+ "Artemis Accessibility Helper > turn it on"
+)
+ACCESSIBILITY_SETTINGS_INTENT = "android.settings.ACCESSIBILITY_SETTINGS"
+
+
+class HelperUnavailable(RuntimeError):
+ """The helper could not be reached on a device (not installed, disabled, or dead)."""
+
+
+def auto_install_allowed() -> bool:
+ """Whether a task may install / upgrade the helper on a device it holds.
+
+ ``ARTEMIS_HELPER_AUTO_INSTALL`` (environment, then settings); default true.
+ """
+ raw = os.environ.get(ENV_ARTEMIS_HELPER_AUTO_INSTALL)
+ if raw is None:
+ try:
+ from artemis.config import settings
+
+ value = getattr(settings, "ARTEMIS_HELPER_AUTO_INSTALL", True)
+ return (
+ bool(value)
+ if not isinstance(value, str)
+ else value.strip().lower()
+ not in (
+ "0",
+ "false",
+ "no",
+ "off",
+ )
+ )
+ except (ImportError, ValueError):
+ return True
+ return raw.strip().lower() not in ("0", "false", "no", "off")
+
+
+@dataclass(frozen=True)
+class BundledHelper:
+ """The APK shipped in the repository, described by ``helper_manifest.json``."""
+
+ apk_path: Path
+ version_code: int
+ version_name: str
+ sha256: str
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "apk_path": str(self.apk_path),
+ "version_code": self.version_code,
+ "version_name": self.version_name,
+ "sha256": self.sha256,
+ }
+
+
+def load_bundled_helper(
+ apk_path: Path = BUNDLED_APK_PATH, manifest_path: Path = BUNDLED_MANIFEST_PATH
+) -> BundledHelper | None:
+ """Read the bundled APK's manifest; ``None`` when the artifact is not shipped."""
+ if not apk_path.is_file() or not manifest_path.is_file():
+ return None
+ try:
+ data = json.loads(manifest_path.read_text(encoding="utf-8"))
+ return BundledHelper(
+ apk_path=apk_path,
+ version_code=int(data["version_code"]),
+ version_name=str(data.get("version_name", "")),
+ sha256=str(data.get("sha256", "")),
+ )
+ except (OSError, ValueError, KeyError, TypeError) as exc:
+ logger.warning(f"Bundled accessibility helper manifest is unreadable: {exc}")
+ return None
+
+
+@dataclass
+class HelperSession:
+ """One process's live tunnel to the helper on one device."""
+
+ serial: str
+ local_port: int
+ transport_id: str | None
+ version_code: int | None
+ version_name: str
+ owns_forward: bool
+ attached_at: float = field(default_factory=time.time)
+ #: Session token the helper expects in ``X-Artemis-Token``; None for pre-token helpers.
+ token: str | None = field(default=None, repr=False)
+ protocol_version: int = 1
+
+ @property
+ def base_url(self) -> str:
+ return f"http://127.0.0.1:{self.local_port}"
+
+ def to_dict(self) -> dict[str, Any]:
+ data = asdict(self)
+ data.pop("token", None)
+ data["base_url"] = self.base_url
+ return data
+
+
+@dataclass
+class ProvisionResult:
+ ok: bool
+ action: str # "up_to_date" | "installed" | "upgraded" | "skipped" | "failed"
+ installed_version: int | None
+ bundled_version: int | None
+ enabled: bool
+ error: str | None = None
+
+ def to_dict(self) -> dict[str, Any]:
+ return asdict(self)
+
+
+def _default_run_adb(args: list[str], timeout: float = 30.0) -> subprocess.CompletedProcess:
+ return subprocess.run(
+ adb_command(args),
+ stdin=subprocess.DEVNULL,
+ capture_output=True,
+ text=True,
+ timeout=timeout,
+ check=False,
+ )
+
+
+def _default_ping(local_port: int, timeout: float = 2.0) -> dict[str, Any] | None:
+ try:
+ with urllib.request.urlopen(f"http://127.0.0.1:{local_port}/ping", timeout=timeout) as resp:
+ data = json.loads(resp.read().decode("utf-8"))
+ except (urllib.error.URLError, OSError, ValueError, TimeoutError):
+ return None
+ if not isinstance(data, dict) or not data.get("success"):
+ return None
+ return data
+
+
+class AccessibilityHelperManager:
+ """Process-wide registry of helper sessions, keyed by device serial."""
+
+ def __init__(
+ self,
+ run_adb: AdbRunner | None = None,
+ ping: Pinger | None = None,
+ bundled: BundledHelper | None | Callable[[], BundledHelper | None] = load_bundled_helper,
+ sleep: Callable[[float], None] = time.sleep,
+ token_path: Path | None = None,
+ auto_install: Callable[[], bool] = auto_install_allowed,
+ ) -> None:
+ self._run_adb = run_adb or _default_run_adb
+ self._ping = ping or _default_ping
+ self._bundled_source = bundled
+ self._sleep = sleep
+ self._token_path = token_path
+ self._auto_install = auto_install
+ self._token: str | None = None
+ self._sessions: dict[str, HelperSession] = {}
+ #: Guards the session registry and the per-device lock table only.
+ self._registry_lock = threading.Lock()
+ self._device_locks: dict[str, threading.RLock] = {}
+
+ def _device_lock(self, serial: str) -> threading.RLock:
+ """One re-entrant lock per device, so devices provision and attach in parallel."""
+ with self._registry_lock:
+ lock = self._device_locks.get(serial)
+ if lock is None:
+ lock = self._device_locks[serial] = threading.RLock()
+ return lock
+
+ # ------------------------------------------------------------------ #
+ # Bundled artifact
+ # ------------------------------------------------------------------ #
+
+ @property
+ def bundled(self) -> BundledHelper | None:
+ source = self._bundled_source
+ return source() if callable(source) else source
+
+ # ------------------------------------------------------------------ #
+ # Session token
+ # ------------------------------------------------------------------ #
+
+ def host_token(self) -> str:
+ """The per-host secret every Artemis process presents to the helper.
+
+ Created on first use (48 hex chars) in the Artemis temp directory with
+ owner-only permissions; shared by the MCP daemon, the CLI and the SDK.
+ """
+ if self._token:
+ return self._token
+ path = self._token_path or (get_temp_dir("helper-token") / TOKEN_FILE_NAME)
+ with self._provision_mutex(f"host-token:{path.resolve()}"):
+ return self._load_or_create_token(path)
+
+ def _load_or_create_token(self, path: Path) -> str:
+ try:
+ text = path.read_text(encoding="utf-8").strip()
+ if re.fullmatch(r"[0-9a-f]{32,128}", text):
+ self._token = text
+ return text
+ except OSError:
+ pass
+ token = secrets.token_hex(24)
+ try:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+ with os.fdopen(fd, "w", encoding="utf-8") as fh:
+ fh.write(token)
+ except OSError as exc:
+ # No persistent file (read-only temp?): a process-local token still
+ # protects the helper; other processes will simply push their own.
+ logger.warning(f"Could not persist the helper session token at {path}: {exc}")
+ self._token = token
+ return token
+
+ def push_token(self, serial: str) -> bool:
+ """Deliver the host token to the helper on ``serial`` (idempotent).
+
+ The broadcast is explicit (component named) so it reaches the manifest
+ receiver on every Android version, and it is sent by the adb shell user,
+ the only sender the receiver's permission guard admits.
+ """
+ result = self._adb(
+ serial,
+ "shell",
+ "am",
+ "broadcast",
+ "-n",
+ TOKEN_RECEIVER,
+ "-a",
+ TOKEN_ACTION,
+ "--es",
+ "token",
+ self.host_token(),
+ )
+ output = f"{result.stdout or ''}\n{result.stderr or ''}"
+ ok = result.returncode == 0 and "Broadcast completed" in output
+ if not ok:
+ logger.warning(
+ f"Pushing the session token to the accessibility helper on {serial} failed: "
+ f"adb exited with code {result.returncode}."
+ )
+ return ok
+
+ # ------------------------------------------------------------------ #
+ # Device queries (no side effects)
+ # ------------------------------------------------------------------ #
+
+ def _adb(self, serial: str, *args: str) -> subprocess.CompletedProcess:
+ return self._run_adb(["-s", serial, *args])
+
+ def transport_id(self, serial: str) -> str | None:
+ """The adb transport id of ``serial``; changes every time the device reconnects."""
+ try:
+ result = self._run_adb(["devices", "-l"])
+ except (OSError, subprocess.SubprocessError) as exc:
+ logger.debug(f"adb devices -l failed while reading transport id: {exc}")
+ return None
+ for line in (result.stdout or "").splitlines():
+ parts = line.split()
+ if len(parts) < 2 or parts[0] != serial:
+ continue
+ for token in parts[2:]:
+ if token.startswith("transport_id:"):
+ return token.split(":", 1)[1]
+ return None
+ return None
+
+ def installed_version(self, serial: str) -> int | None:
+ """versionCode of the helper on the device, or ``None`` when it is not installed."""
+ result = self._adb(serial, "shell", "dumpsys", "package", PACKAGE_NAME)
+ output = result.stdout or ""
+ if result.returncode != 0 or "Unable to find package" in output:
+ return None
+ match = re.search(r"versionCode=(\d+)", output)
+ if match is None:
+ # dumpsys answered but without a version block: the package is absent.
+ return None
+ return int(match.group(1))
+
+ def is_service_enabled(self, serial: str) -> bool:
+ result = self._adb(
+ serial, "shell", "settings", "get", "secure", "enabled_accessibility_services"
+ )
+ enabled = (result.stdout or "").strip()
+ return SERVICE_NAME in enabled.split(":") if enabled and enabled != "null" else False
+
+ def existing_forward(self, serial: str) -> int | None:
+ """Host port of an adb forward that already targets the helper on ``serial``."""
+ result = self._run_adb(["forward", "--list"])
+ for line in (result.stdout or "").splitlines():
+ parts = line.split()
+ if len(parts) != 3 or parts[0] != serial:
+ continue
+ local, remote = parts[1], parts[2]
+ if remote == f"tcp:{DEVICE_PORT}" and local.startswith("tcp:"):
+ try:
+ return int(local[4:])
+ except ValueError:
+ continue
+ return None
+
+ def ping(self, local_port: int) -> dict[str, Any] | None:
+ return self._ping(local_port)
+
+ def uiautomation_holders(self, serial: str) -> dict[str, list[int]]:
+ """Processes on ``serial`` holding a UiAutomation connection that mutes the helper.
+
+ Returns ``{"uiautomator2": [pids], "appium": [pids]}``.
+ """
+ holders: dict[str, list[int]] = {"uiautomator2": [], "appium": []}
+ try:
+ result = self._adb(serial, "shell", "ps", "-A", "-o", "PID,ARGS")
+ except (OSError, subprocess.SubprocessError) as exc:
+ logger.debug(f"ps on {serial} failed while looking for UiAutomation holders: {exc}")
+ return holders
+ for line in (result.stdout or "").splitlines():
+ parts = line.split(None, 1)
+ if len(parts) != 2 or not parts[0].isdigit():
+ continue
+ pid, args = int(parts[0]), parts[1]
+ if args.lstrip().startswith("sh "):
+ continue # the `sh -c` wrapper around app_process, not the holder itself
+ if any(marker in args for marker in _UIAUTOMATOR2_MARKERS):
+ holders["uiautomator2"].append(pid)
+ elif any(marker in args for marker in _APPIUM_MARKERS):
+ holders["appium"].append(pid)
+ return holders
+
+ def stop_uiautomator2_server(self, serial: str, pids: list[int] | None = None) -> bool:
+ """Kill uiautomator2's device server so Android re-binds the helper.
+
+ Only ever aimed at openatx's server (``u2.jar`` / ``com.github.uiautomator``),
+ which Artemis owns as its fallback backend. Returns whether anything was stopped.
+ """
+ if pids is None:
+ pids = self.uiautomation_holders(serial)["uiautomator2"]
+ stopped = False
+ for pid in pids:
+ result = self._adb(serial, "shell", "kill", str(pid))
+ stopped = stopped or result.returncode == 0
+ result = self._adb(serial, "shell", "am", "force-stop", "com.github.uiautomator")
+ stopped = stopped or (result.returncode == 0 and bool(pids))
+ if stopped:
+ logger.info(
+ f"Stopped uiautomator2's device server on {serial} so the accessibility helper "
+ "can bind again (UiAutomation suppresses other accessibility services)."
+ )
+ self._sleep(_UIAUTOMATION_RELEASE_SETTLE_SECONDS)
+ return stopped
+
+ def session(self, serial: str) -> HelperSession | None:
+ with self._registry_lock:
+ return self._sessions.get(serial)
+
+ def status(self, serial: str) -> dict[str, Any]:
+ """Everything a diagnostic wants to know, without changing the device.
+
+ ``reachable`` answers "is the service alive on the phone", not "does
+ this process have a tunnel": when no forward exists yet a temporary one
+ is created for the probe and removed again. ``tunnel`` says which it was.
+ """
+ bundled = self.bundled
+ installed = self.installed_version(serial)
+ enabled = self.is_service_enabled(serial) if installed is not None else False
+ session = self.session(serial)
+ tunnel: str | None
+ forward_port: int | None
+ if session is not None:
+ forward_port, tunnel = session.local_port, "session"
+ else:
+ forward_port = self.existing_forward(serial)
+ tunnel = "shared" if forward_port is not None else None
+ ping = None
+ if forward_port is not None:
+ ping = self.ping(forward_port)
+ elif installed is not None and enabled:
+ ping = self._probe_with_temporary_forward(serial)
+ tunnel = "probe"
+ return {
+ "package": PACKAGE_NAME,
+ "installed": installed is not None,
+ "installed_version": installed,
+ "bundled_version": bundled.version_code if bundled else None,
+ "bundled_apk_present": bundled is not None,
+ "outdated": bool(
+ bundled and installed is not None and installed < bundled.version_code
+ ),
+ "newer_than_bundled": bool(
+ bundled and installed is not None and installed > bundled.version_code
+ ),
+ "enabled": enabled,
+ "forward_port": forward_port,
+ "tunnel": tunnel,
+ "reachable": ping is not None,
+ "reported_version": ping.get("version_code") if ping else None,
+ "protocol_version": ping.get("protocol_version", 1) if ping else None,
+ "protocol_supported": bool(
+ ping and int(ping.get("protocol_version", 1)) >= MIN_PROTOCOL_VERSION
+ ),
+ "token_set": bool(ping.get("token_set")) if ping else None,
+ "auto_install": self._auto_install(),
+ "transport_id": self.transport_id(serial),
+ "session": session.to_dict() if session else None,
+ }
+
+ def _probe_with_temporary_forward(self, serial: str) -> dict[str, Any] | None:
+ try:
+ port = self._create_forward(serial)
+ except HelperUnavailable:
+ return None
+ try:
+ return self.ping(port)
+ finally:
+ self._remove_forward(serial, port)
+
+ # ------------------------------------------------------------------ #
+ # Provision: install / upgrade / enable (persistent device state)
+ # ------------------------------------------------------------------ #
+
+ @contextmanager
+ def _provision_mutex(self, serial: str):
+ """Serialise installs across Artemis processes sharing one device."""
+ root = get_temp_dir("helper-provision")
+ root.mkdir(parents=True, exist_ok=True)
+ digest = hashlib.sha256(serial.encode("utf-8")).hexdigest()[:16]
+ path = root / f"{digest}.mutex"
+ started = time.monotonic()
+ while True:
+ try:
+ fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
+ except (FileExistsError, PermissionError) as exc:
+ # Windows may report access denied while a lock file is being deleted.
+ if isinstance(exc, PermissionError) and os.name != "nt":
+ raise
+ if time.monotonic() - started > _PROVISION_MUTEX_TIMEOUT_SECONDS:
+ raise TimeoutError(
+ f"Timed out waiting for the accessibility helper lock on {serial}."
+ ) from exc
+ try:
+ age = time.time() - path.stat().st_mtime
+ except OSError:
+ self._sleep(0.1)
+ continue
+ if age > _PROVISION_MUTEX_STALE_SECONDS:
+ path.unlink(missing_ok=True)
+ continue
+ if time.monotonic() - started > _PROVISION_MUTEX_TIMEOUT_SECONDS:
+ raise TimeoutError(
+ f"Timed out waiting for another process to finish installing the "
+ f"accessibility helper on {serial}."
+ )
+ self._sleep(0.1)
+ else:
+ try:
+ os.write(fd, str(os.getpid()).encode("ascii"))
+ finally:
+ os.close(fd)
+ break
+ try:
+ yield
+ finally:
+ path.unlink(missing_ok=True)
+
+ def _enable_service(self, serial: str) -> bool:
+ """Add the service to secure settings and confirm the write stuck.
+
+ Right after ``pm install`` (and right after boot) AccessibilityManager
+ may still not resolve the component and prunes it from the setting
+ again, so a single write followed by an immediate read can report
+ success for a value that is gone a moment later. Re-read after a short
+ delay and retry a few times before giving up.
+ """
+ for attempt in range(_ENABLE_ATTEMPTS):
+ result = self._adb(
+ serial, "shell", "settings", "get", "secure", "enabled_accessibility_services"
+ )
+ current = (result.stdout or "").strip()
+ services = [s for s in current.split(":") if s and s != "null"] if current else []
+ if SERVICE_NAME in services:
+ # Already enabled (the common, up-to-date case): no settle wait.
+ self._adb(
+ serial, "shell", "settings", "put", "secure", "accessibility_enabled", "1"
+ )
+ return True
+ services.append(SERVICE_NAME)
+ self._adb(
+ serial,
+ "shell",
+ "settings",
+ "put",
+ "secure",
+ "enabled_accessibility_services",
+ ":".join(services),
+ )
+ self._adb(serial, "shell", "settings", "put", "secure", "accessibility_enabled", "1")
+ self._sleep(_ENABLE_SETTLE_SECONDS)
+ if self.is_service_enabled(serial):
+ return True
+ logger.debug(
+ f"Accessibility service setting on {serial} did not stick "
+ f"(attempt {attempt + 1}/{_ENABLE_ATTEMPTS}); retrying."
+ )
+ return False
+
+ def _revive_service(self, serial: str) -> None:
+ """Make AccessibilityManager rebind a service that stopped answering.
+
+ A force-stopped (or ROM-killed) accessibility service stays dead even
+ though it is still listed as enabled; removing and re-adding it in the
+ secure setting is what makes the system bind it again. Persistent state
+ ends up exactly as before, so this is a repair, not provisioning.
+ """
+ result = self._adb(
+ serial, "shell", "settings", "get", "secure", "enabled_accessibility_services"
+ )
+ current = (result.stdout or "").strip()
+ others = [s for s in current.split(":") if s and s != "null" and s != SERVICE_NAME]
+ self._adb(
+ serial,
+ "shell",
+ "settings",
+ "put",
+ "secure",
+ "enabled_accessibility_services",
+ ":".join(others) if others else "null",
+ )
+ self._sleep(_REVIVE_SETTLE_SECONDS)
+ self._adb(
+ serial,
+ "shell",
+ "settings",
+ "put",
+ "secure",
+ "enabled_accessibility_services",
+ ":".join([*others, SERVICE_NAME]),
+ )
+ self._adb(serial, "shell", "settings", "put", "secure", "accessibility_enabled", "1")
+ logger.info(f"Re-bound the accessibility helper service on {serial}.")
+
+ def provision(
+ self,
+ serial: str,
+ *,
+ force: bool = False,
+ on_event: ProvisionEvent | None = None,
+ install: bool = True,
+ ) -> ProvisionResult:
+ """Install or upgrade the helper to the bundled version and enable the service.
+
+ Idempotent: an up-to-date, enabled helper costs two ``adb shell`` calls.
+ A device running a *newer* helper than the bundle is left alone.
+ ``on_event`` is told ``installing`` / ``upgrading`` before the install.
+ ``install=False`` (auto-install disabled) never runs ``adb install``: a
+ missing helper is reported as a failure that names the manual command,
+ an outdated one is used as is.
+ """
+ bundled = self.bundled
+
+ def emit(event: str, **details: Any) -> None:
+ if on_event is None:
+ return
+ try:
+ on_event(event, {"serial": serial, **details})
+ except Exception as exc: # pylint: disable=broad-exception-caught
+ logger.debug(f"Provision event callback failed: {exc}")
+
+ with self._provision_mutex(serial):
+ installed = self.installed_version(serial)
+ action = "up_to_date"
+ needs_install = (
+ force
+ or installed is None
+ or (bundled is not None and installed < bundled.version_code)
+ )
+ if needs_install and not install:
+ if installed is None:
+ return ProvisionResult(
+ ok=False,
+ action="failed",
+ installed_version=None,
+ bundled_version=bundled.version_code if bundled else None,
+ enabled=False,
+ error=(
+ "The accessibility helper is not installed on this device and "
+ f"{ENV_ARTEMIS_HELPER_AUTO_INSTALL}=false forbids installing it "
+ f"from a task. Install it once with: artemis helper install "
+ f"--serial {serial}"
+ ),
+ )
+ logger.info(
+ f"Accessibility helper on {serial} is v{installed} (bundled "
+ f"v{bundled.version_code if bundled else '?'}); not upgrading because "
+ f"{ENV_ARTEMIS_HELPER_AUTO_INSTALL}=false."
+ )
+ action = "skipped"
+ needs_install = False
+ if needs_install:
+ if bundled is None:
+ if installed is None:
+ return ProvisionResult(
+ ok=False,
+ action="failed",
+ installed_version=None,
+ bundled_version=None,
+ enabled=False,
+ error=(
+ "The accessibility helper is not installed and no bundled APK "
+ f"was found at {BUNDLED_APK_PATH}."
+ ),
+ )
+ action = "skipped"
+ else:
+ # This is the one moment Artemis changes what is installed on the
+ # phone; say so in a way that survives in every log.
+ logger.warning(
+ f"Installing the Artemis accessibility helper v{bundled.version_name} "
+ f"(code {bundled.version_code}) on {serial}. It is an accessibility "
+ "service that reads the screen layout for automated tests, answers only "
+ "on the phone's loopback port to a host holding the session token, and "
+ "stays installed until removed with `artemis helper uninstall`. Set "
+ f"{ENV_ARTEMIS_HELPER_AUTO_INSTALL}=false to install it by hand only."
+ )
+ emit(
+ "installing" if installed is None else "upgrading",
+ from_version=installed,
+ to_version=bundled.version_code,
+ version_name=bundled.version_name,
+ )
+ result = self._adb(serial, "install", "-r", "-g", str(bundled.apk_path))
+ output = f"{result.stdout or ''}\n{result.stderr or ''}"
+ if result.returncode != 0 or "Success" not in output:
+ return ProvisionResult(
+ ok=False,
+ action="failed",
+ installed_version=installed,
+ bundled_version=bundled.version_code,
+ enabled=False,
+ error=f"adb install failed: {output.strip()[-400:]}",
+ )
+ action = "installed" if installed is None else "upgraded"
+ installed = self.installed_version(serial)
+ enabled = self._enable_service(serial)
+ if not enabled:
+ # Secure settings were rejected (OEM ROM or restricted settings):
+ # open the Accessibility settings screen so the person only has to
+ # flip the switch, and say exactly where it is.
+ self._adb(serial, "shell", "am", "start", "-a", ACCESSIBILITY_SETTINGS_INTENT)
+ return ProvisionResult(
+ ok=enabled,
+ action=action,
+ installed_version=installed,
+ bundled_version=bundled.version_code if bundled else None,
+ enabled=enabled,
+ error=None
+ if enabled
+ else (
+ "This device rejected enabling the accessibility service from adb. "
+ f"The Accessibility settings screen was opened on {serial}; enable it by hand: "
+ f"{MANUAL_ENABLE_PATH}."
+ ),
+ )
+
+ def uninstall(self, serial: str) -> bool:
+ """Remove the helper from the device (never done automatically)."""
+ self.detach(serial)
+ result = self._adb(
+ serial, "shell", "settings", "get", "secure", "enabled_accessibility_services"
+ )
+ current = (result.stdout or "").strip()
+ services = [s for s in current.split(":") if s and s != "null" and s != SERVICE_NAME]
+ self._adb(
+ serial,
+ "shell",
+ "settings",
+ "put",
+ "secure",
+ "enabled_accessibility_services",
+ ":".join(services) if services else "null",
+ )
+ result = self._adb(serial, "uninstall", PACKAGE_NAME)
+ return result.returncode == 0 and "Success" in (result.stdout or "")
+
+ # ------------------------------------------------------------------ #
+ # Attach / detach (host-side tunnel)
+ # ------------------------------------------------------------------ #
+
+ def _create_forward(self, serial: str) -> int:
+ result = self._adb(serial, "forward", "--no-rebind", "tcp:0", f"tcp:{DEVICE_PORT}")
+ if result.returncode != 0:
+ raise HelperUnavailable(
+ f"adb forward to the accessibility helper on {serial} failed: "
+ f"{(result.stderr or result.stdout or '').strip()}"
+ )
+ text = (result.stdout or "").strip()
+ match = re.search(r"\b(\d{2,5})\b", text)
+ if match is not None:
+ return int(match.group(1))
+ existing = self.existing_forward(serial)
+ if existing is None:
+ raise HelperUnavailable(
+ f"adb forward on {serial} did not report the allocated host port (output: {text!r})."
+ )
+ return existing
+
+ def _remove_forward(self, serial: str, local_port: int) -> None:
+ try:
+ self._adb(serial, "forward", "--remove", f"tcp:{local_port}")
+ except (OSError, subprocess.SubprocessError) as exc:
+ logger.debug(f"Removing adb forward tcp:{local_port} on {serial} failed: {exc}")
+
+ def _wait_for_ping(self, local_port: int, attempts: int) -> dict[str, Any] | None:
+ for attempt in range(attempts):
+ info = self.ping(local_port)
+ if info is not None:
+ return info
+ if attempt + 1 < attempts:
+ self._sleep(_PING_INTERVAL_SECONDS)
+ return None
+
+ def attach(
+ self,
+ serial: str,
+ *,
+ provision: bool = True,
+ on_event: ProvisionEvent | None = None,
+ revive: bool | None = None,
+ ) -> HelperSession:
+ """Return a live session for ``serial``, building the tunnel when needed.
+
+ ``provision=True`` (task path) installs / upgrades / enables first.
+ ``provision=False`` (observers) only connects to a helper that is
+ already running and raises :class:`HelperUnavailable` otherwise.
+ ``revive`` (defaults to ``provision``) allows re-binding a dead but
+ enabled service; :meth:`reattach` always does, because the caller
+ already owned a working session on this device.
+ """
+ if revive is None:
+ revive = provision
+ with self._device_lock(serial):
+ transport = self.transport_id(serial)
+ existing = self.session(serial)
+ if existing is not None:
+ if existing.transport_id == transport and self.ping(existing.local_port):
+ return existing
+ logger.info(
+ f"Accessibility helper session on {serial} is stale "
+ f"(transport {existing.transport_id} -> {transport}); rebuilding the tunnel."
+ )
+ self._drop_session(serial, remove_forward=existing.transport_id == transport)
+
+ if provision:
+ result = self.provision(serial, on_event=on_event, install=self._auto_install())
+ if not result.ok:
+ raise HelperUnavailable(result.error or "Provisioning the helper failed.")
+
+ reused_port = self.existing_forward(serial)
+ owns_forward = reused_port is None
+ local_port = reused_port if reused_port is not None else self._create_forward(serial)
+
+ attempts = _PING_ATTEMPTS_AFTER_PROVISION if provision else _PING_ATTEMPTS_LAZY
+ info = self._wait_for_ping(local_port, attempts)
+ suppressed_by: str | None = None
+ if info is None and self.installed_version(serial) is not None:
+ # Silent although installed. The usual cause is a UiAutomation
+ # holder (uiautomator2 / Appium) that made Android unbind every
+ # accessibility service; only then a killed service.
+ holders = self.uiautomation_holders(serial)
+ if holders["uiautomator2"] and revive:
+ self.stop_uiautomator2_server(serial, holders["uiautomator2"])
+ info = self._wait_for_ping(local_port, _PING_ATTEMPTS_AFTER_PROVISION)
+ elif holders["uiautomator2"]:
+ suppressed_by = "uiautomator2's device server"
+ if info is None and holders["appium"]:
+ suppressed_by = "Appium's UiAutomator2 server"
+ if info is None and revive and suppressed_by is None:
+ # Installed and enabled but silent: the service was killed
+ # (force-stop, ROM task killer) or has not bound yet after an
+ # upgrade. Re-binding it is cheap and needs no install.
+ if self.is_service_enabled(serial):
+ self._revive_service(serial)
+ else:
+ self._enable_service(serial)
+ info = self._wait_for_ping(local_port, _PING_ATTEMPTS_AFTER_PROVISION)
+ if info is None:
+ if owns_forward:
+ self._remove_forward(serial, local_port)
+ if suppressed_by:
+ raise HelperUnavailable(
+ f"The accessibility helper on {serial} is suppressed by {suppressed_by}: "
+ "Android unbinds accessibility services while a UiAutomation connection "
+ "without FLAG_DONT_SUPPRESS_ACCESSIBILITY_SERVICES is alive. Stop that "
+ "server (for Appium, set the disableSuppressAccessibilityServices "
+ "capability) or run with ARTEMIS_HIERARCHY_BACKEND=uiautomator."
+ )
+ raise HelperUnavailable(
+ f"The accessibility helper on {serial} is not answering on "
+ f"127.0.0.1:{local_port} (device port {DEVICE_PORT})."
+ )
+
+ protocol = _protocol_of(info)
+ if protocol < MIN_PROTOCOL_VERSION:
+ # A helper older than this host can talk to (a dev build with a
+ # newer versionCode but the old contract, or a downgrade). Reinstall
+ # the bundle when this caller may, otherwise say what to run.
+ bundled = self.bundled
+ if provision and self._auto_install() and bundled is not None:
+ logger.warning(
+ f"Accessibility helper on {serial} speaks protocol {protocol} "
+ f"(host needs >= {MIN_PROTOCOL_VERSION}); reinstalling the bundled "
+ f"v{bundled.version_name}."
+ )
+ result = self.provision(serial, force=True, on_event=on_event)
+ if not result.ok:
+ raise HelperUnavailable(result.error or "Reinstalling the helper failed.")
+ info = self._wait_for_ping(local_port, _PING_ATTEMPTS_AFTER_PROVISION)
+ protocol = _protocol_of(info) if info else 0
+ if protocol < MIN_PROTOCOL_VERSION:
+ if owns_forward:
+ self._remove_forward(serial, local_port)
+ raise HelperUnavailable(
+ f"The accessibility helper on {serial} speaks protocol {protocol}, "
+ f"this host needs {MIN_PROTOCOL_VERSION} or newer. Reinstall it with: "
+ f"artemis helper install --serial {serial} --force"
+ )
+
+ token: str | None = None
+ if info.get("auth_required", protocol >= 2):
+ token = self.host_token()
+ self.push_token(serial)
+
+ version = info.get("version_code")
+ session = HelperSession(
+ serial=serial,
+ local_port=local_port,
+ transport_id=transport,
+ version_code=int(version) if isinstance(version, (int, float)) else None,
+ version_name=str(info.get("version_name") or ""),
+ owns_forward=owns_forward,
+ token=token,
+ protocol_version=protocol,
+ )
+ with self._registry_lock:
+ self._sessions[serial] = session
+ logger.info(
+ f"Accessibility helper attached on {serial}: 127.0.0.1:{local_port} "
+ f"(v{session.version_name or '?'}, protocol {protocol}, transport {transport})"
+ )
+ return session
+
+ def reattach(self, serial: str) -> HelperSession:
+ """Rebuild the tunnel after a request failed; never installs anything.
+
+ The caller had a working session, so a dead service is re-bound.
+ """
+ with self._device_lock(serial):
+ self._drop_session(serial, remove_forward=True)
+ return self.attach(serial, provision=False, revive=True)
+
+ def detach(self, serial: str) -> None:
+ with self._device_lock(serial):
+ self._drop_session(serial, remove_forward=True)
+
+ def _drop_session(self, serial: str, *, remove_forward: bool) -> None:
+ with self._registry_lock:
+ session = self._sessions.pop(serial, None)
+ if session is None:
+ return
+ if remove_forward and session.owns_forward:
+ self._remove_forward(serial, session.local_port)
+
+ def detach_all(self) -> None:
+ with self._registry_lock:
+ serials = list(self._sessions)
+ for serial in serials:
+ self.detach(serial)
+
+
+def _protocol_of(info: dict[str, Any] | None) -> int:
+ try:
+ return int((info or {}).get("protocol_version", 1))
+ except (TypeError, ValueError):
+ return 1
+
+
+helper_manager = AccessibilityHelperManager()
+
+__all__ = [
+ "AccessibilityHelperManager",
+ "BundledHelper",
+ "DEVICE_PORT",
+ "HelperSession",
+ "HelperUnavailable",
+ "MANUAL_ENABLE_PATH",
+ "MIN_PROTOCOL_VERSION",
+ "PACKAGE_NAME",
+ "ProvisionEvent",
+ "ProvisionResult",
+ "SERVICE_NAME",
+ "TOKEN_ACTION",
+ "TOKEN_RECEIVER",
+ "auto_install_allowed",
+ "helper_manager",
+ "load_bundled_helper",
+]
diff --git a/artemis/runtime/trace_store.py b/artemis/runtime/trace_store.py
index 09f99c8..7cb1f01 100644
--- a/artemis/runtime/trace_store.py
+++ b/artemis/runtime/trace_store.py
@@ -343,6 +343,20 @@ def update_trace_pid(trace_id: str, pid: int) -> dict[str, Any] | None:
return data
+def update_trace_fields(trace_id: str, **fields: Any) -> dict[str, Any] | None:
+ """Merge ``fields`` into status.json; no-op when the trace has no status file yet."""
+ path = get_status_path(trace_id)
+ if not os.path.exists(path):
+ return None
+ with _status_lock(path):
+ data = read_status(trace_id)
+ if not data:
+ return None
+ data.update(fields)
+ write_status(trace_id, data)
+ return data
+
+
def update_trace_device_serial(trace_id: str, device_serial: str) -> dict[str, Any] | None:
"""Updates the device_serial field of the status.json for a given trace_id."""
path = get_status_path(trace_id)
diff --git a/artemis/sdk/agent.py b/artemis/sdk/agent.py
index 913437c..389e677 100644
--- a/artemis/sdk/agent.py
+++ b/artemis/sdk/agent.py
@@ -13,6 +13,7 @@
# limitations under the License.
import asyncio
+import inspect
import os
import re
@@ -29,7 +30,7 @@ from shutil import which
import sys
import threading
from types import NoneType
-from typing import TypeVar, overload
+from typing import Any, TypeVar, overload
import uuid
from adbutils import AdbClient
@@ -41,7 +42,13 @@ from pydantic import BaseModel
from artemis.agents.flash.runner import FlashRunner
from artemis.agents.outputter.outputter import outputter
-from artemis.clients.ui_automator_client import UIAutomatorClient
+from artemis.clients.screen_client_factory import (
+ ScreenClient,
+ create_screen_client,
+ describe_backend,
+ helper_version,
+ hierarchy_backend_sentence,
+)
from artemis.config import (
CheckerConfig,
OutputConfig,
@@ -60,7 +67,7 @@ from artemis.controllers.controller_factory import get_controller
from artemis.controllers.platform_specific_commands_controller import (
get_first_device,
)
-from artemis.runtime import DeviceExecutionLock
+from artemis.runtime import DeviceExecutionLock, trace_store
from artemis.runtime.cancel_requests import watch_for_cancel_request
from artemis.data_engine.engine import DataEngine
from artemis.data_engine.trace import DataEngineCallbackHandler
@@ -126,7 +133,7 @@ class Agent:
_initialized: bool = False
_device_context: DeviceContext
_adb_client: AdbClient | None
- _ui_adb_client: UIAutomatorClient | None
+ _ui_adb_client: ScreenClient | Any | None
_current_task: asyncio.Task | None = None
_task_lock: asyncio.Lock
@@ -237,6 +244,8 @@ class Agent:
platform=platform,
)
else:
+ # Cloud mode is pinned to UIAutomator2 through the gateway; the
+ # hierarchy backend switch only applies to locally attached devices.
from cloud_service.virtualization import RemoteAdbClient, RemoteUIAutomatorClient
self._adb_client = RemoteAdbClient()
@@ -632,13 +641,7 @@ class Agent:
self._prepare_output_files(task=task)
if os.environ.get("ARTEMIS_CLOUD_MODE") != "1":
if self._ui_adb_client is not None:
- publish_startup_progress(
- "uiautomator", "Connecting to UI Automator", session_id=str(sess_id)
- )
- await asyncio.to_thread(self._ui_adb_client.connect)
- publish_startup_progress(
- "uiautomator_ready", "UI Automator is ready", session_id=str(sess_id)
- )
+ await self._connect_screen_client(context, str(sess_id))
await self._ensure_device_unlocked()
publish_startup_progress(
"environment", "Preparing the device environment", session_id=str(sess_id)
@@ -1277,6 +1280,141 @@ class Agent:
def _get_graph_state(self, task: Task):
return State.initial(task.request.goal)
+ # ------------------------------------------------------------------ #
+ # UI hierarchy backend: connect, and keep the user told which one serves
+ # ------------------------------------------------------------------ #
+
+ async def _connect_screen_client(self, context: ArtemisContext, session_id: str) -> None:
+ """Connect the screen client with visible progress for slow first-time steps."""
+ client = self._ui_adb_client
+ previous_listener = getattr(self, "_hierarchy_backend_listener", None)
+ if previous_listener is not None:
+ previous_client, listener = previous_listener
+ previous_client.remove_backend_listener(listener)
+ self._hierarchy_backend_listener = None
+ publish_startup_progress(
+ "uiautomator", "Connecting to the UI hierarchy service", session_id=session_id
+ )
+
+ def on_provision(event: str, details: dict) -> None:
+ version = details.get("version_name") or details.get("to_version")
+ if event == "installing":
+ publish_startup_progress(
+ "helper_install",
+ "Installing the Artemis accessibility helper on this device for the "
+ f"first time (v{version}, about 3 seconds)",
+ session_id=session_id,
+ **details,
+ )
+ elif event == "upgrading":
+ publish_startup_progress(
+ "helper_upgrade",
+ "Upgrading the Artemis accessibility helper "
+ f"(v{details.get('from_version')} -> v{details.get('to_version')})",
+ session_id=session_id,
+ **details,
+ )
+
+ # Clients without provisioning (UIAutomator2, cloud) take no callback.
+ try:
+ accepts_events = "on_event" in inspect.signature(client.connect).parameters
+ except (TypeError, ValueError):
+ accepts_events = False
+ if accepts_events:
+ await asyncio.to_thread(client.connect, on_event=on_provision)
+ else:
+ await asyncio.to_thread(client.connect)
+
+ backend = describe_backend(client) or "uiautomator"
+ publish_startup_progress(
+ "uiautomator_ready",
+ f"UI hierarchy service is ready ({backend})",
+ session_id=session_id,
+ )
+ self._announce_hierarchy_backend(context, session_id, None, backend, None)
+ add_listener = getattr(client, "add_backend_listener", None)
+ if callable(add_listener):
+
+ def listener(previous, new, reason):
+ self._announce_hierarchy_backend(context, session_id, previous, new, reason)
+
+ add_listener(listener)
+ self._hierarchy_backend_listener = (client, listener)
+
+ def _announce_hierarchy_backend(
+ self,
+ context: ArtemisContext,
+ session_id: str,
+ previous: str | None,
+ backend: str,
+ reason: str | None,
+ ) -> None:
+ """One line everywhere a person or an agent looks for it.
+
+ Startup-progress event (UI timeline), a named log trace, the session's
+ device_info, and status.json for ``mobile_manage_task``. Runs from
+ worker threads too, so every sink is best effort.
+ """
+ label = {"helper": "Artemis accessibility helper", "uiautomator": "UIAutomator2"}
+ version = helper_version(self._ui_adb_client)
+ if previous is None:
+ message = f"UI hierarchy source: {label.get(backend, backend)}"
+ if backend == "helper" and version:
+ message += f" v{version}"
+ stage = "hierarchy_backend"
+ else:
+ message = (
+ f"UI hierarchy source switched from {label.get(previous, previous)} to "
+ f"{label.get(backend, backend)}"
+ )
+ if reason:
+ message += f" because {reason}"
+ stage = "hierarchy_backend_changed"
+ logger.info(message)
+ try:
+ publish_startup_progress(
+ stage,
+ message,
+ session_id=session_id,
+ backend=backend,
+ previous_backend=previous,
+ reason=reason,
+ helper_version=version,
+ )
+ except (OSError, ValueError, RuntimeError) as exc:
+ logger.debug(f"Could not publish hierarchy backend progress: {exc}")
+ engine = getattr(context, "data_engine", None)
+ if engine is not None and getattr(engine, "current_session_id", None):
+ try:
+ engine.record_trace(
+ type="log",
+ name="hierarchy_backend",
+ payload={
+ "message": message,
+ "backend": backend,
+ "previous_backend": previous,
+ "reason": reason,
+ "helper_version": version,
+ "level": "WARNING" if previous is not None else "INFO",
+ },
+ )
+ engine.update_session_device_info(
+ hierarchy_backend=backend,
+ hierarchy_backend_note=hierarchy_backend_sentence(
+ self._ui_adb_client, relative_time=engine.get_relative_time
+ ),
+ )
+ except (OSError, ValueError, RuntimeError) as exc:
+ logger.debug(f"Could not record hierarchy backend in the data engine: {exc}")
+ try:
+ trace_store.update_trace_fields(
+ session_id,
+ hierarchy_backend=backend,
+ hierarchy_backend_note=hierarchy_backend_sentence(self._ui_adb_client),
+ )
+ except (OSError, ValueError, RuntimeError) as exc:
+ logger.debug(f"Could not record hierarchy backend in status.json: {exc}")
+
def _init_clients(
self,
device_id: str,
@@ -1286,7 +1424,7 @@ class Agent:
host=self._config.servers.adb_host,
port=self._config.servers.adb_port,
)
- self._ui_adb_client = UIAutomatorClient(device_id=device_id)
+ self._ui_adb_client = create_screen_client(device_id)
async def _get_device_context(
self,
diff --git a/artemis/utils/ui_filter.py b/artemis/utils/ui_filter.py
index 6f9f5ef..c68d768 100644
--- a/artemis/utils/ui_filter.py
+++ b/artemis/utils/ui_filter.py
@@ -59,7 +59,7 @@ def _parse_bounds(bounds: Any) -> dict[str, int] | None:
return res
if isinstance(bounds, str):
- match = re.match(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", bounds)
+ match = re.match(r"\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]", bounds)
if match:
x1, y1, x2, y2 = map(int, match.groups())
return {"left": x1, "top": y1, "right": x2, "bottom": y2}
@@ -112,6 +112,11 @@ def _clip_bounds(
if is_fully_clipped:
node["is_clipped"] = True
+ # The cached parse is what every later consumer (and _parse_bounds itself)
+ # reads first; leaving it stale would hand out the unclipped rectangle.
+ if isinstance(node.get("parsed_bounds"), dict):
+ node["parsed_bounds"] = new_bounds.copy()
+
# Re-serialize bounds into the node matching original format
orig = node.get("bounds")
if isinstance(orig, str):
diff --git a/artemis/utils/visualization.py b/artemis/utils/visualization.py
index 705a601..c6c2ddd 100644
--- a/artemis/utils/visualization.py
+++ b/artemis/utils/visualization.py
@@ -398,29 +398,42 @@ def format_minimal_list_with_elements(
idx += 1
registered = True
- if not registered and text.strip() and bounds:
+ # An empty input field carries no text but usually a hint ("Search settings");
+ # a rejected one carries an error ("Password too short"). Both come from the
+ # accessibility helper's dump and are exactly what the agent needs to see.
+ hint = str(node.get("hint") or "").strip()
+ error = str(node.get("error") or "").strip()
+ shown = text.strip() or hint
+ if not registered and shown and bounds:
left, top, right, bottom = bounds
cx, cy = get_center_coordinates(left, top, right, bottom)
- if is_duplicate(text, cx, cy):
+ if is_duplicate(shown, cx, cy):
continue
norm_bounds = normalize_bounds(left, top, right, bottom)
- lines.append(f"[{idx}] Text: '{text.strip()}' | Bounds: {norm_bounds}")
+ kind = "Text" if text.strip() else "Hint"
+ line = f"[{idx}] {kind}: '{shown}' | Bounds: {norm_bounds}"
+ if error:
+ line += f" | Error: '{error}'"
+ lines.append(line)
items_bounds.append((idx, (left, top, right, bottom)))
- elements.append(
- {
- "index": idx,
- "center": [cx, cy],
- "text": text.strip(),
- "bounds": [left, top, right, bottom],
- "class": node.get("class"),
- "resource_id": node.get("resource-id"),
- "is_ocr": False,
- }
- )
+ element = {
+ "index": idx,
+ "center": [cx, cy],
+ "text": shown,
+ "bounds": [left, top, right, bottom],
+ "class": node.get("class"),
+ "resource_id": node.get("resource-id"),
+ "is_ocr": False,
+ }
+ if not text.strip():
+ element["is_hint"] = True
+ if error:
+ element["error"] = error
+ elements.append(element)
labels.append(str(idx))
- added_items.append((text.strip(), cx, cy))
+ added_items.append((shown, cx, cy))
idx += 1
lines = _inject_mutual_occlusion_warnings(lines, items_bounds)
diff --git a/mcp_server/tools/diagnose.py b/mcp_server/tools/diagnose.py
index 9f443df..c62b522 100644
--- a/mcp_server/tools/diagnose.py
+++ b/mcp_server/tools/diagnose.py
@@ -24,6 +24,8 @@ to end (screenshot + UI hierarchy) to prove a task could actually start.
from __future__ import annotations
+import subprocess
+
import asyncio
import os
from pathlib import Path
@@ -42,6 +44,7 @@ from artemis.core.diagnostics.readiness import (
)
from artemis.core.diagnostics.schema import ProbeResult, ProbeStatus, SystemReadinessReport
from artemis.runtime import DeviceExecutionLock, trace_store
+from artemis.runtime.helper_manager import helper_manager
from artemis.utils.credentials_validator import validate_api_key
from artemis.utils.logger import get_logger
@@ -499,6 +502,159 @@ def _device_probe_steps(device_probe: dict[str, Any] | None) -> list[str]:
return steps
+# --------------------------------------------------------------------------- #
+# Accessibility helper (UI hierarchy backend)
+# --------------------------------------------------------------------------- #
+
+
+def _hierarchy_backend() -> str:
+ try:
+ from artemis.clients.screen_client_factory import resolve_backend
+
+ return resolve_backend().value
+ except (ImportError, ValueError):
+ return "auto"
+
+
+def _helper_target(adb_result: ProbeResult | None, requested_device: str | None) -> str | None:
+ """The one ready device whose helper state is worth reporting, or None."""
+ devices = (adb_result.metadata.get("devices") if adb_result else None) or []
+ ready = [str(d.get("serial")) for d in devices if d.get("state") == "device"]
+ if requested_device:
+ return requested_device if requested_device in ready else None
+ return ready[0] if len(ready) == 1 else None
+
+
+def _helper_status(serial: str) -> dict[str, Any]:
+ try:
+ status = helper_manager.status(serial)
+ except (OSError, ValueError, RuntimeError, subprocess.SubprocessError) as exc:
+ return {"serial": serial, "error": f"{exc.__class__.__name__}: {exc}"}
+ status["serial"] = serial
+ status["backend"] = _hierarchy_backend()
+ return status
+
+
+def _helper_needs_provision(status: dict[str, Any] | None) -> bool:
+ if not status or status.get("error"):
+ return False
+ return not status.get("installed") or bool(status.get("outdated")) or not status.get("enabled")
+
+
+def _helper_fix(
+ adb_result: ProbeResult | None, requested_device: str | None
+) -> dict[str, Any] | None:
+ """Install / upgrade / enable the helper on the idle target device (attempt_fix only)."""
+ serial = _helper_target(adb_result, requested_device)
+ if serial is None or _hierarchy_backend() == "uiautomator":
+ return None
+ status = _helper_status(serial)
+ if not _helper_needs_provision(status):
+ return None
+ if not status.get("bundled_apk_present"):
+ return {
+ "fix": "install_accessibility_helper",
+ "success": False,
+ "skipped": True,
+ "message": "Skipped: the bundled ArtemisAccessibilityHelper.apk is missing from the checkout.",
+ }
+ wanted = _normalize_serial(serial)
+ if any(_normalize_serial(entry.get("device")) == wanted for entry in _task_state()["active"]):
+ return {
+ "fix": "install_accessibility_helper",
+ "success": False,
+ "skipped": True,
+ "message": f"Skipped: a task is running on {serial}; install it after the task finishes.",
+ }
+ try:
+ result = helper_manager.provision(serial)
+ except (OSError, ValueError, RuntimeError, subprocess.SubprocessError) as exc:
+ return {
+ "fix": "install_accessibility_helper",
+ "success": False,
+ "skipped": False,
+ "message": f"provisioning raised {exc.__class__.__name__}: {exc}",
+ }
+ message = (
+ f"{result.action} (installed version {result.installed_version}, "
+ f"bundled {result.bundled_version}, service enabled={result.enabled})"
+ )
+ if result.error:
+ message += f": {result.error}"
+ return {
+ "fix": "install_accessibility_helper",
+ "success": result.ok,
+ "skipped": False,
+ "message": message,
+ }
+
+
+def _helper_steps(status: dict[str, Any] | None, *, attempt_fix: bool) -> list[str]:
+ if not status:
+ return []
+ serial = status.get("serial") or "the device"
+ backend = status.get("backend") or "auto"
+ if backend == "uiautomator":
+ return []
+ if status.get("error"):
+ return [
+ f"[OPTIONAL] Accessibility helper state on {serial} could not be read: {status['error']}"
+ ]
+ healthy = status.get("installed") and not status.get("outdated") and status.get("enabled")
+ if healthy and status.get("reachable"):
+ return []
+ if not status.get("installed"):
+ problem = "is not installed"
+ elif status.get("outdated"):
+ problem = (
+ f"is outdated (device has version {status.get('installed_version')}, "
+ f"bundled is {status.get('bundled_version')})"
+ )
+ elif not status.get("enabled"):
+ problem = "is installed but its accessibility service is disabled"
+ elif status.get("newer_than_bundled") and status.get("reachable"):
+ return [] # a newer dev build is fine; nothing to do
+ else:
+ problem = "is installed and enabled but not answering on its loopback port"
+ tag = "REQUIRED" if backend == "helper" else "OPTIONAL"
+ consequence = (
+ "tasks cannot read the UI hierarchy until it is fixed"
+ if backend == "helper"
+ else "tasks fall back to UIAutomator2 (slower dumps, conflicts with Appium/Mobly)"
+ )
+ steps = [f"[{tag}] Accessibility helper on {serial} {problem}; {consequence}."]
+ if not status.get("bundled_apk_present"):
+ steps.append(
+ " Guidance: the bundled APK is missing; build it with "
+ "packages/artemis-accessibility-helper/build_apk.sh."
+ )
+ return steps
+ if problem.startswith("is installed and enabled"):
+ steps.append(
+ " Guidance: unlock the phone; if the helper still does not answer, reinstall it with "
+ "the command below (add --force)."
+ )
+ steps.append(f" Run: uv run artemis helper install --serial {serial} --force")
+ return steps
+ if not attempt_fix:
+ steps.append(
+ " Guidance: call mobile_diagnose(attempt_fix=true) to install and enable it "
+ "without touching the phone, or run the command below."
+ )
+ steps.append(f" Run: uv run artemis helper install --serial {serial}")
+ if problem.startswith("is installed but"):
+ # Enabling from adb was already attempted once by whoever installed it;
+ # on ROMs that reject it only a person can flip the switch.
+ from artemis.runtime.helper_manager import MANUAL_ENABLE_PATH
+
+ steps.append(
+ " Guidance: if the command reports that this device rejected enabling the "
+ f"service, ask the user to turn it on by hand: {MANUAL_ENABLE_PATH} "
+ "(the command opens that settings screen on the phone)."
+ )
+ return steps
+
+
# --------------------------------------------------------------------------- #
# next_steps
# --------------------------------------------------------------------------- #
@@ -575,6 +731,7 @@ def _next_steps(
credentials: list[dict[str, Any]] | None,
tasks: dict[str, list[dict[str, Any]]],
device_probe: dict[str, Any] | None,
+ accessibility_helper: dict[str, Any] | None = None,
) -> list[str]:
steps: list[str] = []
needs_restart = False
@@ -628,6 +785,7 @@ def _next_steps(
)
steps.extend(launch_steps)
steps.extend(_device_probe_steps(device_probe))
+ steps.extend(_helper_steps(accessibility_helper, attempt_fix=attempt_fix))
for fix in fixes_applied:
outcome = "applied" if fix.get("success") else "did not help"
@@ -750,7 +908,9 @@ def _cleanup_stale_locks() -> dict[str, Any] | None:
}
-async def _apply_safe_fixes(report: SystemReadinessReport) -> list[dict[str, Any]]:
+async def _apply_safe_fixes(
+ report: SystemReadinessReport, requested_device: str | None = None
+) -> list[dict[str, Any]]:
"""Run the self-heals the console exposes as buttons, only when they cannot disrupt a task."""
applied: list[dict[str, Any]] = []
cleanup = _cleanup_stale_locks()
@@ -768,7 +928,12 @@ async def _apply_safe_fixes(report: SystemReadinessReport) -> list[dict[str, Any
devices = adb.metadata.get("devices") or []
if any(d.get("state") == "device" for d in devices):
- return applied # a ready device exists; nothing to restart
+ # A ready device exists; nothing to restart. Provision the accessibility
+ # helper on it when it is idle and missing, outdated or disabled.
+ helper_fix = await asyncio.to_thread(_helper_fix, adb, requested_device)
+ if helper_fix is not None:
+ applied.append(helper_fix)
+ return applied
if DeviceExecutionLock.get_active_owners():
applied.append(
{
@@ -787,7 +952,9 @@ async def _apply_safe_fixes(report: SystemReadinessReport) -> list[dict[str, Any
def _probes_changed(fixes_applied: list[dict[str, Any]]) -> bool:
"""Only ADB-side fixes can change probe results; lock cleanup does not warrant a re-run."""
return any(
- fix.get("success") and not fix.get("skipped") and fix.get("fix") != "cleanup_stale_locks"
+ fix.get("success")
+ and not fix.get("skipped")
+ and fix.get("fix") not in ("cleanup_stale_locks", "install_accessibility_helper")
for fix in fixes_applied
)
@@ -817,15 +984,18 @@ async def _run_extras(
else:
emulator = _emulator_status()
- credentials, device_probe = await asyncio.gather(
+ helper_serial = _helper_target(adb_result, requested_device)
+ credentials, device_probe, accessibility_helper = await asyncio.gather(
_verify_credentials(_find(results, "gemini_api_key")) if verify_credentials else _none(),
_run_device_probe(adb_result, requested_device) if probe_device else _none(),
+ asyncio.to_thread(_helper_status, helper_serial) if helper_serial else _none(),
)
return {
"emulator": emulator,
"launch_steps": launch_steps,
"credentials": credentials,
"device_probe": device_probe,
+ "accessibility_helper": accessibility_helper,
"tasks": _task_state(),
}
@@ -847,6 +1017,7 @@ def _verdict(
requested_device: str | None,
credentials: list[dict[str, Any]] | None,
device_probe: dict[str, Any] | None,
+ accessibility_helper: dict[str, Any] | None = None,
) -> str:
verdict = base_verdict(results)
if verdict == "blocked" or not _requested_device_ready(results, requested_device):
@@ -856,8 +1027,13 @@ def _verdict(
return "blocked"
if device_probe is not None and not device_probe.get("ok"):
return "blocked"
+ helper_backend = (accessibility_helper or {}).get("backend")
+ if helper_backend == "helper" and not accessibility_helper.get("reachable"):
+ return "blocked"
if any(not entry.get("valid") for entry in credentials or []):
return "degraded"
+ if helper_backend == "auto" and _helper_needs_provision(accessibility_helper):
+ return "degraded" if verdict == "ready" else verdict
return verdict
@@ -949,7 +1125,12 @@ async def mobile_diagnose(
traces_dir, mcp_client (when known), daemon {port, reachable,
port_held_by_other_process, log_path}.
- `device`: the device a task would use ({serial, state, model,
- android_version, is_locked, is_emulator}) or null.
+ android_version, is_locked, is_emulator, accessibility_helper}) or
+ null. `accessibility_helper` describes the Artemis UI-hierarchy
+ helper APK on that device ({installed, installed_version,
+ bundled_version, outdated, enabled, forward_port, reachable,
+ backend}); with backend "auto" a missing helper only degrades
+ (UIAutomator2 fallback), with backend "helper" it blocks.
- `emulator`: background emulator launch state ({avd_name, status,
stage_message, error, serial, elapsed_seconds, progress_percent}) or
null when nothing was launched. `status` is starting /
@@ -970,8 +1151,10 @@ async def mobile_diagnose(
Args:
attempt_fix: When true, applies the safe self-heals the ARTEMIS
console offers: remove device locks left by dead processes,
- regenerate corrupted ADB RSA keys, and restart the ADB server
- (only when no device is ready and no task holds a device). Then
+ regenerate corrupted ADB RSA keys, restart the ADB server (only
+ when no device is ready and no task holds a device), and install,
+ upgrade or enable the Artemis accessibility helper APK on the idle
+ target device when it is missing, outdated or disabled. Then
re-runs the checks. Nothing else is changed.
device_serial: Optional serial the user wants to use; the report
then states explicitly whether that device is attached, authorized
@@ -1004,7 +1187,7 @@ async def mobile_diagnose(
)
if attempt_fix:
fixes_applied = await asyncio.wait_for(
- _apply_safe_fixes(report), timeout=DIAGNOSIS_TIMEOUT_SECONDS
+ _apply_safe_fixes(report, requested_device), timeout=DIAGNOSIS_TIMEOUT_SECONDS
)
if _probes_changed(fixes_applied):
report, host = await asyncio.wait_for(
@@ -1026,12 +1209,17 @@ async def mobile_diagnose(
credentials: list[dict[str, Any]] | None = extras["credentials"]
device_probe: dict[str, Any] | None = extras["device_probe"]
+ accessibility_helper: dict[str, Any] | None = extras.get("accessibility_helper")
verdict = _verdict(
results,
requested_device=requested_device,
credentials=credentials,
device_probe=device_probe,
+ accessibility_helper=accessibility_helper,
)
+ device = _compact_device(report)
+ if device is not None and accessibility_helper is not None:
+ device["accessibility_helper"] = accessibility_helper
env_file = host.metadata.get("env_file")
return {
"verdict": verdict,
@@ -1047,10 +1235,11 @@ async def mobile_diagnose(
credentials=credentials,
tasks=extras["tasks"],
device_probe=device_probe,
+ accessibility_helper=accessibility_helper,
),
"checks": [_render_check(r) for r in sort_by_fix_order(results)],
"host": _compact_host(host.metadata),
- "device": _compact_device(report),
+ "device": device,
"emulator": extras["emulator"],
"tasks": extras["tasks"],
"credentials": credentials,
diff --git a/mcp_server/tools/task_manager.py b/mcp_server/tools/task_manager.py
index b541836..4eb800d 100644
--- a/mcp_server/tools/task_manager.py
+++ b/mcp_server/tools/task_manager.py
@@ -334,6 +334,12 @@ def mobile_manage_task(
"stdout_log": os.path.join(trace_dir, "stdout.log"),
"stderr_log": os.path.join(trace_dir, "stderr.log"),
}
+ # Which UI-hierarchy source served the run ("helper" or "uiautomator"),
+ # plus any mid-run switch; written by the agent once the device connects.
+ if status_data.get("hierarchy_backend"):
+ response["hierarchy_backend"] = status_data.get("hierarchy_backend")
+ if status_data.get("hierarchy_backend_note"):
+ response["hierarchy_backend_note"] = status_data.get("hierarchy_backend_note")
if status_data.get("model", "").lower() != "flash":
response["notes_dir"] = os.path.join(trace_dir, "notes")
diff --git a/packages/artemis-accessibility-helper/ArtemisAccessibilityHelper.apk b/packages/artemis-accessibility-helper/ArtemisAccessibilityHelper.apk
index d8f9b63..16f4480 100644
Binary files a/packages/artemis-accessibility-helper/ArtemisAccessibilityHelper.apk and b/packages/artemis-accessibility-helper/ArtemisAccessibilityHelper.apk differ
diff --git a/packages/artemis-accessibility-helper/app/build.gradle.kts b/packages/artemis-accessibility-helper/app/build.gradle.kts
index eda8fd9..ad802d6 100644
--- a/packages/artemis-accessibility-helper/app/build.gradle.kts
+++ b/packages/artemis-accessibility-helper/app/build.gradle.kts
@@ -10,8 +10,8 @@ android {
applicationId = "com.artemis.helper"
minSdk = 24
targetSdk = 35
- versionCode = 1
- versionName = "1.0.0"
+ versionCode = 6
+ versionName = "1.2.0"
}
buildTypes {
diff --git a/packages/artemis-accessibility-helper/app/src/main/AndroidManifest.xml b/packages/artemis-accessibility-helper/app/src/main/AndroidManifest.xml
index 650b6cf..94b1a1c 100644
--- a/packages/artemis-accessibility-helper/app/src/main/AndroidManifest.xml
+++ b/packages/artemis-accessibility-helper/app/src/main/AndroidManifest.xml
@@ -32,6 +32,18 @@
android:value="Artemis mobile automation test bridge" />
+
+
+
+
+
+
+
diff --git a/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/A11yNode.java b/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/A11yNode.java
index 9dc260b..676728a 100644
--- a/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/A11yNode.java
+++ b/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/A11yNode.java
@@ -1,6 +1,5 @@
package com.artemis.helper;
-import android.graphics.Rect;
import org.json.JSONArray;
import org.json.JSONObject;
@@ -12,6 +11,12 @@ import java.util.List;
* Decoupled from live Android AccessibilityNodeInfo objects to prevent
* Binder proxy memory leaks, avoid stale node crashes, and ensure
* instant thread-safe serialization to both XML and JSON.
+ *
+ * {@code bounds} are the node's
visible bounds: the raw
+ * {@code getBoundsInScreen} rectangle intersected with the display, the
+ * node's window and every scrollable ancestor, exactly like UIAutomator's
+ * {@code getVisibleBoundsInScreen}. They are therefore never negative and
+ * never extend past the screen.
*/
public final class A11yNode {
@@ -32,6 +37,7 @@ public final class A11yNode {
public boolean longClickable = false;
public boolean password = false;
public boolean selected = false;
+ public boolean visibleToUser = true;
// Window metadata for multi-window awareness
public int windowId = -1;
@@ -94,9 +100,12 @@ public final class A11yNode {
XmlUtils.appendBooleanAttribute(sb, "long-clickable", longClickable);
XmlUtils.appendBooleanAttribute(sb, "password", password);
XmlUtils.appendBooleanAttribute(sb, "selected", selected);
+ XmlUtils.appendBooleanAttribute(sb, "visible-to-user", visibleToUser);
XmlUtils.appendAttribute(sb, "bounds", getBoundsString());
XmlUtils.appendIntAttribute(sb, "drawing-order", drawingOrder);
- XmlUtils.appendAttribute(sb, "hint", hint);
+ if (!hint.isEmpty()) {
+ XmlUtils.appendAttribute(sb, "hint", hint);
+ }
// Window metadata (emitted on window root nodes)
if (windowId >= 0) {
@@ -199,6 +208,7 @@ public final class A11yNode {
obj.put("selected", selected);
obj.put("password", password);
obj.put("long-clickable", longClickable);
+ obj.put("visible-to-user", visibleToUser);
obj.put("drawing-order", drawingOrder);
if (!hint.isEmpty()) {
obj.put("hint", hint);
diff --git a/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/ArtemisAccessibilityService.java b/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/ArtemisAccessibilityService.java
index a2067b2..895a59d 100644
--- a/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/ArtemisAccessibilityService.java
+++ b/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/ArtemisAccessibilityService.java
@@ -7,6 +7,7 @@ import android.app.NotificationChannel;
import android.app.NotificationManager;
import android.content.Context;
import android.content.Intent;
+import android.content.pm.PackageInfo;
import android.content.pm.ServiceInfo;
import android.os.Build;
import android.util.Log;
@@ -17,12 +18,24 @@ import android.view.accessibility.AccessibilityEvent;
*
* Runs concurrently with any Android UI testing framework (Mobly, Appium, Espresso)
* without monopolizing Android's singleton UiAutomationService connection.
+ *
+ * The service subscribes to window-state changes only: that is the single event it
+ * uses (to remember the foreground package / activity), and anything wider costs
+ * CPU and battery on a device that is idle between tasks.
*/
public class ArtemisAccessibilityService extends AccessibilityService {
private static final String TAG = "ArtemisA11yService";
public static final int DEFAULT_PORT = 18888;
- private static final String CHANNEL_ID = "artemis_helper_channel";
+ /**
+ * Bumped whenever the HTTP contract changes in a way an older host cannot use.
+ * 2: token authentication, byte-accurate bodies, visible-only dumps, fields=.
+ */
+ public static final int PROTOCOL_VERSION = 2;
+ // Channel importance is frozen by Android once a channel exists, so a quieter
+ // channel needs a new id; the pre-1.1.3 channel is deleted on start.
+ private static final String LEGACY_CHANNEL_ID = "artemis_helper_channel";
+ private static final String CHANNEL_ID = "artemis_helper_quiet";
private static final int NOTIFICATION_ID = 18888;
private static volatile ArtemisAccessibilityService instance;
@@ -43,6 +56,29 @@ public class ArtemisAccessibilityService extends AccessibilityService {
return currentActivityName;
}
+ /** Installed helper versionCode, or -1 when the package manager cannot answer. */
+ @SuppressWarnings("deprecation")
+ public long getVersionCode() {
+ try {
+ PackageInfo info = getPackageManager().getPackageInfo(getPackageName(), 0);
+ if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
+ return info.getLongVersionCode();
+ }
+ return info.versionCode;
+ } catch (Throwable t) {
+ return -1L;
+ }
+ }
+
+ public String getVersionName() {
+ try {
+ PackageInfo info = getPackageManager().getPackageInfo(getPackageName(), 0);
+ return info.versionName == null ? "" : info.versionName;
+ } catch (Throwable t) {
+ return "";
+ }
+ }
+
@Override
public void onServiceConnected() {
super.onServiceConnected();
@@ -58,12 +94,14 @@ public class ArtemisAccessibilityService extends AccessibilityService {
if (info == null) {
info = new AccessibilityServiceInfo();
}
- info.eventTypes = AccessibilityEvent.TYPES_ALL_MASK;
+ info.eventTypes = AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED;
info.feedbackType = AccessibilityServiceInfo.FEEDBACK_GENERIC;
- info.notificationTimeout = 50;
+ info.notificationTimeout = 100;
info.flags |= AccessibilityServiceInfo.FLAG_RETRIEVE_INTERACTIVE_WINDOWS
- | AccessibilityServiceInfo.FLAG_INCLUDE_NOT_IMPORTANT_VIEWS
| AccessibilityServiceInfo.FLAG_REPORT_VIEW_IDS;
+ // Not-important views are what UIAutomator's compressed dump leaves out;
+ // keep them out here too so both backends describe the same tree.
+ info.flags &= ~AccessibilityServiceInfo.FLAG_INCLUDE_NOT_IMPORTANT_VIEWS;
setServiceInfo(info);
Log.i(TAG, "AccessibilityServiceInfo flags dynamically enforced");
} catch (Throwable t) {
@@ -94,12 +132,19 @@ public class ArtemisAccessibilityService extends AccessibilityService {
if (nm == null) return;
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
+ try {
+ nm.deleteNotificationChannel(LEGACY_CHANNEL_ID);
+ } catch (Throwable ignored) {}
+ // Ask for IMPORTANCE_MIN (no sound, collapsed in the shade). Android raises
+ // foreground-service channels to LOW at most, which is still silent. The
+ // notification exists only to keep the service alive on aggressive ROMs;
+ // it must not look like something the person needs to act on.
NotificationChannel channel = new NotificationChannel(
CHANNEL_ID,
- "Artemis Helper Service",
- NotificationManager.IMPORTANCE_LOW
+ "Artemis test helper",
+ NotificationManager.IMPORTANCE_MIN
);
- channel.setDescription("Keeps Artemis Accessibility Helper active for automation testing");
+ channel.setDescription("Shown while the Artemis test helper is installed on this device");
channel.setShowBadge(false);
nm.createNotificationChannel(channel);
}
@@ -111,10 +156,20 @@ public class ArtemisAccessibilityService extends AccessibilityService {
builder = new Notification.Builder(this);
}
- builder.setContentTitle("Artemis Accessibility Helper")
- .setContentText("Active on 127.0.0.1:" + DEFAULT_PORT)
+ builder.setContentTitle("Artemis test helper is running")
+ .setContentText("Lets Artemis read this screen during automated tests. "
+ + "Remove with: artemis helper uninstall")
+ .setStyle(new Notification.BigTextStyle().bigText(
+ "Installed by Artemis to read the screen layout during automated "
+ + "tests. It answers only on this device (127.0.0.1:" + DEFAULT_PORT
+ + "), only to the computer that is connected over USB debugging and "
+ + "holds the session token, and it sends nothing anywhere. Remove it "
+ + "any time with `artemis helper uninstall` or from Settings > Apps."))
.setSmallIcon(android.R.drawable.stat_notify_sync)
.setOngoing(true);
+ if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) {
+ builder.setPriority(Notification.PRIORITY_MIN);
+ }
if (Build.VERSION.SDK_INT >= 34) {
startForeground(NOTIFICATION_ID, builder.build(), ServiceInfo.FOREGROUND_SERVICE_TYPE_SPECIAL_USE);
diff --git a/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/CommandServer.java b/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/CommandServer.java
index 032a303..328f717 100644
--- a/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/CommandServer.java
+++ b/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/CommandServer.java
@@ -3,27 +3,46 @@ package com.artemis.helper;
import android.util.Log;
import org.json.JSONObject;
-import java.io.BufferedReader;
+import java.io.BufferedInputStream;
+import java.io.ByteArrayOutputStream;
import java.io.Closeable;
-import java.io.InputStreamReader;
+import java.io.InputStream;
import java.io.OutputStream;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.Socket;
+import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/**
- * High-performance, multi-threaded command server bound strictly to loopback (127.0.0.1:18888).
- * Supports:
- * - HTTP REST API (/dump, /dump_xml, /hierarchy.xml, /ping, /action)
- * - Line-delimited JSON-RPC over TCP socket
+ * Multi-threaded command server bound strictly to loopback (127.0.0.1:18888).
+ *
+ * Protocols:
+ * - HTTP (/ping, /dump, /dump_xml, /hierarchy.xml, /snapshot, /action)
+ * - Line-delimited JSON-RPC over a raw TCP socket
+ *
+ * Every endpoint except {@code /ping} requires the session token the host pushed
+ * through {@link TokenReceiver} (header {@code X-Artemis-Token}, query parameter
+ * {@code token}, or JSON field {@code token}). Loopback is reachable by every app
+ * on the device, so without the token this service would hand any local app a
+ * full-screen reader and a gesture injector.
+ *
+ * Request bodies are read as bytes (Content-Length is a byte count), so UTF-8
+ * payloads of any script are handled exactly.
*/
public class CommandServer extends Thread {
private static final String TAG = "ArtemisCommandServer";
+ private static final int MAX_HEADER_LINE = 16 * 1024;
+ private static final int MAX_BODY = 4 * 1024 * 1024;
+ private static final String TOKEN_HEADER = "x-artemis-token";
+
private final ArtemisAccessibilityService service;
private final int port;
private volatile boolean isRunning = true;
@@ -67,25 +86,66 @@ public class CommandServer extends Thread {
}
}
+ // ------------------------------------------------------------------ //
+ // Byte-accurate request reading
+ // ------------------------------------------------------------------ //
+
+ /** Reads one line (without CR/LF) as raw bytes; null at end of stream. */
+ private static byte[] readLineBytes(InputStream in) throws Exception {
+ ByteArrayOutputStream line = new ByteArrayOutputStream(256);
+ int b;
+ while ((b = in.read()) >= 0) {
+ if (b == '\n') {
+ break;
+ }
+ if (b != '\r') {
+ line.write(b);
+ }
+ if (line.size() > MAX_HEADER_LINE) {
+ throw new IllegalStateException("Header line too long");
+ }
+ }
+ if (b < 0 && line.size() == 0) {
+ return null;
+ }
+ return line.toByteArray();
+ }
+
+ private static byte[] readExactly(InputStream in, int length) throws Exception {
+ byte[] buf = new byte[length];
+ int total = 0;
+ while (total < length) {
+ int r = in.read(buf, total, length - total);
+ if (r < 0) break;
+ total += r;
+ }
+ if (total < length) {
+ byte[] shorter = new byte[total];
+ System.arraycopy(buf, 0, shorter, 0, total);
+ return shorter;
+ }
+ return buf;
+ }
+
private void handleClient(Socket socket) {
try {
socket.setSoTimeout(10000);
- BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8));
+ InputStream in = new BufferedInputStream(socket.getInputStream());
OutputStream output = socket.getOutputStream();
- String firstLine = reader.readLine();
+ byte[] firstLine = readLineBytes(in);
if (firstLine == null) return;
- String trimmed = firstLine.trim();
+ String ascii = new String(firstLine, StandardCharsets.ISO_8859_1).trim();
- if (trimmed.startsWith("GET ") || trimmed.startsWith("POST ")) {
- handleHttp(trimmed, reader, output);
- } else if (trimmed.startsWith("{")) {
- handleJsonRpc(trimmed, output);
+ if (ascii.startsWith("GET ") || ascii.startsWith("POST ")) {
+ handleHttp(ascii, in, output);
+ } else if (ascii.startsWith("{")) {
+ handleJsonRpc(new String(firstLine, StandardCharsets.UTF_8).trim(), output);
} else {
JSONObject err = new JSONObject();
err.put("success", false);
err.put("error", "Unsupported protocol");
- sendJsonResponse(output, err.toString());
+ sendJson(output, 400, err.toString());
}
} catch (Exception e) {
Log.w(TAG, "Client handling error: " + e.getMessage());
@@ -94,82 +154,131 @@ public class CommandServer extends Thread {
}
}
- private void handleHttp(String requestLine, BufferedReader reader, OutputStream output) {
+ // ------------------------------------------------------------------ //
+ // HTTP
+ // ------------------------------------------------------------------ //
+
+ private static Map
parseQuery(String query) {
+ Map params = new HashMap<>();
+ if (query == null || query.isEmpty()) return params;
+ for (String pair : query.split("&")) {
+ if (pair.isEmpty()) continue;
+ int eq = pair.indexOf('=');
+ try {
+ String key = URLDecoder.decode(eq < 0 ? pair : pair.substring(0, eq), "UTF-8");
+ String value = eq < 0 ? "" : URLDecoder.decode(pair.substring(eq + 1), "UTF-8");
+ params.put(key, value);
+ } catch (Exception ignored) {}
+ }
+ return params;
+ }
+
+ private void handleHttp(String requestLine, InputStream in, OutputStream output) {
try {
String[] parts = requestLine.split(" ");
- String path = parts.length > 1 ? parts[1] : "/";
+ String target = parts.length > 1 ? parts[1] : "/";
+ String path = target;
+ Map query;
+ int q = target.indexOf('?');
+ if (q >= 0) {
+ path = target.substring(0, q);
+ query = parseQuery(target.substring(q + 1));
+ } else {
+ query = new HashMap<>();
+ }
int contentLength = 0;
- String line;
- while ((line = reader.readLine()) != null) {
- if (line.isEmpty()) break;
- String lower = line.toLowerCase();
- if (lower.startsWith("content-length:")) {
+ String headerToken = null;
+ byte[] lineBytes;
+ while ((lineBytes = readLineBytes(in)) != null) {
+ if (lineBytes.length == 0) break;
+ String line = new String(lineBytes, StandardCharsets.ISO_8859_1);
+ int colon = line.indexOf(':');
+ if (colon < 0) continue;
+ String name = line.substring(0, colon).trim().toLowerCase(Locale.ROOT);
+ String value = line.substring(colon + 1).trim();
+ if (name.equals("content-length")) {
try {
- contentLength = Integer.parseInt(lower.substring("content-length:".length()).trim());
+ contentLength = Integer.parseInt(value);
} catch (Exception ignored) {}
+ } else if (name.equals(TOKEN_HEADER)) {
+ headerToken = value;
}
}
+ if (contentLength < 0 || contentLength > MAX_BODY) {
+ sendJson(output, 413, errorJson("Body too large").toString());
+ return;
+ }
String body = "";
if (contentLength > 0) {
- char[] buf = new char[contentLength];
- int total = 0;
- while (total < contentLength) {
- int r = reader.read(buf, total, contentLength - total);
- if (r < 0) break;
- total += r;
- }
- body = new String(buf, 0, total);
+ body = new String(readExactly(in, contentLength), StandardCharsets.UTF_8);
+ }
+
+ String token = headerToken != null ? headerToken : query.get("token");
+ boolean authed = TokenStore.matches(token);
+
+ if (path.equals("/ping") || path.equals("/")) {
+ sendJson(output, 200, buildPing(authed).toString());
+ return;
+ }
+ if (!authed) {
+ sendJson(output, 401, unauthorizedJson().toString());
+ return;
}
// Route endpoints
- if (path.startsWith("/snapshot")) {
- JSONObject json = HierarchyDumper.dumpAtomicSnapshot(service);
- sendJsonResponse(output, json.toString());
- } else if (path.equals("/dump_xml") || path.equals("/hierarchy.xml") || path.startsWith("/dump?format=xml")) {
- String xml = HierarchyDumper.dumpXml(service);
- sendXmlResponse(output, xml);
- } else if (path.startsWith("/dump") || path.startsWith("/hierarchy")) {
- JSONObject json = HierarchyDumper.dump(service);
- sendJsonResponse(output, json.toString());
- } else if (path.startsWith("/ping") || path.equals("/")) {
- JSONObject r = new JSONObject();
- r.put("success", true);
- r.put("service", "ArtemisAccessibilityService");
- r.put("package", service.getCurrentPackageName());
- r.put("activity", service.getCurrentActivityName());
- sendJsonResponse(output, r.toString());
- } else if (path.startsWith("/action") || path.startsWith("/rpc")) {
+ if (path.equals("/snapshot")) {
+ HierarchyDumper.DumpOptions options = HierarchyDumper.DumpOptions.forSnapshot().apply(query);
+ JSONObject json = HierarchyDumper.dumpAtomicSnapshot(service, options);
+ sendJson(output, 200, json.toString());
+ } else if (path.equals("/dump_xml") || path.equals("/hierarchy.xml")
+ || (path.equals("/dump") && "xml".equals(query.get("format")))) {
+ HierarchyDumper.DumpOptions options = HierarchyDumper.DumpOptions.forSnapshot().apply(query);
+ sendXml(output, HierarchyDumper.dumpXml(service, options));
+ } else if (path.equals("/dump") || path.equals("/hierarchy")) {
+ HierarchyDumper.DumpOptions options = HierarchyDumper.DumpOptions.forDump().apply(query);
+ sendJson(output, 200, HierarchyDumper.dump(service, options).toString());
+ } else if (path.equals("/action") || path.equals("/rpc")) {
JSONObject json = body.isEmpty() ? new JSONObject() : new JSONObject(body);
JSONObject resp = executeCommand(json.optString("cmd", ""), json);
- sendJsonResponse(output, resp.toString());
+ sendJson(output, 200, resp.toString());
} else {
- JSONObject r = new JSONObject();
- r.put("success", false);
- r.put("error", "Unknown endpoint: " + path);
- sendJsonResponse(output, r.toString());
+ sendJson(output, 404, errorJson("Unknown endpoint: " + path).toString());
}
} catch (Exception e) {
Log.e(TAG, "HTTP error", e);
+ try {
+ sendJson(output, 500, errorJson("Internal error: " + e.getMessage()).toString());
+ } catch (Throwable ignored) {}
}
}
+ // ------------------------------------------------------------------ //
+ // Raw JSON-RPC
+ // ------------------------------------------------------------------ //
+
private void handleJsonRpc(String line, OutputStream output) {
try {
JSONObject json = new JSONObject(line);
String cmd = json.optString("cmd", "");
- JSONObject resp = executeCommand(cmd, json);
+ boolean authed = TokenStore.matches(json.optString("token", null));
+ JSONObject resp;
+ if (cmd.equalsIgnoreCase("ping")) {
+ resp = buildPing(authed);
+ } else if (!authed) {
+ resp = unauthorizedJson();
+ } else {
+ resp = executeCommand(cmd, json);
+ }
byte[] bytes = resp.toString().getBytes(StandardCharsets.UTF_8);
output.write(bytes);
output.write('\n');
output.flush();
} catch (Exception e) {
try {
- JSONObject err = new JSONObject();
- err.put("success", false);
- err.put("error", "JSON parse error: " + e.getMessage());
+ JSONObject err = errorJson("JSON parse error: " + e.getMessage());
output.write(err.toString().getBytes(StandardCharsets.UTF_8));
output.write('\n');
output.flush();
@@ -177,21 +286,21 @@ public class CommandServer extends Thread {
}
}
+ // ------------------------------------------------------------------ //
+ // Commands
+ // ------------------------------------------------------------------ //
+
private JSONObject executeCommand(String cmd, JSONObject params) {
JSONObject resp = new JSONObject();
try {
- switch (cmd.toLowerCase()) {
+ switch (cmd.toLowerCase(Locale.ROOT)) {
case "ping":
- resp.put("success", true);
- resp.put("service", "ArtemisAccessibilityService");
- resp.put("package", service.getCurrentPackageName());
- resp.put("activity", service.getCurrentActivityName());
- break;
+ return buildPing(true);
case "dump":
case "dump_ui":
- return HierarchyDumper.dump(service);
+ return HierarchyDumper.dump(service, HierarchyDumper.DumpOptions.forDump());
case "snapshot":
- return HierarchyDumper.dumpAtomicSnapshot(service);
+ return HierarchyDumper.dumpAtomicSnapshot(service, HierarchyDumper.DumpOptions.forSnapshot());
case "dump_xml":
resp.put("success", true);
resp.put("xml", HierarchyDumper.dumpXml(service));
@@ -239,11 +348,15 @@ public class CommandServer extends Thread {
break;
case "type":
String text = params.optString("text", "");
- resp.put("success", GestureController.setText(service, text));
+ boolean append = params.optBoolean("append", false);
+ resp.put("success", GestureController.setText(service, text, append));
break;
case "clear":
resp.put("success", GestureController.clearText(service));
break;
+ case "clipboard":
+ resp.put("success", GestureController.setClipboard(service, params.optString("text", "")));
+ break;
case "global":
String action = params.optString("action", "");
resp.put("success", GestureController.performGlobalAction(service, action));
@@ -262,11 +375,64 @@ public class CommandServer extends Thread {
return resp;
}
- private void sendJsonResponse(OutputStream output, String jsonStr) {
+ /**
+ * Health payload shared by GET /ping and the "ping" RPC. The host compares
+ * version_code / protocol_version against the bundled APK to decide whether
+ * to upgrade, and token_set to decide whether to push its token. The
+ * foreground package and activity are only disclosed to an authenticated caller.
+ */
+ private JSONObject buildPing(boolean authed) throws Exception {
+ JSONObject r = new JSONObject();
+ r.put("success", true);
+ r.put("service", "ArtemisAccessibilityService");
+ r.put("version_code", service.getVersionCode());
+ r.put("version_name", service.getVersionName());
+ r.put("protocol_version", ArtemisAccessibilityService.PROTOCOL_VERSION);
+ r.put("port", port);
+ r.put("auth_required", true);
+ r.put("token_set", TokenStore.isSet());
+ r.put("authenticated", authed);
+ if (authed) {
+ r.put("package", service.getCurrentPackageName());
+ r.put("activity", service.getCurrentActivityName());
+ }
+ return r;
+ }
+
+ private static JSONObject errorJson(String message) throws Exception {
+ JSONObject r = new JSONObject();
+ r.put("success", false);
+ r.put("error", message);
+ return r;
+ }
+
+ private static JSONObject unauthorizedJson() throws Exception {
+ JSONObject r = errorJson(TokenStore.isSet()
+ ? "unauthorized: wrong or missing X-Artemis-Token"
+ : "unauthorized: no session token has been pushed to the helper yet");
+ r.put("token_set", TokenStore.isSet());
+ return r;
+ }
+
+ // ------------------------------------------------------------------ //
+ // Responses
+ // ------------------------------------------------------------------ //
+
+ private static String reason(int status) {
+ switch (status) {
+ case 200: return "OK";
+ case 400: return "Bad Request";
+ case 401: return "Unauthorized";
+ case 404: return "Not Found";
+ case 413: return "Payload Too Large";
+ default: return "Internal Server Error";
+ }
+ }
+
+ private static void send(OutputStream output, int status, String contentType, byte[] bytes) {
try {
- byte[] bytes = jsonStr.getBytes(StandardCharsets.UTF_8);
- String header = "HTTP/1.1 200 OK\r\n" +
- "Content-Type: application/json; charset=utf-8\r\n" +
+ String header = "HTTP/1.1 " + status + " " + reason(status) + "\r\n" +
+ "Content-Type: " + contentType + "; charset=utf-8\r\n" +
"Content-Length: " + bytes.length + "\r\n" +
"Connection: close\r\n\r\n";
output.write(header.getBytes(StandardCharsets.UTF_8));
@@ -275,17 +441,12 @@ public class CommandServer extends Thread {
} catch (Exception ignored) {}
}
- private void sendXmlResponse(OutputStream output, String xmlStr) {
- try {
- byte[] bytes = xmlStr.getBytes(StandardCharsets.UTF_8);
- String header = "HTTP/1.1 200 OK\r\n" +
- "Content-Type: application/xml; charset=utf-8\r\n" +
- "Content-Length: " + bytes.length + "\r\n" +
- "Connection: close\r\n\r\n";
- output.write(header.getBytes(StandardCharsets.UTF_8));
- output.write(bytes);
- output.flush();
- } catch (Exception ignored) {}
+ private static void sendJson(OutputStream output, int status, String jsonStr) {
+ send(output, status, "application/json", jsonStr.getBytes(StandardCharsets.UTF_8));
+ }
+
+ private static void sendXml(OutputStream output, String xmlStr) {
+ send(output, 200, "application/xml", xmlStr.getBytes(StandardCharsets.UTF_8));
}
public void shutdown() {
diff --git a/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/GestureController.java b/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/GestureController.java
index 3c29fd2..1714a63 100644
--- a/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/GestureController.java
+++ b/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/GestureController.java
@@ -2,6 +2,9 @@ package com.artemis.helper;
import android.accessibilityservice.AccessibilityService;
import android.accessibilityservice.GestureDescription;
+import android.content.ClipData;
+import android.content.ClipboardManager;
+import android.content.Context;
import android.graphics.Path;
import android.os.Build;
import android.os.Bundle;
@@ -65,14 +68,36 @@ public final class GestureController {
}
public static boolean setText(AccessibilityService service, String text) {
+ return setText(service, text, false);
+ }
+
+ /**
+ * Sets (or, with {@code append}, extends) the text of the focused / first editable field.
+ * ACTION_SET_TEXT replaces the whole content; appending mirrors what typing through an
+ * IME does, which is the contract the host's send_text has always had.
+ */
+ public static boolean setText(AccessibilityService service, String text, boolean append) {
AccessibilityNodeInfo inputNode = HierarchyDumper.findInputNode(service);
if (inputNode == null) {
Log.w(TAG, "No editable/focused input node found for setText");
return false;
}
try {
+ String value = text == null ? "" : text;
+ if (append) {
+ CharSequence existing = inputNode.getText();
+ boolean showingHint = false;
+ if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
+ try {
+ showingHint = inputNode.isShowingHintText();
+ } catch (Throwable ignored) {}
+ }
+ if (existing != null && existing.length() > 0 && !showingHint) {
+ value = existing.toString() + value;
+ }
+ }
Bundle args = new Bundle();
- args.putCharSequence(AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, text);
+ args.putCharSequence(AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, value);
return inputNode.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, args);
} catch (Throwable t) {
Log.w(TAG, "performAction ACTION_SET_TEXT failed", t);
@@ -171,4 +196,37 @@ public final class GestureController {
return false;
}
}
+
+ /**
+ * Write text to the system clipboard. Clipboard writes are allowed from any
+ * app on every API level (only reads are restricted since Android 10), so this
+ * is the IME-free path the host uses for multiline and non-ASCII input:
+ * set the clip here, then KEYCODE_PASTE over adb.
+ */
+ public static boolean setClipboard(final AccessibilityService service, final String text) {
+ final CountDownLatch latch = new CountDownLatch(1);
+ final AtomicBoolean ok = new AtomicBoolean(false);
+ MAIN_HANDLER.post(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ ClipboardManager cm = (ClipboardManager) service.getSystemService(Context.CLIPBOARD_SERVICE);
+ if (cm != null) {
+ cm.setPrimaryClip(ClipData.newPlainText("artemis", text == null ? "" : text));
+ ok.set(true);
+ }
+ } catch (Throwable t) {
+ Log.w(TAG, "setClipboard failed: " + t.getMessage());
+ } finally {
+ latch.countDown();
+ }
+ }
+ });
+ try {
+ latch.await(2000, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ return ok.get();
+ }
}
diff --git a/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/HierarchyDumper.java b/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/HierarchyDumper.java
index 7b43e59..ee1084b 100644
--- a/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/HierarchyDumper.java
+++ b/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/HierarchyDumper.java
@@ -21,11 +21,13 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
/**
@@ -34,73 +36,132 @@ import java.util.concurrent.atomic.AtomicReference;
* Capabilities:
* 1. Zero WaitForIdle hangs: Bypasses UIAutomator's waitForIdle() timeout.
* 2. Multi-window penetration: Captures App, Dialog, System UI, IME Keyboard, Split-Screen.
- * 3. Rich semantic extraction: Captures errorText, isHeading, editable, paneTitle, tooltip.
- * 4. Atomic screenshot snapshot: On API 30+, captures hardware bitmap and DOM tree simultaneously.
- * 5. W3C XML 1.0 compliance: Strict character sanitization prevents Python parse failures.
+ * 3. UIAutomator parity: invisible nodes are skipped and bounds are clipped to the
+ * display, the window and scrollable ancestors, so the tree describes what is on
+ * screen and nothing else. {@code include_invisible=1} keeps everything (debugging).
+ * 4. Rich semantic extraction: Captures errorText, isHeading, editable, paneTitle, tooltip.
+ * 5. Atomic screenshot snapshot: On API 30+, captures hardware bitmap and DOM tree simultaneously.
+ * 6. W3C XML 1.0 compliance: Strict character sanitization prevents Python parse failures.
+ * 7. Batched IPC on API 33+: descendants are prefetched per getChild call instead of one
+ * Binder round trip per node.
*/
public final class HierarchyDumper {
private static final String TAG = "ArtemisHierarchyDumper";
private static final int MAX_DEPTH = 75;
private static final int MAX_NODES = 8000;
- private static final int RETRY_COUNT = 3;
- private static final long RETRY_INTERVAL_MS = 40L;
private static final ExecutorService SCREENSHOT_EXECUTOR = Executors.newSingleThreadExecutor();
+ /** Prefetch strategy for API 33+ ({@code AccessibilityNodeInfo.FLAG_PREFETCH_DESCENDANTS_HYBRID}). */
+ private static final int PREFETCH_DESCENDANTS_HYBRID = 1 << 3;
+
private HierarchyDumper() {}
/**
- * Dumps the complete hierarchy as a JSON response.
+ * What a dump should contain. Every field defaults to the cheapest useful answer.
*/
+ public static final class DumpOptions {
+ public boolean includeInvisible = false;
+ public boolean wantXml = true;
+ public boolean wantElements = false;
+ public boolean wantTree = false;
+
+ public static DumpOptions forDump() {
+ DumpOptions o = new DumpOptions();
+ o.wantElements = true;
+ o.wantTree = true;
+ return o;
+ }
+
+ public static DumpOptions forSnapshot() {
+ return new DumpOptions();
+ }
+
+ /** Applies {@code fields=xml,elements,tree} and {@code include_invisible=1}. */
+ public DumpOptions apply(Map query) {
+ if (query == null) return this;
+ String fields = query.get("fields");
+ if (fields != null && !fields.trim().isEmpty()) {
+ wantXml = wantElements = wantTree = false;
+ for (String f : fields.split(",")) {
+ String key = f.trim().toLowerCase();
+ if (key.equals("xml")) wantXml = true;
+ else if (key.equals("elements")) wantElements = true;
+ else if (key.equals("tree")) wantTree = true;
+ }
+ }
+ String inv = query.get("include_invisible");
+ if (inv != null) {
+ includeInvisible = inv.equals("1") || inv.equalsIgnoreCase("true");
+ }
+ return this;
+ }
+ }
+
+ /** Per-dump counters reported back to the host. */
+ private static final class DumpStats {
+ int nodes = 0;
+ int skippedInvisible = 0;
+ boolean truncated = false;
+ }
+
public static JSONObject dump(AccessibilityService service) {
+ return dump(service, DumpOptions.forDump());
+ }
+
+ /**
+ * Dumps the hierarchy as a JSON response containing the requested representations.
+ */
+ public static JSONObject dump(AccessibilityService service, DumpOptions options) {
long startTime = System.currentTimeMillis();
JSONObject result = new JSONObject();
- JSONArray elementsJson = new JSONArray();
try {
DisplayUtils.DisplayInfo displayInfo = DisplayUtils.getDisplayInfo(service);
- List rootSnapshots = captureRootSnapshots(service);
+ DumpStats stats = new DumpStats();
+ List rootSnapshots = captureRootSnapshots(service, displayInfo, options, stats);
+
+ result.put("rotation", displayInfo.rotation);
+ result.put("width", displayInfo.width);
+ result.put("height", displayInfo.height);
if (rootSnapshots.isEmpty()) {
result.put("success", false);
result.put("error", "No active window or root node found");
result.put("xml", "");
- result.put("elements", elementsJson);
- result.put("rotation", displayInfo.rotation);
- result.put("width", displayInfo.width);
- result.put("height", displayInfo.height);
+ result.put("elements", new JSONArray());
return result;
}
- // 1. Build standard UIAutomator XML
- String xml = buildXml(rootSnapshots, displayInfo.rotation, displayInfo.width, displayInfo.height);
-
- // 2. Build flat elements list
- List flatList = new ArrayList<>();
- for (A11yNode root : rootSnapshots) {
- root.collectFlatElements(flatList);
+ if (options.wantXml) {
+ result.put("xml", buildXml(rootSnapshots, displayInfo.rotation));
}
- for (JSONObject elem : flatList) {
- elementsJson.put(elem);
+ if (options.wantElements) {
+ JSONArray elementsJson = new JSONArray();
+ List flatList = new ArrayList<>();
+ for (A11yNode root : rootSnapshots) {
+ root.collectFlatElements(flatList);
+ }
+ for (JSONObject elem : flatList) {
+ elementsJson.put(elem);
+ }
+ result.put("elements", elementsJson);
}
-
- // 3. Build tree
- JSONArray trees = new JSONArray();
- for (A11yNode root : rootSnapshots) {
- trees.put(root.toTreeJson());
+ if (options.wantTree) {
+ JSONArray trees = new JSONArray();
+ for (A11yNode root : rootSnapshots) {
+ trees.put(root.toTreeJson());
+ }
+ result.put("tree", trees.length() == 1 ? trees.getJSONObject(0) : trees);
}
- long elapsed = System.currentTimeMillis() - startTime;
-
result.put("success", true);
- result.put("xml", xml);
- result.put("elements", elementsJson);
- result.put("tree", trees.length() == 1 ? trees.getJSONObject(0) : trees);
- result.put("rotation", displayInfo.rotation);
- result.put("width", displayInfo.width);
- result.put("height", displayInfo.height);
- result.put("elapsed_ms", elapsed);
+ result.put("node_count", stats.nodes);
+ result.put("skipped_invisible", stats.skippedInvisible);
+ result.put("truncated", stats.truncated);
+ result.put("window_count", rootSnapshots.size());
+ result.put("elapsed_ms", System.currentTimeMillis() - startTime);
if (service instanceof ArtemisAccessibilityService) {
ArtemisAccessibilityService s = (ArtemisAccessibilityService) service;
@@ -114,70 +175,109 @@ public final class HierarchyDumper {
result.put("success", false);
result.put("error", "Dump failed: " + t.getMessage());
result.put("xml", "");
- result.put("elements", elementsJson);
+ result.put("elements", new JSONArray());
} catch (Throwable ignored) {}
}
return result;
}
- /**
- * Dumps an atomic snapshot combining hardware screenshot (JPEG base64) and UI hierarchy XML/JSON
- * at the exact same clock tick, eliminating temporal phase mismatch.
- */
+ private static final int ERROR_TAKE_SCREENSHOT_INTERVAL_TIME_SHORT = 3;
+ private static final long SCREENSHOT_INTERVAL_RETRY_MS = 350L;
+
+ @android.annotation.TargetApi(Build.VERSION_CODES.R)
+ private static void requestScreenshot(
+ final AccessibilityService service,
+ final AtomicReference bitmapRef,
+ final AtomicInteger errorRef,
+ final CountDownLatch latch,
+ final boolean allowRetry) {
+ try {
+ service.takeScreenshot(
+ Display.DEFAULT_DISPLAY,
+ SCREENSHOT_EXECUTOR,
+ new AccessibilityService.TakeScreenshotCallback() {
+ @Override
+ public void onSuccess(AccessibilityService.ScreenshotResult screenshotResult) {
+ try {
+ HardwareBuffer buffer = screenshotResult.getHardwareBuffer();
+ ColorSpace colorSpace = screenshotResult.getColorSpace();
+ Bitmap hwBitmap = Bitmap.wrapHardwareBuffer(buffer, colorSpace);
+ if (hwBitmap != null) {
+ Bitmap swBitmap = hwBitmap.copy(Bitmap.Config.ARGB_8888, false);
+ hwBitmap.recycle();
+ buffer.close();
+ bitmapRef.set(swBitmap);
+ }
+ } catch (Throwable t) {
+ Log.w(TAG, "Error copying screenshot buffer", t);
+ } finally {
+ latch.countDown();
+ }
+ }
+
+ @Override
+ public void onFailure(int errorCode) {
+ errorRef.set(errorCode);
+ if (allowRetry && errorCode == ERROR_TAKE_SCREENSHOT_INTERVAL_TIME_SHORT) {
+ Log.i(TAG, "takeScreenshot rate-limited; retrying after " + SCREENSHOT_INTERVAL_RETRY_MS + " ms");
+ SCREENSHOT_EXECUTOR.execute(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ Thread.sleep(SCREENSHOT_INTERVAL_RETRY_MS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ requestScreenshot(service, bitmapRef, errorRef, latch, false);
+ }
+ });
+ return;
+ }
+ Log.w(TAG, "takeScreenshot failed, errorCode: " + errorCode);
+ latch.countDown();
+ }
+ }
+ );
+ } catch (Throwable t) {
+ Log.w(TAG, "takeScreenshot invocation error", t);
+ latch.countDown();
+ }
+ }
+
public static JSONObject dumpAtomicSnapshot(AccessibilityService service) {
+ return dumpAtomicSnapshot(service, DumpOptions.forSnapshot());
+ }
+
+ /**
+ * Dumps an atomic snapshot combining hardware screenshot (JPEG base64) and UI hierarchy
+ * at the exact same clock tick, eliminating temporal phase mismatch. When the screenshot
+ * is present, {@code width} / {@code height} are the bitmap's own dimensions so the host
+ * normalizes coordinates against the very image it is looking at.
+ */
+ public static JSONObject dumpAtomicSnapshot(AccessibilityService service, DumpOptions options) {
long startTime = System.currentTimeMillis();
- // 1. Trigger hardware screenshot asynchronously
+ // 1. Trigger hardware screenshot asynchronously. The framework rate-limits
+ // takeScreenshot to one call per ~333 ms (ERROR_TAKE_SCREENSHOT_INTERVAL_TIME_SHORT = 3);
+ // back-to-back observations therefore wait out the interval and retry once
+ // here, which is far cheaper than the host falling back to adb screencap.
final AtomicReference bitmapRef = new AtomicReference<>(null);
+ final AtomicInteger errorRef = new AtomicInteger(0);
final CountDownLatch screenshotLatch = new CountDownLatch(1);
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
- try {
- service.takeScreenshot(
- Display.DEFAULT_DISPLAY,
- SCREENSHOT_EXECUTOR,
- new AccessibilityService.TakeScreenshotCallback() {
- @Override
- public void onSuccess(AccessibilityService.ScreenshotResult screenshotResult) {
- try {
- HardwareBuffer buffer = screenshotResult.getHardwareBuffer();
- ColorSpace colorSpace = screenshotResult.getColorSpace();
- Bitmap hwBitmap = Bitmap.wrapHardwareBuffer(buffer, colorSpace);
- if (hwBitmap != null) {
- Bitmap swBitmap = hwBitmap.copy(Bitmap.Config.ARGB_8888, false);
- hwBitmap.recycle();
- buffer.close();
- bitmapRef.set(swBitmap);
- }
- } catch (Throwable t) {
- Log.w(TAG, "Error copying screenshot buffer", t);
- } finally {
- screenshotLatch.countDown();
- }
- }
-
- @Override
- public void onFailure(int errorCode) {
- Log.w(TAG, "takeScreenshot failed, errorCode: " + errorCode);
- screenshotLatch.countDown();
- }
- }
- );
- } catch (Throwable t) {
- Log.w(TAG, "takeScreenshot invocation error", t);
- screenshotLatch.countDown();
- }
+ requestScreenshot(service, bitmapRef, errorRef, screenshotLatch, true);
} else {
screenshotLatch.countDown();
}
// 2. Concurrently capture the UI hierarchy
- JSONObject dumpData = dump(service);
+ JSONObject dumpData = dump(service, options);
- // 3. Wait for the screenshot with a strict 1.5s timeout
+ // 3. Wait for the screenshot (one retry after the rate-limit interval fits inside)
try {
- screenshotLatch.await(1500L, TimeUnit.MILLISECONDS);
+ screenshotLatch.await(2500L, TimeUnit.MILLISECONDS);
} catch (InterruptedException ignored) {}
Bitmap bitmap = bitmapRef.get();
@@ -189,6 +289,8 @@ public final class HierarchyDumper {
String base64Str = Base64.encodeToString(jpegBytes, Base64.NO_WRAP);
dumpData.put("screenshot_base64", base64Str);
dumpData.put("has_screenshot", true);
+ dumpData.put("width", bitmap.getWidth());
+ dumpData.put("height", bitmap.getHeight());
} catch (Throwable t) {
Log.w(TAG, "Failed to compress screenshot to JPEG Base64", t);
try { dumpData.put("has_screenshot", false); } catch (Throwable ignored) {}
@@ -198,8 +300,11 @@ public final class HierarchyDumper {
} else {
try {
dumpData.put("has_screenshot", false);
+ dumpData.put("screenshot_error_code", errorRef.get());
dumpData.put("screenshot_error", Build.VERSION.SDK_INT >= Build.VERSION_CODES.R
- ? "Screenshot capture timed out or failed"
+ ? (errorRef.get() != 0
+ ? "takeScreenshot failed with errorCode " + errorRef.get()
+ : "Screenshot capture timed out")
: "takeScreenshot not supported on Android < 11");
} catch (Throwable ignored) {}
}
@@ -211,23 +316,27 @@ public final class HierarchyDumper {
return dumpData;
}
+ public static String dumpXml(AccessibilityService service) {
+ return dumpXml(service, DumpOptions.forSnapshot());
+ }
+
/**
* Dumps the hierarchy directly as a raw standard UIAutomator XML string.
*/
- public static String dumpXml(AccessibilityService service) {
+ public static String dumpXml(AccessibilityService service, DumpOptions options) {
DisplayUtils.DisplayInfo displayInfo = DisplayUtils.getDisplayInfo(service);
- List rootSnapshots = captureRootSnapshots(service);
+ List rootSnapshots = captureRootSnapshots(service, displayInfo, options, new DumpStats());
if (rootSnapshots.isEmpty()) {
return "\n\n";
}
- return buildXml(rootSnapshots, displayInfo.rotation, displayInfo.width, displayInfo.height);
+ return buildXml(rootSnapshots, displayInfo.rotation);
}
/**
* Builds the standard Android UIAutomator XML string from root snapshots.
*/
- private static String buildXml(List roots, int rotation, int width, int height) {
+ private static String buildXml(List roots, int rotation) {
StringBuilder sb = new StringBuilder(roots.size() * 1024 + 256);
sb.append("\n");
sb.append("");
@@ -249,6 +358,8 @@ public final class HierarchyDumper {
final int windowLayer;
final boolean windowActive;
final boolean windowFocused;
+ /** Screen rectangle of the window, or null when unknown (display bounds are used). */
+ final Rect windowBounds;
RawRootEntry(
AccessibilityNodeInfo root,
@@ -256,7 +367,8 @@ public final class HierarchyDumper {
String windowType,
int windowLayer,
boolean windowActive,
- boolean windowFocused
+ boolean windowFocused,
+ Rect windowBounds
) {
this.root = root;
this.windowId = windowId;
@@ -264,14 +376,25 @@ public final class HierarchyDumper {
this.windowLayer = windowLayer;
this.windowActive = windowActive;
this.windowFocused = windowFocused;
+ this.windowBounds = windowBounds;
}
}
+ public static List captureRootSnapshots(AccessibilityService service) {
+ return captureRootSnapshots(
+ service, DisplayUtils.getDisplayInfo(service), DumpOptions.forSnapshot(), new DumpStats());
+ }
+
/**
* Captures snapshots of all active and interactive windows with adaptive progressive retry support.
* Guaranteed to capture the complete hierarchy even during activity transitions and cold starts.
*/
- public static List captureRootSnapshots(AccessibilityService service) {
+ private static List captureRootSnapshots(
+ AccessibilityService service,
+ DisplayUtils.DisplayInfo displayInfo,
+ DumpOptions options,
+ DumpStats stats
+ ) {
List rawRoots = Collections.emptyList();
long[] retryBackoff = new long[]{40L, 80L, 120L, 160L, 220L, 300L};
@@ -285,13 +408,19 @@ public final class HierarchyDumper {
}
}
+ Rect displayRect = new Rect(0, 0, displayInfo.width, displayInfo.height);
List snapshots = new ArrayList<>(rawRoots.size());
- int[] nodeCounter = new int[]{0};
for (int i = 0; i < rawRoots.size(); i++) {
RawRootEntry entry = rawRoots.get(i);
try {
- A11yNode snapshot = snapshotNode(entry.root, 0, i, nodeCounter);
+ Rect clip = new Rect(displayRect);
+ if (entry.windowBounds != null && !clip.intersect(entry.windowBounds)) {
+ // Window entirely off the display (e.g. a second display): keep the
+ // display rect so the root still serializes with sane bounds.
+ clip = new Rect(displayRect);
+ }
+ A11yNode snapshot = snapshotNode(entry.root, 0, i, clip, options, stats);
if (snapshot != null) {
snapshot.windowId = entry.windowId;
snapshot.windowType = entry.windowType;
@@ -310,6 +439,24 @@ public final class HierarchyDumper {
return snapshots;
}
+ private static AccessibilityNodeInfo windowRoot(AccessibilityWindowInfo window) {
+ if (Build.VERSION.SDK_INT >= 33) {
+ try {
+ return window.getRoot(PREFETCH_DESCENDANTS_HYBRID);
+ } catch (Throwable ignored) {}
+ }
+ return window.getRoot();
+ }
+
+ private static AccessibilityNodeInfo childOf(AccessibilityNodeInfo node, int index) {
+ if (Build.VERSION.SDK_INT >= 33) {
+ try {
+ return node.getChild(index, PREFETCH_DESCENDANTS_HYBRID);
+ } catch (Throwable ignored) {}
+ }
+ return node.getChild(index);
+ }
+
/**
* Multi-tier hierarchy root discovery strategy:
* Tier 1: Multi-window enumeration (sorted by Z-layer descending).
@@ -335,7 +482,7 @@ public final class HierarchyDumper {
for (AccessibilityWindowInfo window : sortedWindows) {
try {
- AccessibilityNodeInfo root = window.getRoot();
+ AccessibilityNodeInfo root = windowRoot(window);
if (root != null) {
int hash = root.hashCode();
if (seenHashes.add(hash)) {
@@ -343,13 +490,16 @@ public final class HierarchyDumper {
if (window.getType() == AccessibilityWindowInfo.TYPE_APPLICATION) {
hasAppWindow = true;
}
+ Rect bounds = new Rect();
+ window.getBoundsInScreen(bounds);
roots.add(new RawRootEntry(
root,
window.getId(),
typeStr,
window.getLayer(),
window.isActive(),
- window.isFocused()
+ window.isFocused(),
+ bounds.isEmpty() ? null : bounds
));
} else {
safeRecycle(root);
@@ -373,7 +523,8 @@ public final class HierarchyDumper {
"application",
0,
true,
- true
+ true,
+ null
));
hasAppWindow = true;
} else {
@@ -414,7 +565,8 @@ public final class HierarchyDumper {
"application",
0,
true,
- true
+ true,
+ null
));
} else {
safeRecycle(current);
@@ -450,6 +602,11 @@ public final class HierarchyDumper {
* Recursively snapshots an AccessibilityNodeInfo into an immutable A11yNode,
* extracting rich semantic properties (error, heading, editable, paneTitle, tooltip, stateDescription).
*
+ * {@code clip} is the rectangle the node may be visible in: the display intersected with the
+ * window and every scrollable ancestor. Children that are not visible to the user are skipped
+ * unless {@code options.includeInvisible} is set; the window root itself is always kept, as in
+ * UIAutomator.
+ *
* Protected against recursion loops via MAX_DEPTH and MAX_NODES without flat hash collision drops.
*/
@SuppressWarnings("deprecation")
@@ -457,20 +614,37 @@ public final class HierarchyDumper {
AccessibilityNodeInfo node,
int depth,
int childIndex,
- int[] nodeCounter
+ Rect clip,
+ DumpOptions options,
+ DumpStats stats
) {
if (node == null) return null;
- if (depth > MAX_DEPTH || nodeCounter[0] >= MAX_NODES) {
+ if (depth > MAX_DEPTH || stats.nodes >= MAX_NODES) {
+ stats.truncated = true;
return null;
}
- nodeCounter[0]++;
+ boolean visible = true;
+ try {
+ visible = node.isVisibleToUser();
+ } catch (Throwable ignored) {}
+ if (!visible && depth > 0 && !options.includeInvisible) {
+ stats.skippedInvisible++;
+ return null;
+ }
+
+ stats.nodes++;
A11yNode snapshot = new A11yNode();
snapshot.index = childIndex;
+ snapshot.visibleToUser = visible;
+ Rect childClip = clip;
try {
Rect bounds = new Rect();
node.getBoundsInScreen(bounds);
+ if (!bounds.intersect(clip)) {
+ bounds.setEmpty();
+ }
snapshot.left = bounds.left;
snapshot.top = bounds.top;
snapshot.right = bounds.right;
@@ -499,6 +673,13 @@ public final class HierarchyDumper {
snapshot.password = node.isPassword();
snapshot.selected = node.isSelected();
+ // Children of a scrollable container are clipped by it (UIAutomator's
+ // trimScrollableParent): a list row half under the toolbar keeps only its
+ // visible part, so its center is a point the user can actually touch.
+ if (snapshot.scrollable && !bounds.isEmpty()) {
+ childClip = new Rect(bounds);
+ }
+
// 1. Editable property
snapshot.editable = node.isEditable();
@@ -515,11 +696,15 @@ public final class HierarchyDumper {
} catch (Throwable ignored) {}
}
- // 4. Hint text (API 26+)
+ // 4. Hint text (API 26+). When the field is empty the framework reports the
+ // hint as the text; keep text and hint apart so "empty input" stays detectable.
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
try {
CharSequence hint = node.getHintText();
if (hint != null) snapshot.hint = hint.toString();
+ if (node.isShowingHintText()) {
+ snapshot.text = "";
+ }
} catch (Throwable ignored) {}
}
@@ -557,9 +742,9 @@ public final class HierarchyDumper {
for (int i = 0; i < childCount; i++) {
AccessibilityNodeInfo childNode = null;
try {
- childNode = node.getChild(i);
+ childNode = childOf(node, i);
if (childNode != null) {
- A11yNode childSnapshot = snapshotNode(childNode, depth + 1, i, nodeCounter);
+ A11yNode childSnapshot = snapshotNode(childNode, depth + 1, i, childClip, options, stats);
if (childSnapshot != null) {
snapshot.children.add(childSnapshot);
}
@@ -594,34 +779,49 @@ public final class HierarchyDumper {
/**
* Finds the currently focused or editable input node for text entry.
+ * Window roots are enumerated once and recycled; only the returned node stays alive.
*/
public static AccessibilityNodeInfo findInputNode(AccessibilityService service) {
- for (RawRootEntry entry : getActiveRawRoots(service)) {
- try {
- AccessibilityNodeInfo focused = entry.root.findFocus(AccessibilityNodeInfo.FOCUS_INPUT);
- if (focused != null) return focused;
- } catch (Throwable ignored) {}
+ List roots = getActiveRawRoots(service);
+ AccessibilityNodeInfo found = null;
+ try {
+ for (RawRootEntry entry : roots) {
+ try {
+ AccessibilityNodeInfo focused = entry.root.findFocus(AccessibilityNodeInfo.FOCUS_INPUT);
+ if (focused != null) {
+ found = focused;
+ return found;
+ }
+ } catch (Throwable ignored) {}
+ }
+ for (RawRootEntry entry : roots) {
+ AccessibilityNodeInfo editable = findFirstEditable(entry.root, 0);
+ if (editable != null) {
+ found = editable;
+ return found;
+ }
+ }
+ return null;
+ } finally {
+ for (RawRootEntry entry : roots) {
+ if (entry.root != found) {
+ safeRecycle(entry.root);
+ }
+ }
}
-
- for (RawRootEntry entry : getActiveRawRoots(service)) {
- AccessibilityNodeInfo editable = findFirstEditable(entry.root);
- if (editable != null) return editable;
- }
-
- return null;
}
- private static AccessibilityNodeInfo findFirstEditable(AccessibilityNodeInfo node) {
- if (node == null) return null;
+ private static AccessibilityNodeInfo findFirstEditable(AccessibilityNodeInfo node, int depth) {
+ if (node == null || depth > MAX_DEPTH) return null;
try {
- if (node.isEditable() && node.isFocusable() && node.isEnabled()) {
+ if (node.isEditable() && node.isFocusable() && node.isEnabled() && node.isVisibleToUser()) {
return node;
}
int count = node.getChildCount();
for (int i = 0; i < count; i++) {
- AccessibilityNodeInfo child = node.getChild(i);
+ AccessibilityNodeInfo child = childOf(node, i);
if (child != null) {
- AccessibilityNodeInfo res = findFirstEditable(child);
+ AccessibilityNodeInfo res = findFirstEditable(child, depth + 1);
if (res != null) return res;
safeRecycle(child);
}
diff --git a/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/TokenReceiver.java b/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/TokenReceiver.java
new file mode 100644
index 0000000..7e70904
--- /dev/null
+++ b/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/TokenReceiver.java
@@ -0,0 +1,36 @@
+package com.artemis.helper;
+
+import android.content.BroadcastReceiver;
+import android.content.Context;
+import android.content.Intent;
+import android.util.Log;
+
+/**
+ * Receives the host's session token.
+ *
+ * The receiver is guarded in the manifest by
+ * {@code android.permission.WRITE_SECURE_SETTINGS}, a permission ordinary apps
+ * cannot hold but the adb shell user does, so only something with adb access to
+ * the device can set (or clear) the token:
+ *
+ *
+ * adb shell am broadcast -n com.artemis.helper/.TokenReceiver \
+ * -a com.artemis.helper.SET_TOKEN --es token <hex>
+ *
+ */
+public class TokenReceiver extends BroadcastReceiver {
+
+ private static final String TAG = "ArtemisTokenReceiver";
+ public static final String ACTION_SET_TOKEN = "com.artemis.helper.SET_TOKEN";
+ public static final String EXTRA_TOKEN = "token";
+
+ @Override
+ public void onReceive(Context context, Intent intent) {
+ if (intent == null || !ACTION_SET_TOKEN.equals(intent.getAction())) {
+ return;
+ }
+ String value = intent.getStringExtra(EXTRA_TOKEN);
+ TokenStore.set(value);
+ Log.i(TAG, TokenStore.isSet() ? "Session token set" : "Session token cleared");
+ }
+}
diff --git a/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/TokenStore.java b/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/TokenStore.java
new file mode 100644
index 0000000..2062fc1
--- /dev/null
+++ b/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/TokenStore.java
@@ -0,0 +1,38 @@
+package com.artemis.helper;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+
+/**
+ * The session token the host pushed for this process lifetime.
+ *
+ * The loopback server is reachable by every app on the device, so requests are
+ * only served when they carry this token. It lives in memory only: a killed or
+ * re-bound service starts without one and answers 401 until the host pushes it
+ * again (which the host does on every attach and on every 401).
+ */
+public final class TokenStore {
+
+ private static volatile String token;
+
+ private TokenStore() {}
+
+ public static void set(String value) {
+ token = (value == null || value.isEmpty()) ? null : value;
+ }
+
+ public static boolean isSet() {
+ return token != null;
+ }
+
+ /** Constant-time comparison so a local app cannot guess the token byte by byte. */
+ public static boolean matches(String candidate) {
+ String current = token;
+ if (current == null || candidate == null) {
+ return false;
+ }
+ return MessageDigest.isEqual(
+ current.getBytes(StandardCharsets.UTF_8),
+ candidate.getBytes(StandardCharsets.UTF_8));
+ }
+}
diff --git a/packages/artemis-accessibility-helper/app/src/main/res/xml/accessibility_service_config.xml b/packages/artemis-accessibility-helper/app/src/main/res/xml/accessibility_service_config.xml
index 359c504..ee07cb5 100644
--- a/packages/artemis-accessibility-helper/app/src/main/res/xml/accessibility_service_config.xml
+++ b/packages/artemis-accessibility-helper/app/src/main/res/xml/accessibility_service_config.xml
@@ -1,9 +1,9 @@
/dev/null 2>&1; then
+ cygpath -u "$1"
+ else
+ printf '%s' "$1"
+ fi
+}
+
# 1. Locate Android SDK
-SDK_ROOT="${ANDROID_HOME:-${ANDROID_SDK_ROOT:-$HOME/Library/Android/sdk}}"
-if [ ! -d "$SDK_ROOT" ]; then
- for candidate in /opt/android-sdk /usr/local/share/android-sdk "$HOME/Android/Sdk"; do
- if [ -d "$candidate" ]; then
+SDK_ROOT="${ANDROID_HOME:-${ANDROID_SDK_ROOT:-}}"
+if [ -n "$SDK_ROOT" ]; then
+ SDK_ROOT="$(_to_posix "$SDK_ROOT")"
+fi
+if [ -z "$SDK_ROOT" ] || [ ! -d "$SDK_ROOT" ]; then
+ for candidate in \
+ "$HOME/Library/Android/sdk" \
+ "$HOME/Android/Sdk" \
+ /opt/android-sdk \
+ /usr/local/share/android-sdk \
+ "$( [ -n "$LOCALAPPDATA" ] && _to_posix "$LOCALAPPDATA/Android/Sdk" )"; do
+ if [ -n "$candidate" ] && [ -d "$candidate" ]; then
SDK_ROOT="$candidate"
break
fi
done
fi
-if [ ! -d "$SDK_ROOT" ]; then
- echo "Error: Android SDK not found at $SDK_ROOT. Set ANDROID_HOME." >&2
+if [ -z "$SDK_ROOT" ] || [ ! -d "$SDK_ROOT" ]; then
+ echo "Error: Android SDK not found. Set ANDROID_HOME." >&2
exit 1
fi
@@ -35,9 +62,15 @@ fi
# 2. Locate Java compiler and runtime
if [ -z "$JAVA_HOME" ]; then
- if [ -d "/Applications/Android Studio.app/Contents/jbr/Contents/Home" ]; then
- export JAVA_HOME="/Applications/Android Studio.app/Contents/jbr/Contents/Home"
- fi
+ for candidate in \
+ "/Applications/Android Studio.app/Contents/jbr/Contents/Home" \
+ "$(_to_posix "${PROGRAMFILES:-/c/Program Files}/Android/Android Studio/jbr")" \
+ "/opt/android-studio/jbr"; do
+ if [ -n "$candidate" ] && [ -x "$candidate/bin/javac" ] || [ -x "$candidate/bin/javac.exe" ]; then
+ export JAVA_HOME="$candidate"
+ break
+ fi
+ done
fi
if [ -n "$JAVA_HOME" ]; then
export PATH="$JAVA_HOME/bin:$PATH"
@@ -48,8 +81,49 @@ if ! command -v javac >/dev/null 2>&1; then
exit 1
fi
+PYTHON_BIN="${PYTHON:-}"
+if [ -z "$PYTHON_BIN" ]; then
+ # The Windows Store "python" stub is on PATH but exits 49; prefer the project venv.
+ for candidate in "$DIR/../../.venv/Scripts/python.exe" "$DIR/../../.venv/bin/python" python3 python; do
+ if "$candidate" -c "import zipfile" >/dev/null 2>&1; then
+ PYTHON_BIN="$candidate"
+ break
+ fi
+ done
+fi
+if [ -z "$PYTHON_BIN" ]; then
+ echo "Error: python3 not found (needed to package the DEX and write helper_manifest.json)" >&2
+ exit 1
+fi
+
+# Build-tools ship as .exe / .bat wrappers on Windows; resolve the right name once.
+_tool() {
+ local base="$BUILD_TOOLS/$1"
+ for suffix in "" ".exe" ".bat"; do
+ if [ -f "$base$suffix" ]; then
+ printf '%s' "$base$suffix"
+ return 0
+ fi
+ done
+ echo "Error: $1 not found under $BUILD_TOOLS" >&2
+ exit 1
+}
+AAPT2="$(_tool aapt2)"
+D8="$(_tool d8)"
+ZIPALIGN="$(_tool zipalign)"
+APKSIGNER="$(_tool apksigner)"
+
+# 3. Version: single source is app/build.gradle.kts
+VERSION_CODE="$(sed -n 's/^[[:space:]]*versionCode[[:space:]]*=[[:space:]]*\([0-9][0-9]*\).*/\1/p' app/build.gradle.kts | head -n 1)"
+VERSION_NAME="$(sed -n 's/^[[:space:]]*versionName[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' app/build.gradle.kts | head -n 1)"
+if [ -z "$VERSION_CODE" ] || [ -z "$VERSION_NAME" ]; then
+ echo "Error: could not read versionCode / versionName from app/build.gradle.kts" >&2
+ exit 1
+fi
+
echo "=================================================="
echo " Building ArtemisAccessibilityHelper.apk"
+echo " Version: $VERSION_NAME (code $VERSION_CODE)"
echo " Build Tools: $(basename "$BUILD_TOOLS")"
echo " Platform: $(basename "$PLATFORM")"
echo " Java: $(javac -version 2>&1)"
@@ -59,14 +133,16 @@ rm -rf build
mkdir -p build/gen build/obj build/apk build/res_compiled
echo "-> Compiling resources..."
-"$BUILD_TOOLS/aapt2" compile --dir app/src/main/res -o build/res_compiled/res.zip
+"$AAPT2" compile --dir app/src/main/res -o build/res_compiled/res.zip
echo "-> Linking APK..."
-"$BUILD_TOOLS/aapt2" link \
+"$AAPT2" link \
-I "$PLATFORM/android.jar" \
--manifest app/src/main/AndroidManifest.xml \
--min-sdk-version 24 \
--target-sdk-version 35 \
+ --version-code "$VERSION_CODE" \
+ --version-name "$VERSION_NAME" \
--java build/gen \
-o build/apk/unaligned.apk \
build/res_compiled/res.zip
@@ -74,25 +150,29 @@ echo "-> Linking APK..."
echo "-> Compiling Java sources..."
JAVA_FILES="$(find build/gen app/src/main/java -name "*.java")"
javac -encoding UTF-8 \
+ -source 8 -target 8 \
+ -Xlint:-options \
-cp "$PLATFORM/android.jar" \
-d build/obj \
$JAVA_FILES
echo "-> Dexing with D8..."
CLASS_FILES="$(find build/obj -name "*.class")"
-"$BUILD_TOOLS/d8" \
+"$D8" \
--lib "$PLATFORM/android.jar" \
--output build/apk/ \
--min-api 24 \
$CLASS_FILES
echo "-> Packaging DEX..."
-cd build/apk
-zip -u -q unaligned.apk classes.dex
-cd "$DIR"
+"$PYTHON_BIN" - <<'PY'
+import zipfile
+with zipfile.ZipFile("build/apk/unaligned.apk", "a", compression=zipfile.ZIP_DEFLATED) as apk:
+ apk.write("build/apk/classes.dex", "classes.dex")
+PY
echo "-> Zipaligning..."
-"$BUILD_TOOLS/zipalign" -f -p 4 build/apk/unaligned.apk build/apk/aligned.apk
+"$ZIPALIGN" -f -p 4 build/apk/unaligned.apk build/apk/aligned.apk
echo "-> Signing APK..."
if [ ! -f debug.keystore ]; then
@@ -103,7 +183,7 @@ if [ ! -f debug.keystore ]; then
-dname "CN=Artemis Debug,O=Artemis,C=US"
fi
-"$BUILD_TOOLS/apksigner" sign \
+"$APKSIGNER" sign \
--ks debug.keystore \
--ks-pass pass:android \
--key-pass pass:android \
@@ -111,6 +191,23 @@ fi
--out ArtemisAccessibilityHelper.apk \
build/apk/aligned.apk
+echo "-> Writing helper_manifest.json..."
+VERSION_CODE="$VERSION_CODE" VERSION_NAME="$VERSION_NAME" "$PYTHON_BIN" - <<'PY'
+import datetime, hashlib, json, os
+digest = hashlib.sha256(open("ArtemisAccessibilityHelper.apk", "rb").read()).hexdigest()
+manifest = {
+ "package": "com.artemis.helper",
+ "version_code": int(os.environ["VERSION_CODE"]),
+ "version_name": os.environ["VERSION_NAME"],
+ "sha256": digest,
+ "built_at": datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat(),
+}
+with open("helper_manifest.json", "w", encoding="utf-8", newline="\n") as fh:
+ json.dump(manifest, fh, indent=2)
+ fh.write("\n")
+print(json.dumps(manifest, indent=2))
+PY
+
echo "=================================================="
echo " Build SUCCESSFUL!"
echo " Artifact: $DIR/ArtemisAccessibilityHelper.apk"
diff --git a/packages/artemis-accessibility-helper/helper_manifest.json b/packages/artemis-accessibility-helper/helper_manifest.json
new file mode 100644
index 0000000..f8d979d
--- /dev/null
+++ b/packages/artemis-accessibility-helper/helper_manifest.json
@@ -0,0 +1,7 @@
+{
+ "package": "com.artemis.helper",
+ "version_code": 6,
+ "version_name": "1.2.0",
+ "sha256": "133e373fe03615c251cd74b03a38539c844d09a074954bca2d2dde6684561969",
+ "built_at": "2026-09-10T23:13:11+00:00"
+}
diff --git a/tests/unit/clients/test_accessibility_client.py b/tests/unit/clients/test_accessibility_client.py
index ccc8546..2d722dc 100644
--- a/tests/unit/clients/test_accessibility_client.py
+++ b/tests/unit/clients/test_accessibility_client.py
@@ -1,87 +1,364 @@
-import unittest
+"""AccessibilityClient: HTTP surface, token handling, element-shape parity, repairs."""
+
+from __future__ import annotations
+
+import base64
+from io import BytesIO
+import json
+import urllib.error
from unittest.mock import MagicMock, patch
-from artemis.clients.accessibility_client import AccessibilityClient, DEFAULT_PORT
+
+from PIL import Image
+import pytest
+
+from artemis.clients.accessibility_client import (
+ TOKEN_HEADER,
+ AccessibilityClient,
+ HelperEmptyHierarchy,
+ HelperRequestError,
+ HelperUnavailable,
+ normalize_helper_elements,
+)
+from artemis.clients.ui_automator_client import _parse_hierarchy_xml_to_elements
+from artemis.runtime.helper_manager import HelperSession
+
+XML = (
+ ''
+ ''
+ ''
+ ""
+)
+TOKEN = "a" * 48
-class TestAccessibilityClient(unittest.TestCase):
- def setUp(self):
- self.client = AccessibilityClient(device_id="dummy_device", local_port=DEFAULT_PORT)
+class FakeManager:
+ def __init__(self):
+ self.session = self._session(41000)
+ self.attach_calls: list[bool] = []
+ self.reattach_calls = 0
+ self.pushed_tokens = 0
+ self.detached = False
+ self.fail_reattach = False
- @patch("artemis.clients.accessibility_client.urllib.request.urlopen")
- def test_ping_success(self, mock_urlopen):
- mock_resp = MagicMock()
- mock_resp.read.return_value = b'{"success": true, "service": "ArtemisAccessibilityService"}'
- mock_resp.__enter__.return_value = mock_resp
- mock_urlopen.return_value = mock_resp
+ @staticmethod
+ def _session(port: int) -> HelperSession:
+ return HelperSession(
+ serial="dev",
+ local_port=port,
+ transport_id="3",
+ version_code=6,
+ version_name="1.2.0",
+ owns_forward=True,
+ token=TOKEN,
+ protocol_version=2,
+ )
- self.assertTrue(self.client.ping())
+ def attach(self, serial, *, provision=True, on_event=None):
+ self.attach_calls.append(provision)
+ if on_event is not None:
+ on_event("installing", {"serial": serial})
+ return self.session
- @patch("artemis.clients.accessibility_client.urllib.request.urlopen")
- def test_get_hierarchy(self, mock_urlopen):
- mock_resp = MagicMock()
- mock_resp.read.return_value = b'{"success": true, "elements": [{"text": "OK"}]}'
- mock_resp.__enter__.return_value = mock_resp
- mock_urlopen.return_value = mock_resp
+ def reattach(self, serial):
+ self.reattach_calls += 1
+ if self.fail_reattach:
+ raise HelperUnavailable("dead")
+ self.session = self._session(41001)
+ return self.session
- data = self.client.get_hierarchy()
- self.assertTrue(data["success"])
- self.assertEqual(data["elements"][0]["text"], "OK")
+ def push_token(self, serial):
+ self.pushed_tokens += 1
+ return True
- @patch("artemis.clients.accessibility_client.urllib.request.urlopen")
- def test_tap_action(self, mock_urlopen):
- mock_resp = MagicMock()
- mock_resp.read.return_value = b'{"success": true}'
- mock_resp.__enter__.return_value = mock_resp
- mock_urlopen.return_value = mock_resp
+ def host_token(self):
+ return TOKEN
- @patch("artemis.clients.accessibility_client.urllib.request.urlopen")
- def test_get_hierarchy_xml(self, mock_urlopen):
- mock_resp = MagicMock()
- mock_resp.read.return_value = b''
- mock_resp.__enter__.return_value = mock_resp
- mock_urlopen.return_value = mock_resp
+ def detach(self, serial):
+ self.detached = True
- xml = self.client.get_hierarchy_xml()
- self.assertIn("',
- "elements": [{"text": "OK"}],
- "width": 1080,
- "height": 2400,
+
+def _response(body: bytes):
+ resp = MagicMock()
+ resp.read.return_value = body
+ resp.__enter__.return_value = resp
+ resp.__exit__.return_value = False
+ return resp
+
+
+def _http_error(code: int, body: bytes = b'{"success": false, "error": "no"}'):
+ return urllib.error.HTTPError("http://x", code, "err", {}, BytesIO(body))
+
+
+def _urlopen():
+ return patch("artemis.clients.accessibility_client.urllib.request.urlopen")
+
+
+@pytest.fixture
+def client():
+ manager = FakeManager()
+ with patch("artemis.clients.accessibility_client.ensure_device_awake", return_value=None):
+ yield AccessibilityClient("dev", manager=manager), manager
+
+
+# --------------------------------------------------------------------------- #
+# Session and transport
+# --------------------------------------------------------------------------- #
+
+
+def test_connect_provisions_and_lazy_path_does_not(client):
+ c, manager = client
+ events = []
+ c.connect(on_event=lambda name, details: events.append(name))
+ assert manager.attach_calls == [True]
+ assert events == ["installing"]
+ assert c.active_backend == "helper"
+
+ lazy = AccessibilityClient("dev", manager=FakeManager())
+ with (
+ patch("artemis.clients.accessibility_client.ensure_device_awake", return_value=None),
+ _urlopen() as urlopen,
+ ):
+ urlopen.return_value = _response(XML.encode())
+ assert lazy.get_hierarchy() == XML
+ assert lazy._manager.attach_calls == [False]
+
+
+def test_every_request_carries_the_session_token(client):
+ c, _ = client
+ with _urlopen() as urlopen:
+ urlopen.return_value = _response(XML.encode())
+ assert c.get_hierarchy() == XML
+ request = urlopen.call_args.args[0]
+ assert request.full_url == "http://127.0.0.1:41000/dump_xml"
+ # urllib stores header names capitalized: "X-artemis-token".
+ assert request.get_header(TOKEN_HEADER.capitalize()) == TOKEN
+
+
+def test_401_pushes_the_token_again_and_retries_once(client):
+ c, manager = client
+ with _urlopen() as urlopen:
+ urlopen.side_effect = [_http_error(401), _response(XML.encode())]
+ assert c.get_hierarchy() == XML
+ assert manager.pushed_tokens == 1
+ assert manager.reattach_calls == 0 # a tunnel rebuild would not fix a token problem
+
+ with _urlopen() as urlopen:
+ urlopen.side_effect = [_http_error(401), _http_error(401)]
+ with pytest.raises(HelperRequestError) as info:
+ c.get_hierarchy()
+ assert info.value.status == 401
+ assert manager.pushed_tokens == 2
+
+
+def test_other_http_errors_surface_without_repairs(client):
+ c, manager = client
+ with _urlopen() as urlopen:
+ urlopen.side_effect = _http_error(404, b"Unknown endpoint")
+ with pytest.raises(HelperRequestError, match="HTTP 404"):
+ c.get_hierarchy()
+ assert manager.reattach_calls == 0 and manager.pushed_tokens == 0
+
+
+def test_transport_failure_reattaches_once_and_retries(client):
+ c, manager = client
+ with _urlopen() as urlopen:
+ urlopen.side_effect = [urllib.error.URLError("connection refused"), _response(XML.encode())]
+ assert c.get_hierarchy() == XML
+ assert manager.reattach_calls == 1
+ assert urlopen.call_args.args[0].full_url == "http://127.0.0.1:41001/dump_xml"
+
+
+def test_transport_failure_after_reattach_propagates(client):
+ c, manager = client
+ manager.fail_reattach = True
+ with _urlopen() as urlopen:
+ urlopen.side_effect = urllib.error.URLError("connection refused")
+ with pytest.raises(HelperUnavailable):
+ c.get_hierarchy()
+ assert manager.reattach_calls == 1
+
+
+def test_action_timeout_does_not_repeat_text_entry(client):
+ c, manager = client
+ with _urlopen() as urlopen:
+ urlopen.side_effect = TimeoutError("response lost after action")
+ assert c.send_text("hello") is False
+ assert urlopen.call_count == 1
+ assert manager.reattach_calls == 0
+
+
+# --------------------------------------------------------------------------- #
+# Screen data: one /snapshot request, UIAutomator element shapes
+# --------------------------------------------------------------------------- #
+
+
+def test_screen_data_is_one_snapshot_request_with_xml_only(client):
+ c, _ = client
+ snapshot = {
+ "success": True,
+ "has_screenshot": True,
+ "screenshot_base64": "QUJD",
+ "xml": XML,
+ "width": 1080,
+ "height": 2424,
+ }
+ with _urlopen() as urlopen:
+ urlopen.return_value = _response(json.dumps(snapshot).encode())
+ data = c.get_screen_data()
+ assert urlopen.call_count == 1
+ assert urlopen.call_args.args[0].full_url == "http://127.0.0.1:41000/snapshot?fields=xml"
+
+ assert data.base64 == "QUJD" and data.width == 1080 and data.height == 2424
+ assert data.hierarchy_xml == XML
+ assert data.elements == _parse_hierarchy_xml_to_elements(XML)
+ element = data.elements[1]
+ assert element["clickable"] == "true"
+ assert element["visible-to-user"] == "true"
+ assert element["accessibilityText"] == "Confirm"
+ assert element["parsed_bounds"] == {"left": 10, "top": 20, "right": 110, "bottom": 70}
+ assert element["state-description"] == "on"
+
+
+def test_screen_data_without_screenshot_reuses_the_hierarchy_and_adds_screencap(client):
+ c, _ = client
+ no_shot = {
+ "success": True,
+ "has_screenshot": False,
+ "screenshot_error": "takeScreenshot not supported on Android < 11",
+ "xml": XML,
+ "width": 1080,
+ "height": 2400,
+ }
+ img = Image.new("RGB", (4, 6), "white")
+ with (
+ _urlopen() as urlopen,
+ patch.object(AccessibilityClient, "get_screenshot", return_value=img),
+ ):
+ urlopen.return_value = _response(json.dumps(no_shot).encode())
+ data = c.get_screen_data()
+ assert urlopen.call_count == 1 # the hierarchy is not dumped a second time
+ assert data.width == 4 and data.height == 6
+ assert Image.open(BytesIO(base64.b64decode(data.base64))).size == (4, 6)
+ assert data.elements == _parse_hierarchy_xml_to_elements(XML)
+
+
+def test_screen_data_falls_back_to_normalized_json_without_xml(client):
+ c, _ = client
+ snapshot = {
+ "success": True,
+ "has_screenshot": True,
+ "screenshot_base64": "QUJD",
+ "xml": "",
+ "elements": [
+ {
+ "text": "",
+ "content-desc": "Menu",
+ "clickable": True,
+ "focused": False,
+ "visible-to-user": True,
+ "parsed_bounds": {"left": 1.0, "top": 2, "right": 3, "bottom": 4},
+ "children": [{"text": "x"}],
+ }
+ ],
+ "width": 1080,
+ "height": 2400,
+ }
+ with _urlopen() as urlopen:
+ urlopen.return_value = _response(json.dumps(snapshot).encode())
+ data = c.get_screen_data()
+ assert data.elements == [
+ {
+ "text": "",
+ "content-desc": "Menu",
+ "accessibilityText": "Menu",
+ "clickable": "true",
+ "focused": "false",
+ "visible-to-user": "true",
+ "parsed_bounds": {"left": 1, "top": 2, "right": 3, "bottom": 4},
}
- screen_data = self.client.get_screen_data()
- self.assertEqual(screen_data.width, 1080)
- self.assertEqual(screen_data.height, 2400)
- self.assertEqual(screen_data.base64, "dummy_b64")
- self.assertIn('rotation="0"', screen_data.hierarchy_xml)
- self.assertEqual(len(screen_data.elements), 1)
-
- @patch.object(AccessibilityClient, "get_atomic_snapshot", return_value=None)
- @patch.object(AccessibilityClient, "get_screenshot")
- @patch.object(AccessibilityClient, "get_hierarchy")
- def test_get_screen_data_fallback(self, mock_get_hierarchy, mock_get_screenshot, _mock_atomic):
- from PIL import Image
-
- mock_get_screenshot.return_value = Image.new("RGB", (1080, 2400))
- mock_get_hierarchy.return_value = {
- "success": True,
- "xml": '',
- "elements": [{"text": "Fallback"}],
- }
-
- screen_data = self.client.get_screen_data()
- self.assertEqual(screen_data.width, 1080)
- self.assertEqual(screen_data.height, 2400)
- self.assertIn("Fallback", screen_data.hierarchy_xml)
- self.assertEqual(len(screen_data.elements), 1)
+ ]
-if __name__ == "__main__":
- unittest.main()
+def test_empty_hierarchy_is_an_error_not_a_blank_screen(client):
+ c, _ = client
+ answer = {"success": False, "error": "No active window or root node found", "xml": ""}
+ with _urlopen() as urlopen:
+ urlopen.return_value = _response(json.dumps(answer).encode())
+ with pytest.raises(HelperEmptyHierarchy, match="No active window"):
+ c.get_screen_data()
+
+
+def test_screenshot_without_dimensions_is_rejected(client):
+ c, _ = client
+ answer = {"success": True, "has_screenshot": True, "screenshot_base64": "QUJD", "xml": XML}
+ with _urlopen() as urlopen:
+ urlopen.return_value = _response(json.dumps(answer).encode())
+ with pytest.raises(HelperRequestError, match="without dimensions"):
+ c.get_screen_data()
+
+
+# --------------------------------------------------------------------------- #
+# Input
+# --------------------------------------------------------------------------- #
+
+
+def test_set_clipboard_sends_raw_utf8(client):
+ c, _ = client
+ with _urlopen() as urlopen:
+ urlopen.return_value = _response(b'{"success": true}')
+ assert c.set_clipboard("héllo\n你好") is True
+ request = urlopen.call_args.args[0]
+ assert request.full_url == "http://127.0.0.1:41000/action"
+ assert request.data == '{"cmd": "clipboard", "text": "héllo\\n你好"}'.encode()
+ assert request.get_header("Content-type") == "application/json; charset=utf-8"
+
+
+def test_send_text_appends_like_an_ime(client):
+ c, _ = client
+ with _urlopen() as urlopen:
+ urlopen.return_value = _response(b'{"success": true}')
+ assert c.send_text("abc") is True
+ assert json.loads(urlopen.call_args.args[0].data) == {
+ "cmd": "type",
+ "text": "abc",
+ "append": True,
+ }
+
+
+def test_rpc_failure_returns_false_instead_of_raising(client):
+ c, manager = client
+ manager.fail_reattach = True
+ with _urlopen() as urlopen:
+ urlopen.side_effect = urllib.error.URLError("refused")
+ assert c.clear_text() is False
+ with _urlopen() as urlopen:
+ urlopen.side_effect = _http_error(500)
+ assert c.clear_text() is False
+
+
+def test_press_key_uses_global_action_for_navigation_keys(client):
+ c, _ = client
+ with _urlopen() as urlopen:
+ urlopen.return_value = _response(b'{"success": true}')
+ assert c.press_key("BACK") is True
+ assert json.loads(urlopen.call_args.args[0].data) == {"cmd": "global", "action": "back"}
+
+
+def test_disconnect_detaches(client):
+ c, manager = client
+ c.connect()
+ c.disconnect()
+ assert manager.detached and c.active_backend is None
+
+
+def test_normalize_helper_elements_tolerates_bad_input():
+ assert normalize_helper_elements(None) == []
+ assert normalize_helper_elements("x") == []
+ assert normalize_helper_elements({"clickable": True}) == [{"clickable": "true"}]
diff --git a/tests/unit/clients/test_screen_client_factory.py b/tests/unit/clients/test_screen_client_factory.py
new file mode 100644
index 0000000..836af3b
--- /dev/null
+++ b/tests/unit/clients/test_screen_client_factory.py
@@ -0,0 +1,227 @@
+"""Backend selection and the helper-first / UIAutomator-fallback composite."""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock
+
+import pytest
+
+from artemis.clients.accessibility_client import AccessibilityClient, HelperUnavailable
+from artemis.clients.screen_client_factory import (
+ DeviceOfflineError,
+ FallbackScreenClient,
+ HierarchyBackend,
+ create_screen_client,
+ describe_backend,
+ hierarchy_backend_sentence,
+ hierarchy_backend_summary,
+ resolve_backend,
+)
+from artemis.clients.ui_automator_client import UIAutomatorClient
+from artemis.config.constants import ENV_ARTEMIS_HIERARCHY_BACKEND
+
+
+# --------------------------------------------------------------------------- #
+# resolve_backend / create_screen_client
+# --------------------------------------------------------------------------- #
+
+
+def test_resolve_backend_prefers_explicit_then_env_then_settings(monkeypatch):
+ monkeypatch.delenv(ENV_ARTEMIS_HIERARCHY_BACKEND, raising=False)
+ from artemis.config import settings
+
+ monkeypatch.setattr(settings, "ARTEMIS_HIERARCHY_BACKEND", "uiautomator", raising=False)
+ assert resolve_backend() is HierarchyBackend.UIAUTOMATOR
+
+ monkeypatch.setenv(ENV_ARTEMIS_HIERARCHY_BACKEND, "Helper")
+ assert resolve_backend() is HierarchyBackend.HELPER
+
+ assert resolve_backend("auto") is HierarchyBackend.AUTO
+ assert resolve_backend(HierarchyBackend.HELPER) is HierarchyBackend.HELPER
+
+
+def test_resolve_backend_unknown_value_falls_back_to_auto(monkeypatch):
+ monkeypatch.setenv(ENV_ARTEMIS_HIERARCHY_BACKEND, "maestro")
+ assert resolve_backend() is HierarchyBackend.AUTO
+
+
+def test_create_screen_client_per_backend():
+ assert isinstance(create_screen_client("d", "uiautomator"), UIAutomatorClient)
+ assert isinstance(create_screen_client("d", "helper"), AccessibilityClient)
+ composite = create_screen_client("d", "auto")
+ assert isinstance(composite, FallbackScreenClient)
+ assert describe_backend(composite) is None # nothing served yet
+ assert describe_backend(UIAutomatorClient("d")) == "uiautomator"
+ assert describe_backend(AccessibilityClient("d")) == "helper"
+
+
+# --------------------------------------------------------------------------- #
+# FallbackScreenClient
+# --------------------------------------------------------------------------- #
+
+
+@pytest.fixture
+def composite():
+ helper = MagicMock(spec=AccessibilityClient)
+ u2 = MagicMock(spec=UIAutomatorClient)
+ factory = MagicMock(return_value=u2)
+ clock = {"now": 0.0, "state": "device"}
+ client = FallbackScreenClient(
+ "dev",
+ helper=helper,
+ uiautomator_factory=factory,
+ retry_after=30.0,
+ clock=lambda: clock["now"],
+ state_probe=lambda _serial: clock["state"],
+ )
+ return client, helper, u2, factory, clock
+
+
+def test_healthy_helper_never_constructs_uiautomator(composite):
+ client, helper, _, factory, _ = composite
+ helper.get_hierarchy.return_value = ""
+ client.connect()
+ assert client.get_hierarchy() == ""
+ assert client.active_backend == "helper"
+ factory.assert_not_called()
+
+
+def test_connect_failure_degrades_to_uiautomator(composite):
+ client, helper, u2, factory, _ = composite
+ helper.connect.side_effect = HelperUnavailable("not installed")
+ client.connect()
+ factory.assert_called_once()
+ u2.connect.assert_called_once()
+ assert client.active_backend == "uiautomator"
+
+
+def test_call_failure_falls_back_then_retries_helper_after_window(composite):
+ client, helper, u2, _, clock = composite
+ helper.get_hierarchy.side_effect = [RuntimeError("tunnel gone"), ""]
+ u2.get_hierarchy.return_value = ""
+
+ assert client.get_hierarchy() == ""
+ assert client.active_backend == "uiautomator"
+
+ # Inside the retry window the helper is not touched again.
+ clock["now"] = 10.0
+ assert client.get_hierarchy() == ""
+ assert helper.get_hierarchy.call_count == 1
+
+ # After the window the helper is tried again and wins.
+ clock["now"] = 31.0
+ assert client.get_hierarchy() == ""
+ assert client.active_backend == "helper"
+
+
+def test_uiautomator_failure_propagates(composite):
+ client, helper, u2, _, _ = composite
+ helper.get_screen_data.side_effect = RuntimeError("helper down")
+ u2.get_screen_data.side_effect = RuntimeError("u2 down too")
+ with pytest.raises(RuntimeError, match="u2 down too"):
+ client.get_screen_data()
+
+
+def test_set_clipboard_goes_to_helper_first(composite):
+ client, helper, u2, _, _ = composite
+ helper.set_clipboard.return_value = True
+ assert client.set_clipboard("x") is True
+ u2.set_clipboard.assert_not_called()
+
+
+def test_disconnect_releases_both_when_uiautomator_was_used(composite):
+ client, helper, u2, _, _ = composite
+ helper.connect.side_effect = HelperUnavailable("nope")
+ client.connect()
+ client.disconnect()
+ helper.disconnect.assert_called_once()
+ u2.disconnect.assert_called_once()
+ assert client.active_backend is None
+
+
+def test_strict_helper_backend_surfaces_errors():
+ manager = MagicMock()
+ manager.attach.side_effect = HelperUnavailable("not installed")
+ client = AccessibilityClient("dev", manager=manager)
+ with pytest.raises(HelperUnavailable):
+ client.connect()
+
+
+def test_offline_device_fails_fast_without_touching_uiautomator(composite):
+ client, helper, u2, factory, clock = composite
+ helper.get_screen_data.side_effect = RuntimeError("connection refused")
+ clock["state"] = "unauthorized"
+ with pytest.raises(DeviceOfflineError, match="unauthorized"):
+ client.get_screen_data()
+ factory.assert_not_called()
+
+ clock["state"] = None
+ with pytest.raises(DeviceOfflineError, match="does not list it"):
+ client.get_screen_data()
+ factory.assert_not_called()
+
+
+def test_backend_history_and_listeners_track_every_switch(composite):
+ client, helper, u2, _, clock = composite
+ seen = []
+ client.add_backend_listener(lambda prev, new, reason: seen.append((prev, new, reason)))
+ helper.get_hierarchy.side_effect = [
+ "",
+ RuntimeError("tunnel gone"),
+ "",
+ ]
+ u2.get_hierarchy.return_value = ""
+
+ client.get_hierarchy() # helper
+ client.get_hierarchy() # helper fails -> uiautomator
+ clock["now"] = 60.0
+ client.get_hierarchy() # helper recovers
+
+ assert [(p, n) for p, n, _ in seen] == [
+ (None, "helper"),
+ ("helper", "uiautomator"),
+ ("uiautomator", "helper"),
+ ]
+ assert seen[1][2] == "RuntimeError: tunnel gone"
+ assert seen[2][2] == "helper recovered"
+ assert [h["backend"] for h in client.backend_history] == ["helper", "uiautomator", "helper"]
+
+ summary = hierarchy_backend_summary(client)
+ assert summary["backend"] == "helper" and len(summary["switches"]) == 2
+ sentence = hierarchy_backend_sentence(client, relative_time=lambda t: "T+00:10")
+ assert sentence.startswith("UI hierarchy source: the Artemis accessibility helper")
+ assert "switched to UIAutomator2 at T+00:10 because RuntimeError: tunnel gone" in sentence
+
+
+def test_listener_errors_do_not_break_the_client(composite):
+ client, helper, _, _, _ = composite
+ client.add_backend_listener(lambda *_: (_ for _ in ()).throw(RuntimeError("boom")))
+ helper.get_hierarchy.return_value = ""
+ assert client.get_hierarchy() == ""
+
+
+def test_sentence_is_none_before_first_use(composite):
+ client, *_ = composite
+ assert hierarchy_backend_sentence(client) is None
+ assert hierarchy_backend_summary(client) is None
+
+
+def test_helper_retry_releases_uiautomation_first(composite):
+ """While our UIAutomator2 server lives the helper cannot bind; stop it before retrying."""
+ client, helper, u2, _, clock = composite
+ helper.get_hierarchy.side_effect = [RuntimeError("tunnel gone"), ""]
+ u2.get_hierarchy.return_value = ""
+ assert client.get_hierarchy() == ""
+ u2.stop_server.assert_not_called() # inside the retry window nothing is touched
+
+ clock["now"] = 31.0
+ assert client.get_hierarchy() == ""
+ u2.stop_server.assert_called_once()
+
+
+def test_disconnect_stops_the_uiautomator_server_it_started(composite):
+ client, helper, u2, _, _ = composite
+ helper.connect.side_effect = HelperUnavailable("nope")
+ client.connect()
+ client.disconnect()
+ u2.disconnect.assert_called_once_with(stop_server=True)
diff --git a/tests/unit/core/test_device_smoke.py b/tests/unit/core/test_device_smoke.py
index 605d54b..8fd6e36 100644
--- a/tests/unit/core/test_device_smoke.py
+++ b/tests/unit/core/test_device_smoke.py
@@ -93,6 +93,7 @@ async def test_success_reports_bytes_elements_serial_and_elapsed(patch_controlle
"elapsed_seconds",
"screenshot_bytes",
"element_count",
+ "hierarchy_backend",
"error",
"fix",
}
diff --git a/tests/unit/mcp/test_diagnose_tool.py b/tests/unit/mcp/test_diagnose_tool.py
index 1efd6dd..cd4627f 100644
--- a/tests/unit/mcp/test_diagnose_tool.py
+++ b/tests/unit/mcp/test_diagnose_tool.py
@@ -279,10 +279,21 @@ def _run(
cleanup=0,
validate=None,
smoke=None,
+ helper_status=None,
+ helper_provision=None,
+ backend="auto",
**kwargs,
):
"""Run the tool with every side-effecting collaborator stubbed."""
with ExitStack() as stack:
+ fake_helper = MagicMock()
+ fake_helper.status = MagicMock(
+ side_effect=lambda serial: dict(helper_status or _helper_healthy(serial))
+ )
+ fake_helper.provision = helper_provision or MagicMock(return_value=_provision_ok())
+ stack.enter_context(patch.object(diagnose, "helper_manager", fake_helper))
+ stack.enter_context(patch.object(diagnose, "_hierarchy_backend", lambda: backend))
+ kwargs["_fake_helper"] = fake_helper
stack.enter_context(
patch.object(
diagnose.readiness_engine,
@@ -342,7 +353,35 @@ def _run(
smoke or AsyncMock(return_value=_smoke_ok()),
)
)
- return asyncio.run(mobile_diagnose(**kwargs))
+ fake_helper = kwargs.pop("_fake_helper")
+ result = asyncio.run(mobile_diagnose(**kwargs))
+ result["_fake_helper"] = fake_helper
+ return result
+
+
+def _helper_healthy(serial: str = "pixel-1") -> dict:
+ return {
+ "package": "com.artemis.helper",
+ "installed": True,
+ "installed_version": 2,
+ "bundled_version": 2,
+ "bundled_apk_present": True,
+ "outdated": False,
+ "enabled": True,
+ "forward_port": 41234,
+ "reachable": True,
+ "reported_version": 2,
+ "transport_id": "7",
+ "session": None,
+ }
+
+
+def _provision_ok():
+ from artemis.runtime.helper_manager import ProvisionResult
+
+ return ProvisionResult(
+ ok=True, action="installed", installed_version=2, bundled_version=2, enabled=True
+ )
def _smoke_ok(serial: str = "pixel-1") -> dict:
@@ -433,6 +472,7 @@ def test_ready_environment_reports_ready_with_slim_shape(temp_trace_env):
"android_version": "15",
"is_locked": False,
"is_emulator": False,
+ "accessibility_helper": {**_helper_healthy(), "serial": "pixel-1", "backend": "auto"},
}
assert result["host"] == {
"server_python": "/proj/.venv/bin/python",
@@ -946,6 +986,11 @@ def test_attempt_fix_heals_corrupted_keys_then_rechecks(temp_trace_env):
patch.object(diagnose.DeviceExecutionLock, "get_active_owners", return_value={}),
patch.object(diagnose.DeviceExecutionLock, "get_queued_tasks", return_value=[]),
patch.object(IntegrationHostProbe, "probe", AsyncMock(return_value=_host())),
+ patch.object(
+ diagnose,
+ "_helper_status",
+ lambda serial: {**_helper_healthy(serial), "serial": serial, "backend": "auto"},
+ ),
):
result = asyncio.run(mobile_diagnose(attempt_fix=True))
@@ -1139,3 +1184,104 @@ def test_logs_surface_recent_errors_and_last_failed_task(temp_trace_env, monkeyp
"Traceback (most recent call last):",
"RuntimeError: device offline",
]
+
+
+# --------------------------------------------------------------------------- #
+# Accessibility helper (UI hierarchy backend)
+# --------------------------------------------------------------------------- #
+
+
+def test_missing_helper_in_auto_mode_degrades_with_optional_step(temp_trace_env):
+ status = {
+ **_helper_healthy(),
+ "installed": False,
+ "installed_version": None,
+ "reachable": False,
+ }
+ result = _run(_healthy_probes(), helper_status=status)
+
+ assert result["verdict"] == "degraded"
+ helper = result["device"] is None or result["device"]["accessibility_helper"]
+ assert helper
+ optional = [s for s in result["next_steps"] if s.startswith("[OPTIONAL] Accessibility helper")]
+ assert len(optional) == 1 and "is not installed" in optional[0]
+ assert " Run: uv run artemis helper install --serial pixel-1" in result["next_steps"]
+ assert any("attempt_fix=true" in s for s in result["next_steps"])
+ result["_fake_helper"].provision.assert_not_called()
+
+
+def test_missing_helper_in_helper_mode_blocks(temp_trace_env):
+ status = {**_helper_healthy(), "installed": False, "reachable": False}
+ result = _run(_healthy_probes(), helper_status=status, backend="helper")
+
+ assert result["verdict"] == "blocked"
+ assert any(s.startswith("[REQUIRED] Accessibility helper") for s in result["next_steps"])
+
+
+def test_uiautomator_backend_ignores_helper_state(temp_trace_env):
+ status = {**_helper_healthy(), "installed": False, "reachable": False}
+ result = _run(_healthy_probes(), helper_status=status, backend="uiautomator")
+
+ assert result["verdict"] == "ready"
+ assert not any("Accessibility helper" in s for s in result["next_steps"])
+
+
+def test_outdated_helper_reports_versions(temp_trace_env):
+ status = {**_helper_healthy(), "installed_version": 1, "outdated": True}
+ result = _run(_healthy_probes(), helper_status=status)
+
+ step = next(s for s in result["next_steps"] if "Accessibility helper" in s)
+ assert "device has version 1, bundled is 2" in step
+
+
+def test_attempt_fix_provisions_missing_helper_on_idle_device(temp_trace_env):
+ status = {
+ **_helper_healthy(),
+ "installed": False,
+ "installed_version": None,
+ "reachable": False,
+ }
+ result = _run(_healthy_probes(), helper_status=status, attempt_fix=True)
+
+ result["_fake_helper"].provision.assert_called_once_with("pixel-1")
+ fix = next(f for f in result["fixes_applied"] if f["fix"] == "install_accessibility_helper")
+ assert fix["success"] is True and fix["skipped"] is False
+ assert "installed" in fix["message"]
+
+
+def test_attempt_fix_skips_helper_install_while_task_holds_device(temp_trace_env):
+ status = {**_helper_healthy(), "installed": False, "reachable": False}
+ result = _run(
+ _healthy_probes(),
+ helper_status=status,
+ attempt_fix=True,
+ active_owners={"pixel-1": _owner()},
+ )
+
+ result["_fake_helper"].provision.assert_not_called()
+ fix = next(f for f in result["fixes_applied"] if f["fix"] == "install_accessibility_helper")
+ assert fix["skipped"] is True
+
+
+def test_attempt_fix_leaves_healthy_helper_alone(temp_trace_env):
+ result = _run(_healthy_probes(), attempt_fix=True)
+
+ result["_fake_helper"].provision.assert_not_called()
+ assert not any(f["fix"] == "install_accessibility_helper" for f in result["fixes_applied"])
+
+
+def test_disabled_helper_step_includes_the_manual_path(temp_trace_env):
+ from artemis.runtime.helper_manager import MANUAL_ENABLE_PATH
+
+ status = {**_helper_healthy(), "enabled": False, "reachable": False}
+ result = _run(_healthy_probes(), helper_status=status)
+ step = next(s for s in result["next_steps"] if "Accessibility helper" in s)
+ assert "service is disabled" in step
+ assert any(MANUAL_ENABLE_PATH in s for s in result["next_steps"])
+
+
+def test_newer_helper_than_bundle_is_not_a_finding(temp_trace_env):
+ status = {**_helper_healthy(), "installed_version": 9, "newer_than_bundled": True}
+ result = _run(_healthy_probes(), helper_status=status)
+ assert result["verdict"] == "ready"
+ assert not any("Accessibility helper" in s for s in result["next_steps"])
diff --git a/tests/unit/runtime/test_helper_bundle.py b/tests/unit/runtime/test_helper_bundle.py
new file mode 100644
index 0000000..45a2c3c
--- /dev/null
+++ b/tests/unit/runtime/test_helper_bundle.py
@@ -0,0 +1,52 @@
+"""The bundled helper APK, its manifest, and the Gradle version must agree.
+
+``helper_manager`` decides installs and upgrades from ``helper_manifest.json``,
+so a rebuilt APK without a regenerated manifest (or a version bump in Gradle
+without a rebuild) would silently ship the wrong version gate.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+
+import pytest
+
+from artemis.runtime.helper_manager import (
+ BUNDLED_APK_PATH,
+ BUNDLED_MANIFEST_PATH,
+ HELPER_PACKAGE_DIR,
+ PACKAGE_NAME,
+ load_bundled_helper,
+)
+
+pytestmark = pytest.mark.skipif(
+ not BUNDLED_APK_PATH.is_file(), reason="bundled helper APK is not present in this checkout"
+)
+
+
+def test_manifest_matches_apk_and_gradle_version():
+ manifest = json.loads(BUNDLED_MANIFEST_PATH.read_text(encoding="utf-8"))
+ assert manifest["package"] == PACKAGE_NAME
+ assert manifest["sha256"] == hashlib.sha256(BUNDLED_APK_PATH.read_bytes()).hexdigest(), (
+ "ArtemisAccessibilityHelper.apk was rebuilt without regenerating helper_manifest.json; "
+ "run packages/artemis-accessibility-helper/build_apk.sh"
+ )
+
+ gradle = (HELPER_PACKAGE_DIR / "app" / "build.gradle.kts").read_text(encoding="utf-8")
+ version_code = int(re.search(r"versionCode\s*=\s*(\d+)", gradle).group(1))
+ version_name = re.search(r'versionName\s*=\s*"([^"]+)"', gradle).group(1)
+ assert manifest["version_code"] == version_code
+ assert manifest["version_name"] == version_name
+
+
+def test_load_bundled_helper_reads_manifest():
+ bundled = load_bundled_helper()
+ assert bundled is not None
+ assert bundled.apk_path == BUNDLED_APK_PATH
+ assert bundled.version_code >= 2 # first version that reports version_code on /ping
+
+
+def test_load_bundled_helper_returns_none_when_missing(tmp_path):
+ assert load_bundled_helper(tmp_path / "x.apk", tmp_path / "x.json") is None
diff --git a/tests/unit/runtime/test_helper_manager.py b/tests/unit/runtime/test_helper_manager.py
new file mode 100644
index 0000000..6acc976
--- /dev/null
+++ b/tests/unit/runtime/test_helper_manager.py
@@ -0,0 +1,716 @@
+"""Lifecycle of the Accessibility Helper: provision, attach, reattach, detach.
+
+Every adb call goes through a scripted fake so the tests assert the exact
+command sequence the manager issues, and the HTTP ping is a stub keyed by host
+port so "the service answers" is a switch the test flips.
+"""
+
+from __future__ import annotations
+
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+import subprocess
+
+import pytest
+
+from artemis.runtime import helper_manager as hm
+from artemis.runtime.helper_manager import (
+ DEVICE_PORT,
+ SERVICE_NAME,
+ AccessibilityHelperManager,
+ BundledHelper,
+ HelperUnavailable,
+)
+
+SERIAL = "pixel-1"
+OTHER = "pixel-2"
+
+
+class FakeAdb:
+ """Scripted adb: records commands and answers from a small device model."""
+
+ def __init__(self):
+ self.calls: list[list[str]] = []
+ self.installed: dict[str, int | None] = {SERIAL: None, OTHER: None}
+ self.enabled_services: dict[str, str] = {SERIAL: "", OTHER: ""}
+ self.transport: dict[str, str] = {SERIAL: "3", OTHER: "4"}
+ self.forwards: list[tuple[str, int, int]] = [] # (serial, local, remote)
+ self.next_port = 40000
+ self.install_result = "Success"
+ # Number of times the system "prunes" a freshly written service setting.
+ self.prune_writes = 0
+ # Serials whose dead service was re-bound by a settings toggle.
+ self.revived: set[str] = set()
+ # HTTP protocol the installed helper speaks (the bundled build speaks 2).
+ self.protocol: dict[str, int] = {SERIAL: 2, OTHER: 2}
+ # Last token delivered by `am broadcast`, per serial.
+ self.tokens: dict[str, str] = {}
+ # Process table lines (PID ARGS) of UiAutomation holders, per serial.
+ self.holders: dict[str, list[str]] = {SERIAL: [], OTHER: []}
+ self.killed: list[tuple[str, int]] = []
+
+ def __call__(self, args: list[str]) -> subprocess.CompletedProcess:
+ self.calls.append(list(args))
+ out, code = "", 0
+ if args[:2] == ["devices", "-l"]:
+ lines = ["List of devices attached"]
+ for serial, tid in self.transport.items():
+ lines.append(f"{serial} device product:p model:m device:d transport_id:{tid}")
+ out = "\n".join(lines) + "\n"
+ elif args[:2] == ["forward", "--list"]:
+ out = "".join(f"{s} tcp:{local} tcp:{r}\n" for s, local, r in self.forwards)
+ elif args[0] == "-s":
+ serial, rest = args[1], args[2:]
+ if rest[:3] == ["shell", "dumpsys", "package"]:
+ version = self.installed.get(serial)
+ out = (
+ f"Package [{rest[3]}]\n versionCode={version} minSdk=24\n"
+ if version is not None
+ else f"Unable to find package: {rest[3]}\n"
+ )
+ elif rest[:4] == ["shell", "settings", "get", "secure"]:
+ key = rest[4]
+ out = (
+ (self.enabled_services.get(serial) or "null") + "\n"
+ if key == "enabled_accessibility_services"
+ else "1\n"
+ )
+ elif rest[:4] == ["shell", "settings", "put", "secure"]:
+ if rest[4] == "enabled_accessibility_services":
+ if self.prune_writes > 0:
+ self.prune_writes -= 1
+ self.enabled_services[serial] = ""
+ else:
+ before = self.enabled_services.get(serial, "")
+ self.enabled_services[serial] = rest[5]
+ # AccessibilityManager binds the service when it is (re)added.
+ if SERVICE_NAME in rest[5] and SERVICE_NAME not in before:
+ self.revived.add(serial)
+ elif rest[0] == "install":
+ if self.install_result == "Success":
+ self.installed[serial] = self.bundled_version
+ self.protocol[serial] = 2
+ out = "Success\n"
+ else:
+ out, code = self.install_result, 1
+ elif rest[0] == "uninstall":
+ self.installed[serial] = None
+ out = "Success\n"
+ elif rest[:2] == ["forward", "--no-rebind"]:
+ port = self.next_port
+ self.next_port += 1
+ self.forwards.append((serial, port, int(rest[3].split(":")[1])))
+ out = f"{port}\n"
+ elif rest[:2] == ["forward", "--remove"]:
+ port = int(rest[2].split(":")[1])
+ self.forwards = [f for f in self.forwards if not (f[0] == serial and f[1] == port)]
+ elif rest[:2] == ["shell", "input"]:
+ pass
+ elif rest[:3] == ["shell", "am", "broadcast"]:
+ assert rest[3:8] == ["-n", hm.TOKEN_RECEIVER, "-a", hm.TOKEN_ACTION, "--es"]
+ self.tokens[serial] = rest[9]
+ out = "Broadcasting: Intent { act=com.artemis.helper.SET_TOKEN }\nBroadcast completed: result=0\n"
+ elif rest[:3] == ["shell", "am", "start"]:
+ self.settings_opened = getattr(self, "settings_opened", 0) + 1
+ elif rest[:2] == ["shell", "ps"]:
+ out = "PID ARGS" + "".join(chr(10) + line for line in self.holders.get(serial, []))
+ elif rest[:2] == ["shell", "kill"]:
+ pid = int(rest[2])
+ self.killed.append((serial, pid))
+ self.holders[serial] = [
+ line for line in self.holders.get(serial, []) if not line.startswith(f"{pid} ")
+ ]
+ elif rest[:3] == ["shell", "am", "force-stop"]:
+ pass
+ else:
+ raise AssertionError(f"unexpected adb call: {args}")
+ else:
+ raise AssertionError(f"unexpected adb call: {args}")
+ return subprocess.CompletedProcess(args, code, out, "")
+
+ bundled_version = 2
+
+
+class FakePing:
+ """Answers on host ports whose forward targets a device running the service."""
+
+ def __init__(self, adb: FakeAdb):
+ self.adb = adb
+ self.dead: set[str] = set() # serials whose service is down
+ self.count = 0
+
+ def __call__(self, port: int):
+ self.count += 1
+ for serial, local, remote in self.adb.forwards:
+ if local == port and remote == DEVICE_PORT:
+ if serial in self.dead and serial not in self.adb.revived:
+ return None
+ if any(
+ not line.split(None, 1)[1].startswith("sh ")
+ for line in self.adb.holders.get(serial, [])
+ ):
+ return None # suppressed by UiAutomation
+ if self.adb.installed.get(serial) is None:
+ return None
+ if SERVICE_NAME not in (self.adb.enabled_services.get(serial) or ""):
+ return None
+ return {
+ "success": True,
+ "version_code": self.adb.installed[serial],
+ "version_name": "1.1.0",
+ "protocol_version": self.adb.protocol.get(serial, 2),
+ "auth_required": self.adb.protocol.get(serial, 2) >= 2,
+ "token_set": serial in self.adb.tokens,
+ }
+ return None
+
+
+@pytest.fixture
+def bundled(tmp_path: Path) -> BundledHelper:
+ apk = tmp_path / "helper.apk"
+ apk.write_bytes(b"apk")
+ return BundledHelper(apk_path=apk, version_code=2, version_name="1.1.0", sha256="x")
+
+
+@pytest.fixture
+def env(tmp_path: Path, monkeypatch, bundled):
+ monkeypatch.setattr(hm, "get_temp_dir", lambda _name: tmp_path / "mutex")
+ adb = FakeAdb()
+ ping = FakePing(adb)
+ auto = {"install": True}
+ manager = AccessibilityHelperManager(
+ run_adb=adb,
+ ping=ping,
+ bundled=bundled,
+ sleep=lambda _s: None,
+ token_path=tmp_path / "token" / "session.token",
+ auto_install=lambda: auto["install"],
+ )
+ manager.auto_install_switch = auto # test handle
+ return adb, ping, manager
+
+
+def _calls(adb: FakeAdb, prefix: list[str]) -> list[list[str]]:
+ return [c for c in adb.calls if c[: len(prefix)] == prefix]
+
+
+def test_status_and_session_repr_do_not_expose_token(env):
+ _, _, manager = env
+ session = manager.attach(SERIAL)
+ status = manager.status(SERIAL)
+ assert session.token
+ assert "token" not in status["session"]
+ assert session.token not in str(status)
+ assert session.token not in repr(session)
+
+
+def test_concurrent_managers_share_the_first_host_token(tmp_path, monkeypatch):
+ monkeypatch.setattr(hm, "get_temp_dir", lambda _name: tmp_path / "mutex")
+ token_path = tmp_path / "token" / "session.token"
+ managers = [AccessibilityHelperManager(token_path=token_path) for _ in range(8)]
+ with ThreadPoolExecutor(max_workers=8) as executor:
+ tokens = list(executor.map(lambda manager: manager.host_token(), managers))
+ assert len(set(tokens)) == 1
+ assert token_path.read_text() == tokens[0]
+
+
+# --------------------------------------------------------------------------- #
+# Provision
+# --------------------------------------------------------------------------- #
+
+
+def test_provision_installs_and_enables_when_missing(env):
+ adb, _, manager = env
+ result = manager.provision(SERIAL)
+
+ assert result.ok and result.action == "installed"
+ assert result.installed_version == 2 and result.bundled_version == 2
+ install = _calls(adb, ["-s", SERIAL, "install"])
+ assert install == [["-s", SERIAL, "install", "-r", "-g", str(manager.bundled.apk_path)]]
+ assert adb.enabled_services[SERIAL] == SERVICE_NAME
+
+
+def test_provision_upgrades_outdated_and_leaves_newer_alone(env):
+ adb, _, manager = env
+ adb.installed[SERIAL] = 1
+ assert manager.provision(SERIAL).action == "upgraded"
+ assert adb.installed[SERIAL] == 2
+
+ adb.installed[SERIAL] = 5
+ adb.calls.clear()
+ result = manager.provision(SERIAL)
+ assert result.action == "up_to_date" and result.installed_version == 5
+ assert not _calls(adb, ["-s", SERIAL, "install"])
+
+
+def test_provision_up_to_date_is_cheap_and_keeps_other_services(env):
+ adb, _, manager = env
+ adb.installed[SERIAL] = 2
+ adb.enabled_services[SERIAL] = "com.android.talkback/.TalkBackService"
+ result = manager.provision(SERIAL)
+
+ assert result.ok and result.action == "up_to_date"
+ assert adb.enabled_services[SERIAL] == f"com.android.talkback/.TalkBackService:{SERVICE_NAME}"
+ assert not _calls(adb, ["-s", SERIAL, "install"])
+
+
+def test_provision_without_bundled_apk_fails_only_when_not_installed(env):
+ adb, ping, _ = env
+ manager = AccessibilityHelperManager(
+ run_adb=adb, ping=ping, bundled=None, sleep=lambda _s: None
+ )
+ result = manager.provision(SERIAL)
+ assert not result.ok and result.action == "failed" and "no bundled APK" in result.error
+
+ adb.installed[SERIAL] = 1
+ result = manager.provision(SERIAL)
+ assert result.ok and result.action == "up_to_date"
+
+
+def test_provision_reports_install_failure(env):
+ adb, _, manager = env
+ adb.install_result = "Failure [INSTALL_FAILED_VERSION_DOWNGRADE]"
+ result = manager.provision(SERIAL)
+ assert not result.ok and result.action == "failed"
+ assert "INSTALL_FAILED_VERSION_DOWNGRADE" in result.error
+
+
+def test_provision_mutex_is_released_and_stale_lock_is_broken(env, tmp_path):
+ adb, _, manager = env
+ manager.provision(SERIAL)
+ mutex_dir = tmp_path / "mutex"
+ assert not list(mutex_dir.glob("*.mutex"))
+
+ # A crashed process left its mutex behind long ago: provisioning proceeds.
+ import hashlib
+ import os
+ import time
+
+ digest = hashlib.sha256(SERIAL.encode()).hexdigest()[:16]
+ stale = mutex_dir / f"{digest}.mutex"
+ stale.write_text("1")
+ old = time.time() - 1000
+ os.utime(stale, (old, old))
+ assert manager.provision(SERIAL).ok
+ assert not stale.exists()
+
+
+# --------------------------------------------------------------------------- #
+# Attach / detach
+# --------------------------------------------------------------------------- #
+
+
+def test_attach_provisions_forwards_and_records_transport(env):
+ adb, _, manager = env
+ session = manager.attach(SERIAL)
+
+ assert session.local_port == 40000 and session.owns_forward
+ assert session.transport_id == "3" and session.version_code == 2
+ assert adb.forwards == [(SERIAL, 40000, DEVICE_PORT)]
+ forward_calls = _calls(adb, ["-s", SERIAL, "forward"])
+ assert forward_calls == [
+ ["-s", SERIAL, "forward", "--no-rebind", "tcp:0", f"tcp:{DEVICE_PORT}"]
+ ]
+
+
+def test_two_devices_get_distinct_host_ports(env):
+ adb, _, manager = env
+ first = manager.attach(SERIAL)
+ second = manager.attach(OTHER)
+
+ assert first.local_port != second.local_port
+ assert {(s, local) for s, local, _ in adb.forwards} == {(SERIAL, 40000), (OTHER, 40001)}
+ assert manager.session(SERIAL) is first and manager.session(OTHER) is second
+
+
+def test_attach_reuses_forward_created_by_another_process(env):
+ adb, _, manager = env
+ adb.installed[SERIAL] = 2
+ adb.enabled_services[SERIAL] = SERVICE_NAME
+ adb.forwards.append((SERIAL, 39999, DEVICE_PORT))
+
+ session = manager.attach(SERIAL)
+ assert session.local_port == 39999 and not session.owns_forward
+ assert not _calls(adb, ["-s", SERIAL, "forward", "--no-rebind"])
+
+ manager.detach(SERIAL)
+ assert adb.forwards == [(SERIAL, 39999, DEVICE_PORT)] # not ours to remove
+
+
+def test_attach_is_idempotent_while_session_is_healthy(env):
+ adb, ping, manager = env
+ first = manager.attach(SERIAL)
+ adb.calls.clear()
+ again = manager.attach(SERIAL)
+ assert again is first
+ assert not _calls(adb, ["-s", SERIAL, "install"])
+ assert not _calls(adb, ["-s", SERIAL, "forward"])
+
+
+def test_lazy_attach_never_installs(env):
+ adb, _, manager = env
+ with pytest.raises(HelperUnavailable):
+ manager.attach(SERIAL, provision=False)
+ assert not _calls(adb, ["-s", SERIAL, "install"])
+ assert not _calls(adb, ["-s", SERIAL, "shell", "settings", "put"])
+ # The dead forward it created for the probe is cleaned up again.
+ assert adb.forwards == []
+ assert manager.session(SERIAL) is None
+
+
+def test_lazy_attach_connects_to_running_helper(env):
+ adb, _, manager = env
+ adb.installed[SERIAL] = 2
+ adb.enabled_services[SERIAL] = SERVICE_NAME
+ session = manager.attach(SERIAL, provision=False)
+ assert session.local_port == 40000
+ assert not _calls(adb, ["-s", SERIAL, "install"])
+
+
+def test_replug_changes_transport_and_rebuilds_tunnel(env):
+ adb, _, manager = env
+ first = manager.attach(SERIAL)
+
+ # Unplug: adb drops the forward and assigns a new transport on replug.
+ adb.forwards.clear()
+ adb.transport[SERIAL] = "9"
+ adb.calls.clear()
+
+ second = manager.attach(SERIAL)
+ assert second is not first
+ assert second.transport_id == "9" and second.local_port == 40001
+ # The vanished forward is not "removed" (adb already dropped it).
+ assert not _calls(adb, ["-s", SERIAL, "forward", "--remove"])
+ assert adb.forwards == [(SERIAL, 40001, DEVICE_PORT)]
+
+
+def test_reattach_after_request_failure_rebuilds_without_install(env):
+ adb, ping, manager = env
+ first = manager.attach(SERIAL)
+ adb.calls.clear()
+
+ session = manager.reattach(SERIAL)
+ assert session.local_port != first.local_port
+ assert _calls(adb, ["-s", SERIAL, "forward", "--remove"]) == [
+ ["-s", SERIAL, "forward", "--remove", f"tcp:{first.local_port}"]
+ ]
+ assert not _calls(adb, ["-s", SERIAL, "install"])
+
+
+def test_reattach_revives_a_killed_service_without_installing(env):
+ adb, ping, manager = env
+ first = manager.attach(SERIAL)
+ ping.dead.add(SERIAL) # force-stopped: enabled in settings, but not running
+ adb.revived.discard(SERIAL)
+ adb.calls.clear()
+
+ session = manager.reattach(SERIAL)
+ assert session.local_port != first.local_port
+ assert not _calls(adb, ["-s", SERIAL, "install"])
+ puts = _calls(
+ adb, ["-s", SERIAL, "shell", "settings", "put", "secure", "enabled_accessibility_services"]
+ )
+ assert [p[-1] for p in puts] == ["null", SERVICE_NAME] # removed, then re-added
+
+
+def test_reattach_raises_when_revive_does_not_help(env):
+ adb, ping, manager = env
+ manager.attach(SERIAL)
+ adb.calls.clear()
+ ping.dead.add(SERIAL)
+ adb.revived.discard(SERIAL)
+ adb.installed[SERIAL] = None # package vanished: nothing to re-bind
+ with pytest.raises(HelperUnavailable):
+ manager.reattach(SERIAL)
+ assert manager.session(SERIAL) is None
+ assert not _calls(adb, ["-s", SERIAL, "install"])
+
+
+def test_lazy_first_attach_does_not_toggle_settings(env):
+ adb, ping, manager = env
+ adb.installed[SERIAL] = 2
+ adb.enabled_services[SERIAL] = SERVICE_NAME
+ ping.dead.add(SERIAL)
+ adb.revived.discard(SERIAL)
+ with pytest.raises(HelperUnavailable):
+ manager.attach(SERIAL, provision=False)
+ assert not _calls(adb, ["-s", SERIAL, "shell", "settings", "put"])
+
+
+def test_detach_removes_only_owned_forward(env):
+ adb, _, manager = env
+ manager.attach(SERIAL)
+ manager.attach(OTHER)
+ manager.detach(SERIAL)
+ assert adb.forwards == [(OTHER, 40001, DEVICE_PORT)]
+ assert manager.session(SERIAL) is None and manager.session(OTHER) is not None
+
+
+def test_status_is_read_only(env):
+ adb, _, manager = env
+ adb.installed[SERIAL] = 1
+ status = manager.status(SERIAL)
+ assert status["installed"] and status["installed_version"] == 1
+ assert status["bundled_version"] == 2 and status["outdated"] is True
+ assert status["enabled"] is False and status["reachable"] is False
+ assert status["transport_id"] == "3"
+ assert not _calls(adb, ["-s", SERIAL, "install"])
+ assert not _calls(adb, ["-s", SERIAL, "forward"])
+
+
+def test_uninstall_disables_service_and_removes_package(env):
+ adb, _, manager = env
+ manager.attach(SERIAL)
+ assert manager.uninstall(SERIAL)
+ assert adb.installed[SERIAL] is None
+ assert SERVICE_NAME not in adb.enabled_services[SERIAL]
+ assert adb.forwards == []
+
+
+def test_transport_id_parsing_handles_missing_device():
+ adb = FakeAdb()
+ manager = AccessibilityHelperManager(run_adb=adb, ping=lambda _p: None, bundled=None)
+ assert manager.transport_id("nope") is None
+ assert manager.transport_id(SERIAL) == "3"
+
+
+def test_enable_retries_when_the_system_prunes_the_setting(env):
+ adb, _, manager = env
+ adb.prune_writes = 2 # AccessibilityManager drops the first two writes
+ result = manager.provision(SERIAL)
+ assert result.ok and result.enabled
+ puts = _calls(
+ adb, ["-s", SERIAL, "shell", "settings", "put", "secure", "enabled_accessibility_services"]
+ )
+ assert len(puts) == 3
+ assert adb.enabled_services[SERIAL] == SERVICE_NAME
+
+
+def test_enable_gives_up_after_bounded_attempts(env):
+ adb, _, manager = env
+ adb.prune_writes = 99
+ result = manager.provision(SERIAL)
+ assert not result.ok and result.action == "installed"
+ # The person gets the exact manual path and the settings screen is opened for them.
+ assert hm.MANUAL_ENABLE_PATH in result.error
+ assert adb.settings_opened == 1
+ puts = _calls(
+ adb, ["-s", SERIAL, "shell", "settings", "put", "secure", "enabled_accessibility_services"]
+ )
+ assert len(puts) == hm._ENABLE_ATTEMPTS
+
+
+def test_provision_emits_install_and_upgrade_events_before_the_install(env):
+ adb, _, manager = env
+ events: list[tuple[str, dict]] = []
+
+ def on_event(name, details):
+ # Fired before adb install ran: nothing installed yet at this point.
+ events.append((name, details, adb.installed[SERIAL]))
+
+ manager.provision(SERIAL, on_event=on_event)
+ assert [(e[0], e[2]) for e in events] == [("installing", None)]
+ assert events[0][1]["to_version"] == 2 and events[0][1]["serial"] == SERIAL
+
+ events.clear()
+ adb.installed[SERIAL] = 1
+ manager.provision(SERIAL, on_event=on_event)
+ assert [(e[0], e[1]["from_version"], e[2]) for e in events] == [("upgrading", 1, 1)]
+
+ events.clear()
+ manager.provision(SERIAL, on_event=on_event) # up to date: silent
+ assert events == []
+
+
+def test_attach_forwards_events_to_provision(env):
+ adb, _, manager = env
+ seen = []
+ manager.attach(SERIAL, on_event=lambda name, details: seen.append(name))
+ assert seen == ["installing"]
+
+
+def test_status_probes_with_a_temporary_forward_when_no_tunnel_exists(env):
+ adb, _, manager = env
+ adb.installed[SERIAL] = 2
+ adb.enabled_services[SERIAL] = SERVICE_NAME
+ status = manager.status(SERIAL)
+ assert status["reachable"] is True and status["tunnel"] == "probe"
+ assert status["forward_port"] is None
+ assert adb.forwards == [] # the probe forward was removed again
+ assert _calls(adb, ["-s", SERIAL, "forward", "--remove"]) == [
+ ["-s", SERIAL, "forward", "--remove", "tcp:40000"]
+ ]
+
+
+def test_status_reports_session_and_shared_tunnels(env):
+ adb, _, manager = env
+ manager.attach(SERIAL)
+ assert manager.status(SERIAL)["tunnel"] == "session"
+ manager.detach(SERIAL)
+ adb.forwards.append((SERIAL, 39999, DEVICE_PORT))
+ status = manager.status(SERIAL)
+ assert status["tunnel"] == "shared" and status["forward_port"] == 39999
+
+
+def test_status_does_not_probe_when_not_installed_or_disabled(env):
+ adb, _, manager = env
+ status = manager.status(SERIAL)
+ assert status["reachable"] is False and status["tunnel"] is None
+ assert not _calls(adb, ["-s", SERIAL, "forward"])
+
+
+def test_status_flags_a_helper_newer_than_the_bundle(env):
+ adb, _, manager = env
+ adb.installed[SERIAL] = 7
+ status = manager.status(SERIAL)
+ assert status["newer_than_bundled"] is True and status["outdated"] is False
+
+
+# --------------------------------------------------------------------------- #
+# Session token, protocol gate, auto-install switch, per-device locking
+# --------------------------------------------------------------------------- #
+
+
+def test_attach_pushes_the_host_token_and_records_it(env, tmp_path):
+ adb, _, manager = env
+ session = manager.attach(SERIAL)
+ token = manager.host_token()
+ assert len(token) == 48 and session.token == token and session.protocol_version == 2
+ assert adb.tokens[SERIAL] == token
+ assert (tmp_path / "token" / "session.token").read_text() == token
+ # A second manager on the same host (another process) presents the same token.
+ other = AccessibilityHelperManager(
+ run_adb=adb,
+ ping=lambda _p: None,
+ bundled=None,
+ token_path=tmp_path / "token" / "session.token",
+ )
+ assert other.host_token() == token
+
+
+def test_push_token_is_repeated_on_demand(env):
+ adb, _, manager = env
+ manager.attach(SERIAL)
+ adb.tokens.clear()
+ assert manager.push_token(SERIAL) is True
+ assert adb.tokens[SERIAL] == manager.host_token()
+
+
+def test_old_protocol_is_reinstalled_when_the_caller_may_provision(env):
+ adb, _, manager = env
+ adb.installed[SERIAL] = 9 # newer versionCode than the bundle, but the old contract
+ adb.protocol[SERIAL] = 1
+ session = manager.attach(SERIAL)
+ assert _calls(adb, ["-s", SERIAL, "install"])
+ assert session.protocol_version == 2 and session.version_code == 2
+
+
+def test_old_protocol_blocks_a_lazy_attach_without_installing(env):
+ adb, _, manager = env
+ adb.installed[SERIAL] = 2
+ adb.enabled_services[SERIAL] = SERVICE_NAME
+ adb.protocol[SERIAL] = 1
+ with pytest.raises(HelperUnavailable, match="protocol 1"):
+ manager.attach(SERIAL, provision=False)
+ assert not _calls(adb, ["-s", SERIAL, "install"])
+ assert adb.forwards == []
+
+
+def test_auto_install_off_never_installs(env):
+ adb, _, manager = env
+ manager.auto_install_switch["install"] = False
+ with pytest.raises(HelperUnavailable, match="artemis helper install"):
+ manager.attach(SERIAL)
+ assert not _calls(adb, ["-s", SERIAL, "install"])
+
+ # An outdated but present helper is used as it is.
+ adb.installed[SERIAL] = 1
+ session = manager.attach(SERIAL)
+ assert session.version_code == 1
+ assert not _calls(adb, ["-s", SERIAL, "install"])
+ assert manager.status(SERIAL)["auto_install"] is False
+
+
+def test_provisioning_one_device_does_not_block_another(env):
+ import threading
+
+ _, _, manager = env
+ lock = manager._device_lock(SERIAL)
+ lock.acquire() # simulate a slow install in progress on SERIAL
+ try:
+ done = threading.Event()
+ result: dict = {}
+
+ def attach_other():
+ result["session"] = manager.attach(OTHER)
+ done.set()
+
+ threading.Thread(target=attach_other, daemon=True).start()
+ assert done.wait(2.0), "attach(OTHER) waited on SERIAL's lock"
+ assert result["session"].serial == OTHER
+ finally:
+ lock.release()
+
+
+def test_status_reports_protocol_and_token(env):
+ adb, _, manager = env
+ manager.attach(SERIAL)
+ status = manager.status(SERIAL)
+ assert status["protocol_version"] == 2 and status["protocol_supported"] is True
+ assert status["token_set"] is True and status["auto_install"] is True
+
+
+# --------------------------------------------------------------------------- #
+# UiAutomation holders (uiautomator2 / Appium) mute the helper
+# --------------------------------------------------------------------------- #
+
+U2_LINES = [
+ "24633 sh -c CLASSPATH=/data/local/tmp/u2.jar app_process / com.wetest.uia2.Main -p 9008",
+ "24635 app_process / com.wetest.uia2.Main -p 9008",
+]
+
+
+def test_attach_stops_a_running_uiautomator2_server_and_binds(env):
+ adb, _, manager = env
+ adb.installed[SERIAL] = 2
+ adb.enabled_services[SERIAL] = SERVICE_NAME
+ adb.holders[SERIAL] = list(U2_LINES)
+ session = manager.attach(SERIAL)
+ assert session.local_port == 40000
+ assert adb.killed == [(SERIAL, 24635)] # the app_process, not its sh wrapper
+ assert ["-s", SERIAL, "shell", "am", "force-stop", "com.github.uiautomator"] in adb.calls
+ # No re-bind toggle was needed once UiAutomation released the services
+ # (provisioning's idempotent accessibility_enabled=1 write is fine).
+ assert not _calls(
+ adb, ["-s", SERIAL, "shell", "settings", "put", "secure", "enabled_accessibility_services"]
+ )
+
+
+def test_lazy_attach_names_uiautomator2_instead_of_killing_it(env):
+ adb, _, manager = env
+ adb.installed[SERIAL] = 2
+ adb.enabled_services[SERIAL] = SERVICE_NAME
+ adb.holders[SERIAL] = list(U2_LINES)
+ with pytest.raises(HelperUnavailable, match="suppressed by uiautomator2's device server"):
+ manager.attach(SERIAL, provision=False)
+ assert adb.killed == []
+
+
+def test_appium_server_is_never_killed(env):
+ adb, _, manager = env
+ adb.installed[SERIAL] = 2
+ adb.enabled_services[SERIAL] = SERVICE_NAME
+ adb.holders[SERIAL] = ["31000 io.appium.uiautomator2.server.test"]
+ with pytest.raises(HelperUnavailable, match="Appium.*disableSuppressAccessibilityServices"):
+ manager.attach(SERIAL)
+ assert adb.killed == []
+ assert not _calls(
+ adb, ["-s", SERIAL, "shell", "settings", "put", "secure", "enabled_accessibility_services"]
+ )
+
+
+def test_uiautomation_holders_classifies_processes(env):
+ adb, _, manager = env
+ adb.holders[SERIAL] = U2_LINES + ["31000 io.appium.uiautomator2.server.test", "7 sh"]
+ holders = manager.uiautomation_holders(SERIAL)
+ assert holders == {"uiautomator2": [24635], "appium": [31000]}
diff --git a/tests/unit/test_hierarchy_backend_visibility.py b/tests/unit/test_hierarchy_backend_visibility.py
new file mode 100644
index 0000000..f6b332d
--- /dev/null
+++ b/tests/unit/test_hierarchy_backend_visibility.py
@@ -0,0 +1,385 @@
+"""The user is told which UI-hierarchy backend serves a run, everywhere they look.
+
+Covers the agent's announcement fan-out (startup progress, log trace, session
+device_info, status.json), the provisioning progress stage, the run_outcome
+environment block, the `mobile_manage_task` field, the `artemis helper` CLI
+(`--all`, status rendering) and the `artemis doctor` helper row.
+"""
+
+from __future__ import annotations
+
+import json
+from types import SimpleNamespace
+from unittest.mock import MagicMock, patch
+
+import pytest
+from typer.testing import CliRunner
+
+from artemis.clients.screen_client_factory import FallbackScreenClient
+from artemis.runtime import trace_store
+from artemis.runtime.helper_manager import HelperSession, ProvisionResult
+from artemis.sdk.agent import Agent
+
+
+# --------------------------------------------------------------------------- #
+# helpers
+# --------------------------------------------------------------------------- #
+
+
+def _composite(helper_version: str = "1.1.2") -> FallbackScreenClient:
+ helper = MagicMock()
+ helper.session = HelperSession(
+ serial="dev",
+ local_port=41000,
+ transport_id="1",
+ version_code=4,
+ version_name=helper_version,
+ owns_forward=True,
+ )
+ helper.get_hierarchy.return_value = ""
+ client = FallbackScreenClient(
+ "dev", helper=helper, uiautomator_factory=MagicMock(), state_probe=lambda _s: "device"
+ )
+ return client
+
+
+def _agent_with(client) -> Agent:
+ agent = object.__new__(Agent)
+ agent._ui_adb_client = client
+ return agent
+
+
+def _engine():
+ engine = MagicMock()
+ engine.current_session_id = "sess-1"
+ engine.get_relative_time = lambda ts: "T+00:05"
+ return engine
+
+
+# --------------------------------------------------------------------------- #
+# Agent announcement fan-out
+# --------------------------------------------------------------------------- #
+
+
+def test_announce_first_backend_reaches_every_sink(tmp_path, monkeypatch):
+ monkeypatch.setattr(trace_store, "TRACES_DIR", str(tmp_path))
+ trace_store.init_trace("sess-1", "goal", "flash")
+ client = _composite()
+ client.get_hierarchy() # helper becomes active
+ agent = _agent_with(client)
+ engine = _engine()
+ context = SimpleNamespace(data_engine=engine)
+
+ with patch("artemis.sdk.agent.publish_startup_progress") as progress:
+ agent._announce_hierarchy_backend(context, "sess-1", None, "helper", None)
+
+ stage, message = progress.call_args.args[:2]
+ assert stage == "hierarchy_backend"
+ assert message == "UI hierarchy source: Artemis accessibility helper v1.1.2"
+ assert progress.call_args.kwargs["backend"] == "helper"
+
+ trace_call = engine.record_trace.call_args.kwargs
+ assert trace_call["name"] == "hierarchy_backend" and trace_call["type"] == "log"
+ assert trace_call["payload"]["helper_version"] == "1.1.2"
+ assert trace_call["payload"]["level"] == "INFO"
+
+ info = engine.update_session_device_info.call_args.kwargs
+ assert info["hierarchy_backend"] == "helper"
+ assert info["hierarchy_backend_note"].startswith("UI hierarchy source: the Artemis")
+
+ status = trace_store.read_status("sess-1")
+ assert status["hierarchy_backend"] == "helper"
+ assert "accessibility helper v1.1.2" in status["hierarchy_backend_note"]
+
+
+def test_announce_switch_is_a_warning_with_the_reason(tmp_path, monkeypatch):
+ monkeypatch.setattr(trace_store, "TRACES_DIR", str(tmp_path))
+ client = _composite()
+ agent = _agent_with(client)
+ engine = _engine()
+ context = SimpleNamespace(data_engine=engine)
+
+ with patch("artemis.sdk.agent.publish_startup_progress") as progress:
+ agent._announce_hierarchy_backend(
+ context, "sess-1", "helper", "uiautomator", "HelperUnavailable: tunnel gone"
+ )
+
+ stage, message = progress.call_args.args[:2]
+ assert stage == "hierarchy_backend_changed"
+ assert message == (
+ "UI hierarchy source switched from Artemis accessibility helper to UIAutomator2 "
+ "because HelperUnavailable: tunnel gone"
+ )
+ assert engine.record_trace.call_args.kwargs["payload"]["level"] == "WARNING"
+ # No status.json for this session: silently skipped, nothing raised.
+ assert trace_store.read_status("sess-1") is None
+
+
+def test_announce_survives_sink_failures(monkeypatch, tmp_path):
+ monkeypatch.setattr(trace_store, "TRACES_DIR", str(tmp_path))
+ client = _composite()
+ agent = _agent_with(client)
+ engine = _engine()
+ engine.record_trace.side_effect = RuntimeError("db closed")
+ with patch("artemis.sdk.agent.publish_startup_progress", side_effect=OSError("no ipc")):
+ agent._announce_hierarchy_backend(
+ SimpleNamespace(data_engine=engine), "sess-1", None, "uiautomator", None
+ )
+
+
+@pytest.mark.asyncio
+async def test_connect_publishes_install_stage_and_registers_listener():
+ client = _composite()
+
+ def fake_connect(on_event=None):
+ on_event("installing", {"serial": "dev", "version_name": "1.1.2", "to_version": 4})
+ client._set_active("helper")
+
+ client.connect = fake_connect # type: ignore[method-assign]
+ agent = _agent_with(client)
+ engine = _engine()
+ context = SimpleNamespace(data_engine=engine)
+
+ with (
+ patch("artemis.sdk.agent.publish_startup_progress") as progress,
+ patch.object(trace_store, "update_trace_fields"),
+ ):
+ await agent._connect_screen_client(context, "sess-1")
+ stages = [c.args[0] for c in progress.call_args_list]
+ assert stages == ["uiautomator", "helper_install", "uiautomator_ready", "hierarchy_backend"]
+ install_msg = progress.call_args_list[1].args[1]
+ assert (
+ "first time" in install_msg and "v1.1.2" in install_msg and "3 seconds" in install_msg
+ )
+
+ # A later fallback goes through the registered listener.
+ progress.reset_mock()
+ client._set_active("uiautomator", "RuntimeError: tunnel gone")
+ assert progress.call_args.args[0] == "hierarchy_backend_changed"
+
+
+@pytest.mark.asyncio
+async def test_connect_without_provisioning_client_takes_no_callback():
+ plain = MagicMock(spec=["connect", "get_hierarchy"])
+ plain.connect = MagicMock() # signature without on_event
+ agent = _agent_with(plain)
+ with (
+ patch("artemis.sdk.agent.publish_startup_progress"),
+ patch.object(trace_store, "update_trace_fields"),
+ ):
+ await agent._connect_screen_client(SimpleNamespace(data_engine=None), "sess-1")
+ plain.connect.assert_called_once_with()
+
+
+@pytest.mark.asyncio
+async def test_reused_agent_records_backend_changes_only_for_current_session():
+ client = _composite()
+ agent = _agent_with(client)
+ context = SimpleNamespace(data_engine=None)
+ with (
+ patch("artemis.sdk.agent.publish_startup_progress") as progress,
+ patch.object(trace_store, "update_trace_fields"),
+ ):
+ await agent._connect_screen_client(context, "sess-1")
+ client._set_active("uiautomator", "tunnel gone")
+ client.disconnect()
+ await agent._connect_screen_client(context, "sess-2")
+ assert len(client.backend_history) == 1
+ progress.reset_mock()
+ client._set_active("uiautomator", "tunnel gone again")
+ assert progress.call_count == 1
+ assert progress.call_args.kwargs["session_id"] == "sess-2"
+
+
+# --------------------------------------------------------------------------- #
+# trace_store / engine / task manager
+# --------------------------------------------------------------------------- #
+
+
+def test_update_trace_fields_merges_and_ignores_missing(tmp_path, monkeypatch):
+ monkeypatch.setattr(trace_store, "TRACES_DIR", str(tmp_path))
+ assert trace_store.update_trace_fields("nope", hierarchy_backend="helper") is None
+ trace_store.init_trace("t1", "goal", "pro")
+ data = trace_store.update_trace_fields("t1", hierarchy_backend="helper", extra=1)
+ assert data["hierarchy_backend"] == "helper" and data["extra"] == 1
+ assert data["task_desc"] == "goal"
+
+
+def test_update_session_device_info_merges_fields():
+ from artemis.data_engine.engine import DataEngine
+
+ engine = object.__new__(DataEngine)
+ engine.current_session_id = "s"
+ session = SimpleNamespace(device_info={"device_id": "dev"})
+ engine.storage = MagicMock()
+ engine.storage.get_session.return_value = session
+ engine.update_session_device_info(hierarchy_backend="helper", hierarchy_backend_note="n")
+ engine.storage.update_session.assert_called_once_with(session)
+ assert session.device_info == {
+ "device_id": "dev",
+ "hierarchy_backend": "helper",
+ "hierarchy_backend_note": "n",
+ }
+
+
+def test_manage_task_status_exposes_hierarchy_backend(tmp_path, monkeypatch):
+ from mcp_server.tools import task_manager
+
+ monkeypatch.setattr(trace_store, "TRACES_DIR", str(tmp_path))
+ trace_store.init_trace("t9", "goal", "flash", device_serial="dev")
+ trace_store.update_trace_fields(
+ "t9", hierarchy_backend="uiautomator", hierarchy_backend_note="switched"
+ )
+ with patch.object(task_manager, "_reconcile_status", lambda *a, **k: None, create=True):
+ result = task_manager.mobile_manage_task(action="status", trace_id="t9")
+ assert result["hierarchy_backend"] == "uiautomator"
+ assert result["hierarchy_backend_note"] == "switched"
+
+
+# --------------------------------------------------------------------------- #
+# run_outcome environment block
+# --------------------------------------------------------------------------- #
+
+
+def test_run_outcome_extra_carries_environment(tmp_path):
+ from artemis.graph.checkpoints import read_run_outcome, write_run_outcome
+ from artemis.clients.screen_client_factory import hierarchy_backend_summary
+
+ client = _composite()
+ client.get_hierarchy()
+ client._set_active("uiautomator", "RuntimeError: gone")
+ outcome = MagicMock()
+ outcome.model_dump.return_value = {"task_status": "completed"}
+ write_run_outcome(tmp_path, outcome, {"environment": hierarchy_backend_summary(client)})
+ data = read_run_outcome(tmp_path)
+ assert data["environment"]["backend"] == "uiautomator"
+ assert data["environment"]["helper_version"] == "1.1.2"
+ assert data["environment"]["switches"][0]["reason"] == "RuntimeError: gone"
+
+
+# --------------------------------------------------------------------------- #
+# CLI: artemis helper
+# --------------------------------------------------------------------------- #
+
+
+def _status(**overrides):
+ base = {
+ "package": "com.artemis.helper",
+ "installed": True,
+ "installed_version": 4,
+ "bundled_version": 4,
+ "bundled_apk_present": True,
+ "outdated": False,
+ "newer_than_bundled": False,
+ "enabled": True,
+ "forward_port": None,
+ "tunnel": "probe",
+ "reachable": True,
+ "reported_version": 4,
+ "transport_id": "1",
+ "session": None,
+ }
+ base.update(overrides)
+ return base
+
+
+def test_helper_status_explains_probe_tunnel_and_newer_build(monkeypatch):
+ from artemis.interfaces.cli.commands import helper as helper_cli
+
+ monkeypatch.setattr(
+ helper_cli.helper_manager,
+ "status",
+ lambda serial: _status(installed_version=9, newer_than_bundled=True),
+ )
+ result = CliRunner().invoke(helper_cli.helper_app, ["status", "--serial", "dev"])
+ assert result.exit_code == 0, result.output
+ assert "temporary forward" in result.output
+ assert "newer than the bundled v4" in result.output
+
+
+def test_helper_install_all_targets_idle_devices_only(monkeypatch):
+ from artemis.interfaces.cli.commands import helper as helper_cli
+
+ devices = [
+ SimpleNamespace(serial="a", is_busy=False),
+ SimpleNamespace(serial="b", is_busy=True),
+ SimpleNamespace(serial="c", is_busy=False),
+ ]
+ monkeypatch.setattr(helper_cli.device_pool, "get_ready_devices", lambda: devices)
+ calls = []
+
+ def provision(serial, force=False, on_event=None):
+ calls.append(serial)
+ on_event("installing", {"version_name": "1.1.2"})
+ return ProvisionResult(
+ ok=True, action="installed", installed_version=4, bundled_version=4, enabled=True
+ )
+
+ monkeypatch.setattr(helper_cli.helper_manager, "provision", provision)
+ result = CliRunner().invoke(helper_cli.helper_app, ["install", "--all"])
+ assert result.exit_code == 0, result.output
+ assert calls == ["a", "c"]
+ assert "a: installing v1.1.2..." in result.output
+
+
+def test_helper_install_prints_manual_path_on_enable_failure(monkeypatch):
+ from artemis.interfaces.cli.commands import helper as helper_cli
+ from artemis.runtime.helper_manager import MANUAL_ENABLE_PATH
+
+ monkeypatch.setattr(
+ helper_cli.helper_manager,
+ "provision",
+ lambda serial, force=False, on_event=None: ProvisionResult(
+ ok=False,
+ action="installed",
+ installed_version=4,
+ bundled_version=4,
+ enabled=False,
+ error=f"This device rejected enabling the accessibility service from adb. {MANUAL_ENABLE_PATH}.",
+ ),
+ )
+ result = CliRunner().invoke(helper_cli.helper_app, ["install", "--serial", "dev"])
+ assert result.exit_code == 1
+ assert "Settings > Accessibility" in result.output
+
+
+# --------------------------------------------------------------------------- #
+# CLI: artemis doctor row
+# --------------------------------------------------------------------------- #
+
+
+def _adb_probe(serials):
+ return SimpleNamespace(
+ id="android_adb",
+ metadata={"devices": [{"serial": s, "state": "device"} for s in serials]},
+ )
+
+
+def test_doctor_row_suggests_preinstall_when_missing(monkeypatch):
+ from artemis.interfaces.cli.commands import doctor
+
+ with patch(
+ "artemis.runtime.helper_manager.helper_manager.status",
+ return_value=_status(installed=False, reachable=False),
+ ):
+ row = doctor._helper_row([_adb_probe(["dev"])])
+ assert row.status == "missing" and row.summary == "Not installed"
+ assert "artemis helper install --serial dev" in row.detail
+ assert "about 3 s" in row.detail
+
+
+def test_doctor_row_ready_mentions_uninstall(monkeypatch):
+ from artemis.interfaces.cli.commands import doctor
+
+ with patch("artemis.runtime.helper_manager.helper_manager.status", return_value=_status()):
+ row = doctor._helper_row([_adb_probe(["dev"])])
+ assert row.status == "pass" and "v4 on dev" in row.summary
+ assert "artemis helper uninstall --serial dev" in row.detail
+
+
+def test_doctor_row_absent_for_multiple_devices_or_uiautomator_backend(monkeypatch):
+ from artemis.interfaces.cli.commands import doctor
+
+ assert doctor._helper_row([_adb_probe(["a", "b"])]) is None
+ monkeypatch.setenv("ARTEMIS_HIERARCHY_BACKEND", "uiautomator")
+ assert doctor._helper_row([_adb_probe(["a"])]) is None
diff --git a/tests/unit/test_hierarchy_fidelity.py b/tests/unit/test_hierarchy_fidelity.py
new file mode 100644
index 0000000..bda15ea
--- /dev/null
+++ b/tests/unit/test_hierarchy_fidelity.py
@@ -0,0 +1,156 @@
+"""What the agent sees must be what is on screen, whichever backend dumped it.
+
+Covers the host-side half of the UIAutomator parity work: negative bounds no
+longer lose an element's coordinates, hints and errors reach the element list
+the model reads, an empty helper answer degrades to UIAutomator2 in ``auto``
+mode, and the parity comparison used by ``artemis helper parity`` flags the
+regressions it exists for.
+"""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock
+
+import pytest
+
+from artemis.clients.accessibility_client import AccessibilityClient, HelperEmptyHierarchy
+from artemis.clients.screen_client_factory import FallbackScreenClient
+from artemis.clients.ui_automator_client import UIAutomatorClient, _parse_hierarchy_xml_to_elements
+from artemis.core.diagnostics.hierarchy_parity import compare_dumps
+from artemis.utils import ui_filter
+from artemis.utils.visualization import format_minimal_list_with_elements
+
+
+def _node(**attrs: str) -> str:
+ base = {
+ "index": "0",
+ "text": "",
+ "resource-id": "",
+ "class": "android.widget.TextView",
+ "package": "app",
+ "content-desc": "",
+ "clickable": "false",
+ "enabled": "true",
+ "bounds": "[0,0][100,100]",
+ }
+ base.update(attrs)
+ return ""
+
+
+def _xml(*nodes: str) -> str:
+ return '' + "".join(nodes) + ""
+
+
+# --------------------------------------------------------------------------- #
+# Negative bounds
+# --------------------------------------------------------------------------- #
+
+
+def test_negative_bounds_keep_their_coordinates_and_get_clipped():
+ elements = _parse_hierarchy_xml_to_elements(_xml(_node(text="Row", bounds="[-20,-40][300,60]")))
+ row = elements[1]
+ assert row["parsed_bounds"] == {"left": -20, "top": -40, "right": 300, "bottom": 60}
+
+ assert ui_filter._parse_bounds("[-20,-40][300,60]") == {
+ "left": -20,
+ "top": -40,
+ "right": 300,
+ "bottom": 60,
+ }
+ filtered = ui_filter.filter_ui_hierarchy([row], screen_width=1080, screen_height=2400)
+ assert len(filtered) == 1
+ assert filtered[0]["parsed_bounds"] == {"left": 0, "top": 0, "right": 300, "bottom": 60}
+
+
+# --------------------------------------------------------------------------- #
+# Hints and errors reach the model
+# --------------------------------------------------------------------------- #
+
+
+def test_minimal_list_shows_hint_for_empty_inputs_and_errors():
+ elements = _parse_hierarchy_xml_to_elements(
+ _xml(
+ _node(text="", hint="Search settings", **{"class": "android.widget.EditText"}),
+ _node(text="abc", error="Password too short", bounds="[0,200][100,300]"),
+ _node(text="", bounds="[0,400][100,500]"), # nothing to say: stays out
+ )
+ )
+ text, items, labels = format_minimal_list_with_elements(elements[1:], 1000, 1000)
+ lines = text.splitlines()
+ assert lines[0] == "[1] Hint: 'Search settings' | Bounds: [0,0][100,100]"
+ assert lines[1] == "[2] Text: 'abc' | Bounds: [0,200][100,300] | Error: 'Password too short'"
+ assert len(lines) == 2 and labels == ["1", "2"]
+ assert items[0]["is_hint"] is True and items[0]["text"] == "Search settings"
+ assert items[1]["error"] == "Password too short"
+
+
+# --------------------------------------------------------------------------- #
+# Empty helper answer degrades instead of showing a blank screen
+# --------------------------------------------------------------------------- #
+
+
+def test_empty_helper_hierarchy_falls_back_to_uiautomator():
+ helper = MagicMock(spec=AccessibilityClient)
+ u2 = MagicMock(spec=UIAutomatorClient)
+ helper.get_screen_data.side_effect = HelperEmptyHierarchy("no window")
+ u2.get_screen_data.return_value = "from-u2"
+ client = FallbackScreenClient(
+ "dev", helper=helper, uiautomator_factory=lambda: u2, state_probe=lambda _s: "device"
+ )
+ assert client.get_screen_data() == "from-u2"
+ assert client.active_backend == "uiautomator"
+ assert "HelperEmptyHierarchy" in client.backend_history[-1]["reason"]
+
+
+# --------------------------------------------------------------------------- #
+# Parity comparison
+# --------------------------------------------------------------------------- #
+
+
+def test_parity_passes_when_both_backends_describe_the_same_screen():
+ u2 = _xml(
+ _node(text="Wi-Fi", bounds="[0,100][1080,200]"),
+ _node(**{"content-desc": "Back", "bounds": "[0,0][120,120]"}),
+ )
+ helper = _xml(
+ _node(text="Wi-Fi", bounds="[2,101][1078,199]"),
+ _node(**{"content-desc": "Back", "bounds": "[0,0][120,120]"}),
+ )
+ report = compare_dumps(helper, u2, 1080, 2400)
+ assert report["ok"], report["problems"]
+ assert report["match"]["recall"] == 1.0 and report["match"]["precision"] == 1.0
+
+
+@pytest.mark.parametrize(
+ "helper_extra, expected",
+ [
+ (_node(text="Hidden drawer item", bounds="[-1080,300][0,400]"), "negative bounds"),
+ (_node(text="Next page", bounds="[1080,300][2160,400]"), "outside the screen"),
+ (
+ "".join(
+ _node(text=f"Ghost {i}", bounds=f"[0,{i * 50}][100,{i * 50 + 40}]")
+ for i in range(8)
+ ),
+ "not on screen",
+ ),
+ ],
+)
+def test_parity_flags_elements_uiautomator_would_not_show(helper_extra, expected):
+ u2 = _xml(_node(text="Wi-Fi", bounds="[0,100][1080,200]"))
+ helper = _xml(_node(text="Wi-Fi", bounds="[0,100][1080,200]"), helper_extra)
+ report = compare_dumps(helper, u2, 1080, 2400)
+ assert not report["ok"]
+ assert any(expected in problem for problem in report["problems"]), report["problems"]
+
+
+def test_parity_flags_missing_elements():
+ u2 = _xml(
+ _node(text="Wi-Fi", bounds="[0,100][1080,200]"),
+ _node(text="Bluetooth", bounds="[0,200][1080,300]"),
+ )
+ helper = _xml(_node(text="Wi-Fi", bounds="[0,100][1080,200]"))
+ report = compare_dumps(helper, u2, 1080, 2400)
+ assert not report["ok"]
+ assert report["match"]["missing_in_helper"] == [
+ "'Bluetooth' at {'left': 0, 'top': 200, 'right': 1080, 'bottom': 300}"
+ ]