feat: add managed accessibility helper with UIAutomator fallback

This commit is contained in:
somew1nd committed 2026-09-10 19:40:26 -07:00
1 parent b86c37127d
commit 52749c7f23
53 files changed
+5935 -536

No files matched your search

+5
View File
@@ -22,6 +22,11 @@ ADB_PORT=5037
# Keep the display awake, falling back to a wake lock or safe heartbeat when needed.
# Set to false if Artemis is used with a personal device whose screen should time out.
ARTEMIS_KEEP_DEVICE_AWAKE=true
# UI hierarchy backend: auto (Accessibility Helper, UIAutomator2 fallback) | helper | uiautomator
ARTEMIS_HIERARCHY_BACKEND=auto
# Let tasks install / upgrade the Accessibility Helper APK on a device they hold.
# Set to false on shared or personal phones: then only `artemis helper install` installs it.
ARTEMIS_HELPER_AUTO_INSTALL=true
# Artemis Execution Defaults (flash or pro)
ARTEMIS_DEFAULT_PROFILE=flash
+20
View File
@@ -251,6 +251,26 @@ if __name__ == "__main__":
* **Developer CLI (`uv run artemis run`)**: Direct terminal execution for automated test cases, exploratory stability inspection, or AndroidWorld benchmarks with high-fidelity structured terminal output;
* **Python SDK**: Integrates as a standard Python library into existing automated testing frameworks (e.g., pytest) or CI/CD pipelines with strongly typed Pydantic structured outputs and assertion support.
<a id="on-device-helper"></a>
## What ARTEMIS Installs on Your Phone
The first task on a device installs the **Artemis Accessibility Helper**, a small
accessibility service that reads the screen layout without taking the
UiAutomation connection. Tools using UiAutomation can suppress the helper unless
they enable `FLAG_DONT_SUPPRESS_ACCESSIBILITY_SERVICES`. You will see
a collapsed "Artemis test helper is running" notification and a new entry under
Settings > Accessibility; both are that helper. It listens only on the phone
itself and sends nothing elsewhere.
* Pre-install it (avoids the ~3 s delay on the first task): `uv run artemis helper install`
* Inspect it: `uv run artemis helper status` / `uv run artemis doctor`
* Remove it any time: `uv run artemis helper uninstall`
* Use UIAutomator2 instead: `ARTEMIS_HIERARCHY_BACKEND=uiautomator` in `.env`
* Prevent automatic installation: `ARTEMIS_HELPER_AUTO_INSTALL=false` in `.env`
If the helper ever fails mid-task, ARTEMIS falls back to UIAutomator2 and says
so in the task timeline, in `mobile_manage_task` status, and in the final report.
<a id="benchmarks"></a>
## Benchmarks: AndroidWorld (SOTA 99%+)
+16
View File
@@ -252,6 +252,22 @@ if __name__ == "__main__":
* **开发者命令行 CLI (`uv run artemis run`)**:支持通过终端直接执行自动化测试用例、探索性稳定性巡检或 AndroidWorld 基准评测,提供高保真结构化终端输出;
* **Python SDK**:作为标准 Python 库集成至现有自动化测试框架(如 pytest)或 CI/CD 流水线,提供基于 Pydantic 的强类型结构化结果与断言支持。
<a id="on-device-helper"></a>
## ARTEMIS 会在手机上安装什么
首次在某台设备上执行任务时,ARTEMIS 会安装 **Artemis Accessibility Helper**:一个用于读取屏幕布局的小型无障碍服务,
不占用 UiAutomation 连接。使用 UiAutomation 的工具需要启用 `FLAG_DONT_SUPPRESS_ACCESSIBILITY_SERVICES`,否则可能使该服务暂停运行。
手机上会出现一条折叠的"Artemis test helper is running"通知,以及"设置 > 无障碍"里的一个新条目,都是它。
它只监听手机本机回环地址,不向外发送任何数据。
* 预先安装(避免首个任务多等约 3 秒):`uv run artemis helper install`
* 查看状态:`uv run artemis helper status` / `uv run artemis doctor`
* 随时移除:`uv run artemis helper uninstall`
* 改用 UIAutomator2:在 `.env` 中设置 `ARTEMIS_HIERARCHY_BACKEND=uiautomator`
* 禁止自动安装:在 `.env` 中设置 `ARTEMIS_HELPER_AUTO_INSTALL=false`
任务中途 helper 失效时,ARTEMIS 会回退到 UIAutomator2,并在任务时间线、`mobile_manage_task` 状态和最终报告里明确说明。
<a id="benchmarks"></a>
<a id="基准评测"></a>
## 基准评测:AndroidWorld (SOTA 99%+)
@@ -40,6 +40,59 @@ describe('startup Work block', () => {
expect(item.isActive).toBeTrue();
});
it('shows the helper install sub-step while the hierarchy service is connecting', () => {
const events: StartupProgressEvent[] = [
{ stage: 'uiautomator', message: 'Connecting to the UI hierarchy service', timestamp: 10 },
{
stage: 'helper_install',
message: 'Installing the Artemis accessibility helper on this device for the first time (v1.1.3, about 3 seconds)',
timestamp: 10.2
}
];
const [item] = buildStartupWorkItems(events, 12, false, true);
expect(item.isActive).toBeTrue();
expect(item.message).toContain('Installing the Artemis accessibility helper');
expect(item.elapsed).toBe('2.0s');
});
it('uses the hierarchy source line as the completion message of the second step', () => {
const events: StartupProgressEvent[] = [
{ stage: 'uiautomator', message: 'Connecting to the UI hierarchy service', timestamp: 10 },
{ stage: 'helper_install', message: 'Installing ...', timestamp: 10.2 },
{ stage: 'uiautomator_ready', message: 'UI hierarchy service is ready (helper)', timestamp: 13 },
{
stage: 'hierarchy_backend',
message: 'UI hierarchy source: Artemis accessibility helper v1.1.3',
timestamp: 13.1
},
// A later mid-run switch must not rewrite the startup line.
{
stage: 'hierarchy_backend_changed',
message: 'UI hierarchy source switched from Artemis accessibility helper to UIAutomator2 because ...',
timestamp: 90
}
];
const [item] = buildStartupWorkItems(events, 100, true, true);
expect(item.isActive).toBeFalse();
expect(item.message).toBe('UI hierarchy source: Artemis accessibility helper v1.1.3');
expect(item.elapsed).toBe('3.0s');
});
it('falls back to the generic completion line for runs without a source event', () => {
const events: StartupProgressEvent[] = [
{ stage: 'uiautomator', message: 'Connecting to the UI hierarchy service', timestamp: 10 },
{ stage: 'environment', message: 'Preparing the device environment', timestamp: 14 }
];
const [item] = buildStartupWorkItems(events, 20, false, true);
expect(item.message).toBe('UI hierarchy service is ready');
});
it('uses old first_response events only as a hidden completion boundary', () => {
const events: StartupProgressEvent[] = [
{ stage: 'environment', message: 'Preparing the device environment', timestamp: 20 },
@@ -962,6 +962,17 @@ limitations under the License.
}
<!-- Generic Tools & LLM Calls (when item is a tool) -->
@if (item.type === 'tool') {
<!-- UI-hierarchy source changed mid-run (helper <-> UIAutomator2). -->
@if (isBackendSwitchNote(item.data)) {
<div class="llm-retry-notes">
<div class="llm-retry-note">
<span class="stream-status-dot"></span>
<div class="llm-retry-note-text">
<span class="llm-retry-status-line">{{ getBackendSwitchMessage(item.data) }}</span>
</div>
</div>
</div>
}
<!-- Only SDK-observed Gemini retries are shown as quiet stream notes. -->
@if (isLLMRetry(item.data)) {
<div class="llm-retry-notes">
@@ -61,6 +61,10 @@ interface StartupWorkStage {
started: string;
completed: string;
completedMessage: string;
/** Sub-steps that replace the live message while the stage is still running. */
liveDetailStages?: string[];
/** A later event whose message is a better completion line than the stage's own. */
completedDetailStage?: string;
}
const STARTUP_WORK_STAGES: StartupWorkStage[] = [
@@ -72,7 +76,12 @@ const STARTUP_WORK_STAGES: StartupWorkStage[] = [
{
started: 'uiautomator',
completed: 'uiautomator_ready',
completedMessage: 'UI Automator is ready'
completedMessage: 'UI hierarchy service is ready',
// First task on a device installs / upgrades the accessibility helper
// (a few seconds): say so instead of a generic "connecting".
liveDetailStages: ['helper_install', 'helper_upgrade'],
// "UI hierarchy source: Artemis accessibility helper v1.1.3" (or UIAutomator2).
completedDetailStage: 'hierarchy_backend'
},
{
started: 'environment',
@@ -119,10 +128,21 @@ export function buildStartupWorkItems(
const endTimestamp = completed?.timestamp
|| (isActive ? nowSeconds : events[events.length - 1]?.timestamp || startTimestamp);
const liveDetail = (stage.liveDetailStages || [])
.map((detailStage) => byStage.get(detailStage))
.filter((event): event is StartupProgressEvent => Boolean(event))
.filter((event) => event.timestamp >= startTimestamp)
.sort((a, b) => b.timestamp - a.timestamp)[0];
const completedDetail = stage.completedDetailStage
? byStage.get(stage.completedDetailStage)
: undefined;
const message = completed
? (completedDetail?.message || explicitlyCompleted?.message || stage.completedMessage)
: (liveDetail?.message || started!.message);
return [{
...(explicitlyCompleted || started!),
message: explicitlyCompleted?.message
|| (completed ? stage.completedMessage : started!.message),
message,
isActive,
elapsed: formatStartupElapsed(endTimestamp - startTimestamp)
}];
@@ -198,6 +218,7 @@ import { drawActionCoordinatesOnOverlay } from '../../utils/image-overlay.util';
import {
consolidateLogsToBlocks,
isBackendSwitchNote,
groupBlocksToPhases,
extractBlockTokens,
formatTokenCount,
@@ -1795,6 +1816,7 @@ export class AgentStreamComponent implements AfterViewInit {
|| this.isDisplayableLLMFailure(t)
|| this.isLLMRetry(t)
|| this.isReportStatusAction(t)
|| this.isBackendSwitchNote(t)
);
const hasAndroidActions = Boolean(block.data?.action_taken) && (this.isAndroidAction(block.data.action_taken) || this.isReportStatusAction(block.data.action_taken));
return hasNative || hasRaw || hasReset || hasVisibleTools || hasAndroidActions;
@@ -1826,6 +1848,15 @@ export class AgentStreamComponent implements AfterViewInit {
&& (tool?.name === 'llm_pause' || tool?.payload?.pause === true);
}
/** Mid-run change of the UI-hierarchy source (helper <-> UIAutomator2). */
public isBackendSwitchNote(tool: any): boolean {
return isBackendSwitchNote(tool);
}
public getBackendSwitchMessage(tool: any): string {
return String(tool?.payload?.message || 'UI hierarchy source changed');
}
public resumePausedTask(event: Event): void {
event.stopPropagation();
this.agentService.resumeTask();
@@ -1,4 +1,4 @@
import { consolidateLogsToBlocks, getSortedStepEvents, persistedStreamToSegments } from './stream-aggregator.util';
import { consolidateLogsToBlocks, getSortedStepEvents, isBackendSwitchNote, persistedStreamToSegments } from './stream-aggregator.util';
describe('stream aggregator timeline ordering', () => {
it('interleaves text, tools, and actions by timestamp', () => {
@@ -413,6 +413,51 @@ describe('stream aggregator step ownership', () => {
expect(blocks.length).toBe(1);
expect(blocks[0].data.generic_tools.map((t: any) => t.trace_id)).toEqual(['trace-note']);
});
it('keeps a mid-run hierarchy source switch in the step where it happened, as a note', () => {
const initial = {
type: 'trace_recorded',
timestamp: '2026-09-02T03:57:20.000Z',
data: {
trace_id: 'trace-backend-initial',
type: 'log',
name: 'hierarchy_backend',
payload: { backend: 'helper', previous_backend: null, message: 'UI hierarchy source: Artemis accessibility helper v1.1.3' }
}
};
const switched = {
type: 'trace_recorded',
timestamp: '2026-09-02T03:57:26.000Z',
data: {
trace_id: 'trace-backend-switch',
type: 'log',
name: 'hierarchy_backend',
timestamp: '2026-09-02T03:57:26.000Z',
payload: {
backend: 'uiautomator',
previous_backend: 'helper',
reason: 'HelperUnavailable: tunnel gone',
message: 'UI hierarchy source switched from Artemis accessibility helper to UIAutomator2 because HelperUnavailable: tunnel gone'
}
}
};
const ordinaryLog = {
type: 'trace_recorded',
timestamp: '2026-09-02T03:57:27.000Z',
data: { trace_id: 'trace-log', type: 'log', name: 'artemis.runtime', payload: { message: 'noise' } }
};
const blocks = consolidateLogsToBlocks([initial, plannerStream, switched, ordinaryLog]);
// The initial line belongs to the startup block; ordinary logs stay out.
expect(blocks.length).toBe(1);
expect(blocks[0].data.generic_tools.map((t: any) => t.trace_id)).toEqual(['trace-backend-switch']);
expect(isBackendSwitchNote(blocks[0].data.generic_tools[0])).toBeTrue();
expect(isBackendSwitchNote(initial.data)).toBeFalse();
const events = getSortedStepEvents(blocks[0].data);
expect(events.map((e) => e.type)).toEqual(['text', 'tool']);
});
});
describe('stream aggregator checker history relocation', () => {
@@ -247,8 +247,17 @@ function relocateCheckerTools(blocks: StepBlock[]): void {
}
/**
* Consolidate raw SSE logs into deduplicated and ordered StepBlocks
* A `hierarchy_backend` log trace that records a *switch* (helper -> UIAutomator2
* or back). The initial "source: ..." line belongs to the startup block; only
* the mid-run change is worth a row in the timeline.
*/
export function isBackendSwitchNote(tool: any): boolean {
return tool?.type === 'log'
&& tool?.name === 'hierarchy_backend'
&& Boolean(tool?.payload?.previous_backend);
}
/** Consolidate raw SSE logs into deduplicated and ordered StepBlocks. */
export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
if (!rawLogs || rawLogs.length === 0) return [];
@@ -487,7 +496,7 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
const isTool = log.data.type === 'tool' || isAction;
const isVisibleLLMEvent = log.data.type === 'llm_call'
&& (log.data.status === 'failed' || log.data.status === 'retrying');
if (!isTool && !isVisibleLLMEvent) return;
if (!isTool && !isVisibleLLMEvent && !isBackendSwitchNote(log.data)) return;
let stepId = log.data.step_id;
let existingIndex = -1;
@@ -279,6 +279,7 @@ export function shouldShowTool(tool: any, _stepData?: any): boolean {
if (!tool || !tool.name) return false;
if (tool.type === 'llm_call') return false;
if (tool.type === 'agent') return false;
if (tool.type === 'log') return false; // system notes render as quiet rows, not tool cards
if (isInternalPlumbingTool(tool)) return false;
return true;
+21
View File
@@ -185,6 +185,27 @@ async def outputter(
except Exception as e:
logger.error(f"Failed to attach verdict ledger to outputter: {e}")
# Run environment facts the model must not guess: which UI-hierarchy source
# served the run and whether it switched mid-way (a fallback to UIAutomator2
# explains slower steps and possible conflicts with other automation tools).
from artemis.clients.screen_client_factory import hierarchy_backend_sentence
relative = ctx.data_engine.get_relative_time if ctx.data_engine else None
environment_line = hierarchy_backend_sentence(
getattr(ctx, "ui_adb_client", None), relative_time=relative
)
if environment_line:
human_message_content.append(
{
"type": "text",
"text": (
"## Run Environment (facts, include verbatim under a short "
"'Environment' note only if the source changed mid-run or the "
"user asked about it)" + chr(10) + environment_line
),
}
)
if screenshot_b64:
human_message_content.append(
{
+358 -179
View File
@@ -12,96 +12,324 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Artemis Accessibility Helper client for Android device automation.
"""HTTP client for the Artemis Accessibility Helper running on a device.
Provides a lightweight, conflict-free alternative to UIAutomator2.
Runs concurrently with Mobly, Appium, and Espresso without monopolizing
the singleton UiAutomationConnection.
A lightweight, conflict-free alternative to UIAutomator2: it never takes the
singleton ``UiAutomationConnection``, so it runs alongside Mobly, Appium and
Espresso. This class only speaks the helper's HTTP API. Installing, enabling,
port forwarding, the session token and hot-plug recovery live in
:mod:`artemis.runtime.helper_manager`; this client asks the manager for a
session lazily and hands transport failures back to it once.
The screen-data contract is byte-for-byte the one :class:`UIAutomatorClient`
exposes (``UIAutomatorScreenData`` with the element dictionaries produced by
``_parse_hierarchy_xml_to_elements``), so every consumer downstream
(``filter_ui_hierarchy``, the Explorer index, the MCP actuators that compare
``clickable == "true"``) sees identical shapes whichever backend is active.
One observation is one request: ``/snapshot?fields=xml`` returns the XML and,
on Android 11+, the screenshot taken at the same instant with the bitmap's own
dimensions. On older Android the same answer carries the XML and the client
only adds an adb screencap; the hierarchy is never dumped twice.
"""
from __future__ import annotations
from io import BytesIO
import json
import os
import subprocess
import time
from typing import Any
import urllib.error
import urllib.request
from PIL import Image
from artemis.clients.ui_automator_client import UIAutomatorScreenData, _pil_to_base64
from artemis.clients.ui_automator_client import (
UIAutomatorScreenData,
_parse_hierarchy_xml_to_elements,
_pil_to_base64,
)
from artemis.runtime.adb_endpoint import adb_command
from artemis.runtime.awake_service import ensure_device_awake
from artemis.runtime.helper_manager import (
DEVICE_PORT,
PACKAGE_NAME,
SERVICE_NAME,
AccessibilityHelperManager,
HelperSession,
HelperUnavailable,
ProvisionEvent,
helper_manager,
)
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
PACKAGE_NAME = "com.artemis.helper"
SERVICE_NAME = f"{PACKAGE_NAME}/.ArtemisAccessibilityService"
DEFAULT_PORT = 18888
# Kept for callers that imported these from here.
DEFAULT_PORT = DEVICE_PORT
TOKEN_HEADER = "X-Artemis-Token"
__all__ = [
"AccessibilityClient",
"DEFAULT_PORT",
"HelperEmptyHierarchy",
"HelperRequestError",
"HelperUnavailable",
"PACKAGE_NAME",
"SERVICE_NAME",
"TOKEN_HEADER",
"normalize_helper_elements",
]
_BOOL_ATTRIBUTES = (
"checkable",
"checked",
"clickable",
"enabled",
"focusable",
"focused",
"scrollable",
"long-clickable",
"password",
"selected",
"visible-to-user",
)
_GLOBAL_KEYS = ("back", "home", "recents", "notifications", "quick_settings")
class HelperEmptyHierarchy(RuntimeError):
"""The helper answered but found no window to dump.
Raised instead of returning a screen with zero elements, so ``auto`` mode
can try UIAutomator2 and ``helper`` mode reports the real cause instead of
showing the agent an empty screen.
"""
class HelperRequestError(RuntimeError):
"""The helper answered with an HTTP error that a new tunnel cannot fix."""
def __init__(self, status: int, path: str, body: str):
self.status = status
self.path = path
self.body = body
super().__init__(f"accessibility helper {path} answered HTTP {status}: {body[:200]}")
def normalize_helper_elements(elements: Any) -> list[dict[str, Any]]:
"""Coerce the helper's JSON element list into the UIAutomator element shape.
Used only when a dump carries elements but no XML. The JSON encodes flags
as booleans and omits ``accessibilityText``; the XML parser emits string
flags and mirrors ``content-desc`` into ``accessibilityText``, and several
consumers compare against the string ``"true"``.
"""
if isinstance(elements, dict):
elements = [elements]
if not isinstance(elements, list):
return []
normalized: list[dict[str, Any]] = []
for raw in elements:
if not isinstance(raw, dict):
continue
element = dict(raw)
for key in _BOOL_ATTRIBUTES:
if key in element and isinstance(element[key], bool):
element[key] = "true" if element[key] else "false"
desc = element.get("content-desc")
if desc is not None and "accessibilityText" not in element:
element["accessibilityText"] = desc
bounds = element.get("parsed_bounds")
if isinstance(bounds, dict):
element["parsed_bounds"] = {
side: int(bounds.get(side, 0)) for side in ("left", "top", "right", "bottom")
}
element.pop("children", None)
normalized.append(element)
return normalized
class AccessibilityClient:
"""Non-exclusive accessibility client for Artemis UI automation."""
"""Screen-data client over the helper's loopback HTTP API."""
def __init__(self, device_id: str, local_port: int = DEFAULT_PORT):
backend_name = "helper"
def __init__(
self,
device_id: str,
manager: AccessibilityHelperManager | None = None,
*,
provision_on_connect: bool = True,
request_timeout: float = 6.0,
) -> None:
self._device_id = device_id
self._local_port = local_port
self._base_url = f"http://127.0.0.1:{local_port}"
self._manager = manager or helper_manager
self._provision_on_connect = provision_on_connect
self._request_timeout = request_timeout
self._session: HelperSession | None = None
self._awake_strategy: str | None = None
def ensure_service_ready(self, apk_path: str | None = None) -> bool:
"""Ensure ArtemisAccessibilityHelper is installed, enabled, and port-forwarded."""
# 1. Check if package is installed
if not self._is_installed():
if apk_path is None:
default_apk = os.path.abspath(
os.path.join(
os.path.dirname(__file__),
"../../packages/artemis-accessibility-helper/ArtemisAccessibilityHelper.apk",
)
)
if os.path.exists(default_apk):
apk_path = default_apk
# ------------------------------------------------------------------ #
# Session lifecycle
# ------------------------------------------------------------------ #
if apk_path and os.path.exists(apk_path):
logger.info(f"Installing ArtemisAccessibilityHelper on {self._device_id}...")
self._run_adb(["install", "-r", "-g", apk_path])
else:
logger.warning(
f"ArtemisAccessibilityHelper not installed on {self._device_id} and no APK provided."
)
@property
def device_id(self) -> str:
return self._device_id
# 2. Ensure accessibility service is enabled via secure settings
self._enable_accessibility_service()
@property
def session(self) -> HelperSession | None:
return self._session
# 3. Setup port forward
self._setup_port_forward()
@property
def active_backend(self) -> str | None:
return self.backend_name if self._session is not None else None
# 4. Verify connection
for attempt in range(5):
if self.ping():
logger.info(
f"ArtemisAccessibilityHelper connected successfully on {self._device_id}"
)
return True
time.sleep(0.5)
def connect(self, on_event: ProvisionEvent | None = None) -> None:
"""Task-path entry: provision (install / upgrade / enable) and attach.
logger.warning(
f"Failed to connect to ArtemisAccessibilityHelper on port {self._local_port}"
``on_event`` receives ``installing`` / ``upgrading`` before the slow step.
"""
self._session = self._manager.attach(
self._device_id, provision=self._provision_on_connect, on_event=on_event
)
return False
if self._awake_strategy is None:
self._awake_strategy = ensure_device_awake(self._device_id)
def _ensure_session(self) -> HelperSession:
"""Observer-path entry: attach to a helper that is already running."""
if self._session is None:
self._session = self._manager.attach(self._device_id, provision=False)
if self._awake_strategy is None:
self._awake_strategy = ensure_device_awake(self._device_id)
return self._session
def disconnect(self) -> None:
self._manager.detach(self._device_id)
self._session = None
self._awake_strategy = None
def ping(self) -> bool:
"""Health check probe."""
try:
req = urllib.request.Request(f"{self._base_url}/ping")
with urllib.request.urlopen(req, timeout=2.0) as resp:
data = json.loads(resp.read().decode())
return data.get("success", False)
except Exception:
session = self._ensure_session()
except HelperUnavailable:
return False
return self._manager.ping(session.local_port) is not None
# ------------------------------------------------------------------ #
# Transport
# ------------------------------------------------------------------ #
def _http(
self, path: str, payload: dict[str, Any] | None = None, timeout: float | None = None
) -> bytes:
"""One request with two single-shot repairs.
* HTTP 401: the helper lost its token (re-bound service, reboot): push it
again and retry once.
* Transport failure on a read: rebuild the tunnel once and retry.
Actions are not replayed because they may already have executed.
Any other HTTP status is a :class:`HelperRequestError`; a new tunnel
would not change the answer.
"""
session = self._ensure_session()
try:
return self._http_once(session, path, payload, timeout)
except urllib.error.HTTPError as exc:
body = _read_error_body(exc)
if exc.code != 401:
raise HelperRequestError(exc.code, path, body) from exc
logger.warning(
f"Accessibility helper on {self._device_id} rejected the session token "
f"({body[:120]}); pushing it again."
)
self._manager.push_token(self._device_id)
try:
return self._http_once(session, path, payload, timeout)
except urllib.error.HTTPError as again:
raise HelperRequestError(again.code, path, _read_error_body(again)) from again
except (urllib.error.URLError, OSError, TimeoutError) as exc:
if payload is not None:
raise
logger.warning(
f"Accessibility helper request {path} on {self._device_id} failed ({exc}); "
"rebuilding the tunnel once."
)
self._session = self._manager.reattach(self._device_id)
try:
return self._http_once(self._session, path, payload, timeout)
except urllib.error.HTTPError as again:
raise HelperRequestError(again.code, path, _read_error_body(again)) from again
def _http_once(
self,
session: HelperSession,
path: str,
payload: dict[str, Any] | None,
timeout: float | None,
) -> bytes:
data = (
json.dumps(payload, ensure_ascii=False).encode("utf-8") if payload is not None else None
)
headers: dict[str, str] = {}
token = session.token or self._manager.host_token()
if token:
headers[TOKEN_HEADER] = token
if data is not None:
headers["Content-Type"] = "application/json; charset=utf-8"
request = urllib.request.Request(f"{session.base_url}{path}", data=data, headers=headers)
with urllib.request.urlopen(request, timeout=timeout or self._request_timeout) as resp:
return resp.read()
def _rpc(self, cmd: str, params: dict[str, Any] | None = None) -> bool:
payload = {"cmd": cmd, **(params or {})}
try:
data = json.loads(self._http("/action", payload).decode("utf-8"))
except (
HelperUnavailable,
HelperRequestError,
urllib.error.URLError,
OSError,
ValueError,
TimeoutError,
) as exc:
logger.warning(f"Accessibility helper command '{cmd}' failed: {exc}")
return False
return bool(data.get("success", False))
# ------------------------------------------------------------------ #
# Screen data
# ------------------------------------------------------------------ #
def get_hierarchy(self) -> str:
"""Raw UIAutomator-format XML, the same contract as ``UIAutomatorClient``."""
return self._http("/dump_xml").decode("utf-8")
def get_hierarchy_json(self, *, include_invisible: bool = False) -> dict[str, Any]:
"""The helper's JSON dump (XML plus its own element list); diagnostics only."""
query = "fields=xml,elements" + ("&include_invisible=1" if include_invisible else "")
return json.loads(self._http(f"/dump?{query}").decode("utf-8"))
def _snapshot(self) -> dict[str, Any]:
"""``/snapshot?fields=xml``: hierarchy plus, on Android 11+, the screenshot."""
data = json.loads(self._http("/snapshot?fields=xml", timeout=8.0).decode("utf-8"))
if not isinstance(data, dict):
raise HelperRequestError(200, "/snapshot", "non-object answer")
if not data.get("success"):
raise HelperEmptyHierarchy(
f"accessibility helper on {self._device_id} returned no hierarchy: "
f"{data.get('error') or 'unknown reason'}"
)
return data
def get_atomic_snapshot(self) -> dict[str, Any] | None:
"""Screenshot + hierarchy captured in one call on the device (Android 11+), else None."""
data = self._snapshot()
if data.get("has_screenshot") and data.get("screenshot_base64"):
return data
return None
def get_screenshot(self) -> Image.Image | None:
"""Capture screenshot via high-speed ADB exec-out."""
try:
result = subprocess.run(
adb_command(["-s", self._device_id, "exec-out", "screencap", "-p"]),
@@ -111,160 +339,111 @@ class AccessibilityClient:
check=True,
)
return Image.open(BytesIO(result.stdout))
except Exception as e:
logger.error(f"Failed to capture screenshot via adb: {e}")
except (OSError, subprocess.SubprocessError) as exc:
logger.error(f"Failed to capture screenshot via adb: {exc}")
return None
def get_hierarchy(self) -> dict:
"""Fetch hierarchy JSON directly from ArtemisAccessibilityHelper."""
req = urllib.request.Request(f"{self._base_url}/dump")
with urllib.request.urlopen(req, timeout=5.0) as resp:
return json.loads(resp.read().decode())
def get_screenshot_base64(self) -> str | None:
screenshot = self.get_screenshot()
return None if screenshot is None else _pil_to_base64(screenshot, format="JPEG")
def get_hierarchy_xml(self) -> str:
"""Fetch raw standard UIAutomator XML string directly from helper."""
req = urllib.request.Request(f"{self._base_url}/dump_xml")
with urllib.request.urlopen(req, timeout=5.0) as resp:
return resp.read().decode("utf-8")
def get_atomic_snapshot(self) -> dict | None:
"""Fetch atomic snapshot (screenshot Base64 + hierarchy) directly in one call."""
try:
req = urllib.request.Request(f"{self._base_url}/snapshot")
with urllib.request.urlopen(req, timeout=6.0) as resp:
data = json.loads(resp.read().decode())
if (
data.get("success")
and data.get("has_screenshot")
and data.get("screenshot_base64")
):
return data
except Exception as e:
logger.debug(f"Atomic snapshot request failed (falling back to dual path): {e}")
return None
@staticmethod
def _elements_from_dump(dump: dict[str, Any]) -> tuple[str, list[dict[str, Any]]]:
xml = dump.get("xml") or ""
if isinstance(xml, str) and xml.strip():
return xml, _parse_hierarchy_xml_to_elements(xml)
return "", normalize_helper_elements(dump.get("elements"))
def get_screen_data(self) -> UIAutomatorScreenData:
"""Retrieve complete screen data (screenshot + parsed elements + standard XML).
Prioritizes the atomic hardware snapshot API (Android 11+) to eliminate
any temporal phase mismatch between visual frame and layout DOM tree.
Falls back to separate ADB screencap + HTTP dump on older platforms.
"""
# 1. Try atomic snapshot
atomic = self.get_atomic_snapshot()
if atomic is not None:
snapshot = self._snapshot()
xml, elements = self._elements_from_dump(snapshot)
if snapshot.get("has_screenshot") and snapshot.get("screenshot_base64"):
width, height = _dimensions(snapshot)
return UIAutomatorScreenData(
base64=atomic.get("screenshot_base64", ""),
hierarchy_xml=atomic.get("xml", ""),
elements=atomic.get("elements", []),
width=atomic.get("width", 1080),
height=atomic.get("height", 2400),
base64=str(snapshot["screenshot_base64"]),
hierarchy_xml=xml,
elements=elements,
width=width,
height=height,
)
# 2. Fallback: separate screenshot and hierarchy dump
# Android 10 and below (or a rate-limited capture that did not recover):
# keep the hierarchy we already have and only add an adb screencap.
reason = snapshot.get("screenshot_error")
if reason:
logger.debug(f"Helper snapshot without screenshot ({reason}); using adb screencap")
screenshot = self.get_screenshot()
if screenshot is None:
raise RuntimeError("Failed to capture screenshot")
hierarchy_data = self.get_hierarchy()
elements = hierarchy_data.get("elements", [])
hierarchy_xml = hierarchy_data.get("xml", "")
return UIAutomatorScreenData(
base64=_pil_to_base64(screenshot, format="JPEG"),
hierarchy_xml=hierarchy_xml,
hierarchy_xml=xml,
elements=elements,
width=screenshot.width,
height=screenshot.height,
)
def tap(self, x: float, y: float) -> bool:
"""Perform tap gesture via AccessibilityService."""
return self._send_rpc("tap", {"x": x, "y": y})
# ------------------------------------------------------------------ #
# Input
# ------------------------------------------------------------------ #
def swipe(self, x1: float, y1: float, x2: float, y2: float, duration_ms: int = 300) -> bool:
"""Perform swipe gesture via AccessibilityService."""
return self._send_rpc(
"swipe",
{"x1": x1, "y1": y1, "x2": x2, "y2": y2, "duration": duration_ms},
)
def set_clipboard(self, text: str) -> bool:
"""Put ``text`` on the device clipboard so the driver can paste it."""
return self._rpc("clipboard", {"text": text})
def send_text(self, text: str) -> bool:
"""Send text input via AccessibilityNodeInfo.ACTION_SET_TEXT."""
success = self._send_rpc("type", {"text": text})
if not success:
# Fallback to adb shell input text
safe_text = text.replace(" ", "%s")
self._run_adb(["shell", "input", "text", safe_text])
return True
return success
"""Append ``text`` to the focused field, the way typing through an IME does."""
return self._rpc("type", {"text": text, "append": True})
def clear_text(self) -> bool:
"""Clear text of focused input."""
return self._send_rpc("clear", {})
return self._rpc("clear", {})
def press_key(self, key: str) -> bool:
"""Perform global navigation action (back, home, recents)."""
key_lower = key.lower()
if key_lower in ("back", "home", "recents", "notifications", "quick_settings"):
return self._send_rpc("global", {"action": key_lower})
# Fallback to adb keyevent
self._run_adb(["shell", "input", "keyevent", f"KEYCODE_{key.upper()}"])
return True
def _send_rpc(self, cmd: str, params: dict) -> bool:
"""Send JSON-RPC payload to helper server."""
payload = {"cmd": cmd, **params}
try:
req = urllib.request.Request(
f"{self._base_url}/action",
data=json.dumps(payload).encode("utf-8"),
headers={"Content-Type": "application/json"},
)
with urllib.request.urlopen(req, timeout=5.0) as resp:
data = json.loads(resp.read().decode())
return data.get("success", False)
except Exception as e:
logger.warning(f"RPC command '{cmd}' failed: {e}")
return False
def _is_installed(self) -> bool:
res = self._run_adb(["shell", "pm", "list", "packages", PACKAGE_NAME])
return f"package:{PACKAGE_NAME}" in res.stdout
def _enable_accessibility_service(self) -> None:
"""Silently enable the accessibility service via adb settings without UI prompts."""
res = self._run_adb(
["shell", "settings", "get", "secure", "enabled_accessibility_services"]
)
current_services = res.stdout.strip()
if SERVICE_NAME not in current_services:
new_services = (
f"{current_services}:{SERVICE_NAME}"
if current_services and current_services != "null"
else SERVICE_NAME
)
self._run_adb(
key_lower = str(key).lower()
if key_lower in _GLOBAL_KEYS:
return self._rpc("global", {"action": key_lower})
result = subprocess.run(
adb_command(
[
"-s",
self._device_id,
"shell",
"settings",
"put",
"secure",
"enabled_accessibility_services",
new_services,
"input",
"keyevent",
f"KEYCODE_{key_lower.upper()}",
]
)
self._run_adb(["shell", "settings", "put", "secure", "accessibility_enabled", "1"])
def _setup_port_forward(self) -> None:
self._run_adb(["forward", f"tcp:{self._local_port}", f"tcp:{DEFAULT_PORT}"])
def _run_adb(self, args: list[str]) -> subprocess.CompletedProcess:
return subprocess.run(
adb_command(["-s", self._device_id] + args),
),
stdin=subprocess.DEVNULL,
capture_output=True,
text=True,
timeout=15,
check=False,
)
return result.returncode == 0
def tap(self, x: float, y: float) -> bool:
return self._rpc("tap", {"x": x, "y": y})
def swipe(self, x1: float, y1: float, x2: float, y2: float, duration_ms: int = 300) -> bool:
return self._rpc("swipe", {"x1": x1, "y1": y1, "x2": x2, "y2": y2, "duration": duration_ms})
def _read_error_body(exc: urllib.error.HTTPError) -> str:
try:
return exc.read().decode("utf-8", "replace")
except OSError:
return str(exc)
def _dimensions(snapshot: dict[str, Any]) -> tuple[int, int]:
"""The screenshot's own size; a snapshot without it is a broken answer, not a 1080x2400 one."""
try:
width = int(snapshot.get("width") or 0)
height = int(snapshot.get("height") or 0)
except (TypeError, ValueError):
width = height = 0
if width <= 0 or height <= 0:
raise HelperRequestError(
200, "/snapshot", f"screenshot without dimensions (width={width}, height={height})"
)
return width, height
+400
View File
@@ -0,0 +1,400 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Select the UI-hierarchy backend for a device and build its screen client.
Three backends, chosen by ``ARTEMIS_HIERARCHY_BACKEND`` (environment or
``.env``):
* ``auto`` (default): the Accessibility Helper, with UIAutomator2 as a
per-call fallback. UIAutomator2 is only constructed when the helper fails,
so a healthy helper never triggers ``u2.connect`` (which pushes its own APKs
and uninstalls Maestro).
* ``helper``: the helper only. Failures surface as errors; use it in CI to
prove the helper is really what serves the hierarchy.
* ``uiautomator``: the pre-helper path, unchanged.
Every entrypoint that used to build ``UIAutomatorClient`` directly (the SDK
agent, the MCP device controller, and through it the device smoke test) goes
through :func:`create_screen_client`, so the switch applies everywhere.
"""
from __future__ import annotations
from collections.abc import Callable
from enum import Enum
import os
import subprocess
import time
from typing import Any
from PIL import Image
from artemis.clients.accessibility_client import AccessibilityClient, HelperUnavailable
from artemis.clients.ui_automator_client import UIAutomatorClient, UIAutomatorScreenData
from artemis.config.constants import ENV_ARTEMIS_HIERARCHY_BACKEND
from artemis.runtime.adb_endpoint import adb_command
from artemis.runtime.helper_manager import ProvisionEvent
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
#: ``(previous_backend, new_backend, reason)``; ``previous_backend`` is None on first use.
BackendChangeListener = Callable[[str | None, str, str | None], None]
class DeviceOfflineError(RuntimeError):
"""The device is not attached / authorized; no backend can serve it.
Raised instead of trying UIAutomator2 when the helper fails because the
phone itself is gone, so the failure names the real cause and does not
burn UIAutomator2's own multi-attempt connect timeout.
"""
def __init__(self, device_id: str, state: str | None):
self.device_id = device_id
self.state = state
detail = f"adb reports state '{state}'" if state else "adb does not list it"
super().__init__(
f"Device {device_id} is not available ({detail}). Reconnect the device or "
"accept the USB debugging prompt, then retry."
)
def device_state(device_id: str, timeout: float = 5.0) -> str | None:
"""``adb get-state`` for one serial: 'device', 'offline', 'unauthorized', or None."""
try:
result = subprocess.run(
adb_command(["-s", device_id, "get-state"]),
stdin=subprocess.DEVNULL,
capture_output=True,
text=True,
timeout=timeout,
check=False,
)
except (OSError, subprocess.SubprocessError):
return None
text = (result.stdout or "").strip()
if result.returncode != 0 or not text:
# "error: device unauthorized" / "device offline" arrive on stderr.
err = (result.stderr or "").lower()
for known in ("unauthorized", "offline"):
if known in err:
return known
return None
return text
class HierarchyBackend(str, Enum):
AUTO = "auto"
HELPER = "helper"
UIAUTOMATOR = "uiautomator"
def resolve_backend(value: str | HierarchyBackend | None = None) -> HierarchyBackend:
"""Backend from an explicit value, else the environment, else settings, else ``auto``."""
if isinstance(value, HierarchyBackend):
return value
text = value
if text is None:
text = os.environ.get(ENV_ARTEMIS_HIERARCHY_BACKEND)
if text is None:
try:
from artemis.config import settings
text = getattr(settings, "ARTEMIS_HIERARCHY_BACKEND", None)
except (ImportError, ValueError):
text = None
normalized = (text or "auto").strip().lower()
try:
return HierarchyBackend(normalized)
except ValueError:
logger.warning(
f"Unknown hierarchy backend {text!r}; valid values are "
f"{', '.join(b.value for b in HierarchyBackend)}. Using 'auto'."
)
return HierarchyBackend.AUTO
class FallbackScreenClient:
"""Helper first, UIAutomator2 on failure, with a retry window for the helper.
After a helper failure the client serves from UIAutomator2 for
``retry_after`` seconds before trying the helper again, so one dead tunnel
does not cost a 5-second timeout on every step.
"""
def __init__(
self,
device_id: str,
helper: AccessibilityClient | None = None,
uiautomator_factory: Callable[[], UIAutomatorClient] | None = None,
*,
retry_after: float = 30.0,
clock: Callable[[], float] = time.monotonic,
state_probe: Callable[[str], str | None] = device_state,
) -> None:
self._device_id = device_id
self._helper = helper or AccessibilityClient(device_id)
self._uiautomator_factory = uiautomator_factory or (lambda: UIAutomatorClient(device_id))
self._uiautomator: UIAutomatorClient | None = None
self._retry_after = retry_after
self._clock = clock
self._state_probe = state_probe
self._helper_down_until: float | None = None
self._active_backend: str | None = None
self._last_failure: str | None = None
#: Every backend switch, oldest first: {time, backend, reason}.
self.backend_history: list[dict[str, Any]] = []
self._listeners: list[BackendChangeListener] = []
# ------------------------------------------------------------------ #
@property
def device_id(self) -> str:
return self._device_id
@property
def helper(self) -> AccessibilityClient:
return self._helper
@property
def active_backend(self) -> str | None:
"""Backend that served the last call: ``"helper"``, ``"uiautomator"`` or ``None``."""
return self._active_backend
@property
def uiautomator(self) -> UIAutomatorClient:
if self._uiautomator is None:
self._uiautomator = self._uiautomator_factory()
return self._uiautomator
def add_backend_listener(self, listener: BackendChangeListener) -> None:
"""Be told whenever the serving backend changes (first use included)."""
self._listeners.append(listener)
def remove_backend_listener(self, listener: BackendChangeListener) -> None:
if listener in self._listeners:
self._listeners.remove(listener)
def _set_active(self, backend: str, reason: str | None = None) -> None:
previous = self._active_backend
if previous == backend:
return
self._active_backend = backend
self.backend_history.append({"time": time.time(), "backend": backend, "reason": reason})
for listener in list(self._listeners):
try:
listener(previous, backend, reason)
except Exception as exc: # pylint: disable=broad-exception-caught
logger.debug(f"Backend listener failed: {exc}")
def _ensure_device_online(self) -> None:
"""Fail fast with the real cause when the phone itself is gone."""
state = self._state_probe(self._device_id)
if state != "device":
raise DeviceOfflineError(self._device_id, state)
def _helper_allowed(self) -> bool:
if self._helper_down_until is None:
return True
if self._clock() >= self._helper_down_until:
self._helper_down_until = None
return True
return False
def _degrade(self, reason: BaseException) -> None:
# Never fall back to UIAutomator2 for a phone that is simply not there.
self._ensure_device_online()
self._last_failure = f"{reason.__class__.__name__}: {reason}"
if self._helper_down_until is None:
logger.warning(
f"Accessibility helper unavailable on {self._device_id} ({reason}); "
f"serving from UIAutomator2 and retrying the helper in {self._retry_after:g}s."
)
self._helper_down_until = self._clock() + self._retry_after
def _release_uiautomation(self) -> None:
"""Stop our UIAutomator2 server before giving the helper another chance.
Its UiAutomation connection keeps every accessibility service unbound,
so while it lives the helper cannot answer and the retry would be a
guaranteed failure. UIAutomator2 restarts its server by itself on the
next call if the helper turns out to be still dead.
"""
if self._uiautomator is not None and self._active_backend == "uiautomator":
self._uiautomator.stop_server()
def _call(self, name: str, *args: Any, **kwargs: Any) -> Any:
if self._helper_allowed():
self._release_uiautomation()
try:
result = getattr(self._helper, name)(*args, **kwargs)
except (RuntimeError, OSError, ValueError, subprocess.SubprocessError) as exc:
self._degrade(exc)
else:
self._set_active("helper", "helper recovered" if self._last_failure else None)
self._last_failure = None
return result
result = getattr(self.uiautomator, name)(*args, **kwargs)
self._set_active("uiautomator", self._last_failure)
return result
# ------------------------------------------------------------------ #
# Lifecycle
# ------------------------------------------------------------------ #
def connect(self, on_event: ProvisionEvent | None = None) -> None:
self._release_uiautomation()
self.backend_history.clear()
self._active_backend = None
if self._helper_allowed():
self._release_uiautomation()
try:
self._helper.connect(on_event=on_event)
except (RuntimeError, OSError, ValueError, subprocess.SubprocessError) as exc:
self._degrade(exc)
else:
self._set_active("helper")
return
self.uiautomator.connect()
self._set_active("uiautomator", self._last_failure)
def disconnect(self) -> None:
try:
self._helper.disconnect()
finally:
if self._uiautomator is not None:
# Release UiAutomation so the helper is back for the next task
# on this device (possibly served by another process).
self._uiautomator.disconnect(stop_server=True)
self._active_backend = None
# ------------------------------------------------------------------ #
# Screen data and input, same surface as UIAutomatorClient
# ------------------------------------------------------------------ #
def get_screen_data(self) -> UIAutomatorScreenData:
return self._call("get_screen_data")
def get_hierarchy(self) -> str:
return self._call("get_hierarchy")
def get_screenshot(self) -> Image.Image | None:
return self._call("get_screenshot")
def get_screenshot_base64(self) -> str | None:
return self._call("get_screenshot_base64")
def set_clipboard(self, text: str) -> bool:
return self._call("set_clipboard", text)
def send_text(self, text: str) -> Any:
return self._call("send_text", text)
def clear_text(self) -> bool:
return self._call("clear_text")
def press_key(self, key: str) -> Any:
return self._call("press_key", key)
ScreenClient = AccessibilityClient | UIAutomatorClient | FallbackScreenClient
def create_screen_client(
device_id: str, backend: str | HierarchyBackend | None = None
) -> ScreenClient:
"""Build the screen client for ``device_id`` according to the configured backend."""
resolved = resolve_backend(backend)
if resolved is HierarchyBackend.UIAUTOMATOR:
return UIAutomatorClient(device_id=device_id)
if resolved is HierarchyBackend.HELPER:
return AccessibilityClient(device_id)
return FallbackScreenClient(device_id)
def describe_backend(client: Any) -> str | None:
"""Best-effort name of the backend a client is serving from, for diagnostics."""
active = getattr(client, "active_backend", None)
if active:
return str(active)
if isinstance(client, UIAutomatorClient):
return "uiautomator"
if isinstance(client, AccessibilityClient):
return "helper"
return None
def helper_version(client: Any) -> str | None:
"""Version name of the helper a client is attached to, if any."""
helper = getattr(client, "helper", None) or client
session = getattr(helper, "session", None)
name = getattr(session, "version_name", None)
return str(name) if name else None
def hierarchy_backend_summary(client: Any) -> dict[str, Any] | None:
"""Machine-readable account of which backend served a run, for reports."""
backend = describe_backend(client)
if backend is None:
return None
history = list(getattr(client, "backend_history", None) or [])
return {
"backend": backend,
"helper_version": helper_version(client),
"switches": history[1:] if len(history) > 1 else [],
}
_BACKEND_LABEL = {"helper": "the Artemis accessibility helper", "uiautomator": "UIAutomator2"}
def hierarchy_backend_sentence(
client: Any, relative_time: Callable[[float], str] | None = None
) -> str | None:
"""One plain sentence for reports: what served the hierarchy and whether it changed."""
summary = hierarchy_backend_summary(client)
if summary is None:
return None
history = getattr(client, "backend_history", None) or []
first = (history[0].get("backend") if history else None) or summary["backend"]
text = f"UI hierarchy source: {_BACKEND_LABEL.get(first, first)}"
if first == "helper" and summary.get("helper_version"):
text += f" v{summary['helper_version']}"
for switch in summary["switches"]:
when = relative_time(switch["time"]) if relative_time else None
text += f"; switched to {_BACKEND_LABEL.get(switch['backend'], switch['backend'])}"
if when:
text += f" at {when}"
if switch.get("reason"):
text += f" because {str(switch['reason']).rstrip('. ')}"
return text + "."
__all__ = [
"BackendChangeListener",
"DeviceOfflineError",
"FallbackScreenClient",
"HelperUnavailable",
"HierarchyBackend",
"ScreenClient",
"create_screen_client",
"describe_backend",
"device_state",
"helper_version",
"hierarchy_backend_sentence",
"hierarchy_backend_summary",
"resolve_backend",
]
+32 -5
View File
@@ -117,7 +117,7 @@ def _parse_hierarchy_xml_to_elements(hierarchy_xml: str) -> list[dict]:
elif attr_name == "bounds":
element["bounds"] = attr_value
match = _re.match(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", attr_value)
match = _re.match(r"\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]", attr_value)
if match:
x1, y1, x2, y2 = map(int, match.groups())
element["parsed_bounds"] = {
@@ -190,7 +190,7 @@ def _is_package_installed(device_id: str, pkg: str) -> bool:
except subprocess.TimeoutExpired:
logger.warning("Timeout checking installed packages")
return False
except Exception as e:
except (OSError, subprocess.SubprocessError) as e:
logger.warning(f"Error checking installed packages: {e}")
return False
@@ -400,7 +400,7 @@ class UIAutomatorClient:
check=True,
)
return Image.open(BytesIO(result.stdout))
except Exception as e:
except (OSError, subprocess.SubprocessError) as e:
logger.warning(
f"Failed to capture screenshot via adb: {e}, falling back to uiautomator2"
)
@@ -446,12 +446,39 @@ class UIAutomatorClient:
height=screenshot.height,
)
def disconnect(self) -> None:
"""Disconnect this client without ending the host's awake lifetime."""
def disconnect(self, stop_server: bool = False) -> None:
"""Disconnect this client without ending the host's awake lifetime.
``stop_server`` also kills uiautomator2's device server. Its UiAutomation
connection suppresses every accessibility service (the Artemis helper
included) for as long as it lives, so a caller that prefers the helper
must release it; the pure UIAutomator2 backend keeps it for fast reconnects.
"""
device = self._device
self._awake_strategy = None
self._device = None
if stop_server and device is not None:
self.stop_server(device)
logger.info("UIAutomator2 client disconnected")
def stop_server(self, device: "Device | None" = None) -> bool:
"""Kill uiautomator2's on-device server (see :meth:`disconnect`)."""
device = device or self._device
if device is None:
return False
try:
device.stop_uiautomator()
logger.info(f"UIAutomator2 device server stopped on {self._device_id}")
return True
except (
OSError,
RuntimeError,
subprocess.SubprocessError,
u2.exceptions.DeviceError,
) as exc:
logger.debug(f"Stopping the UIAutomator2 server on {self._device_id} failed: {exc}")
return False
def get_client(device_id: str) -> UIAutomatorClient:
"""Factory function to create a UIAutomatorClient.
+5
View File
@@ -46,6 +46,11 @@ ENV_ARTEMIS_DEVICE_ID = "ARTEMIS_DEVICE_ID"
ENV_ADB_HOST = "ADB_HOST"
ENV_ADB_PORT = "ADB_PORT"
ENV_ADB_SERVER_SOCKET = "ADB_SERVER_SOCKET"
# UI hierarchy backend: "auto" (helper, UIAutomator2 fallback), "helper", "uiautomator"
ENV_ARTEMIS_HIERARCHY_BACKEND = "ARTEMIS_HIERARCHY_BACKEND"
# "true" (default) lets a task install / upgrade the Accessibility Helper APK on a
# device it holds; "false" only attaches to a helper someone installed by hand.
ENV_ARTEMIS_HELPER_AUTO_INSTALL = "ARTEMIS_HELPER_AUTO_INSTALL"
# LLM & Vision OCR API Keys
ENV_GOOGLE_API_KEY = "GOOGLE_API_KEY"
+6
View File
@@ -113,6 +113,12 @@ class Settings(BaseSettings):
ADB_HOST: str | None = Field(default=DEFAULT_ADB_HOST)
ADB_PORT: int | None = Field(default=DEFAULT_ADB_PORT)
ADB_DEVICE_SERIAL: str | None = None
# UI hierarchy backend: "auto" = Accessibility Helper with UIAutomator2
# fallback, "helper" = helper only, "uiautomator" = UIAutomator2 only.
ARTEMIS_HIERARCHY_BACKEND: str = Field(default="auto")
# Whether a task may install / upgrade the Accessibility Helper APK on a device
# it holds. False = only attach to a helper installed by `artemis helper install`.
ARTEMIS_HELPER_AUTO_INSTALL: bool = Field(default=True)
# Execution Defaults
PROJECT_NAME: str | None = None
+5
View File
@@ -268,6 +268,7 @@ async def smoke_test_device(
{"ok": bool, "serial": str | None, "elapsed_seconds": float,
"screenshot_bytes": int | None, "element_count": int | None,
"hierarchy_backend": "helper" | "uiautomator" | None,
"error": str | None, "fix": list[str]}
"""
started = time.monotonic()
@@ -278,6 +279,7 @@ async def smoke_test_device(
"elapsed_seconds": 0.0,
"screenshot_bytes": None,
"element_count": None,
"hierarchy_backend": None,
"error": None,
"fix": [],
}
@@ -342,6 +344,9 @@ async def smoke_test_device(
result["screenshot_bytes"] = _decoded_length(getattr(device_data, "base64", None))
result["element_count"] = _count_elements(getattr(device_data, "elements", None))
from artemis.clients.screen_client_factory import describe_backend
result["hierarchy_backend"] = describe_backend(getattr(controller_ctx, "ui_adb_client", None))
if result["screenshot_bytes"] is None:
return _finish("Screen capture returned no screenshot data")
@@ -0,0 +1,216 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Real-device parity check between the Accessibility Helper and UIAutomator2.
The unit tests prove the two backends produce the same element *shapes*; only
a real screen can prove they describe the same *content*. ``compare_backends``
dumps the current screen with both, then checks what the agent depends on:
* every labelled UIAutomator element (text or content-desc) has a helper
element with the same label at the same place (bounds IoU >= 0.5), and vice
versa, so neither backend invents or loses what is on screen;
* the helper emits no negative and no off-screen bounds (its bounds are
clipped to the display, like UIAutomator's);
* dump latency of both, for the record.
``artemis helper parity`` is the CLI front end.
"""
from __future__ import annotations
from statistics import median
import time
from typing import Any
from artemis.clients.ui_automator_client import _parse_hierarchy_xml_to_elements
MIN_RECALL = 0.9
MIN_PRECISION = 0.8
def _label(element: dict[str, Any]) -> str:
text = str(element.get("text") or "").strip()
if text:
return text
return str(element.get("content-desc") or "").strip()
def _iou(a: dict[str, int], b: dict[str, int]) -> float:
left = max(a["left"], b["left"])
top = max(a["top"], b["top"])
right = min(a["right"], b["right"])
bottom = min(a["bottom"], b["bottom"])
if right <= left or bottom <= top:
return 0.0
inter = (right - left) * (bottom - top)
area_a = max(1, (a["right"] - a["left"]) * (a["bottom"] - a["top"]))
area_b = max(1, (b["right"] - b["left"]) * (b["bottom"] - b["top"]))
return inter / (area_a + area_b - inter)
def analyze_xml(xml: str, width: int, height: int) -> dict[str, Any]:
"""Counts and labelled elements of one dump, plus the bounds problems the helper must not have."""
elements = _parse_hierarchy_xml_to_elements(xml)
labelled: list[tuple[str, dict[str, int]]] = []
negative = offscreen = 0
package = None
for element in elements:
bounds = element.get("parsed_bounds")
if package is None and element.get("package"):
package = element["package"]
if not isinstance(bounds, dict):
continue
if min(bounds.values()) < 0:
negative += 1
if (
bounds["left"] >= width
or bounds["top"] >= height
or bounds["right"] > width
or bounds["bottom"] > height
):
offscreen += 1
label = _label(element)
if label:
labelled.append((label, bounds))
return {
"nodes": len(elements),
"labelled": len(labelled),
"labels": labelled,
"negative_bounds": negative,
"offscreen_bounds": offscreen,
"package": package,
}
def match_labels(
source: list[tuple[str, dict[str, int]]], target: list[tuple[str, dict[str, int]]]
) -> tuple[int, list[str]]:
"""How many labelled ``source`` elements have a same-label, same-place element in ``target``."""
used = [False] * len(target)
found = 0
missing: list[str] = []
for label, bounds in source:
hit = False
for i, (other_label, other_bounds) in enumerate(target):
if used[i] or other_label != label:
continue
if _iou(bounds, other_bounds) >= 0.5:
used[i] = True
hit = True
break
if hit:
found += 1
else:
missing.append(f"{label!r} at {bounds}")
return found, missing
def compare_dumps(helper_xml: str, uiautomator_xml: str, width: int, height: int) -> dict[str, Any]:
"""Pure comparison of two dumps of the same screen."""
helper = analyze_xml(helper_xml, width, height)
u2 = analyze_xml(uiautomator_xml, width, height)
u2_found, missing_in_helper = match_labels(u2["labels"], helper["labels"])
helper_found, extra_in_helper = match_labels(helper["labels"], u2["labels"])
recall = u2_found / u2["labelled"] if u2["labelled"] else 1.0
precision = helper_found / helper["labelled"] if helper["labelled"] else 1.0
problems: list[str] = []
if helper["negative_bounds"]:
problems.append(
f"helper emitted {helper['negative_bounds']} element(s) with negative bounds"
)
if helper["offscreen_bounds"]:
problems.append(
f"helper emitted {helper['offscreen_bounds']} element(s) outside the screen"
)
if recall < MIN_RECALL:
problems.append(
f"only {recall:.0%} of UIAutomator's labelled elements appear in the helper dump "
f"(need {MIN_RECALL:.0%})"
)
if precision < MIN_PRECISION:
problems.append(
f"only {precision:.0%} of the helper's labelled elements appear in the UIAutomator dump "
f"(need {MIN_PRECISION:.0%}); the helper is showing things that are not on screen"
)
for side in (helper, u2):
side.pop("labels", None)
return {
"ok": not problems,
"problems": problems,
"helper": helper,
"uiautomator": u2,
"match": {
"uiautomator_labelled": u2["labelled"],
"uiautomator_in_helper": u2_found,
"recall": recall,
"helper_labelled": helper["labelled"],
"helper_in_uiautomator": helper_found,
"precision": precision,
"missing_in_helper": missing_in_helper,
"extra_in_helper": extra_in_helper,
},
}
def compare_backends(serial: str, rounds: int = 3) -> dict[str, Any]:
"""Dump the current screen of ``serial`` with both backends and compare (device required)."""
from artemis.clients.accessibility_client import AccessibilityClient
from artemis.clients.ui_automator_client import UIAutomatorClient
helper = AccessibilityClient(serial)
u2 = UIAutomatorClient(serial)
def timed(dump) -> tuple[str, float]:
times: list[float] = []
xml = ""
for _ in range(rounds):
started = time.perf_counter()
xml = dump()
times.append((time.perf_counter() - started) * 1000.0)
return xml, median(times)
# The two cannot run at once: UIAutomator2's UiAutomation connection unbinds
# the helper. Dump with the helper first, then with UIAutomator2, then stop
# its server so the helper is back for whoever comes next.
helper_version = None
try:
helper.connect()
helper_version = helper.session.version_name if helper.session else None
info = helper.get_hierarchy_json()
width, height = int(info.get("width") or 0), int(info.get("height") or 0)
if width <= 0 or height <= 0:
screen = helper.get_screen_data()
width, height = screen.width, screen.height
helper_xml, helper_ms = timed(helper.get_hierarchy)
finally:
helper.disconnect()
try:
u2.connect()
u2_xml, u2_ms = timed(u2.get_hierarchy)
finally:
u2.disconnect(stop_server=True)
report = compare_dumps(helper_xml, u2_xml, width, height)
report["serial"] = serial
report["screen"] = {"width": width, "height": height}
report["helper"]["median_ms"] = helper_ms
report["helper"]["version"] = helper_version
report["uiautomator"]["median_ms"] = u2_ms
report["xml"] = {"helper": helper_xml, "uiautomator": u2_xml}
return report
__all__ = ["analyze_xml", "compare_backends", "compare_dumps", "match_labels"]
+14
View File
@@ -21,6 +21,7 @@ import os
from pathlib import Path
import re
import socket
import sqlite3
import threading
import time
from typing import Any
@@ -785,6 +786,19 @@ class DataEngine:
except Exception as e:
logger.error(f"Failed to record video failure in DataEngine: {e}")
def update_session_device_info(self, **fields: Any) -> None:
"""Merge ``fields`` into the running session's ``device_info`` record."""
if not self.storage or not self.current_session_id:
return
try:
session = self.storage.get_session(self.current_session_id)
if session is None:
return
session.device_info = {**(session.device_info or {}), **fields}
self.storage.update_session(session)
except (OSError, ValueError, sqlite3.Error) as e:
logger.warning(f"Failed to update session device_info in DataEngine: {e}")
def update_video_path(self, local_video_path: str | Path):
"""Update the video path across tables when traces or videos are moved."""
if not self.storage or not self.current_session_id:
+1 -1
View File
@@ -33,7 +33,7 @@ def parse_ui_hierarchy(xml_string: str) -> list[dict[str, Any]]:
for node in root.iter():
attribs = node.attrib
bounds_str = attribs.get("bounds", "")
match = re.match(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", bounds_str)
match = re.match(r"\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]", bounds_str)
if not match:
continue
+3 -1
View File
@@ -33,7 +33,9 @@ logger = get_logger(__name__)
def create_driver(ctx: "ArtemisContext") -> BaseDeviceDriver:
"""Instantiates the appropriate BaseDeviceDriver based on the runtime context."""
# 1. Cloud mode check
# 1. Cloud mode check. Cloud devices are reached through the gateway's
# RemoteUIAutomatorClient; ARTEMIS_HIERARCHY_BACKEND does not apply there
# because the Accessibility Helper needs a local adb forward.
if os.environ.get("ARTEMIS_CLOUD_MODE") == "1":
if ctx.adb_client is None:
from cloud_service.virtualization import RemoteAdbClient
+9 -2
View File
@@ -447,8 +447,15 @@ async def exit_settlement_node(state: State, ctx: ArtemisContext):
if snapshot.all_check_items or records or verify_blocked:
update["run_outcome"] = outcome.model_dump()
# BLOCKED/partial wrap-ups carry the last findings in the metadata file.
extra = {"last_findings": blocked_findings} if blocked_findings else None
write_run_outcome(base_dir, outcome, extra)
extra: dict = {"last_findings": blocked_findings} if blocked_findings else {}
# Which UI-hierarchy source served the run, so a report reader can tell a
# helper run from a UIAutomator2 fallback without reading logs.
from artemis.clients.screen_client_factory import hierarchy_backend_summary
environment = hierarchy_backend_summary(getattr(ctx, "ui_adb_client", None))
if environment:
extra["environment"] = environment
write_run_outcome(base_dir, outcome, extra or None)
publish_checker_event(
ctx,
{
+64
View File
@@ -27,6 +27,7 @@ from dataclasses import dataclass
import json
import platform as sys_platform
import shutil
import subprocess
from typing import Any
from rich.console import Console
@@ -194,6 +195,66 @@ def _showcase_row() -> ExtraRow:
)
def _helper_row(results: list[ProbeResult]) -> ExtraRow | None:
"""State of the Artemis accessibility helper on the one ready, idle device.
Tasks install the helper themselves when they take a device; this row lets a
person pre-install it so the first task does not pay the install delay, and
tells them the helper is a thing they can remove.
"""
from artemis.clients.screen_client_factory import resolve_backend
from artemis.runtime.helper_manager import helper_manager
if resolve_backend().value == "uiautomator":
return None
adb = next((r for r in results if r.id == "android_adb"), None)
devices = (adb.metadata.get("devices") if adb else None) or []
ready = [str(d.get("serial")) for d in devices if d.get("state") == "device"]
if len(ready) != 1:
return None
serial = ready[0]
try:
status = helper_manager.status(serial)
except (OSError, ValueError, subprocess.SubprocessError) as exc:
return ExtraRow(
key="accessibility_helper",
title="Artemis Accessibility Helper",
status="missing",
status_markup="[dim]⚪ Unknown[/dim]",
summary="Could not read",
detail=f"{serial}: {exc}",
)
healthy = status["installed"] and status["enabled"] and not status["outdated"]
if healthy and status["reachable"]:
version = status.get("installed_version")
note = " (newer than the bundled build)" if status.get("newer_than_bundled") else ""
return ExtraRow(
key="accessibility_helper",
title="Artemis Accessibility Helper",
status="pass",
status_markup="[bold green]✔ Ready[/bold green]",
summary=f"v{version} on {serial}{note}",
detail=f"Remove any time with: artemis helper uninstall --serial {serial}",
)
if not status["installed"]:
summary, why = "Not installed", "the first task on this device will install it (about 3 s)"
elif status["outdated"]:
summary = f"Outdated (v{status['installed_version']} < v{status['bundled_version']})"
why = "the next task will upgrade it"
elif not status["enabled"]:
summary, why = "Service disabled", "the next task will try to enable it"
else:
summary, why = "Not answering", "unlock the phone or reinstall with --force"
return ExtraRow(
key="accessibility_helper",
title="Artemis Accessibility Helper",
status="missing",
status_markup="[bold yellow]○ Pending[/bold yellow]",
summary=summary,
detail=f"{why}; pre-install now: artemis helper install --serial {serial}",
)
# ---------------------------------------------------------------------------
# Rendering
# ---------------------------------------------------------------------------
@@ -373,6 +434,9 @@ def doctor_command(
"""Run diagnostics to inspect system dependencies, device connectivity, and configuration."""
results, fixes = asyncio.run(_diagnose(fix))
extras = [_npm_row(), _showcase_row()]
helper_row = _helper_row(results)
if helper_row is not None:
extras.append(helper_row)
verdict = base_verdict(results)
if json_output:
+226
View File
@@ -0,0 +1,226 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""``artemis helper``: manage the Accessibility Helper APK on attached devices.
Tasks provision the helper themselves when they take a device, so these
commands exist for the cases the automatic path deliberately leaves alone:
inspecting a device, forcing a reinstall, and removing the helper (which is
never done automatically).
"""
from __future__ import annotations
import json
from typing import Annotated
from rich.console import Console
from rich.table import Table
import typer
from artemis.runtime.device_pool import device_pool
from artemis.runtime.helper_manager import helper_manager
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
helper_app = typer.Typer(help="Install, inspect, or remove the Artemis Accessibility Helper APK.")
SerialOption = Annotated[
str | None,
typer.Option("--serial", "-s", help="Device serial. Defaults to the only ready device."),
]
def _resolve_serial(serial: str | None) -> str:
if serial:
return serial
ready = [d.serial for d in device_pool.get_ready_devices()]
if len(ready) == 1:
return ready[0]
if not ready:
typer.secho("No authorized device is attached.", fg=typer.colors.RED)
else:
typer.secho(
"Several devices are ready; pass --serial, or --all to target every idle one: "
+ ", ".join(ready),
fg=typer.colors.RED,
)
raise typer.Exit(code=1)
def _idle_ready_serials() -> list[str]:
"""Every authorized device that no Artemis task currently holds."""
return [d.serial for d in device_pool.get_ready_devices() if not d.is_busy]
def _bool(value: object) -> str:
return "[green]yes[/green]" if value else "[red]no[/red]"
@helper_app.command("status")
def helper_status(
serial: SerialOption = None,
as_json: Annotated[bool, typer.Option("--json", help="Print raw JSON.")] = False,
) -> None:
"""Show install, version, service and tunnel state of the helper on a device."""
target = _resolve_serial(serial)
status = helper_manager.status(target)
if as_json:
typer.echo(json.dumps(status, indent=2))
return
table = Table(title=f"Accessibility helper on {target}")
table.add_column("Field", style="cyan")
table.add_column("Value")
table.add_row("Installed", _bool(status["installed"]))
version = str(status["installed_version"])
if status.get("newer_than_bundled"):
version += f" [yellow](newer than the bundled v{status['bundled_version']}; not downgraded)[/yellow]"
elif status.get("outdated"):
version += f" [yellow](bundled is v{status['bundled_version']}; the next task upgrades it)[/yellow]"
table.add_row("Installed version", version)
table.add_row("Bundled version", str(status["bundled_version"]))
table.add_row("Service enabled", _bool(status["enabled"]))
tunnel = {
"session": f"this process, host port {status['forward_port']}",
"shared": f"another Artemis process, host port {status['forward_port']}",
"probe": "none (probed through a temporary forward)",
None: "none",
}[status.get("tunnel")]
table.add_row("Tunnel", tunnel)
table.add_row("Service answering", _bool(status["reachable"]))
if status.get("reachable"):
protocol = status.get("protocol_version")
proto_text = str(protocol)
if not status.get("protocol_supported"):
proto_text += " [red](too old for this host; reinstall with --force)[/red]"
table.add_row("Protocol", proto_text)
table.add_row("Session token pushed", _bool(status.get("token_set")))
table.add_row("Auto-install by tasks", _bool(status.get("auto_install", True)))
table.add_row("Transport id", str(status["transport_id"]))
Console().print(table)
def _install_one(target: str, force: bool) -> bool:
def on_event(event: str, details: dict) -> None:
if event == "installing":
typer.echo(f"{target}: installing v{details.get('version_name')}...")
elif event == "upgrading":
typer.echo(
f"{target}: upgrading v{details.get('from_version')} -> "
f"v{details.get('to_version')}..."
)
result = helper_manager.provision(target, force=force, on_event=on_event)
if result.ok:
typer.secho(
f"{target}: {result.action} (version {result.installed_version}, service enabled).",
fg=typer.colors.GREEN,
)
return True
typer.secho(f"{target}: {result.action}: {result.error}", fg=typer.colors.RED)
return False
@helper_app.command("install")
def helper_install(
serial: SerialOption = None,
force: Annotated[bool, typer.Option("--force", help="Reinstall even when up to date.")] = False,
all_devices: Annotated[
bool, typer.Option("--all", help="Every authorized device that no task is using.")
] = False,
) -> None:
"""Install or upgrade the helper to the bundled version and enable its service."""
if all_devices:
targets = _idle_ready_serials()
if not targets:
typer.secho("No idle, authorized device is attached.", fg=typer.colors.RED)
raise typer.Exit(code=1)
else:
targets = [_resolve_serial(serial)]
outcomes = [_install_one(target, force) for target in targets]
if not all(outcomes):
raise typer.Exit(code=1)
@helper_app.command("uninstall")
def helper_uninstall(
serial: SerialOption = None,
yes: Annotated[bool, typer.Option("--yes", "-y", help="Do not ask for confirmation.")] = False,
) -> None:
"""Disable the service and remove the helper package from a device."""
target = _resolve_serial(serial)
if not yes and not typer.confirm(f"Remove the Artemis accessibility helper from {target}?"):
raise typer.Exit(code=0)
if helper_manager.uninstall(target):
typer.secho(f"{target}: helper removed.", fg=typer.colors.GREEN)
return
typer.secho(f"{target}: uninstall failed (is the package installed?).", fg=typer.colors.RED)
raise typer.Exit(code=1)
@helper_app.command("parity")
def helper_parity(
serial: SerialOption = None,
rounds: Annotated[int, typer.Option("--rounds", help="Timed dumps per backend.")] = 3,
as_json: Annotated[bool, typer.Option("--json", help="Print raw JSON.")] = False,
) -> None:
"""Dump the current screen with the helper and with UIAutomator2 and compare them.
The two backends must describe the same visible screen: every labelled
UIAutomator element should have a helper element with the same label at
the same place, the helper must emit no off-screen or negative bounds, and
its dump should not be slower. Run it on a busy screen (a scrolled list, a
dialog over an app) to catch regressions the unit tests cannot see.
"""
from artemis.core.diagnostics.hierarchy_parity import compare_backends
target = _resolve_serial(serial)
report = compare_backends(target, rounds=max(1, rounds))
if as_json:
typer.echo(json.dumps(report, indent=2, ensure_ascii=False))
raise typer.Exit(code=0 if report["ok"] else 1)
table = Table(title=f"Hierarchy parity on {target}")
table.add_column("Metric", style="cyan")
table.add_column("Helper")
table.add_column("UIAutomator2")
h, u = report["helper"], report["uiautomator"]
table.add_row("Dump time, median", f"{h['median_ms']:.0f} ms", f"{u['median_ms']:.0f} ms")
table.add_row("Nodes", str(h["nodes"]), str(u["nodes"]))
table.add_row("Labelled nodes (text / desc)", str(h["labelled"]), str(u["labelled"]))
table.add_row("Negative bounds", str(h["negative_bounds"]), str(u["negative_bounds"]))
table.add_row("Off-screen bounds", str(h["offscreen_bounds"]), str(u["offscreen_bounds"]))
table.add_row("Foreground package", str(h["package"]), str(u["package"]))
Console().print(table)
m = report["match"]
typer.echo(
f"UIAutomator labels found in helper dump: {m['uiautomator_in_helper']}/{m['uiautomator_labelled']}"
f" ({m['recall']:.0%}); helper labels found in UIAutomator dump: "
f"{m['helper_in_uiautomator']}/{m['helper_labelled']} ({m['precision']:.0%})"
)
for label in m["missing_in_helper"][:10]:
typer.echo(f" helper lacks: {label}")
for label in m["extra_in_helper"][:10]:
typer.echo(f" helper adds: {label}")
for problem in report["problems"]:
typer.secho(f" ! {problem}", fg=typer.colors.RED)
typer.secho(
"PARITY OK" if report["ok"] else "PARITY FAILED",
fg=typer.colors.GREEN if report["ok"] else typer.colors.RED,
)
raise typer.Exit(code=0 if report["ok"] else 1)
__all__ = ["helper_app"]
+4
View File
@@ -19,6 +19,7 @@ from typing import Annotated
from artemis._version import __version__
from artemis.interfaces.cli.commands.batch import batch_command
from artemis.interfaces.cli.commands.doctor import doctor_command
from artemis.interfaces.cli.commands.helper import helper_app
from artemis.interfaces.cli.commands.init import init_command
from artemis.interfaces.cli.commands.mcp import mcp_command
from artemis.interfaces.cli.commands.run import run_command
@@ -63,6 +64,9 @@ app.command(name="batch", help="Execute a batch sequence of automation tasks.")(
app.command(name="mcp", help="Start the Artemis Model Context Protocol (MCP) server.")(mcp_command)
app.add_typer(server_app, name="server", help="Cloud Run proxy and web dashboard server.")
app.add_typer(trace_app, name="trace", help="Inspect and query execution traces.")
app.add_typer(
helper_app, name="helper", help="Manage the Accessibility Helper APK on attached devices."
)
def version_callback(value: bool):
+5 -2
View File
@@ -39,7 +39,7 @@ except Exception as exc: # pylint: disable=broad-exception-caught
# Best-effort compatibility shim for FastMCP/pydantic version drift.
logger.debug("FastMCP Settings model_rebuild skipped: %s", exc, exc_info=True)
from artemis.clients.ui_automator_client import UIAutomatorClient
from artemis.clients.screen_client_factory import create_screen_client
from artemis.context import ArtemisContext, DeviceContext, DevicePlatform
from artemis.controllers.unified_controller import UnifiedMobileController
from artemis.platform import platform
@@ -146,7 +146,10 @@ def _get_controller(device_serial: str | None = None):
device = devices[0]
device_id = device.serial
ui_client = UIAutomatorClient(device_id=device_id)
# Observer path: the helper is used only when it is already installed and
# running; nothing is installed from here (that happens inside a task's
# device-lock boundary or via `artemis helper install`).
ui_client = create_screen_client(device_id)
try:
ui_data = ui_client.get_screen_data()
width, height = ui_data.width, ui_data.height
File diff suppressed because it is too large. Load diff
+14
View File
@@ -343,6 +343,20 @@ def update_trace_pid(trace_id: str, pid: int) -> dict[str, Any] | None:
return data
def update_trace_fields(trace_id: str, **fields: Any) -> dict[str, Any] | None:
"""Merge ``fields`` into status.json; no-op when the trace has no status file yet."""
path = get_status_path(trace_id)
if not os.path.exists(path):
return None
with _status_lock(path):
data = read_status(trace_id)
if not data:
return None
data.update(fields)
write_status(trace_id, data)
return data
def update_trace_device_serial(trace_id: str, device_serial: str) -> dict[str, Any] | None:
"""Updates the device_serial field of the status.json for a given trace_id."""
path = get_status_path(trace_id)
+150 -12
View File
@@ -13,6 +13,7 @@
# limitations under the License.
import asyncio
import inspect
import os
import re
@@ -29,7 +30,7 @@ from shutil import which
import sys
import threading
from types import NoneType
from typing import TypeVar, overload
from typing import Any, TypeVar, overload
import uuid
from adbutils import AdbClient
@@ -41,7 +42,13 @@ from pydantic import BaseModel
from artemis.agents.flash.runner import FlashRunner
from artemis.agents.outputter.outputter import outputter
from artemis.clients.ui_automator_client import UIAutomatorClient
from artemis.clients.screen_client_factory import (
ScreenClient,
create_screen_client,
describe_backend,
helper_version,
hierarchy_backend_sentence,
)
from artemis.config import (
CheckerConfig,
OutputConfig,
@@ -60,7 +67,7 @@ from artemis.controllers.controller_factory import get_controller
from artemis.controllers.platform_specific_commands_controller import (
get_first_device,
)
from artemis.runtime import DeviceExecutionLock
from artemis.runtime import DeviceExecutionLock, trace_store
from artemis.runtime.cancel_requests import watch_for_cancel_request
from artemis.data_engine.engine import DataEngine
from artemis.data_engine.trace import DataEngineCallbackHandler
@@ -126,7 +133,7 @@ class Agent:
_initialized: bool = False
_device_context: DeviceContext
_adb_client: AdbClient | None
_ui_adb_client: UIAutomatorClient | None
_ui_adb_client: ScreenClient | Any | None
_current_task: asyncio.Task | None = None
_task_lock: asyncio.Lock
@@ -237,6 +244,8 @@ class Agent:
platform=platform,
)
else:
# Cloud mode is pinned to UIAutomator2 through the gateway; the
# hierarchy backend switch only applies to locally attached devices.
from cloud_service.virtualization import RemoteAdbClient, RemoteUIAutomatorClient
self._adb_client = RemoteAdbClient()
@@ -632,13 +641,7 @@ class Agent:
self._prepare_output_files(task=task)
if os.environ.get("ARTEMIS_CLOUD_MODE") != "1":
if self._ui_adb_client is not None:
publish_startup_progress(
"uiautomator", "Connecting to UI Automator", session_id=str(sess_id)
)
await asyncio.to_thread(self._ui_adb_client.connect)
publish_startup_progress(
"uiautomator_ready", "UI Automator is ready", session_id=str(sess_id)
)
await self._connect_screen_client(context, str(sess_id))
await self._ensure_device_unlocked()
publish_startup_progress(
"environment", "Preparing the device environment", session_id=str(sess_id)
@@ -1277,6 +1280,141 @@ class Agent:
def _get_graph_state(self, task: Task):
return State.initial(task.request.goal)
# ------------------------------------------------------------------ #
# UI hierarchy backend: connect, and keep the user told which one serves
# ------------------------------------------------------------------ #
async def _connect_screen_client(self, context: ArtemisContext, session_id: str) -> None:
"""Connect the screen client with visible progress for slow first-time steps."""
client = self._ui_adb_client
previous_listener = getattr(self, "_hierarchy_backend_listener", None)
if previous_listener is not None:
previous_client, listener = previous_listener
previous_client.remove_backend_listener(listener)
self._hierarchy_backend_listener = None
publish_startup_progress(
"uiautomator", "Connecting to the UI hierarchy service", session_id=session_id
)
def on_provision(event: str, details: dict) -> None:
version = details.get("version_name") or details.get("to_version")
if event == "installing":
publish_startup_progress(
"helper_install",
"Installing the Artemis accessibility helper on this device for the "
f"first time (v{version}, about 3 seconds)",
session_id=session_id,
**details,
)
elif event == "upgrading":
publish_startup_progress(
"helper_upgrade",
"Upgrading the Artemis accessibility helper "
f"(v{details.get('from_version')} -> v{details.get('to_version')})",
session_id=session_id,
**details,
)
# Clients without provisioning (UIAutomator2, cloud) take no callback.
try:
accepts_events = "on_event" in inspect.signature(client.connect).parameters
except (TypeError, ValueError):
accepts_events = False
if accepts_events:
await asyncio.to_thread(client.connect, on_event=on_provision)
else:
await asyncio.to_thread(client.connect)
backend = describe_backend(client) or "uiautomator"
publish_startup_progress(
"uiautomator_ready",
f"UI hierarchy service is ready ({backend})",
session_id=session_id,
)
self._announce_hierarchy_backend(context, session_id, None, backend, None)
add_listener = getattr(client, "add_backend_listener", None)
if callable(add_listener):
def listener(previous, new, reason):
self._announce_hierarchy_backend(context, session_id, previous, new, reason)
add_listener(listener)
self._hierarchy_backend_listener = (client, listener)
def _announce_hierarchy_backend(
self,
context: ArtemisContext,
session_id: str,
previous: str | None,
backend: str,
reason: str | None,
) -> None:
"""One line everywhere a person or an agent looks for it.
Startup-progress event (UI timeline), a named log trace, the session's
device_info, and status.json for ``mobile_manage_task``. Runs from
worker threads too, so every sink is best effort.
"""
label = {"helper": "Artemis accessibility helper", "uiautomator": "UIAutomator2"}
version = helper_version(self._ui_adb_client)
if previous is None:
message = f"UI hierarchy source: {label.get(backend, backend)}"
if backend == "helper" and version:
message += f" v{version}"
stage = "hierarchy_backend"
else:
message = (
f"UI hierarchy source switched from {label.get(previous, previous)} to "
f"{label.get(backend, backend)}"
)
if reason:
message += f" because {reason}"
stage = "hierarchy_backend_changed"
logger.info(message)
try:
publish_startup_progress(
stage,
message,
session_id=session_id,
backend=backend,
previous_backend=previous,
reason=reason,
helper_version=version,
)
except (OSError, ValueError, RuntimeError) as exc:
logger.debug(f"Could not publish hierarchy backend progress: {exc}")
engine = getattr(context, "data_engine", None)
if engine is not None and getattr(engine, "current_session_id", None):
try:
engine.record_trace(
type="log",
name="hierarchy_backend",
payload={
"message": message,
"backend": backend,
"previous_backend": previous,
"reason": reason,
"helper_version": version,
"level": "WARNING" if previous is not None else "INFO",
},
)
engine.update_session_device_info(
hierarchy_backend=backend,
hierarchy_backend_note=hierarchy_backend_sentence(
self._ui_adb_client, relative_time=engine.get_relative_time
),
)
except (OSError, ValueError, RuntimeError) as exc:
logger.debug(f"Could not record hierarchy backend in the data engine: {exc}")
try:
trace_store.update_trace_fields(
session_id,
hierarchy_backend=backend,
hierarchy_backend_note=hierarchy_backend_sentence(self._ui_adb_client),
)
except (OSError, ValueError, RuntimeError) as exc:
logger.debug(f"Could not record hierarchy backend in status.json: {exc}")
def _init_clients(
self,
device_id: str,
@@ -1286,7 +1424,7 @@ class Agent:
host=self._config.servers.adb_host,
port=self._config.servers.adb_port,
)
self._ui_adb_client = UIAutomatorClient(device_id=device_id)
self._ui_adb_client = create_screen_client(device_id)
async def _get_device_context(
self,
+6 -1
View File
@@ -59,7 +59,7 @@ def _parse_bounds(bounds: Any) -> dict[str, int] | None:
return res
if isinstance(bounds, str):
match = re.match(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", bounds)
match = re.match(r"\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]", bounds)
if match:
x1, y1, x2, y2 = map(int, match.groups())
return {"left": x1, "top": y1, "right": x2, "bottom": y2}
@@ -112,6 +112,11 @@ def _clip_bounds(
if is_fully_clipped:
node["is_clipped"] = True
# The cached parse is what every later consumer (and _parse_bounds itself)
# reads first; leaving it stale would hand out the unclipped rectangle.
if isinstance(node.get("parsed_bounds"), dict):
node["parsed_bounds"] = new_bounds.copy()
# Re-serialize bounds into the node matching original format
orig = node.get("bounds")
if isinstance(orig, str):
+28 -15
View File
@@ -398,29 +398,42 @@ def format_minimal_list_with_elements(
idx += 1
registered = True
if not registered and text.strip() and bounds:
# An empty input field carries no text but usually a hint ("Search settings");
# a rejected one carries an error ("Password too short"). Both come from the
# accessibility helper's dump and are exactly what the agent needs to see.
hint = str(node.get("hint") or "").strip()
error = str(node.get("error") or "").strip()
shown = text.strip() or hint
if not registered and shown and bounds:
left, top, right, bottom = bounds
cx, cy = get_center_coordinates(left, top, right, bottom)
if is_duplicate(text, cx, cy):
if is_duplicate(shown, cx, cy):
continue
norm_bounds = normalize_bounds(left, top, right, bottom)
lines.append(f"[{idx}] Text: '{text.strip()}' | Bounds: {norm_bounds}")
kind = "Text" if text.strip() else "Hint"
line = f"[{idx}] {kind}: '{shown}' | Bounds: {norm_bounds}"
if error:
line += f" | Error: '{error}'"
lines.append(line)
items_bounds.append((idx, (left, top, right, bottom)))
elements.append(
{
"index": idx,
"center": [cx, cy],
"text": text.strip(),
"bounds": [left, top, right, bottom],
"class": node.get("class"),
"resource_id": node.get("resource-id"),
"is_ocr": False,
}
)
element = {
"index": idx,
"center": [cx, cy],
"text": shown,
"bounds": [left, top, right, bottom],
"class": node.get("class"),
"resource_id": node.get("resource-id"),
"is_ocr": False,
}
if not text.strip():
element["is_hint"] = True
if error:
element["error"] = error
elements.append(element)
labels.append(str(idx))
added_items.append((text.strip(), cx, cy))
added_items.append((shown, cx, cy))
idx += 1
lines = _inject_mutual_occlusion_warnings(lines, items_bounds)
+198 -9
View File
@@ -24,6 +24,8 @@ to end (screenshot + UI hierarchy) to prove a task could actually start.
from __future__ import annotations
import subprocess
import asyncio
import os
from pathlib import Path
@@ -42,6 +44,7 @@ from artemis.core.diagnostics.readiness import (
)
from artemis.core.diagnostics.schema import ProbeResult, ProbeStatus, SystemReadinessReport
from artemis.runtime import DeviceExecutionLock, trace_store
from artemis.runtime.helper_manager import helper_manager
from artemis.utils.credentials_validator import validate_api_key
from artemis.utils.logger import get_logger
@@ -499,6 +502,159 @@ def _device_probe_steps(device_probe: dict[str, Any] | None) -> list[str]:
return steps
# --------------------------------------------------------------------------- #
# Accessibility helper (UI hierarchy backend)
# --------------------------------------------------------------------------- #
def _hierarchy_backend() -> str:
try:
from artemis.clients.screen_client_factory import resolve_backend
return resolve_backend().value
except (ImportError, ValueError):
return "auto"
def _helper_target(adb_result: ProbeResult | None, requested_device: str | None) -> str | None:
"""The one ready device whose helper state is worth reporting, or None."""
devices = (adb_result.metadata.get("devices") if adb_result else None) or []
ready = [str(d.get("serial")) for d in devices if d.get("state") == "device"]
if requested_device:
return requested_device if requested_device in ready else None
return ready[0] if len(ready) == 1 else None
def _helper_status(serial: str) -> dict[str, Any]:
try:
status = helper_manager.status(serial)
except (OSError, ValueError, RuntimeError, subprocess.SubprocessError) as exc:
return {"serial": serial, "error": f"{exc.__class__.__name__}: {exc}"}
status["serial"] = serial
status["backend"] = _hierarchy_backend()
return status
def _helper_needs_provision(status: dict[str, Any] | None) -> bool:
if not status or status.get("error"):
return False
return not status.get("installed") or bool(status.get("outdated")) or not status.get("enabled")
def _helper_fix(
adb_result: ProbeResult | None, requested_device: str | None
) -> dict[str, Any] | None:
"""Install / upgrade / enable the helper on the idle target device (attempt_fix only)."""
serial = _helper_target(adb_result, requested_device)
if serial is None or _hierarchy_backend() == "uiautomator":
return None
status = _helper_status(serial)
if not _helper_needs_provision(status):
return None
if not status.get("bundled_apk_present"):
return {
"fix": "install_accessibility_helper",
"success": False,
"skipped": True,
"message": "Skipped: the bundled ArtemisAccessibilityHelper.apk is missing from the checkout.",
}
wanted = _normalize_serial(serial)
if any(_normalize_serial(entry.get("device")) == wanted for entry in _task_state()["active"]):
return {
"fix": "install_accessibility_helper",
"success": False,
"skipped": True,
"message": f"Skipped: a task is running on {serial}; install it after the task finishes.",
}
try:
result = helper_manager.provision(serial)
except (OSError, ValueError, RuntimeError, subprocess.SubprocessError) as exc:
return {
"fix": "install_accessibility_helper",
"success": False,
"skipped": False,
"message": f"provisioning raised {exc.__class__.__name__}: {exc}",
}
message = (
f"{result.action} (installed version {result.installed_version}, "
f"bundled {result.bundled_version}, service enabled={result.enabled})"
)
if result.error:
message += f": {result.error}"
return {
"fix": "install_accessibility_helper",
"success": result.ok,
"skipped": False,
"message": message,
}
def _helper_steps(status: dict[str, Any] | None, *, attempt_fix: bool) -> list[str]:
if not status:
return []
serial = status.get("serial") or "the device"
backend = status.get("backend") or "auto"
if backend == "uiautomator":
return []
if status.get("error"):
return [
f"[OPTIONAL] Accessibility helper state on {serial} could not be read: {status['error']}"
]
healthy = status.get("installed") and not status.get("outdated") and status.get("enabled")
if healthy and status.get("reachable"):
return []
if not status.get("installed"):
problem = "is not installed"
elif status.get("outdated"):
problem = (
f"is outdated (device has version {status.get('installed_version')}, "
f"bundled is {status.get('bundled_version')})"
)
elif not status.get("enabled"):
problem = "is installed but its accessibility service is disabled"
elif status.get("newer_than_bundled") and status.get("reachable"):
return [] # a newer dev build is fine; nothing to do
else:
problem = "is installed and enabled but not answering on its loopback port"
tag = "REQUIRED" if backend == "helper" else "OPTIONAL"
consequence = (
"tasks cannot read the UI hierarchy until it is fixed"
if backend == "helper"
else "tasks fall back to UIAutomator2 (slower dumps, conflicts with Appium/Mobly)"
)
steps = [f"[{tag}] Accessibility helper on {serial} {problem}; {consequence}."]
if not status.get("bundled_apk_present"):
steps.append(
" Guidance: the bundled APK is missing; build it with "
"packages/artemis-accessibility-helper/build_apk.sh."
)
return steps
if problem.startswith("is installed and enabled"):
steps.append(
" Guidance: unlock the phone; if the helper still does not answer, reinstall it with "
"the command below (add --force)."
)
steps.append(f" Run: uv run artemis helper install --serial {serial} --force")
return steps
if not attempt_fix:
steps.append(
" Guidance: call mobile_diagnose(attempt_fix=true) to install and enable it "
"without touching the phone, or run the command below."
)
steps.append(f" Run: uv run artemis helper install --serial {serial}")
if problem.startswith("is installed but"):
# Enabling from adb was already attempted once by whoever installed it;
# on ROMs that reject it only a person can flip the switch.
from artemis.runtime.helper_manager import MANUAL_ENABLE_PATH
steps.append(
" Guidance: if the command reports that this device rejected enabling the "
f"service, ask the user to turn it on by hand: {MANUAL_ENABLE_PATH} "
"(the command opens that settings screen on the phone)."
)
return steps
# --------------------------------------------------------------------------- #
# next_steps
# --------------------------------------------------------------------------- #
@@ -575,6 +731,7 @@ def _next_steps(
credentials: list[dict[str, Any]] | None,
tasks: dict[str, list[dict[str, Any]]],
device_probe: dict[str, Any] | None,
accessibility_helper: dict[str, Any] | None = None,
) -> list[str]:
steps: list[str] = []
needs_restart = False
@@ -628,6 +785,7 @@ def _next_steps(
)
steps.extend(launch_steps)
steps.extend(_device_probe_steps(device_probe))
steps.extend(_helper_steps(accessibility_helper, attempt_fix=attempt_fix))
for fix in fixes_applied:
outcome = "applied" if fix.get("success") else "did not help"
@@ -750,7 +908,9 @@ def _cleanup_stale_locks() -> dict[str, Any] | None:
}
async def _apply_safe_fixes(report: SystemReadinessReport) -> list[dict[str, Any]]:
async def _apply_safe_fixes(
report: SystemReadinessReport, requested_device: str | None = None
) -> list[dict[str, Any]]:
"""Run the self-heals the console exposes as buttons, only when they cannot disrupt a task."""
applied: list[dict[str, Any]] = []
cleanup = _cleanup_stale_locks()
@@ -768,7 +928,12 @@ async def _apply_safe_fixes(report: SystemReadinessReport) -> list[dict[str, Any
devices = adb.metadata.get("devices") or []
if any(d.get("state") == "device" for d in devices):
return applied # a ready device exists; nothing to restart
# A ready device exists; nothing to restart. Provision the accessibility
# helper on it when it is idle and missing, outdated or disabled.
helper_fix = await asyncio.to_thread(_helper_fix, adb, requested_device)
if helper_fix is not None:
applied.append(helper_fix)
return applied
if DeviceExecutionLock.get_active_owners():
applied.append(
{
@@ -787,7 +952,9 @@ async def _apply_safe_fixes(report: SystemReadinessReport) -> list[dict[str, Any
def _probes_changed(fixes_applied: list[dict[str, Any]]) -> bool:
"""Only ADB-side fixes can change probe results; lock cleanup does not warrant a re-run."""
return any(
fix.get("success") and not fix.get("skipped") and fix.get("fix") != "cleanup_stale_locks"
fix.get("success")
and not fix.get("skipped")
and fix.get("fix") not in ("cleanup_stale_locks", "install_accessibility_helper")
for fix in fixes_applied
)
@@ -817,15 +984,18 @@ async def _run_extras(
else:
emulator = _emulator_status()
credentials, device_probe = await asyncio.gather(
helper_serial = _helper_target(adb_result, requested_device)
credentials, device_probe, accessibility_helper = await asyncio.gather(
_verify_credentials(_find(results, "gemini_api_key")) if verify_credentials else _none(),
_run_device_probe(adb_result, requested_device) if probe_device else _none(),
asyncio.to_thread(_helper_status, helper_serial) if helper_serial else _none(),
)
return {
"emulator": emulator,
"launch_steps": launch_steps,
"credentials": credentials,
"device_probe": device_probe,
"accessibility_helper": accessibility_helper,
"tasks": _task_state(),
}
@@ -847,6 +1017,7 @@ def _verdict(
requested_device: str | None,
credentials: list[dict[str, Any]] | None,
device_probe: dict[str, Any] | None,
accessibility_helper: dict[str, Any] | None = None,
) -> str:
verdict = base_verdict(results)
if verdict == "blocked" or not _requested_device_ready(results, requested_device):
@@ -856,8 +1027,13 @@ def _verdict(
return "blocked"
if device_probe is not None and not device_probe.get("ok"):
return "blocked"
helper_backend = (accessibility_helper or {}).get("backend")
if helper_backend == "helper" and not accessibility_helper.get("reachable"):
return "blocked"
if any(not entry.get("valid") for entry in credentials or []):
return "degraded"
if helper_backend == "auto" and _helper_needs_provision(accessibility_helper):
return "degraded" if verdict == "ready" else verdict
return verdict
@@ -949,7 +1125,12 @@ async def mobile_diagnose(
traces_dir, mcp_client (when known), daemon {port, reachable,
port_held_by_other_process, log_path}.
- `device`: the device a task would use ({serial, state, model,
android_version, is_locked, is_emulator}) or null.
android_version, is_locked, is_emulator, accessibility_helper}) or
null. `accessibility_helper` describes the Artemis UI-hierarchy
helper APK on that device ({installed, installed_version,
bundled_version, outdated, enabled, forward_port, reachable,
backend}); with backend "auto" a missing helper only degrades
(UIAutomator2 fallback), with backend "helper" it blocks.
- `emulator`: background emulator launch state ({avd_name, status,
stage_message, error, serial, elapsed_seconds, progress_percent}) or
null when nothing was launched. `status` is starting /
@@ -970,8 +1151,10 @@ async def mobile_diagnose(
Args:
attempt_fix: When true, applies the safe self-heals the ARTEMIS
console offers: remove device locks left by dead processes,
regenerate corrupted ADB RSA keys, and restart the ADB server
(only when no device is ready and no task holds a device). Then
regenerate corrupted ADB RSA keys, restart the ADB server (only
when no device is ready and no task holds a device), and install,
upgrade or enable the Artemis accessibility helper APK on the idle
target device when it is missing, outdated or disabled. Then
re-runs the checks. Nothing else is changed.
device_serial: Optional serial the user wants to use; the report
then states explicitly whether that device is attached, authorized
@@ -1004,7 +1187,7 @@ async def mobile_diagnose(
)
if attempt_fix:
fixes_applied = await asyncio.wait_for(
_apply_safe_fixes(report), timeout=DIAGNOSIS_TIMEOUT_SECONDS
_apply_safe_fixes(report, requested_device), timeout=DIAGNOSIS_TIMEOUT_SECONDS
)
if _probes_changed(fixes_applied):
report, host = await asyncio.wait_for(
@@ -1026,12 +1209,17 @@ async def mobile_diagnose(
credentials: list[dict[str, Any]] | None = extras["credentials"]
device_probe: dict[str, Any] | None = extras["device_probe"]
accessibility_helper: dict[str, Any] | None = extras.get("accessibility_helper")
verdict = _verdict(
results,
requested_device=requested_device,
credentials=credentials,
device_probe=device_probe,
accessibility_helper=accessibility_helper,
)
device = _compact_device(report)
if device is not None and accessibility_helper is not None:
device["accessibility_helper"] = accessibility_helper
env_file = host.metadata.get("env_file")
return {
"verdict": verdict,
@@ -1047,10 +1235,11 @@ async def mobile_diagnose(
credentials=credentials,
tasks=extras["tasks"],
device_probe=device_probe,
accessibility_helper=accessibility_helper,
),
"checks": [_render_check(r) for r in sort_by_fix_order(results)],
"host": _compact_host(host.metadata),
"device": _compact_device(report),
"device": device,
"emulator": extras["emulator"],
"tasks": extras["tasks"],
"credentials": credentials,
+6
View File
@@ -334,6 +334,12 @@ def mobile_manage_task(
"stdout_log": os.path.join(trace_dir, "stdout.log"),
"stderr_log": os.path.join(trace_dir, "stderr.log"),
}
# Which UI-hierarchy source served the run ("helper" or "uiautomator"),
# plus any mid-run switch; written by the agent once the device connects.
if status_data.get("hierarchy_backend"):
response["hierarchy_backend"] = status_data.get("hierarchy_backend")
if status_data.get("hierarchy_backend_note"):
response["hierarchy_backend_note"] = status_data.get("hierarchy_backend_note")
if status_data.get("model", "").lower() != "flash":
response["notes_dir"] = os.path.join(trace_dir, "notes")
@@ -10,8 +10,8 @@ android {
applicationId = "com.artemis.helper"
minSdk = 24
targetSdk = 35
versionCode = 1
versionName = "1.0.0"
versionCode = 6
versionName = "1.2.0"
}
buildTypes {
@@ -32,6 +32,18 @@
android:value="Artemis mobile automation test bridge" />
</service>
<!-- Session token drop: only a sender holding WRITE_SECURE_SETTINGS (the adb
shell user, the system) can deliver this broadcast, so no ordinary app on
the device can set a token of its own and then talk to the loopback server. -->
<receiver
android:name=".TokenReceiver"
android:exported="true"
android:permission="android.permission.WRITE_SECURE_SETTINGS">
<intent-filter>
<action android:name="com.artemis.helper.SET_TOKEN" />
</intent-filter>
</receiver>
</application>
</manifest>
@@ -1,6 +1,5 @@
package com.artemis.helper;
import android.graphics.Rect;
import org.json.JSONArray;
import org.json.JSONObject;
@@ -12,6 +11,12 @@ import java.util.List;
* Decoupled from live Android AccessibilityNodeInfo objects to prevent
* Binder proxy memory leaks, avoid stale node crashes, and ensure
* instant thread-safe serialization to both XML and JSON.
*
* {@code bounds} are the node's <em>visible</em> bounds: the raw
* {@code getBoundsInScreen} rectangle intersected with the display, the
* node's window and every scrollable ancestor, exactly like UIAutomator's
* {@code getVisibleBoundsInScreen}. They are therefore never negative and
* never extend past the screen.
*/
public final class A11yNode {
@@ -32,6 +37,7 @@ public final class A11yNode {
public boolean longClickable = false;
public boolean password = false;
public boolean selected = false;
public boolean visibleToUser = true;
// Window metadata for multi-window awareness
public int windowId = -1;
@@ -94,9 +100,12 @@ public final class A11yNode {
XmlUtils.appendBooleanAttribute(sb, "long-clickable", longClickable);
XmlUtils.appendBooleanAttribute(sb, "password", password);
XmlUtils.appendBooleanAttribute(sb, "selected", selected);
XmlUtils.appendBooleanAttribute(sb, "visible-to-user", visibleToUser);
XmlUtils.appendAttribute(sb, "bounds", getBoundsString());
XmlUtils.appendIntAttribute(sb, "drawing-order", drawingOrder);
XmlUtils.appendAttribute(sb, "hint", hint);
if (!hint.isEmpty()) {
XmlUtils.appendAttribute(sb, "hint", hint);
}
// Window metadata (emitted on window root nodes)
if (windowId >= 0) {
@@ -199,6 +208,7 @@ public final class A11yNode {
obj.put("selected", selected);
obj.put("password", password);
obj.put("long-clickable", longClickable);
obj.put("visible-to-user", visibleToUser);
obj.put("drawing-order", drawingOrder);
if (!hint.isEmpty()) {
obj.put("hint", hint);
@@ -7,6 +7,7 @@ import android.app.NotificationChannel;
import android.app.NotificationManager;
import android.content.Context;
import android.content.Intent;
import android.content.pm.PackageInfo;
import android.content.pm.ServiceInfo;
import android.os.Build;
import android.util.Log;
@@ -17,12 +18,24 @@ import android.view.accessibility.AccessibilityEvent;
*
* Runs concurrently with any Android UI testing framework (Mobly, Appium, Espresso)
* without monopolizing Android's singleton UiAutomationService connection.
*
* The service subscribes to window-state changes only: that is the single event it
* uses (to remember the foreground package / activity), and anything wider costs
* CPU and battery on a device that is idle between tasks.
*/
public class ArtemisAccessibilityService extends AccessibilityService {
private static final String TAG = "ArtemisA11yService";
public static final int DEFAULT_PORT = 18888;
private static final String CHANNEL_ID = "artemis_helper_channel";
/**
* Bumped whenever the HTTP contract changes in a way an older host cannot use.
* 2: token authentication, byte-accurate bodies, visible-only dumps, fields=.
*/
public static final int PROTOCOL_VERSION = 2;
// Channel importance is frozen by Android once a channel exists, so a quieter
// channel needs a new id; the pre-1.1.3 channel is deleted on start.
private static final String LEGACY_CHANNEL_ID = "artemis_helper_channel";
private static final String CHANNEL_ID = "artemis_helper_quiet";
private static final int NOTIFICATION_ID = 18888;
private static volatile ArtemisAccessibilityService instance;
@@ -43,6 +56,29 @@ public class ArtemisAccessibilityService extends AccessibilityService {
return currentActivityName;
}
/** Installed helper versionCode, or -1 when the package manager cannot answer. */
@SuppressWarnings("deprecation")
public long getVersionCode() {
try {
PackageInfo info = getPackageManager().getPackageInfo(getPackageName(), 0);
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
return info.getLongVersionCode();
}
return info.versionCode;
} catch (Throwable t) {
return -1L;
}
}
public String getVersionName() {
try {
PackageInfo info = getPackageManager().getPackageInfo(getPackageName(), 0);
return info.versionName == null ? "" : info.versionName;
} catch (Throwable t) {
return "";
}
}
@Override
public void onServiceConnected() {
super.onServiceConnected();
@@ -58,12 +94,14 @@ public class ArtemisAccessibilityService extends AccessibilityService {
if (info == null) {
info = new AccessibilityServiceInfo();
}
info.eventTypes = AccessibilityEvent.TYPES_ALL_MASK;
info.eventTypes = AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED;
info.feedbackType = AccessibilityServiceInfo.FEEDBACK_GENERIC;
info.notificationTimeout = 50;
info.notificationTimeout = 100;
info.flags |= AccessibilityServiceInfo.FLAG_RETRIEVE_INTERACTIVE_WINDOWS
| AccessibilityServiceInfo.FLAG_INCLUDE_NOT_IMPORTANT_VIEWS
| AccessibilityServiceInfo.FLAG_REPORT_VIEW_IDS;
// Not-important views are what UIAutomator's compressed dump leaves out;
// keep them out here too so both backends describe the same tree.
info.flags &= ~AccessibilityServiceInfo.FLAG_INCLUDE_NOT_IMPORTANT_VIEWS;
setServiceInfo(info);
Log.i(TAG, "AccessibilityServiceInfo flags dynamically enforced");
} catch (Throwable t) {
@@ -94,12 +132,19 @@ public class ArtemisAccessibilityService extends AccessibilityService {
if (nm == null) return;
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
try {
nm.deleteNotificationChannel(LEGACY_CHANNEL_ID);
} catch (Throwable ignored) {}
// Ask for IMPORTANCE_MIN (no sound, collapsed in the shade). Android raises
// foreground-service channels to LOW at most, which is still silent. The
// notification exists only to keep the service alive on aggressive ROMs;
// it must not look like something the person needs to act on.
NotificationChannel channel = new NotificationChannel(
CHANNEL_ID,
"Artemis Helper Service",
NotificationManager.IMPORTANCE_LOW
"Artemis test helper",
NotificationManager.IMPORTANCE_MIN
);
channel.setDescription("Keeps Artemis Accessibility Helper active for automation testing");
channel.setDescription("Shown while the Artemis test helper is installed on this device");
channel.setShowBadge(false);
nm.createNotificationChannel(channel);
}
@@ -111,10 +156,20 @@ public class ArtemisAccessibilityService extends AccessibilityService {
builder = new Notification.Builder(this);
}
builder.setContentTitle("Artemis Accessibility Helper")
.setContentText("Active on 127.0.0.1:" + DEFAULT_PORT)
builder.setContentTitle("Artemis test helper is running")
.setContentText("Lets Artemis read this screen during automated tests. "
+ "Remove with: artemis helper uninstall")
.setStyle(new Notification.BigTextStyle().bigText(
"Installed by Artemis to read the screen layout during automated "
+ "tests. It answers only on this device (127.0.0.1:" + DEFAULT_PORT
+ "), only to the computer that is connected over USB debugging and "
+ "holds the session token, and it sends nothing anywhere. Remove it "
+ "any time with `artemis helper uninstall` or from Settings > Apps."))
.setSmallIcon(android.R.drawable.stat_notify_sync)
.setOngoing(true);
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) {
builder.setPriority(Notification.PRIORITY_MIN);
}
if (Build.VERSION.SDK_INT >= 34) {
startForeground(NOTIFICATION_ID, builder.build(), ServiceInfo.FOREGROUND_SERVICE_TYPE_SPECIAL_USE);
@@ -3,27 +3,46 @@ package com.artemis.helper;
import android.util.Log;
import org.json.JSONObject;
import java.io.BufferedReader;
import java.io.BufferedInputStream;
import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.InputStreamReader;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.Socket;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/**
* High-performance, multi-threaded command server bound strictly to loopback (127.0.0.1:18888).
* Supports:
* - HTTP REST API (/dump, /dump_xml, /hierarchy.xml, /ping, /action)
* - Line-delimited JSON-RPC over TCP socket
* Multi-threaded command server bound strictly to loopback (127.0.0.1:18888).
*
* Protocols:
* - HTTP (/ping, /dump, /dump_xml, /hierarchy.xml, /snapshot, /action)
* - Line-delimited JSON-RPC over a raw TCP socket
*
* Every endpoint except {@code /ping} requires the session token the host pushed
* through {@link TokenReceiver} (header {@code X-Artemis-Token}, query parameter
* {@code token}, or JSON field {@code token}). Loopback is reachable by every app
* on the device, so without the token this service would hand any local app a
* full-screen reader and a gesture injector.
*
* Request bodies are read as bytes (Content-Length is a byte count), so UTF-8
* payloads of any script are handled exactly.
*/
public class CommandServer extends Thread {
private static final String TAG = "ArtemisCommandServer";
private static final int MAX_HEADER_LINE = 16 * 1024;
private static final int MAX_BODY = 4 * 1024 * 1024;
private static final String TOKEN_HEADER = "x-artemis-token";
private final ArtemisAccessibilityService service;
private final int port;
private volatile boolean isRunning = true;
@@ -67,25 +86,66 @@ public class CommandServer extends Thread {
}
}
// ------------------------------------------------------------------ //
// Byte-accurate request reading
// ------------------------------------------------------------------ //
/** Reads one line (without CR/LF) as raw bytes; null at end of stream. */
private static byte[] readLineBytes(InputStream in) throws Exception {
ByteArrayOutputStream line = new ByteArrayOutputStream(256);
int b;
while ((b = in.read()) >= 0) {
if (b == '\n') {
break;
}
if (b != '\r') {
line.write(b);
}
if (line.size() > MAX_HEADER_LINE) {
throw new IllegalStateException("Header line too long");
}
}
if (b < 0 && line.size() == 0) {
return null;
}
return line.toByteArray();
}
private static byte[] readExactly(InputStream in, int length) throws Exception {
byte[] buf = new byte[length];
int total = 0;
while (total < length) {
int r = in.read(buf, total, length - total);
if (r < 0) break;
total += r;
}
if (total < length) {
byte[] shorter = new byte[total];
System.arraycopy(buf, 0, shorter, 0, total);
return shorter;
}
return buf;
}
private void handleClient(Socket socket) {
try {
socket.setSoTimeout(10000);
BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8));
InputStream in = new BufferedInputStream(socket.getInputStream());
OutputStream output = socket.getOutputStream();
String firstLine = reader.readLine();
byte[] firstLine = readLineBytes(in);
if (firstLine == null) return;
String trimmed = firstLine.trim();
String ascii = new String(firstLine, StandardCharsets.ISO_8859_1).trim();
if (trimmed.startsWith("GET ") || trimmed.startsWith("POST ")) {
handleHttp(trimmed, reader, output);
} else if (trimmed.startsWith("{")) {
handleJsonRpc(trimmed, output);
if (ascii.startsWith("GET ") || ascii.startsWith("POST ")) {
handleHttp(ascii, in, output);
} else if (ascii.startsWith("{")) {
handleJsonRpc(new String(firstLine, StandardCharsets.UTF_8).trim(), output);
} else {
JSONObject err = new JSONObject();
err.put("success", false);
err.put("error", "Unsupported protocol");
sendJsonResponse(output, err.toString());
sendJson(output, 400, err.toString());
}
} catch (Exception e) {
Log.w(TAG, "Client handling error: " + e.getMessage());
@@ -94,82 +154,131 @@ public class CommandServer extends Thread {
}
}
private void handleHttp(String requestLine, BufferedReader reader, OutputStream output) {
// ------------------------------------------------------------------ //
// HTTP
// ------------------------------------------------------------------ //
private static Map<String, String> parseQuery(String query) {
Map<String, String> params = new HashMap<>();
if (query == null || query.isEmpty()) return params;
for (String pair : query.split("&")) {
if (pair.isEmpty()) continue;
int eq = pair.indexOf('=');
try {
String key = URLDecoder.decode(eq < 0 ? pair : pair.substring(0, eq), "UTF-8");
String value = eq < 0 ? "" : URLDecoder.decode(pair.substring(eq + 1), "UTF-8");
params.put(key, value);
} catch (Exception ignored) {}
}
return params;
}
private void handleHttp(String requestLine, InputStream in, OutputStream output) {
try {
String[] parts = requestLine.split(" ");
String path = parts.length > 1 ? parts[1] : "/";
String target = parts.length > 1 ? parts[1] : "/";
String path = target;
Map<String, String> query;
int q = target.indexOf('?');
if (q >= 0) {
path = target.substring(0, q);
query = parseQuery(target.substring(q + 1));
} else {
query = new HashMap<>();
}
int contentLength = 0;
String line;
while ((line = reader.readLine()) != null) {
if (line.isEmpty()) break;
String lower = line.toLowerCase();
if (lower.startsWith("content-length:")) {
String headerToken = null;
byte[] lineBytes;
while ((lineBytes = readLineBytes(in)) != null) {
if (lineBytes.length == 0) break;
String line = new String(lineBytes, StandardCharsets.ISO_8859_1);
int colon = line.indexOf(':');
if (colon < 0) continue;
String name = line.substring(0, colon).trim().toLowerCase(Locale.ROOT);
String value = line.substring(colon + 1).trim();
if (name.equals("content-length")) {
try {
contentLength = Integer.parseInt(lower.substring("content-length:".length()).trim());
contentLength = Integer.parseInt(value);
} catch (Exception ignored) {}
} else if (name.equals(TOKEN_HEADER)) {
headerToken = value;
}
}
if (contentLength < 0 || contentLength > MAX_BODY) {
sendJson(output, 413, errorJson("Body too large").toString());
return;
}
String body = "";
if (contentLength > 0) {
char[] buf = new char[contentLength];
int total = 0;
while (total < contentLength) {
int r = reader.read(buf, total, contentLength - total);
if (r < 0) break;
total += r;
}
body = new String(buf, 0, total);
body = new String(readExactly(in, contentLength), StandardCharsets.UTF_8);
}
String token = headerToken != null ? headerToken : query.get("token");
boolean authed = TokenStore.matches(token);
if (path.equals("/ping") || path.equals("/")) {
sendJson(output, 200, buildPing(authed).toString());
return;
}
if (!authed) {
sendJson(output, 401, unauthorizedJson().toString());
return;
}
// Route endpoints
if (path.startsWith("/snapshot")) {
JSONObject json = HierarchyDumper.dumpAtomicSnapshot(service);
sendJsonResponse(output, json.toString());
} else if (path.equals("/dump_xml") || path.equals("/hierarchy.xml") || path.startsWith("/dump?format=xml")) {
String xml = HierarchyDumper.dumpXml(service);
sendXmlResponse(output, xml);
} else if (path.startsWith("/dump") || path.startsWith("/hierarchy")) {
JSONObject json = HierarchyDumper.dump(service);
sendJsonResponse(output, json.toString());
} else if (path.startsWith("/ping") || path.equals("/")) {
JSONObject r = new JSONObject();
r.put("success", true);
r.put("service", "ArtemisAccessibilityService");
r.put("package", service.getCurrentPackageName());
r.put("activity", service.getCurrentActivityName());
sendJsonResponse(output, r.toString());
} else if (path.startsWith("/action") || path.startsWith("/rpc")) {
if (path.equals("/snapshot")) {
HierarchyDumper.DumpOptions options = HierarchyDumper.DumpOptions.forSnapshot().apply(query);
JSONObject json = HierarchyDumper.dumpAtomicSnapshot(service, options);
sendJson(output, 200, json.toString());
} else if (path.equals("/dump_xml") || path.equals("/hierarchy.xml")
|| (path.equals("/dump") && "xml".equals(query.get("format")))) {
HierarchyDumper.DumpOptions options = HierarchyDumper.DumpOptions.forSnapshot().apply(query);
sendXml(output, HierarchyDumper.dumpXml(service, options));
} else if (path.equals("/dump") || path.equals("/hierarchy")) {
HierarchyDumper.DumpOptions options = HierarchyDumper.DumpOptions.forDump().apply(query);
sendJson(output, 200, HierarchyDumper.dump(service, options).toString());
} else if (path.equals("/action") || path.equals("/rpc")) {
JSONObject json = body.isEmpty() ? new JSONObject() : new JSONObject(body);
JSONObject resp = executeCommand(json.optString("cmd", ""), json);
sendJsonResponse(output, resp.toString());
sendJson(output, 200, resp.toString());
} else {
JSONObject r = new JSONObject();
r.put("success", false);
r.put("error", "Unknown endpoint: " + path);
sendJsonResponse(output, r.toString());
sendJson(output, 404, errorJson("Unknown endpoint: " + path).toString());
}
} catch (Exception e) {
Log.e(TAG, "HTTP error", e);
try {
sendJson(output, 500, errorJson("Internal error: " + e.getMessage()).toString());
} catch (Throwable ignored) {}
}
}
// ------------------------------------------------------------------ //
// Raw JSON-RPC
// ------------------------------------------------------------------ //
private void handleJsonRpc(String line, OutputStream output) {
try {
JSONObject json = new JSONObject(line);
String cmd = json.optString("cmd", "");
JSONObject resp = executeCommand(cmd, json);
boolean authed = TokenStore.matches(json.optString("token", null));
JSONObject resp;
if (cmd.equalsIgnoreCase("ping")) {
resp = buildPing(authed);
} else if (!authed) {
resp = unauthorizedJson();
} else {
resp = executeCommand(cmd, json);
}
byte[] bytes = resp.toString().getBytes(StandardCharsets.UTF_8);
output.write(bytes);
output.write('\n');
output.flush();
} catch (Exception e) {
try {
JSONObject err = new JSONObject();
err.put("success", false);
err.put("error", "JSON parse error: " + e.getMessage());
JSONObject err = errorJson("JSON parse error: " + e.getMessage());
output.write(err.toString().getBytes(StandardCharsets.UTF_8));
output.write('\n');
output.flush();
@@ -177,21 +286,21 @@ public class CommandServer extends Thread {
}
}
// ------------------------------------------------------------------ //
// Commands
// ------------------------------------------------------------------ //
private JSONObject executeCommand(String cmd, JSONObject params) {
JSONObject resp = new JSONObject();
try {
switch (cmd.toLowerCase()) {
switch (cmd.toLowerCase(Locale.ROOT)) {
case "ping":
resp.put("success", true);
resp.put("service", "ArtemisAccessibilityService");
resp.put("package", service.getCurrentPackageName());
resp.put("activity", service.getCurrentActivityName());
break;
return buildPing(true);
case "dump":
case "dump_ui":
return HierarchyDumper.dump(service);
return HierarchyDumper.dump(service, HierarchyDumper.DumpOptions.forDump());
case "snapshot":
return HierarchyDumper.dumpAtomicSnapshot(service);
return HierarchyDumper.dumpAtomicSnapshot(service, HierarchyDumper.DumpOptions.forSnapshot());
case "dump_xml":
resp.put("success", true);
resp.put("xml", HierarchyDumper.dumpXml(service));
@@ -239,11 +348,15 @@ public class CommandServer extends Thread {
break;
case "type":
String text = params.optString("text", "");
resp.put("success", GestureController.setText(service, text));
boolean append = params.optBoolean("append", false);
resp.put("success", GestureController.setText(service, text, append));
break;
case "clear":
resp.put("success", GestureController.clearText(service));
break;
case "clipboard":
resp.put("success", GestureController.setClipboard(service, params.optString("text", "")));
break;
case "global":
String action = params.optString("action", "");
resp.put("success", GestureController.performGlobalAction(service, action));
@@ -262,11 +375,64 @@ public class CommandServer extends Thread {
return resp;
}
private void sendJsonResponse(OutputStream output, String jsonStr) {
/**
* Health payload shared by GET /ping and the "ping" RPC. The host compares
* version_code / protocol_version against the bundled APK to decide whether
* to upgrade, and token_set to decide whether to push its token. The
* foreground package and activity are only disclosed to an authenticated caller.
*/
private JSONObject buildPing(boolean authed) throws Exception {
JSONObject r = new JSONObject();
r.put("success", true);
r.put("service", "ArtemisAccessibilityService");
r.put("version_code", service.getVersionCode());
r.put("version_name", service.getVersionName());
r.put("protocol_version", ArtemisAccessibilityService.PROTOCOL_VERSION);
r.put("port", port);
r.put("auth_required", true);
r.put("token_set", TokenStore.isSet());
r.put("authenticated", authed);
if (authed) {
r.put("package", service.getCurrentPackageName());
r.put("activity", service.getCurrentActivityName());
}
return r;
}
private static JSONObject errorJson(String message) throws Exception {
JSONObject r = new JSONObject();
r.put("success", false);
r.put("error", message);
return r;
}
private static JSONObject unauthorizedJson() throws Exception {
JSONObject r = errorJson(TokenStore.isSet()
? "unauthorized: wrong or missing X-Artemis-Token"
: "unauthorized: no session token has been pushed to the helper yet");
r.put("token_set", TokenStore.isSet());
return r;
}
// ------------------------------------------------------------------ //
// Responses
// ------------------------------------------------------------------ //
private static String reason(int status) {
switch (status) {
case 200: return "OK";
case 400: return "Bad Request";
case 401: return "Unauthorized";
case 404: return "Not Found";
case 413: return "Payload Too Large";
default: return "Internal Server Error";
}
}
private static void send(OutputStream output, int status, String contentType, byte[] bytes) {
try {
byte[] bytes = jsonStr.getBytes(StandardCharsets.UTF_8);
String header = "HTTP/1.1 200 OK\r\n" +
"Content-Type: application/json; charset=utf-8\r\n" +
String header = "HTTP/1.1 " + status + " " + reason(status) + "\r\n" +
"Content-Type: " + contentType + "; charset=utf-8\r\n" +
"Content-Length: " + bytes.length + "\r\n" +
"Connection: close\r\n\r\n";
output.write(header.getBytes(StandardCharsets.UTF_8));
@@ -275,17 +441,12 @@ public class CommandServer extends Thread {
} catch (Exception ignored) {}
}
private void sendXmlResponse(OutputStream output, String xmlStr) {
try {
byte[] bytes = xmlStr.getBytes(StandardCharsets.UTF_8);
String header = "HTTP/1.1 200 OK\r\n" +
"Content-Type: application/xml; charset=utf-8\r\n" +
"Content-Length: " + bytes.length + "\r\n" +
"Connection: close\r\n\r\n";
output.write(header.getBytes(StandardCharsets.UTF_8));
output.write(bytes);
output.flush();
} catch (Exception ignored) {}
private static void sendJson(OutputStream output, int status, String jsonStr) {
send(output, status, "application/json", jsonStr.getBytes(StandardCharsets.UTF_8));
}
private static void sendXml(OutputStream output, String xmlStr) {
send(output, 200, "application/xml", xmlStr.getBytes(StandardCharsets.UTF_8));
}
public void shutdown() {
@@ -2,6 +2,9 @@ package com.artemis.helper;
import android.accessibilityservice.AccessibilityService;
import android.accessibilityservice.GestureDescription;
import android.content.ClipData;
import android.content.ClipboardManager;
import android.content.Context;
import android.graphics.Path;
import android.os.Build;
import android.os.Bundle;
@@ -65,14 +68,36 @@ public final class GestureController {
}
public static boolean setText(AccessibilityService service, String text) {
return setText(service, text, false);
}
/**
* Sets (or, with {@code append}, extends) the text of the focused / first editable field.
* ACTION_SET_TEXT replaces the whole content; appending mirrors what typing through an
* IME does, which is the contract the host's send_text has always had.
*/
public static boolean setText(AccessibilityService service, String text, boolean append) {
AccessibilityNodeInfo inputNode = HierarchyDumper.findInputNode(service);
if (inputNode == null) {
Log.w(TAG, "No editable/focused input node found for setText");
return false;
}
try {
String value = text == null ? "" : text;
if (append) {
CharSequence existing = inputNode.getText();
boolean showingHint = false;
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
try {
showingHint = inputNode.isShowingHintText();
} catch (Throwable ignored) {}
}
if (existing != null && existing.length() > 0 && !showingHint) {
value = existing.toString() + value;
}
}
Bundle args = new Bundle();
args.putCharSequence(AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, text);
args.putCharSequence(AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, value);
return inputNode.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, args);
} catch (Throwable t) {
Log.w(TAG, "performAction ACTION_SET_TEXT failed", t);
@@ -171,4 +196,37 @@ public final class GestureController {
return false;
}
}
/**
* Write text to the system clipboard. Clipboard writes are allowed from any
* app on every API level (only reads are restricted since Android 10), so this
* is the IME-free path the host uses for multiline and non-ASCII input:
* set the clip here, then KEYCODE_PASTE over adb.
*/
public static boolean setClipboard(final AccessibilityService service, final String text) {
final CountDownLatch latch = new CountDownLatch(1);
final AtomicBoolean ok = new AtomicBoolean(false);
MAIN_HANDLER.post(new Runnable() {
@Override
public void run() {
try {
ClipboardManager cm = (ClipboardManager) service.getSystemService(Context.CLIPBOARD_SERVICE);
if (cm != null) {
cm.setPrimaryClip(ClipData.newPlainText("artemis", text == null ? "" : text));
ok.set(true);
}
} catch (Throwable t) {
Log.w(TAG, "setClipboard failed: " + t.getMessage());
} finally {
latch.countDown();
}
}
});
try {
latch.await(2000, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
return ok.get();
}
}
@@ -21,11 +21,13 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
/**
@@ -34,73 +36,132 @@ import java.util.concurrent.atomic.AtomicReference;
* Capabilities:
* 1. Zero WaitForIdle hangs: Bypasses UIAutomator's waitForIdle() timeout.
* 2. Multi-window penetration: Captures App, Dialog, System UI, IME Keyboard, Split-Screen.
* 3. Rich semantic extraction: Captures errorText, isHeading, editable, paneTitle, tooltip.
* 4. Atomic screenshot snapshot: On API 30+, captures hardware bitmap and DOM tree simultaneously.
* 5. W3C XML 1.0 compliance: Strict character sanitization prevents Python parse failures.
* 3. UIAutomator parity: invisible nodes are skipped and bounds are clipped to the
* display, the window and scrollable ancestors, so the tree describes what is on
* screen and nothing else. {@code include_invisible=1} keeps everything (debugging).
* 4. Rich semantic extraction: Captures errorText, isHeading, editable, paneTitle, tooltip.
* 5. Atomic screenshot snapshot: On API 30+, captures hardware bitmap and DOM tree simultaneously.
* 6. W3C XML 1.0 compliance: Strict character sanitization prevents Python parse failures.
* 7. Batched IPC on API 33+: descendants are prefetched per getChild call instead of one
* Binder round trip per node.
*/
public final class HierarchyDumper {
private static final String TAG = "ArtemisHierarchyDumper";
private static final int MAX_DEPTH = 75;
private static final int MAX_NODES = 8000;
private static final int RETRY_COUNT = 3;
private static final long RETRY_INTERVAL_MS = 40L;
private static final ExecutorService SCREENSHOT_EXECUTOR = Executors.newSingleThreadExecutor();
/** Prefetch strategy for API 33+ ({@code AccessibilityNodeInfo.FLAG_PREFETCH_DESCENDANTS_HYBRID}). */
private static final int PREFETCH_DESCENDANTS_HYBRID = 1 << 3;
private HierarchyDumper() {}
/**
* Dumps the complete hierarchy as a JSON response.
* What a dump should contain. Every field defaults to the cheapest useful answer.
*/
public static final class DumpOptions {
public boolean includeInvisible = false;
public boolean wantXml = true;
public boolean wantElements = false;
public boolean wantTree = false;
public static DumpOptions forDump() {
DumpOptions o = new DumpOptions();
o.wantElements = true;
o.wantTree = true;
return o;
}
public static DumpOptions forSnapshot() {
return new DumpOptions();
}
/** Applies {@code fields=xml,elements,tree} and {@code include_invisible=1}. */
public DumpOptions apply(Map<String, String> query) {
if (query == null) return this;
String fields = query.get("fields");
if (fields != null && !fields.trim().isEmpty()) {
wantXml = wantElements = wantTree = false;
for (String f : fields.split(",")) {
String key = f.trim().toLowerCase();
if (key.equals("xml")) wantXml = true;
else if (key.equals("elements")) wantElements = true;
else if (key.equals("tree")) wantTree = true;
}
}
String inv = query.get("include_invisible");
if (inv != null) {
includeInvisible = inv.equals("1") || inv.equalsIgnoreCase("true");
}
return this;
}
}
/** Per-dump counters reported back to the host. */
private static final class DumpStats {
int nodes = 0;
int skippedInvisible = 0;
boolean truncated = false;
}
public static JSONObject dump(AccessibilityService service) {
return dump(service, DumpOptions.forDump());
}
/**
* Dumps the hierarchy as a JSON response containing the requested representations.
*/
public static JSONObject dump(AccessibilityService service, DumpOptions options) {
long startTime = System.currentTimeMillis();
JSONObject result = new JSONObject();
JSONArray elementsJson = new JSONArray();
try {
DisplayUtils.DisplayInfo displayInfo = DisplayUtils.getDisplayInfo(service);
List<A11yNode> rootSnapshots = captureRootSnapshots(service);
DumpStats stats = new DumpStats();
List<A11yNode> rootSnapshots = captureRootSnapshots(service, displayInfo, options, stats);
result.put("rotation", displayInfo.rotation);
result.put("width", displayInfo.width);
result.put("height", displayInfo.height);
if (rootSnapshots.isEmpty()) {
result.put("success", false);
result.put("error", "No active window or root node found");
result.put("xml", "");
result.put("elements", elementsJson);
result.put("rotation", displayInfo.rotation);
result.put("width", displayInfo.width);
result.put("height", displayInfo.height);
result.put("elements", new JSONArray());
return result;
}
// 1. Build standard UIAutomator XML
String xml = buildXml(rootSnapshots, displayInfo.rotation, displayInfo.width, displayInfo.height);
// 2. Build flat elements list
List<JSONObject> flatList = new ArrayList<>();
for (A11yNode root : rootSnapshots) {
root.collectFlatElements(flatList);
if (options.wantXml) {
result.put("xml", buildXml(rootSnapshots, displayInfo.rotation));
}
for (JSONObject elem : flatList) {
elementsJson.put(elem);
if (options.wantElements) {
JSONArray elementsJson = new JSONArray();
List<JSONObject> flatList = new ArrayList<>();
for (A11yNode root : rootSnapshots) {
root.collectFlatElements(flatList);
}
for (JSONObject elem : flatList) {
elementsJson.put(elem);
}
result.put("elements", elementsJson);
}
// 3. Build tree
JSONArray trees = new JSONArray();
for (A11yNode root : rootSnapshots) {
trees.put(root.toTreeJson());
if (options.wantTree) {
JSONArray trees = new JSONArray();
for (A11yNode root : rootSnapshots) {
trees.put(root.toTreeJson());
}
result.put("tree", trees.length() == 1 ? trees.getJSONObject(0) : trees);
}
long elapsed = System.currentTimeMillis() - startTime;
result.put("success", true);
result.put("xml", xml);
result.put("elements", elementsJson);
result.put("tree", trees.length() == 1 ? trees.getJSONObject(0) : trees);
result.put("rotation", displayInfo.rotation);
result.put("width", displayInfo.width);
result.put("height", displayInfo.height);
result.put("elapsed_ms", elapsed);
result.put("node_count", stats.nodes);
result.put("skipped_invisible", stats.skippedInvisible);
result.put("truncated", stats.truncated);
result.put("window_count", rootSnapshots.size());
result.put("elapsed_ms", System.currentTimeMillis() - startTime);
if (service instanceof ArtemisAccessibilityService) {
ArtemisAccessibilityService s = (ArtemisAccessibilityService) service;
@@ -114,70 +175,109 @@ public final class HierarchyDumper {
result.put("success", false);
result.put("error", "Dump failed: " + t.getMessage());
result.put("xml", "");
result.put("elements", elementsJson);
result.put("elements", new JSONArray());
} catch (Throwable ignored) {}
}
return result;
}
/**
* Dumps an atomic snapshot combining hardware screenshot (JPEG base64) and UI hierarchy XML/JSON
* at the exact same clock tick, eliminating temporal phase mismatch.
*/
private static final int ERROR_TAKE_SCREENSHOT_INTERVAL_TIME_SHORT = 3;
private static final long SCREENSHOT_INTERVAL_RETRY_MS = 350L;
@android.annotation.TargetApi(Build.VERSION_CODES.R)
private static void requestScreenshot(
final AccessibilityService service,
final AtomicReference<Bitmap> bitmapRef,
final AtomicInteger errorRef,
final CountDownLatch latch,
final boolean allowRetry) {
try {
service.takeScreenshot(
Display.DEFAULT_DISPLAY,
SCREENSHOT_EXECUTOR,
new AccessibilityService.TakeScreenshotCallback() {
@Override
public void onSuccess(AccessibilityService.ScreenshotResult screenshotResult) {
try {
HardwareBuffer buffer = screenshotResult.getHardwareBuffer();
ColorSpace colorSpace = screenshotResult.getColorSpace();
Bitmap hwBitmap = Bitmap.wrapHardwareBuffer(buffer, colorSpace);
if (hwBitmap != null) {
Bitmap swBitmap = hwBitmap.copy(Bitmap.Config.ARGB_8888, false);
hwBitmap.recycle();
buffer.close();
bitmapRef.set(swBitmap);
}
} catch (Throwable t) {
Log.w(TAG, "Error copying screenshot buffer", t);
} finally {
latch.countDown();
}
}
@Override
public void onFailure(int errorCode) {
errorRef.set(errorCode);
if (allowRetry && errorCode == ERROR_TAKE_SCREENSHOT_INTERVAL_TIME_SHORT) {
Log.i(TAG, "takeScreenshot rate-limited; retrying after " + SCREENSHOT_INTERVAL_RETRY_MS + " ms");
SCREENSHOT_EXECUTOR.execute(new Runnable() {
@Override
public void run() {
try {
Thread.sleep(SCREENSHOT_INTERVAL_RETRY_MS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
requestScreenshot(service, bitmapRef, errorRef, latch, false);
}
});
return;
}
Log.w(TAG, "takeScreenshot failed, errorCode: " + errorCode);
latch.countDown();
}
}
);
} catch (Throwable t) {
Log.w(TAG, "takeScreenshot invocation error", t);
latch.countDown();
}
}
public static JSONObject dumpAtomicSnapshot(AccessibilityService service) {
return dumpAtomicSnapshot(service, DumpOptions.forSnapshot());
}
/**
* Dumps an atomic snapshot combining hardware screenshot (JPEG base64) and UI hierarchy
* at the exact same clock tick, eliminating temporal phase mismatch. When the screenshot
* is present, {@code width} / {@code height} are the bitmap's own dimensions so the host
* normalizes coordinates against the very image it is looking at.
*/
public static JSONObject dumpAtomicSnapshot(AccessibilityService service, DumpOptions options) {
long startTime = System.currentTimeMillis();
// 1. Trigger hardware screenshot asynchronously
// 1. Trigger hardware screenshot asynchronously. The framework rate-limits
// takeScreenshot to one call per ~333 ms (ERROR_TAKE_SCREENSHOT_INTERVAL_TIME_SHORT = 3);
// back-to-back observations therefore wait out the interval and retry once
// here, which is far cheaper than the host falling back to adb screencap.
final AtomicReference<Bitmap> bitmapRef = new AtomicReference<>(null);
final AtomicInteger errorRef = new AtomicInteger(0);
final CountDownLatch screenshotLatch = new CountDownLatch(1);
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
try {
service.takeScreenshot(
Display.DEFAULT_DISPLAY,
SCREENSHOT_EXECUTOR,
new AccessibilityService.TakeScreenshotCallback() {
@Override
public void onSuccess(AccessibilityService.ScreenshotResult screenshotResult) {
try {
HardwareBuffer buffer = screenshotResult.getHardwareBuffer();
ColorSpace colorSpace = screenshotResult.getColorSpace();
Bitmap hwBitmap = Bitmap.wrapHardwareBuffer(buffer, colorSpace);
if (hwBitmap != null) {
Bitmap swBitmap = hwBitmap.copy(Bitmap.Config.ARGB_8888, false);
hwBitmap.recycle();
buffer.close();
bitmapRef.set(swBitmap);
}
} catch (Throwable t) {
Log.w(TAG, "Error copying screenshot buffer", t);
} finally {
screenshotLatch.countDown();
}
}
@Override
public void onFailure(int errorCode) {
Log.w(TAG, "takeScreenshot failed, errorCode: " + errorCode);
screenshotLatch.countDown();
}
}
);
} catch (Throwable t) {
Log.w(TAG, "takeScreenshot invocation error", t);
screenshotLatch.countDown();
}
requestScreenshot(service, bitmapRef, errorRef, screenshotLatch, true);
} else {
screenshotLatch.countDown();
}
// 2. Concurrently capture the UI hierarchy
JSONObject dumpData = dump(service);
JSONObject dumpData = dump(service, options);
// 3. Wait for the screenshot with a strict 1.5s timeout
// 3. Wait for the screenshot (one retry after the rate-limit interval fits inside)
try {
screenshotLatch.await(1500L, TimeUnit.MILLISECONDS);
screenshotLatch.await(2500L, TimeUnit.MILLISECONDS);
} catch (InterruptedException ignored) {}
Bitmap bitmap = bitmapRef.get();
@@ -189,6 +289,8 @@ public final class HierarchyDumper {
String base64Str = Base64.encodeToString(jpegBytes, Base64.NO_WRAP);
dumpData.put("screenshot_base64", base64Str);
dumpData.put("has_screenshot", true);
dumpData.put("width", bitmap.getWidth());
dumpData.put("height", bitmap.getHeight());
} catch (Throwable t) {
Log.w(TAG, "Failed to compress screenshot to JPEG Base64", t);
try { dumpData.put("has_screenshot", false); } catch (Throwable ignored) {}
@@ -198,8 +300,11 @@ public final class HierarchyDumper {
} else {
try {
dumpData.put("has_screenshot", false);
dumpData.put("screenshot_error_code", errorRef.get());
dumpData.put("screenshot_error", Build.VERSION.SDK_INT >= Build.VERSION_CODES.R
? "Screenshot capture timed out or failed"
? (errorRef.get() != 0
? "takeScreenshot failed with errorCode " + errorRef.get()
: "Screenshot capture timed out")
: "takeScreenshot not supported on Android < 11");
} catch (Throwable ignored) {}
}
@@ -211,23 +316,27 @@ public final class HierarchyDumper {
return dumpData;
}
public static String dumpXml(AccessibilityService service) {
return dumpXml(service, DumpOptions.forSnapshot());
}
/**
* Dumps the hierarchy directly as a raw standard UIAutomator XML string.
*/
public static String dumpXml(AccessibilityService service) {
public static String dumpXml(AccessibilityService service, DumpOptions options) {
DisplayUtils.DisplayInfo displayInfo = DisplayUtils.getDisplayInfo(service);
List<A11yNode> rootSnapshots = captureRootSnapshots(service);
List<A11yNode> rootSnapshots = captureRootSnapshots(service, displayInfo, options, new DumpStats());
if (rootSnapshots.isEmpty()) {
return "<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>\n<hierarchy rotation=\""
+ displayInfo.rotation + "\" />\n";
}
return buildXml(rootSnapshots, displayInfo.rotation, displayInfo.width, displayInfo.height);
return buildXml(rootSnapshots, displayInfo.rotation);
}
/**
* Builds the standard Android UIAutomator XML string from root snapshots.
*/
private static String buildXml(List<A11yNode> roots, int rotation, int width, int height) {
private static String buildXml(List<A11yNode> roots, int rotation) {
StringBuilder sb = new StringBuilder(roots.size() * 1024 + 256);
sb.append("<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>\n");
sb.append("<hierarchy rotation=\"").append(rotation).append("\">");
@@ -249,6 +358,8 @@ public final class HierarchyDumper {
final int windowLayer;
final boolean windowActive;
final boolean windowFocused;
/** Screen rectangle of the window, or null when unknown (display bounds are used). */
final Rect windowBounds;
RawRootEntry(
AccessibilityNodeInfo root,
@@ -256,7 +367,8 @@ public final class HierarchyDumper {
String windowType,
int windowLayer,
boolean windowActive,
boolean windowFocused
boolean windowFocused,
Rect windowBounds
) {
this.root = root;
this.windowId = windowId;
@@ -264,14 +376,25 @@ public final class HierarchyDumper {
this.windowLayer = windowLayer;
this.windowActive = windowActive;
this.windowFocused = windowFocused;
this.windowBounds = windowBounds;
}
}
public static List<A11yNode> captureRootSnapshots(AccessibilityService service) {
return captureRootSnapshots(
service, DisplayUtils.getDisplayInfo(service), DumpOptions.forSnapshot(), new DumpStats());
}
/**
* Captures snapshots of all active and interactive windows with adaptive progressive retry support.
* Guaranteed to capture the complete hierarchy even during activity transitions and cold starts.
*/
public static List<A11yNode> captureRootSnapshots(AccessibilityService service) {
private static List<A11yNode> captureRootSnapshots(
AccessibilityService service,
DisplayUtils.DisplayInfo displayInfo,
DumpOptions options,
DumpStats stats
) {
List<RawRootEntry> rawRoots = Collections.emptyList();
long[] retryBackoff = new long[]{40L, 80L, 120L, 160L, 220L, 300L};
@@ -285,13 +408,19 @@ public final class HierarchyDumper {
}
}
Rect displayRect = new Rect(0, 0, displayInfo.width, displayInfo.height);
List<A11yNode> snapshots = new ArrayList<>(rawRoots.size());
int[] nodeCounter = new int[]{0};
for (int i = 0; i < rawRoots.size(); i++) {
RawRootEntry entry = rawRoots.get(i);
try {
A11yNode snapshot = snapshotNode(entry.root, 0, i, nodeCounter);
Rect clip = new Rect(displayRect);
if (entry.windowBounds != null && !clip.intersect(entry.windowBounds)) {
// Window entirely off the display (e.g. a second display): keep the
// display rect so the root still serializes with sane bounds.
clip = new Rect(displayRect);
}
A11yNode snapshot = snapshotNode(entry.root, 0, i, clip, options, stats);
if (snapshot != null) {
snapshot.windowId = entry.windowId;
snapshot.windowType = entry.windowType;
@@ -310,6 +439,24 @@ public final class HierarchyDumper {
return snapshots;
}
private static AccessibilityNodeInfo windowRoot(AccessibilityWindowInfo window) {
if (Build.VERSION.SDK_INT >= 33) {
try {
return window.getRoot(PREFETCH_DESCENDANTS_HYBRID);
} catch (Throwable ignored) {}
}
return window.getRoot();
}
private static AccessibilityNodeInfo childOf(AccessibilityNodeInfo node, int index) {
if (Build.VERSION.SDK_INT >= 33) {
try {
return node.getChild(index, PREFETCH_DESCENDANTS_HYBRID);
} catch (Throwable ignored) {}
}
return node.getChild(index);
}
/**
* Multi-tier hierarchy root discovery strategy:
* Tier 1: Multi-window enumeration (sorted by Z-layer descending).
@@ -335,7 +482,7 @@ public final class HierarchyDumper {
for (AccessibilityWindowInfo window : sortedWindows) {
try {
AccessibilityNodeInfo root = window.getRoot();
AccessibilityNodeInfo root = windowRoot(window);
if (root != null) {
int hash = root.hashCode();
if (seenHashes.add(hash)) {
@@ -343,13 +490,16 @@ public final class HierarchyDumper {
if (window.getType() == AccessibilityWindowInfo.TYPE_APPLICATION) {
hasAppWindow = true;
}
Rect bounds = new Rect();
window.getBoundsInScreen(bounds);
roots.add(new RawRootEntry(
root,
window.getId(),
typeStr,
window.getLayer(),
window.isActive(),
window.isFocused()
window.isFocused(),
bounds.isEmpty() ? null : bounds
));
} else {
safeRecycle(root);
@@ -373,7 +523,8 @@ public final class HierarchyDumper {
"application",
0,
true,
true
true,
null
));
hasAppWindow = true;
} else {
@@ -414,7 +565,8 @@ public final class HierarchyDumper {
"application",
0,
true,
true
true,
null
));
} else {
safeRecycle(current);
@@ -450,6 +602,11 @@ public final class HierarchyDumper {
* Recursively snapshots an AccessibilityNodeInfo into an immutable A11yNode,
* extracting rich semantic properties (error, heading, editable, paneTitle, tooltip, stateDescription).
*
* {@code clip} is the rectangle the node may be visible in: the display intersected with the
* window and every scrollable ancestor. Children that are not visible to the user are skipped
* unless {@code options.includeInvisible} is set; the window root itself is always kept, as in
* UIAutomator.
*
* Protected against recursion loops via MAX_DEPTH and MAX_NODES without flat hash collision drops.
*/
@SuppressWarnings("deprecation")
@@ -457,20 +614,37 @@ public final class HierarchyDumper {
AccessibilityNodeInfo node,
int depth,
int childIndex,
int[] nodeCounter
Rect clip,
DumpOptions options,
DumpStats stats
) {
if (node == null) return null;
if (depth > MAX_DEPTH || nodeCounter[0] >= MAX_NODES) {
if (depth > MAX_DEPTH || stats.nodes >= MAX_NODES) {
stats.truncated = true;
return null;
}
nodeCounter[0]++;
boolean visible = true;
try {
visible = node.isVisibleToUser();
} catch (Throwable ignored) {}
if (!visible && depth > 0 && !options.includeInvisible) {
stats.skippedInvisible++;
return null;
}
stats.nodes++;
A11yNode snapshot = new A11yNode();
snapshot.index = childIndex;
snapshot.visibleToUser = visible;
Rect childClip = clip;
try {
Rect bounds = new Rect();
node.getBoundsInScreen(bounds);
if (!bounds.intersect(clip)) {
bounds.setEmpty();
}
snapshot.left = bounds.left;
snapshot.top = bounds.top;
snapshot.right = bounds.right;
@@ -499,6 +673,13 @@ public final class HierarchyDumper {
snapshot.password = node.isPassword();
snapshot.selected = node.isSelected();
// Children of a scrollable container are clipped by it (UIAutomator's
// trimScrollableParent): a list row half under the toolbar keeps only its
// visible part, so its center is a point the user can actually touch.
if (snapshot.scrollable && !bounds.isEmpty()) {
childClip = new Rect(bounds);
}
// 1. Editable property
snapshot.editable = node.isEditable();
@@ -515,11 +696,15 @@ public final class HierarchyDumper {
} catch (Throwable ignored) {}
}
// 4. Hint text (API 26+)
// 4. Hint text (API 26+). When the field is empty the framework reports the
// hint as the text; keep text and hint apart so "empty input" stays detectable.
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
try {
CharSequence hint = node.getHintText();
if (hint != null) snapshot.hint = hint.toString();
if (node.isShowingHintText()) {
snapshot.text = "";
}
} catch (Throwable ignored) {}
}
@@ -557,9 +742,9 @@ public final class HierarchyDumper {
for (int i = 0; i < childCount; i++) {
AccessibilityNodeInfo childNode = null;
try {
childNode = node.getChild(i);
childNode = childOf(node, i);
if (childNode != null) {
A11yNode childSnapshot = snapshotNode(childNode, depth + 1, i, nodeCounter);
A11yNode childSnapshot = snapshotNode(childNode, depth + 1, i, childClip, options, stats);
if (childSnapshot != null) {
snapshot.children.add(childSnapshot);
}
@@ -594,34 +779,49 @@ public final class HierarchyDumper {
/**
* Finds the currently focused or editable input node for text entry.
* Window roots are enumerated once and recycled; only the returned node stays alive.
*/
public static AccessibilityNodeInfo findInputNode(AccessibilityService service) {
for (RawRootEntry entry : getActiveRawRoots(service)) {
try {
AccessibilityNodeInfo focused = entry.root.findFocus(AccessibilityNodeInfo.FOCUS_INPUT);
if (focused != null) return focused;
} catch (Throwable ignored) {}
List<RawRootEntry> roots = getActiveRawRoots(service);
AccessibilityNodeInfo found = null;
try {
for (RawRootEntry entry : roots) {
try {
AccessibilityNodeInfo focused = entry.root.findFocus(AccessibilityNodeInfo.FOCUS_INPUT);
if (focused != null) {
found = focused;
return found;
}
} catch (Throwable ignored) {}
}
for (RawRootEntry entry : roots) {
AccessibilityNodeInfo editable = findFirstEditable(entry.root, 0);
if (editable != null) {
found = editable;
return found;
}
}
return null;
} finally {
for (RawRootEntry entry : roots) {
if (entry.root != found) {
safeRecycle(entry.root);
}
}
}
for (RawRootEntry entry : getActiveRawRoots(service)) {
AccessibilityNodeInfo editable = findFirstEditable(entry.root);
if (editable != null) return editable;
}
return null;
}
private static AccessibilityNodeInfo findFirstEditable(AccessibilityNodeInfo node) {
if (node == null) return null;
private static AccessibilityNodeInfo findFirstEditable(AccessibilityNodeInfo node, int depth) {
if (node == null || depth > MAX_DEPTH) return null;
try {
if (node.isEditable() && node.isFocusable() && node.isEnabled()) {
if (node.isEditable() && node.isFocusable() && node.isEnabled() && node.isVisibleToUser()) {
return node;
}
int count = node.getChildCount();
for (int i = 0; i < count; i++) {
AccessibilityNodeInfo child = node.getChild(i);
AccessibilityNodeInfo child = childOf(node, i);
if (child != null) {
AccessibilityNodeInfo res = findFirstEditable(child);
AccessibilityNodeInfo res = findFirstEditable(child, depth + 1);
if (res != null) return res;
safeRecycle(child);
}
@@ -0,0 +1,36 @@
package com.artemis.helper;
import android.content.BroadcastReceiver;
import android.content.Context;
import android.content.Intent;
import android.util.Log;
/**
* Receives the host's session token.
*
* The receiver is guarded in the manifest by
* {@code android.permission.WRITE_SECURE_SETTINGS}, a permission ordinary apps
* cannot hold but the adb shell user does, so only something with adb access to
* the device can set (or clear) the token:
*
* <pre>
* adb shell am broadcast -n com.artemis.helper/.TokenReceiver \
* -a com.artemis.helper.SET_TOKEN --es token &lt;hex&gt;
* </pre>
*/
public class TokenReceiver extends BroadcastReceiver {
private static final String TAG = "ArtemisTokenReceiver";
public static final String ACTION_SET_TOKEN = "com.artemis.helper.SET_TOKEN";
public static final String EXTRA_TOKEN = "token";
@Override
public void onReceive(Context context, Intent intent) {
if (intent == null || !ACTION_SET_TOKEN.equals(intent.getAction())) {
return;
}
String value = intent.getStringExtra(EXTRA_TOKEN);
TokenStore.set(value);
Log.i(TAG, TokenStore.isSet() ? "Session token set" : "Session token cleared");
}
}
@@ -0,0 +1,38 @@
package com.artemis.helper;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
/**
* The session token the host pushed for this process lifetime.
*
* The loopback server is reachable by every app on the device, so requests are
* only served when they carry this token. It lives in memory only: a killed or
* re-bound service starts without one and answers 401 until the host pushes it
* again (which the host does on every attach and on every 401).
*/
public final class TokenStore {
private static volatile String token;
private TokenStore() {}
public static void set(String value) {
token = (value == null || value.isEmpty()) ? null : value;
}
public static boolean isSet() {
return token != null;
}
/** Constant-time comparison so a local app cannot guess the token byte by byte. */
public static boolean matches(String candidate) {
String current = token;
if (current == null || candidate == null) {
return false;
}
return MessageDigest.isEqual(
current.getBytes(StandardCharsets.UTF_8),
candidate.getBytes(StandardCharsets.UTF_8));
}
}
@@ -1,9 +1,9 @@
<?xml version="1.0" encoding="utf-8"?>
<accessibility-service xmlns:android="http://schemas.android.com/apk/res/android"
android:accessibilityEventTypes="typeAllMask"
android:accessibilityEventTypes="typeWindowStateChanged"
android:accessibilityFeedbackType="feedbackGeneric"
android:notificationTimeout="50"
android:accessibilityFlags="flagDefault|flagRetrieveInteractiveWindows|flagIncludeNotImportantViews|flagReportViewIds"
android:notificationTimeout="100"
android:accessibilityFlags="flagDefault|flagRetrieveInteractiveWindows|flagReportViewIds"
android:canRetrieveWindowContent="true"
android:canPerformGestures="true"
android:canTakeScreenshot="true"
@@ -1,22 +1,49 @@
#!/usr/bin/env bash
# Standalone build for ArtemisAccessibilityHelper.apk (no Gradle daemon).
#
# Outputs, next to this script:
# ArtemisAccessibilityHelper.apk the signed, zipaligned artifact
# helper_manifest.json {version_code, version_name, sha256, built_at}
#
# The Python runtime (artemis/runtime/helper_manager.py) reads helper_manifest.json
# to decide whether a device needs an install or an upgrade, so the manifest must
# always be regenerated together with the APK. The version is sourced from
# app/build.gradle.kts (defaultConfig.versionCode / versionName) so the Gradle
# build and this script cannot drift.
set -e
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "$DIR"
_to_posix() {
if command -v cygpath >/dev/null 2>&1; then
cygpath -u "$1"
else
printf '%s' "$1"
fi
}
# 1. Locate Android SDK
SDK_ROOT="${ANDROID_HOME:-${ANDROID_SDK_ROOT:-$HOME/Library/Android/sdk}}"
if [ ! -d "$SDK_ROOT" ]; then
for candidate in /opt/android-sdk /usr/local/share/android-sdk "$HOME/Android/Sdk"; do
if [ -d "$candidate" ]; then
SDK_ROOT="${ANDROID_HOME:-${ANDROID_SDK_ROOT:-}}"
if [ -n "$SDK_ROOT" ]; then
SDK_ROOT="$(_to_posix "$SDK_ROOT")"
fi
if [ -z "$SDK_ROOT" ] || [ ! -d "$SDK_ROOT" ]; then
for candidate in \
"$HOME/Library/Android/sdk" \
"$HOME/Android/Sdk" \
/opt/android-sdk \
/usr/local/share/android-sdk \
"$( [ -n "$LOCALAPPDATA" ] && _to_posix "$LOCALAPPDATA/Android/Sdk" )"; do
if [ -n "$candidate" ] && [ -d "$candidate" ]; then
SDK_ROOT="$candidate"
break
fi
done
fi
if [ ! -d "$SDK_ROOT" ]; then
echo "Error: Android SDK not found at $SDK_ROOT. Set ANDROID_HOME." >&2
if [ -z "$SDK_ROOT" ] || [ ! -d "$SDK_ROOT" ]; then
echo "Error: Android SDK not found. Set ANDROID_HOME." >&2
exit 1
fi
@@ -35,9 +62,15 @@ fi
# 2. Locate Java compiler and runtime
if [ -z "$JAVA_HOME" ]; then
if [ -d "/Applications/Android Studio.app/Contents/jbr/Contents/Home" ]; then
export JAVA_HOME="/Applications/Android Studio.app/Contents/jbr/Contents/Home"
fi
for candidate in \
"/Applications/Android Studio.app/Contents/jbr/Contents/Home" \
"$(_to_posix "${PROGRAMFILES:-/c/Program Files}/Android/Android Studio/jbr")" \
"/opt/android-studio/jbr"; do
if [ -n "$candidate" ] && [ -x "$candidate/bin/javac" ] || [ -x "$candidate/bin/javac.exe" ]; then
export JAVA_HOME="$candidate"
break
fi
done
fi
if [ -n "$JAVA_HOME" ]; then
export PATH="$JAVA_HOME/bin:$PATH"
@@ -48,8 +81,49 @@ if ! command -v javac >/dev/null 2>&1; then
exit 1
fi
PYTHON_BIN="${PYTHON:-}"
if [ -z "$PYTHON_BIN" ]; then
# The Windows Store "python" stub is on PATH but exits 49; prefer the project venv.
for candidate in "$DIR/../../.venv/Scripts/python.exe" "$DIR/../../.venv/bin/python" python3 python; do
if "$candidate" -c "import zipfile" >/dev/null 2>&1; then
PYTHON_BIN="$candidate"
break
fi
done
fi
if [ -z "$PYTHON_BIN" ]; then
echo "Error: python3 not found (needed to package the DEX and write helper_manifest.json)" >&2
exit 1
fi
# Build-tools ship as .exe / .bat wrappers on Windows; resolve the right name once.
_tool() {
local base="$BUILD_TOOLS/$1"
for suffix in "" ".exe" ".bat"; do
if [ -f "$base$suffix" ]; then
printf '%s' "$base$suffix"
return 0
fi
done
echo "Error: $1 not found under $BUILD_TOOLS" >&2
exit 1
}
AAPT2="$(_tool aapt2)"
D8="$(_tool d8)"
ZIPALIGN="$(_tool zipalign)"
APKSIGNER="$(_tool apksigner)"
# 3. Version: single source is app/build.gradle.kts
VERSION_CODE="$(sed -n 's/^[[:space:]]*versionCode[[:space:]]*=[[:space:]]*\([0-9][0-9]*\).*/\1/p' app/build.gradle.kts | head -n 1)"
VERSION_NAME="$(sed -n 's/^[[:space:]]*versionName[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' app/build.gradle.kts | head -n 1)"
if [ -z "$VERSION_CODE" ] || [ -z "$VERSION_NAME" ]; then
echo "Error: could not read versionCode / versionName from app/build.gradle.kts" >&2
exit 1
fi
echo "=================================================="
echo " Building ArtemisAccessibilityHelper.apk"
echo " Version: $VERSION_NAME (code $VERSION_CODE)"
echo " Build Tools: $(basename "$BUILD_TOOLS")"
echo " Platform: $(basename "$PLATFORM")"
echo " Java: $(javac -version 2>&1)"
@@ -59,14 +133,16 @@ rm -rf build
mkdir -p build/gen build/obj build/apk build/res_compiled
echo "-> Compiling resources..."
"$BUILD_TOOLS/aapt2" compile --dir app/src/main/res -o build/res_compiled/res.zip
"$AAPT2" compile --dir app/src/main/res -o build/res_compiled/res.zip
echo "-> Linking APK..."
"$BUILD_TOOLS/aapt2" link \
"$AAPT2" link \
-I "$PLATFORM/android.jar" \
--manifest app/src/main/AndroidManifest.xml \
--min-sdk-version 24 \
--target-sdk-version 35 \
--version-code "$VERSION_CODE" \
--version-name "$VERSION_NAME" \
--java build/gen \
-o build/apk/unaligned.apk \
build/res_compiled/res.zip
@@ -74,25 +150,29 @@ echo "-> Linking APK..."
echo "-> Compiling Java sources..."
JAVA_FILES="$(find build/gen app/src/main/java -name "*.java")"
javac -encoding UTF-8 \
-source 8 -target 8 \
-Xlint:-options \
-cp "$PLATFORM/android.jar" \
-d build/obj \
$JAVA_FILES
echo "-> Dexing with D8..."
CLASS_FILES="$(find build/obj -name "*.class")"
"$BUILD_TOOLS/d8" \
"$D8" \
--lib "$PLATFORM/android.jar" \
--output build/apk/ \
--min-api 24 \
$CLASS_FILES
echo "-> Packaging DEX..."
cd build/apk
zip -u -q unaligned.apk classes.dex
cd "$DIR"
"$PYTHON_BIN" - <<'PY'
import zipfile
with zipfile.ZipFile("build/apk/unaligned.apk", "a", compression=zipfile.ZIP_DEFLATED) as apk:
apk.write("build/apk/classes.dex", "classes.dex")
PY
echo "-> Zipaligning..."
"$BUILD_TOOLS/zipalign" -f -p 4 build/apk/unaligned.apk build/apk/aligned.apk
"$ZIPALIGN" -f -p 4 build/apk/unaligned.apk build/apk/aligned.apk
echo "-> Signing APK..."
if [ ! -f debug.keystore ]; then
@@ -103,7 +183,7 @@ if [ ! -f debug.keystore ]; then
-dname "CN=Artemis Debug,O=Artemis,C=US"
fi
"$BUILD_TOOLS/apksigner" sign \
"$APKSIGNER" sign \
--ks debug.keystore \
--ks-pass pass:android \
--key-pass pass:android \
@@ -111,6 +191,23 @@ fi
--out ArtemisAccessibilityHelper.apk \
build/apk/aligned.apk
echo "-> Writing helper_manifest.json..."
VERSION_CODE="$VERSION_CODE" VERSION_NAME="$VERSION_NAME" "$PYTHON_BIN" - <<'PY'
import datetime, hashlib, json, os
digest = hashlib.sha256(open("ArtemisAccessibilityHelper.apk", "rb").read()).hexdigest()
manifest = {
"package": "com.artemis.helper",
"version_code": int(os.environ["VERSION_CODE"]),
"version_name": os.environ["VERSION_NAME"],
"sha256": digest,
"built_at": datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat(),
}
with open("helper_manifest.json", "w", encoding="utf-8", newline="\n") as fh:
json.dump(manifest, fh, indent=2)
fh.write("\n")
print(json.dumps(manifest, indent=2))
PY
echo "=================================================="
echo " Build SUCCESSFUL!"
echo " Artifact: $DIR/ArtemisAccessibilityHelper.apk"
@@ -0,0 +1,7 @@
{
"package": "com.artemis.helper",
"version_code": 6,
"version_name": "1.2.0",
"sha256": "133e373fe03615c251cd74b03a38539c844d09a074954bca2d2dde6684561969",
"built_at": "2026-09-10T23:13:11+00:00"
}
+350 -73
View File
@@ -1,87 +1,364 @@
import unittest
"""AccessibilityClient: HTTP surface, token handling, element-shape parity, repairs."""
from __future__ import annotations
import base64
from io import BytesIO
import json
import urllib.error
from unittest.mock import MagicMock, patch
from artemis.clients.accessibility_client import AccessibilityClient, DEFAULT_PORT
from PIL import Image
import pytest
from artemis.clients.accessibility_client import (
TOKEN_HEADER,
AccessibilityClient,
HelperEmptyHierarchy,
HelperRequestError,
HelperUnavailable,
normalize_helper_elements,
)
from artemis.clients.ui_automator_client import _parse_hierarchy_xml_to_elements
from artemis.runtime.helper_manager import HelperSession
XML = (
'<?xml version="1.0" encoding="UTF-8"?>'
'<hierarchy rotation="0">'
'<node index="0" text="OK" resource-id="app:id/ok" class="android.widget.Button" '
'package="app" content-desc="Confirm" checkable="false" checked="false" clickable="true" '
'enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" '
'password="false" selected="false" visible-to-user="true" bounds="[10,20][110,70]" '
'state-description="on" />'
"</hierarchy>"
)
TOKEN = "a" * 48
class TestAccessibilityClient(unittest.TestCase):
def setUp(self):
self.client = AccessibilityClient(device_id="dummy_device", local_port=DEFAULT_PORT)
class FakeManager:
def __init__(self):
self.session = self._session(41000)
self.attach_calls: list[bool] = []
self.reattach_calls = 0
self.pushed_tokens = 0
self.detached = False
self.fail_reattach = False
@patch("artemis.clients.accessibility_client.urllib.request.urlopen")
def test_ping_success(self, mock_urlopen):
mock_resp = MagicMock()
mock_resp.read.return_value = b'{"success": true, "service": "ArtemisAccessibilityService"}'
mock_resp.__enter__.return_value = mock_resp
mock_urlopen.return_value = mock_resp
@staticmethod
def _session(port: int) -> HelperSession:
return HelperSession(
serial="dev",
local_port=port,
transport_id="3",
version_code=6,
version_name="1.2.0",
owns_forward=True,
token=TOKEN,
protocol_version=2,
)
self.assertTrue(self.client.ping())
def attach(self, serial, *, provision=True, on_event=None):
self.attach_calls.append(provision)
if on_event is not None:
on_event("installing", {"serial": serial})
return self.session
@patch("artemis.clients.accessibility_client.urllib.request.urlopen")
def test_get_hierarchy(self, mock_urlopen):
mock_resp = MagicMock()
mock_resp.read.return_value = b'{"success": true, "elements": [{"text": "OK"}]}'
mock_resp.__enter__.return_value = mock_resp
mock_urlopen.return_value = mock_resp
def reattach(self, serial):
self.reattach_calls += 1
if self.fail_reattach:
raise HelperUnavailable("dead")
self.session = self._session(41001)
return self.session
data = self.client.get_hierarchy()
self.assertTrue(data["success"])
self.assertEqual(data["elements"][0]["text"], "OK")
def push_token(self, serial):
self.pushed_tokens += 1
return True
@patch("artemis.clients.accessibility_client.urllib.request.urlopen")
def test_tap_action(self, mock_urlopen):
mock_resp = MagicMock()
mock_resp.read.return_value = b'{"success": true}'
mock_resp.__enter__.return_value = mock_resp
mock_urlopen.return_value = mock_resp
def host_token(self):
return TOKEN
@patch("artemis.clients.accessibility_client.urllib.request.urlopen")
def test_get_hierarchy_xml(self, mock_urlopen):
mock_resp = MagicMock()
mock_resp.read.return_value = b'<hierarchy rotation="0"><node text="Save" /></hierarchy>'
mock_resp.__enter__.return_value = mock_resp
mock_urlopen.return_value = mock_resp
def detach(self, serial):
self.detached = True
xml = self.client.get_hierarchy_xml()
self.assertIn("<hierarchy", xml)
self.assertIn("Save", xml)
def ping(self, port):
return {"success": True}
@patch.object(AccessibilityClient, "get_atomic_snapshot")
def test_get_screen_data_atomic(self, mock_atomic):
mock_atomic.return_value = {
"success": True,
"has_screenshot": True,
"screenshot_base64": "dummy_b64",
"xml": '<hierarchy rotation="0"><node text="OK" /></hierarchy>',
"elements": [{"text": "OK"}],
"width": 1080,
"height": 2400,
def _response(body: bytes):
resp = MagicMock()
resp.read.return_value = body
resp.__enter__.return_value = resp
resp.__exit__.return_value = False
return resp
def _http_error(code: int, body: bytes = b'{"success": false, "error": "no"}'):
return urllib.error.HTTPError("http://x", code, "err", {}, BytesIO(body))
def _urlopen():
return patch("artemis.clients.accessibility_client.urllib.request.urlopen")
@pytest.fixture
def client():
manager = FakeManager()
with patch("artemis.clients.accessibility_client.ensure_device_awake", return_value=None):
yield AccessibilityClient("dev", manager=manager), manager
# --------------------------------------------------------------------------- #
# Session and transport
# --------------------------------------------------------------------------- #
def test_connect_provisions_and_lazy_path_does_not(client):
c, manager = client
events = []
c.connect(on_event=lambda name, details: events.append(name))
assert manager.attach_calls == [True]
assert events == ["installing"]
assert c.active_backend == "helper"
lazy = AccessibilityClient("dev", manager=FakeManager())
with (
patch("artemis.clients.accessibility_client.ensure_device_awake", return_value=None),
_urlopen() as urlopen,
):
urlopen.return_value = _response(XML.encode())
assert lazy.get_hierarchy() == XML
assert lazy._manager.attach_calls == [False]
def test_every_request_carries_the_session_token(client):
c, _ = client
with _urlopen() as urlopen:
urlopen.return_value = _response(XML.encode())
assert c.get_hierarchy() == XML
request = urlopen.call_args.args[0]
assert request.full_url == "http://127.0.0.1:41000/dump_xml"
# urllib stores header names capitalized: "X-artemis-token".
assert request.get_header(TOKEN_HEADER.capitalize()) == TOKEN
def test_401_pushes_the_token_again_and_retries_once(client):
c, manager = client
with _urlopen() as urlopen:
urlopen.side_effect = [_http_error(401), _response(XML.encode())]
assert c.get_hierarchy() == XML
assert manager.pushed_tokens == 1
assert manager.reattach_calls == 0 # a tunnel rebuild would not fix a token problem
with _urlopen() as urlopen:
urlopen.side_effect = [_http_error(401), _http_error(401)]
with pytest.raises(HelperRequestError) as info:
c.get_hierarchy()
assert info.value.status == 401
assert manager.pushed_tokens == 2
def test_other_http_errors_surface_without_repairs(client):
c, manager = client
with _urlopen() as urlopen:
urlopen.side_effect = _http_error(404, b"Unknown endpoint")
with pytest.raises(HelperRequestError, match="HTTP 404"):
c.get_hierarchy()
assert manager.reattach_calls == 0 and manager.pushed_tokens == 0
def test_transport_failure_reattaches_once_and_retries(client):
c, manager = client
with _urlopen() as urlopen:
urlopen.side_effect = [urllib.error.URLError("connection refused"), _response(XML.encode())]
assert c.get_hierarchy() == XML
assert manager.reattach_calls == 1
assert urlopen.call_args.args[0].full_url == "http://127.0.0.1:41001/dump_xml"
def test_transport_failure_after_reattach_propagates(client):
c, manager = client
manager.fail_reattach = True
with _urlopen() as urlopen:
urlopen.side_effect = urllib.error.URLError("connection refused")
with pytest.raises(HelperUnavailable):
c.get_hierarchy()
assert manager.reattach_calls == 1
def test_action_timeout_does_not_repeat_text_entry(client):
c, manager = client
with _urlopen() as urlopen:
urlopen.side_effect = TimeoutError("response lost after action")
assert c.send_text("hello") is False
assert urlopen.call_count == 1
assert manager.reattach_calls == 0
# --------------------------------------------------------------------------- #
# Screen data: one /snapshot request, UIAutomator element shapes
# --------------------------------------------------------------------------- #
def test_screen_data_is_one_snapshot_request_with_xml_only(client):
c, _ = client
snapshot = {
"success": True,
"has_screenshot": True,
"screenshot_base64": "QUJD",
"xml": XML,
"width": 1080,
"height": 2424,
}
with _urlopen() as urlopen:
urlopen.return_value = _response(json.dumps(snapshot).encode())
data = c.get_screen_data()
assert urlopen.call_count == 1
assert urlopen.call_args.args[0].full_url == "http://127.0.0.1:41000/snapshot?fields=xml"
assert data.base64 == "QUJD" and data.width == 1080 and data.height == 2424
assert data.hierarchy_xml == XML
assert data.elements == _parse_hierarchy_xml_to_elements(XML)
element = data.elements[1]
assert element["clickable"] == "true"
assert element["visible-to-user"] == "true"
assert element["accessibilityText"] == "Confirm"
assert element["parsed_bounds"] == {"left": 10, "top": 20, "right": 110, "bottom": 70}
assert element["state-description"] == "on"
def test_screen_data_without_screenshot_reuses_the_hierarchy_and_adds_screencap(client):
c, _ = client
no_shot = {
"success": True,
"has_screenshot": False,
"screenshot_error": "takeScreenshot not supported on Android < 11",
"xml": XML,
"width": 1080,
"height": 2400,
}
img = Image.new("RGB", (4, 6), "white")
with (
_urlopen() as urlopen,
patch.object(AccessibilityClient, "get_screenshot", return_value=img),
):
urlopen.return_value = _response(json.dumps(no_shot).encode())
data = c.get_screen_data()
assert urlopen.call_count == 1 # the hierarchy is not dumped a second time
assert data.width == 4 and data.height == 6
assert Image.open(BytesIO(base64.b64decode(data.base64))).size == (4, 6)
assert data.elements == _parse_hierarchy_xml_to_elements(XML)
def test_screen_data_falls_back_to_normalized_json_without_xml(client):
c, _ = client
snapshot = {
"success": True,
"has_screenshot": True,
"screenshot_base64": "QUJD",
"xml": "",
"elements": [
{
"text": "",
"content-desc": "Menu",
"clickable": True,
"focused": False,
"visible-to-user": True,
"parsed_bounds": {"left": 1.0, "top": 2, "right": 3, "bottom": 4},
"children": [{"text": "x"}],
}
],
"width": 1080,
"height": 2400,
}
with _urlopen() as urlopen:
urlopen.return_value = _response(json.dumps(snapshot).encode())
data = c.get_screen_data()
assert data.elements == [
{
"text": "",
"content-desc": "Menu",
"accessibilityText": "Menu",
"clickable": "true",
"focused": "false",
"visible-to-user": "true",
"parsed_bounds": {"left": 1, "top": 2, "right": 3, "bottom": 4},
}
screen_data = self.client.get_screen_data()
self.assertEqual(screen_data.width, 1080)
self.assertEqual(screen_data.height, 2400)
self.assertEqual(screen_data.base64, "dummy_b64")
self.assertIn('rotation="0"', screen_data.hierarchy_xml)
self.assertEqual(len(screen_data.elements), 1)
@patch.object(AccessibilityClient, "get_atomic_snapshot", return_value=None)
@patch.object(AccessibilityClient, "get_screenshot")
@patch.object(AccessibilityClient, "get_hierarchy")
def test_get_screen_data_fallback(self, mock_get_hierarchy, mock_get_screenshot, _mock_atomic):
from PIL import Image
mock_get_screenshot.return_value = Image.new("RGB", (1080, 2400))
mock_get_hierarchy.return_value = {
"success": True,
"xml": '<hierarchy rotation="0"><node text="Fallback" /></hierarchy>',
"elements": [{"text": "Fallback"}],
}
screen_data = self.client.get_screen_data()
self.assertEqual(screen_data.width, 1080)
self.assertEqual(screen_data.height, 2400)
self.assertIn("Fallback", screen_data.hierarchy_xml)
self.assertEqual(len(screen_data.elements), 1)
]
if __name__ == "__main__":
unittest.main()
def test_empty_hierarchy_is_an_error_not_a_blank_screen(client):
c, _ = client
answer = {"success": False, "error": "No active window or root node found", "xml": ""}
with _urlopen() as urlopen:
urlopen.return_value = _response(json.dumps(answer).encode())
with pytest.raises(HelperEmptyHierarchy, match="No active window"):
c.get_screen_data()
def test_screenshot_without_dimensions_is_rejected(client):
c, _ = client
answer = {"success": True, "has_screenshot": True, "screenshot_base64": "QUJD", "xml": XML}
with _urlopen() as urlopen:
urlopen.return_value = _response(json.dumps(answer).encode())
with pytest.raises(HelperRequestError, match="without dimensions"):
c.get_screen_data()
# --------------------------------------------------------------------------- #
# Input
# --------------------------------------------------------------------------- #
def test_set_clipboard_sends_raw_utf8(client):
c, _ = client
with _urlopen() as urlopen:
urlopen.return_value = _response(b'{"success": true}')
assert c.set_clipboard("héllo\n你好") is True
request = urlopen.call_args.args[0]
assert request.full_url == "http://127.0.0.1:41000/action"
assert request.data == '{"cmd": "clipboard", "text": "héllo\\n你好"}'.encode()
assert request.get_header("Content-type") == "application/json; charset=utf-8"
def test_send_text_appends_like_an_ime(client):
c, _ = client
with _urlopen() as urlopen:
urlopen.return_value = _response(b'{"success": true}')
assert c.send_text("abc") is True
assert json.loads(urlopen.call_args.args[0].data) == {
"cmd": "type",
"text": "abc",
"append": True,
}
def test_rpc_failure_returns_false_instead_of_raising(client):
c, manager = client
manager.fail_reattach = True
with _urlopen() as urlopen:
urlopen.side_effect = urllib.error.URLError("refused")
assert c.clear_text() is False
with _urlopen() as urlopen:
urlopen.side_effect = _http_error(500)
assert c.clear_text() is False
def test_press_key_uses_global_action_for_navigation_keys(client):
c, _ = client
with _urlopen() as urlopen:
urlopen.return_value = _response(b'{"success": true}')
assert c.press_key("BACK") is True
assert json.loads(urlopen.call_args.args[0].data) == {"cmd": "global", "action": "back"}
def test_disconnect_detaches(client):
c, manager = client
c.connect()
c.disconnect()
assert manager.detached and c.active_backend is None
def test_normalize_helper_elements_tolerates_bad_input():
assert normalize_helper_elements(None) == []
assert normalize_helper_elements("x") == []
assert normalize_helper_elements({"clickable": True}) == [{"clickable": "true"}]
@@ -0,0 +1,227 @@
"""Backend selection and the helper-first / UIAutomator-fallback composite."""
from __future__ import annotations
from unittest.mock import MagicMock
import pytest
from artemis.clients.accessibility_client import AccessibilityClient, HelperUnavailable
from artemis.clients.screen_client_factory import (
DeviceOfflineError,
FallbackScreenClient,
HierarchyBackend,
create_screen_client,
describe_backend,
hierarchy_backend_sentence,
hierarchy_backend_summary,
resolve_backend,
)
from artemis.clients.ui_automator_client import UIAutomatorClient
from artemis.config.constants import ENV_ARTEMIS_HIERARCHY_BACKEND
# --------------------------------------------------------------------------- #
# resolve_backend / create_screen_client
# --------------------------------------------------------------------------- #
def test_resolve_backend_prefers_explicit_then_env_then_settings(monkeypatch):
monkeypatch.delenv(ENV_ARTEMIS_HIERARCHY_BACKEND, raising=False)
from artemis.config import settings
monkeypatch.setattr(settings, "ARTEMIS_HIERARCHY_BACKEND", "uiautomator", raising=False)
assert resolve_backend() is HierarchyBackend.UIAUTOMATOR
monkeypatch.setenv(ENV_ARTEMIS_HIERARCHY_BACKEND, "Helper")
assert resolve_backend() is HierarchyBackend.HELPER
assert resolve_backend("auto") is HierarchyBackend.AUTO
assert resolve_backend(HierarchyBackend.HELPER) is HierarchyBackend.HELPER
def test_resolve_backend_unknown_value_falls_back_to_auto(monkeypatch):
monkeypatch.setenv(ENV_ARTEMIS_HIERARCHY_BACKEND, "maestro")
assert resolve_backend() is HierarchyBackend.AUTO
def test_create_screen_client_per_backend():
assert isinstance(create_screen_client("d", "uiautomator"), UIAutomatorClient)
assert isinstance(create_screen_client("d", "helper"), AccessibilityClient)
composite = create_screen_client("d", "auto")
assert isinstance(composite, FallbackScreenClient)
assert describe_backend(composite) is None # nothing served yet
assert describe_backend(UIAutomatorClient("d")) == "uiautomator"
assert describe_backend(AccessibilityClient("d")) == "helper"
# --------------------------------------------------------------------------- #
# FallbackScreenClient
# --------------------------------------------------------------------------- #
@pytest.fixture
def composite():
helper = MagicMock(spec=AccessibilityClient)
u2 = MagicMock(spec=UIAutomatorClient)
factory = MagicMock(return_value=u2)
clock = {"now": 0.0, "state": "device"}
client = FallbackScreenClient(
"dev",
helper=helper,
uiautomator_factory=factory,
retry_after=30.0,
clock=lambda: clock["now"],
state_probe=lambda _serial: clock["state"],
)
return client, helper, u2, factory, clock
def test_healthy_helper_never_constructs_uiautomator(composite):
client, helper, _, factory, _ = composite
helper.get_hierarchy.return_value = "<hierarchy/>"
client.connect()
assert client.get_hierarchy() == "<hierarchy/>"
assert client.active_backend == "helper"
factory.assert_not_called()
def test_connect_failure_degrades_to_uiautomator(composite):
client, helper, u2, factory, _ = composite
helper.connect.side_effect = HelperUnavailable("not installed")
client.connect()
factory.assert_called_once()
u2.connect.assert_called_once()
assert client.active_backend == "uiautomator"
def test_call_failure_falls_back_then_retries_helper_after_window(composite):
client, helper, u2, _, clock = composite
helper.get_hierarchy.side_effect = [RuntimeError("tunnel gone"), "<from-helper/>"]
u2.get_hierarchy.return_value = "<from-u2/>"
assert client.get_hierarchy() == "<from-u2/>"
assert client.active_backend == "uiautomator"
# Inside the retry window the helper is not touched again.
clock["now"] = 10.0
assert client.get_hierarchy() == "<from-u2/>"
assert helper.get_hierarchy.call_count == 1
# After the window the helper is tried again and wins.
clock["now"] = 31.0
assert client.get_hierarchy() == "<from-helper/>"
assert client.active_backend == "helper"
def test_uiautomator_failure_propagates(composite):
client, helper, u2, _, _ = composite
helper.get_screen_data.side_effect = RuntimeError("helper down")
u2.get_screen_data.side_effect = RuntimeError("u2 down too")
with pytest.raises(RuntimeError, match="u2 down too"):
client.get_screen_data()
def test_set_clipboard_goes_to_helper_first(composite):
client, helper, u2, _, _ = composite
helper.set_clipboard.return_value = True
assert client.set_clipboard("x") is True
u2.set_clipboard.assert_not_called()
def test_disconnect_releases_both_when_uiautomator_was_used(composite):
client, helper, u2, _, _ = composite
helper.connect.side_effect = HelperUnavailable("nope")
client.connect()
client.disconnect()
helper.disconnect.assert_called_once()
u2.disconnect.assert_called_once()
assert client.active_backend is None
def test_strict_helper_backend_surfaces_errors():
manager = MagicMock()
manager.attach.side_effect = HelperUnavailable("not installed")
client = AccessibilityClient("dev", manager=manager)
with pytest.raises(HelperUnavailable):
client.connect()
def test_offline_device_fails_fast_without_touching_uiautomator(composite):
client, helper, u2, factory, clock = composite
helper.get_screen_data.side_effect = RuntimeError("connection refused")
clock["state"] = "unauthorized"
with pytest.raises(DeviceOfflineError, match="unauthorized"):
client.get_screen_data()
factory.assert_not_called()
clock["state"] = None
with pytest.raises(DeviceOfflineError, match="does not list it"):
client.get_screen_data()
factory.assert_not_called()
def test_backend_history_and_listeners_track_every_switch(composite):
client, helper, u2, _, clock = composite
seen = []
client.add_backend_listener(lambda prev, new, reason: seen.append((prev, new, reason)))
helper.get_hierarchy.side_effect = [
"<h/>",
RuntimeError("tunnel gone"),
"<h/>",
]
u2.get_hierarchy.return_value = "<u2/>"
client.get_hierarchy() # helper
client.get_hierarchy() # helper fails -> uiautomator
clock["now"] = 60.0
client.get_hierarchy() # helper recovers
assert [(p, n) for p, n, _ in seen] == [
(None, "helper"),
("helper", "uiautomator"),
("uiautomator", "helper"),
]
assert seen[1][2] == "RuntimeError: tunnel gone"
assert seen[2][2] == "helper recovered"
assert [h["backend"] for h in client.backend_history] == ["helper", "uiautomator", "helper"]
summary = hierarchy_backend_summary(client)
assert summary["backend"] == "helper" and len(summary["switches"]) == 2
sentence = hierarchy_backend_sentence(client, relative_time=lambda t: "T+00:10")
assert sentence.startswith("UI hierarchy source: the Artemis accessibility helper")
assert "switched to UIAutomator2 at T+00:10 because RuntimeError: tunnel gone" in sentence
def test_listener_errors_do_not_break_the_client(composite):
client, helper, _, _, _ = composite
client.add_backend_listener(lambda *_: (_ for _ in ()).throw(RuntimeError("boom")))
helper.get_hierarchy.return_value = "<h/>"
assert client.get_hierarchy() == "<h/>"
def test_sentence_is_none_before_first_use(composite):
client, *_ = composite
assert hierarchy_backend_sentence(client) is None
assert hierarchy_backend_summary(client) is None
def test_helper_retry_releases_uiautomation_first(composite):
"""While our UIAutomator2 server lives the helper cannot bind; stop it before retrying."""
client, helper, u2, _, clock = composite
helper.get_hierarchy.side_effect = [RuntimeError("tunnel gone"), "<from-helper/>"]
u2.get_hierarchy.return_value = "<from-u2/>"
assert client.get_hierarchy() == "<from-u2/>"
u2.stop_server.assert_not_called() # inside the retry window nothing is touched
clock["now"] = 31.0
assert client.get_hierarchy() == "<from-helper/>"
u2.stop_server.assert_called_once()
def test_disconnect_stops_the_uiautomator_server_it_started(composite):
client, helper, u2, _, _ = composite
helper.connect.side_effect = HelperUnavailable("nope")
client.connect()
client.disconnect()
u2.disconnect.assert_called_once_with(stop_server=True)
+1
View File
@@ -93,6 +93,7 @@ async def test_success_reports_bytes_elements_serial_and_elapsed(patch_controlle
"elapsed_seconds",
"screenshot_bytes",
"element_count",
"hierarchy_backend",
"error",
"fix",
}
+147 -1
View File
@@ -279,10 +279,21 @@ def _run(
cleanup=0,
validate=None,
smoke=None,
helper_status=None,
helper_provision=None,
backend="auto",
**kwargs,
):
"""Run the tool with every side-effecting collaborator stubbed."""
with ExitStack() as stack:
fake_helper = MagicMock()
fake_helper.status = MagicMock(
side_effect=lambda serial: dict(helper_status or _helper_healthy(serial))
)
fake_helper.provision = helper_provision or MagicMock(return_value=_provision_ok())
stack.enter_context(patch.object(diagnose, "helper_manager", fake_helper))
stack.enter_context(patch.object(diagnose, "_hierarchy_backend", lambda: backend))
kwargs["_fake_helper"] = fake_helper
stack.enter_context(
patch.object(
diagnose.readiness_engine,
@@ -342,7 +353,35 @@ def _run(
smoke or AsyncMock(return_value=_smoke_ok()),
)
)
return asyncio.run(mobile_diagnose(**kwargs))
fake_helper = kwargs.pop("_fake_helper")
result = asyncio.run(mobile_diagnose(**kwargs))
result["_fake_helper"] = fake_helper
return result
def _helper_healthy(serial: str = "pixel-1") -> dict:
return {
"package": "com.artemis.helper",
"installed": True,
"installed_version": 2,
"bundled_version": 2,
"bundled_apk_present": True,
"outdated": False,
"enabled": True,
"forward_port": 41234,
"reachable": True,
"reported_version": 2,
"transport_id": "7",
"session": None,
}
def _provision_ok():
from artemis.runtime.helper_manager import ProvisionResult
return ProvisionResult(
ok=True, action="installed", installed_version=2, bundled_version=2, enabled=True
)
def _smoke_ok(serial: str = "pixel-1") -> dict:
@@ -433,6 +472,7 @@ def test_ready_environment_reports_ready_with_slim_shape(temp_trace_env):
"android_version": "15",
"is_locked": False,
"is_emulator": False,
"accessibility_helper": {**_helper_healthy(), "serial": "pixel-1", "backend": "auto"},
}
assert result["host"] == {
"server_python": "/proj/.venv/bin/python",
@@ -946,6 +986,11 @@ def test_attempt_fix_heals_corrupted_keys_then_rechecks(temp_trace_env):
patch.object(diagnose.DeviceExecutionLock, "get_active_owners", return_value={}),
patch.object(diagnose.DeviceExecutionLock, "get_queued_tasks", return_value=[]),
patch.object(IntegrationHostProbe, "probe", AsyncMock(return_value=_host())),
patch.object(
diagnose,
"_helper_status",
lambda serial: {**_helper_healthy(serial), "serial": serial, "backend": "auto"},
),
):
result = asyncio.run(mobile_diagnose(attempt_fix=True))
@@ -1139,3 +1184,104 @@ def test_logs_surface_recent_errors_and_last_failed_task(temp_trace_env, monkeyp
"Traceback (most recent call last):",
"RuntimeError: device offline",
]
# --------------------------------------------------------------------------- #
# Accessibility helper (UI hierarchy backend)
# --------------------------------------------------------------------------- #
def test_missing_helper_in_auto_mode_degrades_with_optional_step(temp_trace_env):
status = {
**_helper_healthy(),
"installed": False,
"installed_version": None,
"reachable": False,
}
result = _run(_healthy_probes(), helper_status=status)
assert result["verdict"] == "degraded"
helper = result["device"] is None or result["device"]["accessibility_helper"]
assert helper
optional = [s for s in result["next_steps"] if s.startswith("[OPTIONAL] Accessibility helper")]
assert len(optional) == 1 and "is not installed" in optional[0]
assert " Run: uv run artemis helper install --serial pixel-1" in result["next_steps"]
assert any("attempt_fix=true" in s for s in result["next_steps"])
result["_fake_helper"].provision.assert_not_called()
def test_missing_helper_in_helper_mode_blocks(temp_trace_env):
status = {**_helper_healthy(), "installed": False, "reachable": False}
result = _run(_healthy_probes(), helper_status=status, backend="helper")
assert result["verdict"] == "blocked"
assert any(s.startswith("[REQUIRED] Accessibility helper") for s in result["next_steps"])
def test_uiautomator_backend_ignores_helper_state(temp_trace_env):
status = {**_helper_healthy(), "installed": False, "reachable": False}
result = _run(_healthy_probes(), helper_status=status, backend="uiautomator")
assert result["verdict"] == "ready"
assert not any("Accessibility helper" in s for s in result["next_steps"])
def test_outdated_helper_reports_versions(temp_trace_env):
status = {**_helper_healthy(), "installed_version": 1, "outdated": True}
result = _run(_healthy_probes(), helper_status=status)
step = next(s for s in result["next_steps"] if "Accessibility helper" in s)
assert "device has version 1, bundled is 2" in step
def test_attempt_fix_provisions_missing_helper_on_idle_device(temp_trace_env):
status = {
**_helper_healthy(),
"installed": False,
"installed_version": None,
"reachable": False,
}
result = _run(_healthy_probes(), helper_status=status, attempt_fix=True)
result["_fake_helper"].provision.assert_called_once_with("pixel-1")
fix = next(f for f in result["fixes_applied"] if f["fix"] == "install_accessibility_helper")
assert fix["success"] is True and fix["skipped"] is False
assert "installed" in fix["message"]
def test_attempt_fix_skips_helper_install_while_task_holds_device(temp_trace_env):
status = {**_helper_healthy(), "installed": False, "reachable": False}
result = _run(
_healthy_probes(),
helper_status=status,
attempt_fix=True,
active_owners={"pixel-1": _owner()},
)
result["_fake_helper"].provision.assert_not_called()
fix = next(f for f in result["fixes_applied"] if f["fix"] == "install_accessibility_helper")
assert fix["skipped"] is True
def test_attempt_fix_leaves_healthy_helper_alone(temp_trace_env):
result = _run(_healthy_probes(), attempt_fix=True)
result["_fake_helper"].provision.assert_not_called()
assert not any(f["fix"] == "install_accessibility_helper" for f in result["fixes_applied"])
def test_disabled_helper_step_includes_the_manual_path(temp_trace_env):
from artemis.runtime.helper_manager import MANUAL_ENABLE_PATH
status = {**_helper_healthy(), "enabled": False, "reachable": False}
result = _run(_healthy_probes(), helper_status=status)
step = next(s for s in result["next_steps"] if "Accessibility helper" in s)
assert "service is disabled" in step
assert any(MANUAL_ENABLE_PATH in s for s in result["next_steps"])
def test_newer_helper_than_bundle_is_not_a_finding(temp_trace_env):
status = {**_helper_healthy(), "installed_version": 9, "newer_than_bundled": True}
result = _run(_healthy_probes(), helper_status=status)
assert result["verdict"] == "ready"
assert not any("Accessibility helper" in s for s in result["next_steps"])
+52
View File
@@ -0,0 +1,52 @@
"""The bundled helper APK, its manifest, and the Gradle version must agree.
``helper_manager`` decides installs and upgrades from ``helper_manifest.json``,
so a rebuilt APK without a regenerated manifest (or a version bump in Gradle
without a rebuild) would silently ship the wrong version gate.
"""
from __future__ import annotations
import hashlib
import json
import re
import pytest
from artemis.runtime.helper_manager import (
BUNDLED_APK_PATH,
BUNDLED_MANIFEST_PATH,
HELPER_PACKAGE_DIR,
PACKAGE_NAME,
load_bundled_helper,
)
pytestmark = pytest.mark.skipif(
not BUNDLED_APK_PATH.is_file(), reason="bundled helper APK is not present in this checkout"
)
def test_manifest_matches_apk_and_gradle_version():
manifest = json.loads(BUNDLED_MANIFEST_PATH.read_text(encoding="utf-8"))
assert manifest["package"] == PACKAGE_NAME
assert manifest["sha256"] == hashlib.sha256(BUNDLED_APK_PATH.read_bytes()).hexdigest(), (
"ArtemisAccessibilityHelper.apk was rebuilt without regenerating helper_manifest.json; "
"run packages/artemis-accessibility-helper/build_apk.sh"
)
gradle = (HELPER_PACKAGE_DIR / "app" / "build.gradle.kts").read_text(encoding="utf-8")
version_code = int(re.search(r"versionCode\s*=\s*(\d+)", gradle).group(1))
version_name = re.search(r'versionName\s*=\s*"([^"]+)"', gradle).group(1)
assert manifest["version_code"] == version_code
assert manifest["version_name"] == version_name
def test_load_bundled_helper_reads_manifest():
bundled = load_bundled_helper()
assert bundled is not None
assert bundled.apk_path == BUNDLED_APK_PATH
assert bundled.version_code >= 2 # first version that reports version_code on /ping
def test_load_bundled_helper_returns_none_when_missing(tmp_path):
assert load_bundled_helper(tmp_path / "x.apk", tmp_path / "x.json") is None
+716
View File
@@ -0,0 +1,716 @@
"""Lifecycle of the Accessibility Helper: provision, attach, reattach, detach.
Every adb call goes through a scripted fake so the tests assert the exact
command sequence the manager issues, and the HTTP ping is a stub keyed by host
port so "the service answers" is a switch the test flips.
"""
from __future__ import annotations
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
import subprocess
import pytest
from artemis.runtime import helper_manager as hm
from artemis.runtime.helper_manager import (
DEVICE_PORT,
SERVICE_NAME,
AccessibilityHelperManager,
BundledHelper,
HelperUnavailable,
)
SERIAL = "pixel-1"
OTHER = "pixel-2"
class FakeAdb:
"""Scripted adb: records commands and answers from a small device model."""
def __init__(self):
self.calls: list[list[str]] = []
self.installed: dict[str, int | None] = {SERIAL: None, OTHER: None}
self.enabled_services: dict[str, str] = {SERIAL: "", OTHER: ""}
self.transport: dict[str, str] = {SERIAL: "3", OTHER: "4"}
self.forwards: list[tuple[str, int, int]] = [] # (serial, local, remote)
self.next_port = 40000
self.install_result = "Success"
# Number of times the system "prunes" a freshly written service setting.
self.prune_writes = 0
# Serials whose dead service was re-bound by a settings toggle.
self.revived: set[str] = set()
# HTTP protocol the installed helper speaks (the bundled build speaks 2).
self.protocol: dict[str, int] = {SERIAL: 2, OTHER: 2}
# Last token delivered by `am broadcast`, per serial.
self.tokens: dict[str, str] = {}
# Process table lines (PID ARGS) of UiAutomation holders, per serial.
self.holders: dict[str, list[str]] = {SERIAL: [], OTHER: []}
self.killed: list[tuple[str, int]] = []
def __call__(self, args: list[str]) -> subprocess.CompletedProcess:
self.calls.append(list(args))
out, code = "", 0
if args[:2] == ["devices", "-l"]:
lines = ["List of devices attached"]
for serial, tid in self.transport.items():
lines.append(f"{serial} device product:p model:m device:d transport_id:{tid}")
out = "\n".join(lines) + "\n"
elif args[:2] == ["forward", "--list"]:
out = "".join(f"{s} tcp:{local} tcp:{r}\n" for s, local, r in self.forwards)
elif args[0] == "-s":
serial, rest = args[1], args[2:]
if rest[:3] == ["shell", "dumpsys", "package"]:
version = self.installed.get(serial)
out = (
f"Package [{rest[3]}]\n versionCode={version} minSdk=24\n"
if version is not None
else f"Unable to find package: {rest[3]}\n"
)
elif rest[:4] == ["shell", "settings", "get", "secure"]:
key = rest[4]
out = (
(self.enabled_services.get(serial) or "null") + "\n"
if key == "enabled_accessibility_services"
else "1\n"
)
elif rest[:4] == ["shell", "settings", "put", "secure"]:
if rest[4] == "enabled_accessibility_services":
if self.prune_writes > 0:
self.prune_writes -= 1
self.enabled_services[serial] = ""
else:
before = self.enabled_services.get(serial, "")
self.enabled_services[serial] = rest[5]
# AccessibilityManager binds the service when it is (re)added.
if SERVICE_NAME in rest[5] and SERVICE_NAME not in before:
self.revived.add(serial)
elif rest[0] == "install":
if self.install_result == "Success":
self.installed[serial] = self.bundled_version
self.protocol[serial] = 2
out = "Success\n"
else:
out, code = self.install_result, 1
elif rest[0] == "uninstall":
self.installed[serial] = None
out = "Success\n"
elif rest[:2] == ["forward", "--no-rebind"]:
port = self.next_port
self.next_port += 1
self.forwards.append((serial, port, int(rest[3].split(":")[1])))
out = f"{port}\n"
elif rest[:2] == ["forward", "--remove"]:
port = int(rest[2].split(":")[1])
self.forwards = [f for f in self.forwards if not (f[0] == serial and f[1] == port)]
elif rest[:2] == ["shell", "input"]:
pass
elif rest[:3] == ["shell", "am", "broadcast"]:
assert rest[3:8] == ["-n", hm.TOKEN_RECEIVER, "-a", hm.TOKEN_ACTION, "--es"]
self.tokens[serial] = rest[9]
out = "Broadcasting: Intent { act=com.artemis.helper.SET_TOKEN }\nBroadcast completed: result=0\n"
elif rest[:3] == ["shell", "am", "start"]:
self.settings_opened = getattr(self, "settings_opened", 0) + 1
elif rest[:2] == ["shell", "ps"]:
out = "PID ARGS" + "".join(chr(10) + line for line in self.holders.get(serial, []))
elif rest[:2] == ["shell", "kill"]:
pid = int(rest[2])
self.killed.append((serial, pid))
self.holders[serial] = [
line for line in self.holders.get(serial, []) if not line.startswith(f"{pid} ")
]
elif rest[:3] == ["shell", "am", "force-stop"]:
pass
else:
raise AssertionError(f"unexpected adb call: {args}")
else:
raise AssertionError(f"unexpected adb call: {args}")
return subprocess.CompletedProcess(args, code, out, "")
bundled_version = 2
class FakePing:
"""Answers on host ports whose forward targets a device running the service."""
def __init__(self, adb: FakeAdb):
self.adb = adb
self.dead: set[str] = set() # serials whose service is down
self.count = 0
def __call__(self, port: int):
self.count += 1
for serial, local, remote in self.adb.forwards:
if local == port and remote == DEVICE_PORT:
if serial in self.dead and serial not in self.adb.revived:
return None
if any(
not line.split(None, 1)[1].startswith("sh ")
for line in self.adb.holders.get(serial, [])
):
return None # suppressed by UiAutomation
if self.adb.installed.get(serial) is None:
return None
if SERVICE_NAME not in (self.adb.enabled_services.get(serial) or ""):
return None
return {
"success": True,
"version_code": self.adb.installed[serial],
"version_name": "1.1.0",
"protocol_version": self.adb.protocol.get(serial, 2),
"auth_required": self.adb.protocol.get(serial, 2) >= 2,
"token_set": serial in self.adb.tokens,
}
return None
@pytest.fixture
def bundled(tmp_path: Path) -> BundledHelper:
apk = tmp_path / "helper.apk"
apk.write_bytes(b"apk")
return BundledHelper(apk_path=apk, version_code=2, version_name="1.1.0", sha256="x")
@pytest.fixture
def env(tmp_path: Path, monkeypatch, bundled):
monkeypatch.setattr(hm, "get_temp_dir", lambda _name: tmp_path / "mutex")
adb = FakeAdb()
ping = FakePing(adb)
auto = {"install": True}
manager = AccessibilityHelperManager(
run_adb=adb,
ping=ping,
bundled=bundled,
sleep=lambda _s: None,
token_path=tmp_path / "token" / "session.token",
auto_install=lambda: auto["install"],
)
manager.auto_install_switch = auto # test handle
return adb, ping, manager
def _calls(adb: FakeAdb, prefix: list[str]) -> list[list[str]]:
return [c for c in adb.calls if c[: len(prefix)] == prefix]
def test_status_and_session_repr_do_not_expose_token(env):
_, _, manager = env
session = manager.attach(SERIAL)
status = manager.status(SERIAL)
assert session.token
assert "token" not in status["session"]
assert session.token not in str(status)
assert session.token not in repr(session)
def test_concurrent_managers_share_the_first_host_token(tmp_path, monkeypatch):
monkeypatch.setattr(hm, "get_temp_dir", lambda _name: tmp_path / "mutex")
token_path = tmp_path / "token" / "session.token"
managers = [AccessibilityHelperManager(token_path=token_path) for _ in range(8)]
with ThreadPoolExecutor(max_workers=8) as executor:
tokens = list(executor.map(lambda manager: manager.host_token(), managers))
assert len(set(tokens)) == 1
assert token_path.read_text() == tokens[0]
# --------------------------------------------------------------------------- #
# Provision
# --------------------------------------------------------------------------- #
def test_provision_installs_and_enables_when_missing(env):
adb, _, manager = env
result = manager.provision(SERIAL)
assert result.ok and result.action == "installed"
assert result.installed_version == 2 and result.bundled_version == 2
install = _calls(adb, ["-s", SERIAL, "install"])
assert install == [["-s", SERIAL, "install", "-r", "-g", str(manager.bundled.apk_path)]]
assert adb.enabled_services[SERIAL] == SERVICE_NAME
def test_provision_upgrades_outdated_and_leaves_newer_alone(env):
adb, _, manager = env
adb.installed[SERIAL] = 1
assert manager.provision(SERIAL).action == "upgraded"
assert adb.installed[SERIAL] == 2
adb.installed[SERIAL] = 5
adb.calls.clear()
result = manager.provision(SERIAL)
assert result.action == "up_to_date" and result.installed_version == 5
assert not _calls(adb, ["-s", SERIAL, "install"])
def test_provision_up_to_date_is_cheap_and_keeps_other_services(env):
adb, _, manager = env
adb.installed[SERIAL] = 2
adb.enabled_services[SERIAL] = "com.android.talkback/.TalkBackService"
result = manager.provision(SERIAL)
assert result.ok and result.action == "up_to_date"
assert adb.enabled_services[SERIAL] == f"com.android.talkback/.TalkBackService:{SERVICE_NAME}"
assert not _calls(adb, ["-s", SERIAL, "install"])
def test_provision_without_bundled_apk_fails_only_when_not_installed(env):
adb, ping, _ = env
manager = AccessibilityHelperManager(
run_adb=adb, ping=ping, bundled=None, sleep=lambda _s: None
)
result = manager.provision(SERIAL)
assert not result.ok and result.action == "failed" and "no bundled APK" in result.error
adb.installed[SERIAL] = 1
result = manager.provision(SERIAL)
assert result.ok and result.action == "up_to_date"
def test_provision_reports_install_failure(env):
adb, _, manager = env
adb.install_result = "Failure [INSTALL_FAILED_VERSION_DOWNGRADE]"
result = manager.provision(SERIAL)
assert not result.ok and result.action == "failed"
assert "INSTALL_FAILED_VERSION_DOWNGRADE" in result.error
def test_provision_mutex_is_released_and_stale_lock_is_broken(env, tmp_path):
adb, _, manager = env
manager.provision(SERIAL)
mutex_dir = tmp_path / "mutex"
assert not list(mutex_dir.glob("*.mutex"))
# A crashed process left its mutex behind long ago: provisioning proceeds.
import hashlib
import os
import time
digest = hashlib.sha256(SERIAL.encode()).hexdigest()[:16]
stale = mutex_dir / f"{digest}.mutex"
stale.write_text("1")
old = time.time() - 1000
os.utime(stale, (old, old))
assert manager.provision(SERIAL).ok
assert not stale.exists()
# --------------------------------------------------------------------------- #
# Attach / detach
# --------------------------------------------------------------------------- #
def test_attach_provisions_forwards_and_records_transport(env):
adb, _, manager = env
session = manager.attach(SERIAL)
assert session.local_port == 40000 and session.owns_forward
assert session.transport_id == "3" and session.version_code == 2
assert adb.forwards == [(SERIAL, 40000, DEVICE_PORT)]
forward_calls = _calls(adb, ["-s", SERIAL, "forward"])
assert forward_calls == [
["-s", SERIAL, "forward", "--no-rebind", "tcp:0", f"tcp:{DEVICE_PORT}"]
]
def test_two_devices_get_distinct_host_ports(env):
adb, _, manager = env
first = manager.attach(SERIAL)
second = manager.attach(OTHER)
assert first.local_port != second.local_port
assert {(s, local) for s, local, _ in adb.forwards} == {(SERIAL, 40000), (OTHER, 40001)}
assert manager.session(SERIAL) is first and manager.session(OTHER) is second
def test_attach_reuses_forward_created_by_another_process(env):
adb, _, manager = env
adb.installed[SERIAL] = 2
adb.enabled_services[SERIAL] = SERVICE_NAME
adb.forwards.append((SERIAL, 39999, DEVICE_PORT))
session = manager.attach(SERIAL)
assert session.local_port == 39999 and not session.owns_forward
assert not _calls(adb, ["-s", SERIAL, "forward", "--no-rebind"])
manager.detach(SERIAL)
assert adb.forwards == [(SERIAL, 39999, DEVICE_PORT)] # not ours to remove
def test_attach_is_idempotent_while_session_is_healthy(env):
adb, ping, manager = env
first = manager.attach(SERIAL)
adb.calls.clear()
again = manager.attach(SERIAL)
assert again is first
assert not _calls(adb, ["-s", SERIAL, "install"])
assert not _calls(adb, ["-s", SERIAL, "forward"])
def test_lazy_attach_never_installs(env):
adb, _, manager = env
with pytest.raises(HelperUnavailable):
manager.attach(SERIAL, provision=False)
assert not _calls(adb, ["-s", SERIAL, "install"])
assert not _calls(adb, ["-s", SERIAL, "shell", "settings", "put"])
# The dead forward it created for the probe is cleaned up again.
assert adb.forwards == []
assert manager.session(SERIAL) is None
def test_lazy_attach_connects_to_running_helper(env):
adb, _, manager = env
adb.installed[SERIAL] = 2
adb.enabled_services[SERIAL] = SERVICE_NAME
session = manager.attach(SERIAL, provision=False)
assert session.local_port == 40000
assert not _calls(adb, ["-s", SERIAL, "install"])
def test_replug_changes_transport_and_rebuilds_tunnel(env):
adb, _, manager = env
first = manager.attach(SERIAL)
# Unplug: adb drops the forward and assigns a new transport on replug.
adb.forwards.clear()
adb.transport[SERIAL] = "9"
adb.calls.clear()
second = manager.attach(SERIAL)
assert second is not first
assert second.transport_id == "9" and second.local_port == 40001
# The vanished forward is not "removed" (adb already dropped it).
assert not _calls(adb, ["-s", SERIAL, "forward", "--remove"])
assert adb.forwards == [(SERIAL, 40001, DEVICE_PORT)]
def test_reattach_after_request_failure_rebuilds_without_install(env):
adb, ping, manager = env
first = manager.attach(SERIAL)
adb.calls.clear()
session = manager.reattach(SERIAL)
assert session.local_port != first.local_port
assert _calls(adb, ["-s", SERIAL, "forward", "--remove"]) == [
["-s", SERIAL, "forward", "--remove", f"tcp:{first.local_port}"]
]
assert not _calls(adb, ["-s", SERIAL, "install"])
def test_reattach_revives_a_killed_service_without_installing(env):
adb, ping, manager = env
first = manager.attach(SERIAL)
ping.dead.add(SERIAL) # force-stopped: enabled in settings, but not running
adb.revived.discard(SERIAL)
adb.calls.clear()
session = manager.reattach(SERIAL)
assert session.local_port != first.local_port
assert not _calls(adb, ["-s", SERIAL, "install"])
puts = _calls(
adb, ["-s", SERIAL, "shell", "settings", "put", "secure", "enabled_accessibility_services"]
)
assert [p[-1] for p in puts] == ["null", SERVICE_NAME] # removed, then re-added
def test_reattach_raises_when_revive_does_not_help(env):
adb, ping, manager = env
manager.attach(SERIAL)
adb.calls.clear()
ping.dead.add(SERIAL)
adb.revived.discard(SERIAL)
adb.installed[SERIAL] = None # package vanished: nothing to re-bind
with pytest.raises(HelperUnavailable):
manager.reattach(SERIAL)
assert manager.session(SERIAL) is None
assert not _calls(adb, ["-s", SERIAL, "install"])
def test_lazy_first_attach_does_not_toggle_settings(env):
adb, ping, manager = env
adb.installed[SERIAL] = 2
adb.enabled_services[SERIAL] = SERVICE_NAME
ping.dead.add(SERIAL)
adb.revived.discard(SERIAL)
with pytest.raises(HelperUnavailable):
manager.attach(SERIAL, provision=False)
assert not _calls(adb, ["-s", SERIAL, "shell", "settings", "put"])
def test_detach_removes_only_owned_forward(env):
adb, _, manager = env
manager.attach(SERIAL)
manager.attach(OTHER)
manager.detach(SERIAL)
assert adb.forwards == [(OTHER, 40001, DEVICE_PORT)]
assert manager.session(SERIAL) is None and manager.session(OTHER) is not None
def test_status_is_read_only(env):
adb, _, manager = env
adb.installed[SERIAL] = 1
status = manager.status(SERIAL)
assert status["installed"] and status["installed_version"] == 1
assert status["bundled_version"] == 2 and status["outdated"] is True
assert status["enabled"] is False and status["reachable"] is False
assert status["transport_id"] == "3"
assert not _calls(adb, ["-s", SERIAL, "install"])
assert not _calls(adb, ["-s", SERIAL, "forward"])
def test_uninstall_disables_service_and_removes_package(env):
adb, _, manager = env
manager.attach(SERIAL)
assert manager.uninstall(SERIAL)
assert adb.installed[SERIAL] is None
assert SERVICE_NAME not in adb.enabled_services[SERIAL]
assert adb.forwards == []
def test_transport_id_parsing_handles_missing_device():
adb = FakeAdb()
manager = AccessibilityHelperManager(run_adb=adb, ping=lambda _p: None, bundled=None)
assert manager.transport_id("nope") is None
assert manager.transport_id(SERIAL) == "3"
def test_enable_retries_when_the_system_prunes_the_setting(env):
adb, _, manager = env
adb.prune_writes = 2 # AccessibilityManager drops the first two writes
result = manager.provision(SERIAL)
assert result.ok and result.enabled
puts = _calls(
adb, ["-s", SERIAL, "shell", "settings", "put", "secure", "enabled_accessibility_services"]
)
assert len(puts) == 3
assert adb.enabled_services[SERIAL] == SERVICE_NAME
def test_enable_gives_up_after_bounded_attempts(env):
adb, _, manager = env
adb.prune_writes = 99
result = manager.provision(SERIAL)
assert not result.ok and result.action == "installed"
# The person gets the exact manual path and the settings screen is opened for them.
assert hm.MANUAL_ENABLE_PATH in result.error
assert adb.settings_opened == 1
puts = _calls(
adb, ["-s", SERIAL, "shell", "settings", "put", "secure", "enabled_accessibility_services"]
)
assert len(puts) == hm._ENABLE_ATTEMPTS
def test_provision_emits_install_and_upgrade_events_before_the_install(env):
adb, _, manager = env
events: list[tuple[str, dict]] = []
def on_event(name, details):
# Fired before adb install ran: nothing installed yet at this point.
events.append((name, details, adb.installed[SERIAL]))
manager.provision(SERIAL, on_event=on_event)
assert [(e[0], e[2]) for e in events] == [("installing", None)]
assert events[0][1]["to_version"] == 2 and events[0][1]["serial"] == SERIAL
events.clear()
adb.installed[SERIAL] = 1
manager.provision(SERIAL, on_event=on_event)
assert [(e[0], e[1]["from_version"], e[2]) for e in events] == [("upgrading", 1, 1)]
events.clear()
manager.provision(SERIAL, on_event=on_event) # up to date: silent
assert events == []
def test_attach_forwards_events_to_provision(env):
adb, _, manager = env
seen = []
manager.attach(SERIAL, on_event=lambda name, details: seen.append(name))
assert seen == ["installing"]
def test_status_probes_with_a_temporary_forward_when_no_tunnel_exists(env):
adb, _, manager = env
adb.installed[SERIAL] = 2
adb.enabled_services[SERIAL] = SERVICE_NAME
status = manager.status(SERIAL)
assert status["reachable"] is True and status["tunnel"] == "probe"
assert status["forward_port"] is None
assert adb.forwards == [] # the probe forward was removed again
assert _calls(adb, ["-s", SERIAL, "forward", "--remove"]) == [
["-s", SERIAL, "forward", "--remove", "tcp:40000"]
]
def test_status_reports_session_and_shared_tunnels(env):
adb, _, manager = env
manager.attach(SERIAL)
assert manager.status(SERIAL)["tunnel"] == "session"
manager.detach(SERIAL)
adb.forwards.append((SERIAL, 39999, DEVICE_PORT))
status = manager.status(SERIAL)
assert status["tunnel"] == "shared" and status["forward_port"] == 39999
def test_status_does_not_probe_when_not_installed_or_disabled(env):
adb, _, manager = env
status = manager.status(SERIAL)
assert status["reachable"] is False and status["tunnel"] is None
assert not _calls(adb, ["-s", SERIAL, "forward"])
def test_status_flags_a_helper_newer_than_the_bundle(env):
adb, _, manager = env
adb.installed[SERIAL] = 7
status = manager.status(SERIAL)
assert status["newer_than_bundled"] is True and status["outdated"] is False
# --------------------------------------------------------------------------- #
# Session token, protocol gate, auto-install switch, per-device locking
# --------------------------------------------------------------------------- #
def test_attach_pushes_the_host_token_and_records_it(env, tmp_path):
adb, _, manager = env
session = manager.attach(SERIAL)
token = manager.host_token()
assert len(token) == 48 and session.token == token and session.protocol_version == 2
assert adb.tokens[SERIAL] == token
assert (tmp_path / "token" / "session.token").read_text() == token
# A second manager on the same host (another process) presents the same token.
other = AccessibilityHelperManager(
run_adb=adb,
ping=lambda _p: None,
bundled=None,
token_path=tmp_path / "token" / "session.token",
)
assert other.host_token() == token
def test_push_token_is_repeated_on_demand(env):
adb, _, manager = env
manager.attach(SERIAL)
adb.tokens.clear()
assert manager.push_token(SERIAL) is True
assert adb.tokens[SERIAL] == manager.host_token()
def test_old_protocol_is_reinstalled_when_the_caller_may_provision(env):
adb, _, manager = env
adb.installed[SERIAL] = 9 # newer versionCode than the bundle, but the old contract
adb.protocol[SERIAL] = 1
session = manager.attach(SERIAL)
assert _calls(adb, ["-s", SERIAL, "install"])
assert session.protocol_version == 2 and session.version_code == 2
def test_old_protocol_blocks_a_lazy_attach_without_installing(env):
adb, _, manager = env
adb.installed[SERIAL] = 2
adb.enabled_services[SERIAL] = SERVICE_NAME
adb.protocol[SERIAL] = 1
with pytest.raises(HelperUnavailable, match="protocol 1"):
manager.attach(SERIAL, provision=False)
assert not _calls(adb, ["-s", SERIAL, "install"])
assert adb.forwards == []
def test_auto_install_off_never_installs(env):
adb, _, manager = env
manager.auto_install_switch["install"] = False
with pytest.raises(HelperUnavailable, match="artemis helper install"):
manager.attach(SERIAL)
assert not _calls(adb, ["-s", SERIAL, "install"])
# An outdated but present helper is used as it is.
adb.installed[SERIAL] = 1
session = manager.attach(SERIAL)
assert session.version_code == 1
assert not _calls(adb, ["-s", SERIAL, "install"])
assert manager.status(SERIAL)["auto_install"] is False
def test_provisioning_one_device_does_not_block_another(env):
import threading
_, _, manager = env
lock = manager._device_lock(SERIAL)
lock.acquire() # simulate a slow install in progress on SERIAL
try:
done = threading.Event()
result: dict = {}
def attach_other():
result["session"] = manager.attach(OTHER)
done.set()
threading.Thread(target=attach_other, daemon=True).start()
assert done.wait(2.0), "attach(OTHER) waited on SERIAL's lock"
assert result["session"].serial == OTHER
finally:
lock.release()
def test_status_reports_protocol_and_token(env):
adb, _, manager = env
manager.attach(SERIAL)
status = manager.status(SERIAL)
assert status["protocol_version"] == 2 and status["protocol_supported"] is True
assert status["token_set"] is True and status["auto_install"] is True
# --------------------------------------------------------------------------- #
# UiAutomation holders (uiautomator2 / Appium) mute the helper
# --------------------------------------------------------------------------- #
U2_LINES = [
"24633 sh -c CLASSPATH=/data/local/tmp/u2.jar app_process / com.wetest.uia2.Main -p 9008",
"24635 app_process / com.wetest.uia2.Main -p 9008",
]
def test_attach_stops_a_running_uiautomator2_server_and_binds(env):
adb, _, manager = env
adb.installed[SERIAL] = 2
adb.enabled_services[SERIAL] = SERVICE_NAME
adb.holders[SERIAL] = list(U2_LINES)
session = manager.attach(SERIAL)
assert session.local_port == 40000
assert adb.killed == [(SERIAL, 24635)] # the app_process, not its sh wrapper
assert ["-s", SERIAL, "shell", "am", "force-stop", "com.github.uiautomator"] in adb.calls
# No re-bind toggle was needed once UiAutomation released the services
# (provisioning's idempotent accessibility_enabled=1 write is fine).
assert not _calls(
adb, ["-s", SERIAL, "shell", "settings", "put", "secure", "enabled_accessibility_services"]
)
def test_lazy_attach_names_uiautomator2_instead_of_killing_it(env):
adb, _, manager = env
adb.installed[SERIAL] = 2
adb.enabled_services[SERIAL] = SERVICE_NAME
adb.holders[SERIAL] = list(U2_LINES)
with pytest.raises(HelperUnavailable, match="suppressed by uiautomator2's device server"):
manager.attach(SERIAL, provision=False)
assert adb.killed == []
def test_appium_server_is_never_killed(env):
adb, _, manager = env
adb.installed[SERIAL] = 2
adb.enabled_services[SERIAL] = SERVICE_NAME
adb.holders[SERIAL] = ["31000 io.appium.uiautomator2.server.test"]
with pytest.raises(HelperUnavailable, match="Appium.*disableSuppressAccessibilityServices"):
manager.attach(SERIAL)
assert adb.killed == []
assert not _calls(
adb, ["-s", SERIAL, "shell", "settings", "put", "secure", "enabled_accessibility_services"]
)
def test_uiautomation_holders_classifies_processes(env):
adb, _, manager = env
adb.holders[SERIAL] = U2_LINES + ["31000 io.appium.uiautomator2.server.test", "7 sh"]
holders = manager.uiautomation_holders(SERIAL)
assert holders == {"uiautomator2": [24635], "appium": [31000]}
@@ -0,0 +1,385 @@
"""The user is told which UI-hierarchy backend serves a run, everywhere they look.
Covers the agent's announcement fan-out (startup progress, log trace, session
device_info, status.json), the provisioning progress stage, the run_outcome
environment block, the `mobile_manage_task` field, the `artemis helper` CLI
(`--all`, status rendering) and the `artemis doctor` helper row.
"""
from __future__ import annotations
import json
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
from typer.testing import CliRunner
from artemis.clients.screen_client_factory import FallbackScreenClient
from artemis.runtime import trace_store
from artemis.runtime.helper_manager import HelperSession, ProvisionResult
from artemis.sdk.agent import Agent
# --------------------------------------------------------------------------- #
# helpers
# --------------------------------------------------------------------------- #
def _composite(helper_version: str = "1.1.2") -> FallbackScreenClient:
helper = MagicMock()
helper.session = HelperSession(
serial="dev",
local_port=41000,
transport_id="1",
version_code=4,
version_name=helper_version,
owns_forward=True,
)
helper.get_hierarchy.return_value = "<h/>"
client = FallbackScreenClient(
"dev", helper=helper, uiautomator_factory=MagicMock(), state_probe=lambda _s: "device"
)
return client
def _agent_with(client) -> Agent:
agent = object.__new__(Agent)
agent._ui_adb_client = client
return agent
def _engine():
engine = MagicMock()
engine.current_session_id = "sess-1"
engine.get_relative_time = lambda ts: "T+00:05"
return engine
# --------------------------------------------------------------------------- #
# Agent announcement fan-out
# --------------------------------------------------------------------------- #
def test_announce_first_backend_reaches_every_sink(tmp_path, monkeypatch):
monkeypatch.setattr(trace_store, "TRACES_DIR", str(tmp_path))
trace_store.init_trace("sess-1", "goal", "flash")
client = _composite()
client.get_hierarchy() # helper becomes active
agent = _agent_with(client)
engine = _engine()
context = SimpleNamespace(data_engine=engine)
with patch("artemis.sdk.agent.publish_startup_progress") as progress:
agent._announce_hierarchy_backend(context, "sess-1", None, "helper", None)
stage, message = progress.call_args.args[:2]
assert stage == "hierarchy_backend"
assert message == "UI hierarchy source: Artemis accessibility helper v1.1.2"
assert progress.call_args.kwargs["backend"] == "helper"
trace_call = engine.record_trace.call_args.kwargs
assert trace_call["name"] == "hierarchy_backend" and trace_call["type"] == "log"
assert trace_call["payload"]["helper_version"] == "1.1.2"
assert trace_call["payload"]["level"] == "INFO"
info = engine.update_session_device_info.call_args.kwargs
assert info["hierarchy_backend"] == "helper"
assert info["hierarchy_backend_note"].startswith("UI hierarchy source: the Artemis")
status = trace_store.read_status("sess-1")
assert status["hierarchy_backend"] == "helper"
assert "accessibility helper v1.1.2" in status["hierarchy_backend_note"]
def test_announce_switch_is_a_warning_with_the_reason(tmp_path, monkeypatch):
monkeypatch.setattr(trace_store, "TRACES_DIR", str(tmp_path))
client = _composite()
agent = _agent_with(client)
engine = _engine()
context = SimpleNamespace(data_engine=engine)
with patch("artemis.sdk.agent.publish_startup_progress") as progress:
agent._announce_hierarchy_backend(
context, "sess-1", "helper", "uiautomator", "HelperUnavailable: tunnel gone"
)
stage, message = progress.call_args.args[:2]
assert stage == "hierarchy_backend_changed"
assert message == (
"UI hierarchy source switched from Artemis accessibility helper to UIAutomator2 "
"because HelperUnavailable: tunnel gone"
)
assert engine.record_trace.call_args.kwargs["payload"]["level"] == "WARNING"
# No status.json for this session: silently skipped, nothing raised.
assert trace_store.read_status("sess-1") is None
def test_announce_survives_sink_failures(monkeypatch, tmp_path):
monkeypatch.setattr(trace_store, "TRACES_DIR", str(tmp_path))
client = _composite()
agent = _agent_with(client)
engine = _engine()
engine.record_trace.side_effect = RuntimeError("db closed")
with patch("artemis.sdk.agent.publish_startup_progress", side_effect=OSError("no ipc")):
agent._announce_hierarchy_backend(
SimpleNamespace(data_engine=engine), "sess-1", None, "uiautomator", None
)
@pytest.mark.asyncio
async def test_connect_publishes_install_stage_and_registers_listener():
client = _composite()
def fake_connect(on_event=None):
on_event("installing", {"serial": "dev", "version_name": "1.1.2", "to_version": 4})
client._set_active("helper")
client.connect = fake_connect # type: ignore[method-assign]
agent = _agent_with(client)
engine = _engine()
context = SimpleNamespace(data_engine=engine)
with (
patch("artemis.sdk.agent.publish_startup_progress") as progress,
patch.object(trace_store, "update_trace_fields"),
):
await agent._connect_screen_client(context, "sess-1")
stages = [c.args[0] for c in progress.call_args_list]
assert stages == ["uiautomator", "helper_install", "uiautomator_ready", "hierarchy_backend"]
install_msg = progress.call_args_list[1].args[1]
assert (
"first time" in install_msg and "v1.1.2" in install_msg and "3 seconds" in install_msg
)
# A later fallback goes through the registered listener.
progress.reset_mock()
client._set_active("uiautomator", "RuntimeError: tunnel gone")
assert progress.call_args.args[0] == "hierarchy_backend_changed"
@pytest.mark.asyncio
async def test_connect_without_provisioning_client_takes_no_callback():
plain = MagicMock(spec=["connect", "get_hierarchy"])
plain.connect = MagicMock() # signature without on_event
agent = _agent_with(plain)
with (
patch("artemis.sdk.agent.publish_startup_progress"),
patch.object(trace_store, "update_trace_fields"),
):
await agent._connect_screen_client(SimpleNamespace(data_engine=None), "sess-1")
plain.connect.assert_called_once_with()
@pytest.mark.asyncio
async def test_reused_agent_records_backend_changes_only_for_current_session():
client = _composite()
agent = _agent_with(client)
context = SimpleNamespace(data_engine=None)
with (
patch("artemis.sdk.agent.publish_startup_progress") as progress,
patch.object(trace_store, "update_trace_fields"),
):
await agent._connect_screen_client(context, "sess-1")
client._set_active("uiautomator", "tunnel gone")
client.disconnect()
await agent._connect_screen_client(context, "sess-2")
assert len(client.backend_history) == 1
progress.reset_mock()
client._set_active("uiautomator", "tunnel gone again")
assert progress.call_count == 1
assert progress.call_args.kwargs["session_id"] == "sess-2"
# --------------------------------------------------------------------------- #
# trace_store / engine / task manager
# --------------------------------------------------------------------------- #
def test_update_trace_fields_merges_and_ignores_missing(tmp_path, monkeypatch):
monkeypatch.setattr(trace_store, "TRACES_DIR", str(tmp_path))
assert trace_store.update_trace_fields("nope", hierarchy_backend="helper") is None
trace_store.init_trace("t1", "goal", "pro")
data = trace_store.update_trace_fields("t1", hierarchy_backend="helper", extra=1)
assert data["hierarchy_backend"] == "helper" and data["extra"] == 1
assert data["task_desc"] == "goal"
def test_update_session_device_info_merges_fields():
from artemis.data_engine.engine import DataEngine
engine = object.__new__(DataEngine)
engine.current_session_id = "s"
session = SimpleNamespace(device_info={"device_id": "dev"})
engine.storage = MagicMock()
engine.storage.get_session.return_value = session
engine.update_session_device_info(hierarchy_backend="helper", hierarchy_backend_note="n")
engine.storage.update_session.assert_called_once_with(session)
assert session.device_info == {
"device_id": "dev",
"hierarchy_backend": "helper",
"hierarchy_backend_note": "n",
}
def test_manage_task_status_exposes_hierarchy_backend(tmp_path, monkeypatch):
from mcp_server.tools import task_manager
monkeypatch.setattr(trace_store, "TRACES_DIR", str(tmp_path))
trace_store.init_trace("t9", "goal", "flash", device_serial="dev")
trace_store.update_trace_fields(
"t9", hierarchy_backend="uiautomator", hierarchy_backend_note="switched"
)
with patch.object(task_manager, "_reconcile_status", lambda *a, **k: None, create=True):
result = task_manager.mobile_manage_task(action="status", trace_id="t9")
assert result["hierarchy_backend"] == "uiautomator"
assert result["hierarchy_backend_note"] == "switched"
# --------------------------------------------------------------------------- #
# run_outcome environment block
# --------------------------------------------------------------------------- #
def test_run_outcome_extra_carries_environment(tmp_path):
from artemis.graph.checkpoints import read_run_outcome, write_run_outcome
from artemis.clients.screen_client_factory import hierarchy_backend_summary
client = _composite()
client.get_hierarchy()
client._set_active("uiautomator", "RuntimeError: gone")
outcome = MagicMock()
outcome.model_dump.return_value = {"task_status": "completed"}
write_run_outcome(tmp_path, outcome, {"environment": hierarchy_backend_summary(client)})
data = read_run_outcome(tmp_path)
assert data["environment"]["backend"] == "uiautomator"
assert data["environment"]["helper_version"] == "1.1.2"
assert data["environment"]["switches"][0]["reason"] == "RuntimeError: gone"
# --------------------------------------------------------------------------- #
# CLI: artemis helper
# --------------------------------------------------------------------------- #
def _status(**overrides):
base = {
"package": "com.artemis.helper",
"installed": True,
"installed_version": 4,
"bundled_version": 4,
"bundled_apk_present": True,
"outdated": False,
"newer_than_bundled": False,
"enabled": True,
"forward_port": None,
"tunnel": "probe",
"reachable": True,
"reported_version": 4,
"transport_id": "1",
"session": None,
}
base.update(overrides)
return base
def test_helper_status_explains_probe_tunnel_and_newer_build(monkeypatch):
from artemis.interfaces.cli.commands import helper as helper_cli
monkeypatch.setattr(
helper_cli.helper_manager,
"status",
lambda serial: _status(installed_version=9, newer_than_bundled=True),
)
result = CliRunner().invoke(helper_cli.helper_app, ["status", "--serial", "dev"])
assert result.exit_code == 0, result.output
assert "temporary forward" in result.output
assert "newer than the bundled v4" in result.output
def test_helper_install_all_targets_idle_devices_only(monkeypatch):
from artemis.interfaces.cli.commands import helper as helper_cli
devices = [
SimpleNamespace(serial="a", is_busy=False),
SimpleNamespace(serial="b", is_busy=True),
SimpleNamespace(serial="c", is_busy=False),
]
monkeypatch.setattr(helper_cli.device_pool, "get_ready_devices", lambda: devices)
calls = []
def provision(serial, force=False, on_event=None):
calls.append(serial)
on_event("installing", {"version_name": "1.1.2"})
return ProvisionResult(
ok=True, action="installed", installed_version=4, bundled_version=4, enabled=True
)
monkeypatch.setattr(helper_cli.helper_manager, "provision", provision)
result = CliRunner().invoke(helper_cli.helper_app, ["install", "--all"])
assert result.exit_code == 0, result.output
assert calls == ["a", "c"]
assert "a: installing v1.1.2..." in result.output
def test_helper_install_prints_manual_path_on_enable_failure(monkeypatch):
from artemis.interfaces.cli.commands import helper as helper_cli
from artemis.runtime.helper_manager import MANUAL_ENABLE_PATH
monkeypatch.setattr(
helper_cli.helper_manager,
"provision",
lambda serial, force=False, on_event=None: ProvisionResult(
ok=False,
action="installed",
installed_version=4,
bundled_version=4,
enabled=False,
error=f"This device rejected enabling the accessibility service from adb. {MANUAL_ENABLE_PATH}.",
),
)
result = CliRunner().invoke(helper_cli.helper_app, ["install", "--serial", "dev"])
assert result.exit_code == 1
assert "Settings > Accessibility" in result.output
# --------------------------------------------------------------------------- #
# CLI: artemis doctor row
# --------------------------------------------------------------------------- #
def _adb_probe(serials):
return SimpleNamespace(
id="android_adb",
metadata={"devices": [{"serial": s, "state": "device"} for s in serials]},
)
def test_doctor_row_suggests_preinstall_when_missing(monkeypatch):
from artemis.interfaces.cli.commands import doctor
with patch(
"artemis.runtime.helper_manager.helper_manager.status",
return_value=_status(installed=False, reachable=False),
):
row = doctor._helper_row([_adb_probe(["dev"])])
assert row.status == "missing" and row.summary == "Not installed"
assert "artemis helper install --serial dev" in row.detail
assert "about 3 s" in row.detail
def test_doctor_row_ready_mentions_uninstall(monkeypatch):
from artemis.interfaces.cli.commands import doctor
with patch("artemis.runtime.helper_manager.helper_manager.status", return_value=_status()):
row = doctor._helper_row([_adb_probe(["dev"])])
assert row.status == "pass" and "v4 on dev" in row.summary
assert "artemis helper uninstall --serial dev" in row.detail
def test_doctor_row_absent_for_multiple_devices_or_uiautomator_backend(monkeypatch):
from artemis.interfaces.cli.commands import doctor
assert doctor._helper_row([_adb_probe(["a", "b"])]) is None
monkeypatch.setenv("ARTEMIS_HIERARCHY_BACKEND", "uiautomator")
assert doctor._helper_row([_adb_probe(["a"])]) is None
+156
View File
@@ -0,0 +1,156 @@
"""What the agent sees must be what is on screen, whichever backend dumped it.
Covers the host-side half of the UIAutomator parity work: negative bounds no
longer lose an element's coordinates, hints and errors reach the element list
the model reads, an empty helper answer degrades to UIAutomator2 in ``auto``
mode, and the parity comparison used by ``artemis helper parity`` flags the
regressions it exists for.
"""
from __future__ import annotations
from unittest.mock import MagicMock
import pytest
from artemis.clients.accessibility_client import AccessibilityClient, HelperEmptyHierarchy
from artemis.clients.screen_client_factory import FallbackScreenClient
from artemis.clients.ui_automator_client import UIAutomatorClient, _parse_hierarchy_xml_to_elements
from artemis.core.diagnostics.hierarchy_parity import compare_dumps
from artemis.utils import ui_filter
from artemis.utils.visualization import format_minimal_list_with_elements
def _node(**attrs: str) -> str:
base = {
"index": "0",
"text": "",
"resource-id": "",
"class": "android.widget.TextView",
"package": "app",
"content-desc": "",
"clickable": "false",
"enabled": "true",
"bounds": "[0,0][100,100]",
}
base.update(attrs)
return "<node " + " ".join(f'{k}="{v}"' for k, v in base.items()) + " />"
def _xml(*nodes: str) -> str:
return '<?xml version="1.0"?><hierarchy rotation="0">' + "".join(nodes) + "</hierarchy>"
# --------------------------------------------------------------------------- #
# Negative bounds
# --------------------------------------------------------------------------- #
def test_negative_bounds_keep_their_coordinates_and_get_clipped():
elements = _parse_hierarchy_xml_to_elements(_xml(_node(text="Row", bounds="[-20,-40][300,60]")))
row = elements[1]
assert row["parsed_bounds"] == {"left": -20, "top": -40, "right": 300, "bottom": 60}
assert ui_filter._parse_bounds("[-20,-40][300,60]") == {
"left": -20,
"top": -40,
"right": 300,
"bottom": 60,
}
filtered = ui_filter.filter_ui_hierarchy([row], screen_width=1080, screen_height=2400)
assert len(filtered) == 1
assert filtered[0]["parsed_bounds"] == {"left": 0, "top": 0, "right": 300, "bottom": 60}
# --------------------------------------------------------------------------- #
# Hints and errors reach the model
# --------------------------------------------------------------------------- #
def test_minimal_list_shows_hint_for_empty_inputs_and_errors():
elements = _parse_hierarchy_xml_to_elements(
_xml(
_node(text="", hint="Search settings", **{"class": "android.widget.EditText"}),
_node(text="abc", error="Password too short", bounds="[0,200][100,300]"),
_node(text="", bounds="[0,400][100,500]"), # nothing to say: stays out
)
)
text, items, labels = format_minimal_list_with_elements(elements[1:], 1000, 1000)
lines = text.splitlines()
assert lines[0] == "[1] Hint: 'Search settings' | Bounds: [0,0][100,100]"
assert lines[1] == "[2] Text: 'abc' | Bounds: [0,200][100,300] | Error: 'Password too short'"
assert len(lines) == 2 and labels == ["1", "2"]
assert items[0]["is_hint"] is True and items[0]["text"] == "Search settings"
assert items[1]["error"] == "Password too short"
# --------------------------------------------------------------------------- #
# Empty helper answer degrades instead of showing a blank screen
# --------------------------------------------------------------------------- #
def test_empty_helper_hierarchy_falls_back_to_uiautomator():
helper = MagicMock(spec=AccessibilityClient)
u2 = MagicMock(spec=UIAutomatorClient)
helper.get_screen_data.side_effect = HelperEmptyHierarchy("no window")
u2.get_screen_data.return_value = "from-u2"
client = FallbackScreenClient(
"dev", helper=helper, uiautomator_factory=lambda: u2, state_probe=lambda _s: "device"
)
assert client.get_screen_data() == "from-u2"
assert client.active_backend == "uiautomator"
assert "HelperEmptyHierarchy" in client.backend_history[-1]["reason"]
# --------------------------------------------------------------------------- #
# Parity comparison
# --------------------------------------------------------------------------- #
def test_parity_passes_when_both_backends_describe_the_same_screen():
u2 = _xml(
_node(text="Wi-Fi", bounds="[0,100][1080,200]"),
_node(**{"content-desc": "Back", "bounds": "[0,0][120,120]"}),
)
helper = _xml(
_node(text="Wi-Fi", bounds="[2,101][1078,199]"),
_node(**{"content-desc": "Back", "bounds": "[0,0][120,120]"}),
)
report = compare_dumps(helper, u2, 1080, 2400)
assert report["ok"], report["problems"]
assert report["match"]["recall"] == 1.0 and report["match"]["precision"] == 1.0
@pytest.mark.parametrize(
"helper_extra, expected",
[
(_node(text="Hidden drawer item", bounds="[-1080,300][0,400]"), "negative bounds"),
(_node(text="Next page", bounds="[1080,300][2160,400]"), "outside the screen"),
(
"".join(
_node(text=f"Ghost {i}", bounds=f"[0,{i * 50}][100,{i * 50 + 40}]")
for i in range(8)
),
"not on screen",
),
],
)
def test_parity_flags_elements_uiautomator_would_not_show(helper_extra, expected):
u2 = _xml(_node(text="Wi-Fi", bounds="[0,100][1080,200]"))
helper = _xml(_node(text="Wi-Fi", bounds="[0,100][1080,200]"), helper_extra)
report = compare_dumps(helper, u2, 1080, 2400)
assert not report["ok"]
assert any(expected in problem for problem in report["problems"]), report["problems"]
def test_parity_flags_missing_elements():
u2 = _xml(
_node(text="Wi-Fi", bounds="[0,100][1080,200]"),
_node(text="Bluetooth", bounds="[0,200][1080,300]"),
)
helper = _xml(_node(text="Wi-Fi", bounds="[0,100][1080,200]"))
report = compare_dumps(helper, u2, 1080, 2400)
assert not report["ok"]
assert report["match"]["missing_in_helper"] == [
"'Bluetooth' at {'left': 0, 'top': 200, 'right': 1080, 'bottom': 300}"
]