feat: implement robust multi-tier text input architecture and ADB diagnostics

- Android Text Input & IME Normalization:
  - Implement progressive 3-tier input architecture in AndroidAdbDriver and AndroidDeviceController:
    - Tier 1 (Primary): Zero-dependency clipboard injection + KEYCODE_PASTE (immune to IME Pinyin interception, full Unicode/Emoji support, preserves multiline formatting).
    - Tier 2 (Broadcast): Base64 broadcast injection when ADBKeyboard is active.
    - Tier 3 (Fallback): Native ADB physical keyevent fallback.
  - Add literal newline normalization (\r\n, \n, \r) to prevent JSON escape string corruption (e.g. '10086\n' -> '10086、n').
  - Add set_clipboard method to UIAutomatorClient without auxiliary APK requirements.

- Diagnostics & System Probes:
  - Add adb_keys diagnostics probe for ADB key auth checks.
  - Integrate diagnostics into CLI doctor command.

- Admin Console & Showcase UI:
  - Improve model service and system router in admin console.
  - Enhance workspace stream view and session state in showcase UI.
This commit is contained in:
somew1nd committed 2026-08-24 16:48:45 -07:00
1 parent 42b8988661
commit ea3708a98f
19 files changed
+785 -89

No files matched your search

+17
View File
@@ -341,6 +341,23 @@ class UIAutomatorClient:
logger.error(f"UIAutomator2 clear_text failed: {e}")
return False
def set_clipboard(self, text: str) -> bool:
"""Set text to system clipboard via UIAutomator2.
Args:
text: The string to set in the device clipboard.
Returns:
True if successful, False otherwise.
"""
try:
device = self._ensure_connected()
device.set_clipboard(text)
return True
except Exception as e:
logger.debug(f"UIAutomator2 set_clipboard failed: {e}")
return False
def get_hierarchy(self) -> str:
"""Get the UI hierarchy XML from the device.
+34 -10
View File
@@ -156,22 +156,46 @@ class AndroidDeviceController(MobileDeviceController):
async def input_text(self, text: str) -> bool:
try:
# Fast path for ASCII text: Use ADB directly.
# This avoids toggling FastInputIME, which hides the keyboard and can cause focus loss.
is_ascii = all(ord(c) < 128 for c in text)
if is_ascii:
return self._input_text_adb_fallback(text)
norm_text = (
text.replace(r"\r\n", "\n")
.replace(r"\n", "\n")
.replace(r"\r", "\n")
)
self.ui_adb_client.send_text(text)
return True
# 1. Tier 1: Try clipboard injection + KEYCODE_PASTE (Zero IME interference, preserves multiline, works for all charsets)
if self.ui_adb_client:
try:
set_clip_ok = False
if hasattr(self.ui_adb_client, "set_clipboard"):
set_clip_ok = self.ui_adb_client.set_clipboard(norm_text)
elif hasattr(self.ui_adb_client, "_device") and self.ui_adb_client._device:
self.ui_adb_client._device.set_clipboard(norm_text)
set_clip_ok = True
elif hasattr(self.ui_adb_client, "_ensure_connected"):
dev = self.ui_adb_client._ensure_connected()
dev.set_clipboard(norm_text)
set_clip_ok = True
if set_clip_ok:
self.device.shell("input keyevent 279")
return True
except Exception as e:
logger.debug(f"Clipboard paste fallback: {e}")
return self._input_text_adb_fallback(norm_text)
except Exception as e:
logger.warning(f"UIAutomator2 send_text failed: {e}, falling back to ADB shell")
return self._input_text_adb_fallback(text)
logger.error(f"Failed to input text: {e}")
return False
def _input_text_adb_fallback(self, text: str) -> bool:
"""Fallback method using ADB shell input text command."""
try:
lines = text.split("\n")
norm_text = (
text.replace(r"\r\n", "\n")
.replace(r"\n", "\n")
.replace(r"\r", "\n")
)
lines = norm_text.split("\n")
for line_idx, line in enumerate(lines):
if line_idx > 0:
self.device.shell("input keyevent 66")
+10
View File
@@ -14,6 +14,12 @@
"""Artemis Diagnostics & System Readiness Package."""
from artemis.core.diagnostics.adb_keys import (
AdbKeyStatus,
get_adb_key_paths,
heal_adb_keys,
inspect_adb_keys,
)
from artemis.core.diagnostics.emulator_manager import (
EmulatorLaunchStage,
EmulatorLaunchState,
@@ -59,4 +65,8 @@ __all__ = [
"ProbeResult",
"ProbeStatus",
"SystemReadinessReport",
"AdbKeyStatus",
"get_adb_key_paths",
"inspect_adb_keys",
"heal_adb_keys",
]
+267
View File
@@ -0,0 +1,267 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""ADB Authentication Key Diagnostic and Automated Self-Healing Module.
Android 4.2.2+ requires RSA authentication keys (~/.android/adbkey) for USB debugging.
When key files become 0-bytes or corrupted (due to aborted writes, touch commands,
or disk sync issues), ADB fails to send the public key, causing connected devices
to silently remain in an 'unauthorized' state without ever displaying the authorization prompt.
This module provides detection and self-healing for corrupted ADB keys.
"""
from dataclasses import asdict, dataclass
import os
from pathlib import Path
import shutil
import subprocess
import time
from typing import Any
from artemis.toolchain import toolchain
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
@dataclass
class AdbKeyStatus:
"""Status details for ADB authentication keys."""
is_valid: bool
is_corrupted: bool
exists: bool
key_path: str
pub_key_path: str
key_size_bytes: int = 0
pub_key_size_bytes: int = 0
error_reason: str | None = None
def to_dict(self) -> dict[str, Any]:
return asdict(self)
def get_adb_key_paths() -> tuple[Path, Path]:
"""Resolve the paths to the active ADB private and public key files."""
adb_vendor_keys = os.environ.get("ADB_VENDOR_KEYS")
if adb_vendor_keys:
custom_priv = Path(adb_vendor_keys.split(os.pathsep)[0])
custom_pub = Path(f"{custom_priv}.pub")
return custom_priv, custom_pub
android_dir = Path.home() / ".android"
return android_dir / "adbkey", android_dir / "adbkey.pub"
def inspect_adb_keys() -> AdbKeyStatus:
"""Inspect ADB authentication RSA key files for corruption or invalid state."""
priv_path, pub_path = get_adb_key_paths()
if not priv_path.exists() and not pub_path.exists():
# Fresh environment: ADB will automatically generate keys on first start
return AdbKeyStatus(
is_valid=True,
is_corrupted=False,
exists=False,
key_path=str(priv_path),
pub_key_path=str(pub_path),
key_size_bytes=0,
pub_key_size_bytes=0,
error_reason=None,
)
priv_size = priv_path.stat().st_size if priv_path.exists() else 0
pub_size = pub_path.stat().st_size if pub_path.exists() else 0
# 1. Check for 0-byte or truncated private key file
if priv_path.exists():
if priv_size == 0:
return AdbKeyStatus(
is_valid=False,
is_corrupted=True,
exists=True,
key_path=str(priv_path),
pub_key_path=str(pub_path),
key_size_bytes=0,
pub_key_size_bytes=pub_size,
error_reason="Private key file is 0 bytes (empty). ADB cannot authenticate devices.",
)
if priv_size < 100:
return AdbKeyStatus(
is_valid=False,
is_corrupted=True,
exists=True,
key_path=str(priv_path),
pub_key_path=str(pub_path),
key_size_bytes=priv_size,
pub_key_size_bytes=pub_size,
error_reason=f"Private key file is suspiciously small ({priv_size} bytes).",
)
# Validate PEM header structure
try:
content = priv_path.read_text(encoding="utf-8", errors="replace")
if "BEGIN" not in content or "PRIVATE KEY" not in content:
return AdbKeyStatus(
is_valid=False,
is_corrupted=True,
exists=True,
key_path=str(priv_path),
pub_key_path=str(pub_path),
key_size_bytes=priv_size,
pub_key_size_bytes=pub_size,
error_reason="Private key does not contain a valid PEM private key header.",
)
except Exception as e:
return AdbKeyStatus(
is_valid=False,
is_corrupted=True,
exists=True,
key_path=str(priv_path),
pub_key_path=str(pub_path),
key_size_bytes=priv_size,
pub_key_size_bytes=pub_size,
error_reason=f"Cannot read private key: {e}",
)
# 2. Check for 0-byte public key if it exists
if pub_path.exists() and pub_size == 0:
return AdbKeyStatus(
is_valid=False,
is_corrupted=True,
exists=True,
key_path=str(priv_path),
pub_key_path=str(pub_path),
key_size_bytes=priv_size,
pub_key_size_bytes=0,
error_reason="Public key file (adbkey.pub) is 0 bytes (empty).",
)
return AdbKeyStatus(
is_valid=True,
is_corrupted=False,
exists=True,
key_path=str(priv_path),
pub_key_path=str(pub_path),
key_size_bytes=priv_size,
pub_key_size_bytes=pub_size,
error_reason=None,
)
def heal_adb_keys(
adb_path: str | None = None, force: bool = False
) -> dict[str, Any]:
"""Auto-heal corrupted ADB authentication keys.
If keys are corrupted (or if `force=True`), this function:
1. Backs up or removes corrupted key files.
2. Restarts the ADB server to cleanly generate a fresh RSA 2048-bit key pair.
3. Verifies that the new key pair is healthy and valid.
"""
status = inspect_adb_keys()
if not status.is_corrupted and not force:
return {
"success": True,
"repaired": False,
"message": "ADB keys are healthy, no repair required.",
"status": status.to_dict(),
}
priv_path, pub_path = get_adb_key_paths()
priv_path.parent.mkdir(parents=True, exist_ok=True)
# Backup corrupted non-empty files before removal
timestamp = int(time.time())
if priv_path.exists() and priv_path.stat().st_size > 0:
backup_path = priv_path.with_name(f"adbkey.corrupted.{timestamp}")
try:
shutil.copy2(priv_path, backup_path)
logger.info(f"Backed up corrupted adbkey to {backup_path}")
except Exception as e:
logger.warning(f"Could not backup corrupted adbkey: {e}")
# Remove corrupted key files
try:
if priv_path.exists():
priv_path.unlink(missing_ok=True)
if pub_path.exists():
pub_path.unlink(missing_ok=True)
except Exception as e:
logger.error(f"Failed to remove corrupted adb keys: {e}")
return {
"success": False,
"repaired": False,
"message": f"Failed to remove corrupted ADB key files: {e}",
"status": status.to_dict(),
}
# Resolve ADB executable
resolved_adb = adb_path or toolchain.resolve("adb") or shutil.which("adb") or "adb"
# Restart ADB server to trigger key generation
try:
subprocess.run(
[resolved_adb, "kill-server"],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
timeout=5,
)
subprocess.run(
[resolved_adb, "start-server"],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
timeout=5,
)
# Execute adb devices to trigger initial handshake & key material generation if needed
subprocess.run(
[resolved_adb, "devices"],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
timeout=5,
)
except Exception as e:
logger.error(f"Error restarting ADB server during key healing: {e}")
return {
"success": False,
"repaired": False,
"message": f"Failed to restart ADB server: {e}",
"status": status.to_dict(),
}
# Re-inspect to verify health
new_status = inspect_adb_keys()
if new_status.is_valid and new_status.key_size_bytes > 0:
logger.info("Successfully healed corrupted ADB keys.")
return {
"success": True,
"repaired": True,
"message": "Corrupted ADB keys were successfully repaired and regenerated.",
"status": new_status.to_dict(),
}
elif new_status.is_valid and not new_status.is_corrupted:
return {
"success": True,
"repaired": True,
"message": "Corrupted ADB keys cleared. Fresh keys will be created on first device connection.",
"status": new_status.to_dict(),
}
else:
return {
"success": False,
"repaired": False,
"message": f"Key healing completed, but keys remain invalid: {new_status.error_reason}",
"status": new_status.to_dict(),
}
+18 -1
View File
@@ -133,11 +133,28 @@ class ReadinessEngine:
timestamp=time.time(),
)
async def heal_adb_keys(self, force: bool = False) -> dict[str, Any]:
"""Auto-heal corrupted or 0-byte ADB authentication keys."""
from artemis.core.diagnostics.adb_keys import heal_adb_keys
logger.info("[ReadinessEngine] Auto-healing ADB authentication keys...")
return await asyncio.to_thread(heal_adb_keys, None, force)
async def restart_adb_server(self) -> dict[str, Any]:
"""Execute adb kill-server && adb start-server to recover connectivity."""
"""Execute adb kill-server && adb start-server to recover connectivity, auto-healing corrupted keys if needed."""
from artemis.core.diagnostics.adb_keys import heal_adb_keys, inspect_adb_keys
adb_path = shutil.which("adb") or "adb"
def _restart_sync():
# If keys are corrupted, heal them first
key_status = inspect_adb_keys()
if key_status.is_corrupted:
logger.warning(
f"[ReadinessEngine] Corrupted ADB keys detected ({key_status.error_reason}). Auto-healing..."
)
return heal_adb_keys(adb_path=adb_path)
try:
subprocess.run(
[adb_path, "kill-server"],
@@ -546,6 +546,11 @@ class AdbDeviceProbe(BaseProbe):
adb_ver = await self._get_adb_version(adb_path)
devices = await self._parse_adb_devices(adb_path)
# Inspect ADB authentication key health
from artemis.core.diagnostics.adb_keys import inspect_adb_keys
key_status = inspect_adb_keys()
# Normalize emulator command display
is_emu_in_path = shutil.which("emulator") is not None
emu_display_cmd = "emulator" if is_emu_in_path else (emulator_path or "emulator")
@@ -554,6 +559,7 @@ class AdbDeviceProbe(BaseProbe):
"installed": True,
"adb_path": adb_path,
"adb_version": adb_ver,
"adb_keys": key_status.to_dict(),
"emulator_path": emulator_path,
"is_emulator_in_path": is_emu_in_path,
"installed_avds": installed_avds,
@@ -565,6 +571,15 @@ class AdbDeviceProbe(BaseProbe):
if not devices:
actions: list[ProbeAction] = []
if key_status.is_corrupted:
actions.append(
ProbeAction(
action_type="command",
label="Auto-Heal ADB Keys",
payload="artemis doctor --fix",
)
)
# If user has installed AVDs, generate exact launch commands for each
if installed_avds:
for avd in installed_avds:
@@ -593,6 +608,8 @@ class AdbDeviceProbe(BaseProbe):
)
desc = "ADB is ready, but no active Android device or emulator was detected."
if key_status.is_corrupted:
desc += f" Warning: Corrupted ADB RSA key detected ({key_status.error_reason})."
if installed_avds:
desc += f" Found {len(installed_avds)} installed emulator(s): {', '.join(installed_avds)}."
@@ -615,6 +632,33 @@ class AdbDeviceProbe(BaseProbe):
if not ready_devices:
if unauthorized_devices:
unauth_serial = unauthorized_devices[0].serial
if key_status.is_corrupted:
return ProbeResult(
id=self.probe_id,
category=self.category,
title="Device / Emulator Connected",
status=ProbeStatus.WARN,
is_blocker=self.is_blocker,
summary="ADB Key Corrupted",
description=(
f"Device detected ({unauth_serial}), but ADB authentication keys are corrupted "
f"({key_status.error_reason}). This prevents the device from displaying the USB Debugging prompt."
),
metadata=metadata,
actions=[
ProbeAction(
action_type="command",
label="Auto-Heal ADB Keys",
payload="artemis doctor --fix",
),
ProbeAction(
action_type="command",
label="Restart ADB Server",
payload=f"{adb_path} kill-server && {adb_path} start-server",
),
],
)
return ProbeResult(
id=self.probe_id,
category=self.category,
+25 -24
View File
@@ -322,46 +322,47 @@ class AndroidAdbDriver(BaseDeviceDriver):
"input keyevent 123", # KEYCODE_MOVE_END
)
# 1. Fast path for ASCII text: Use ADB input text directly.
# This avoids toggling IME, preserves keyboard focus, and works on all Android devices.
if text.isascii():
lines = text.split("\n")
for i, line in enumerate(lines):
if i > 0:
# Send Enter key between lines
await asyncio.to_thread(self.device.shell, "input keyevent 66")
if line:
escaped = _escape_for_adb_text(line)
await asyncio.to_thread(self.device.shell, f"input text {escaped}")
return True
# Normalize literal escaped newlines from LLM / tool call serialization
norm_text = (
text.replace(r"\r\n", "\n")
.replace(r"\n", "\n")
.replace(r"\r", "\n")
)
# 2. For non-ASCII / Unicode text: Try UIAutomator2 FastInputIME
if self._ui_adb_client and hasattr(self._ui_adb_client, "send_text"):
# 1. Tier 1: Try clipboard injection + KEYCODE_PASTE (Zero IME interference, preserves multiline, works for all charsets)
if self._ui_adb_client:
try:
res = self._ui_adb_client.send_text(text)
if asyncio.iscoroutine(res):
await res
return True
except Exception as e:
logger.warning(f"UIAutomator2 send_text failed: {e}")
set_clip_ok = False
if hasattr(self._ui_adb_client, "set_clipboard"):
set_clip_ok = self._ui_adb_client.set_clipboard(norm_text)
elif hasattr(self._ui_adb_client, "_device") and self._ui_adb_client._device:
self._ui_adb_client._device.set_clipboard(norm_text)
set_clip_ok = True
# 3. Check if ADBKeyboard is currently active
if set_clip_ok:
await asyncio.to_thread(self.device.shell, "input keyevent 279")
return True
except Exception as e:
logger.debug(f"Clipboard paste fallback to ADB input: {e}")
# 2. Tier 2: Check if ADBKeyboard is currently active
try:
default_ime = await asyncio.to_thread(
self.device.shell, "settings get secure default_input_method"
)
if "adbkeyboard" in str(default_ime).lower():
b64_text = base64.b64encode(text.encode("utf-8")).decode("utf-8")
b64_text = base64.b64encode(norm_text.encode("utf-8")).decode("utf-8")
broadcast_cmd = f"am broadcast -a ADB_INPUT_B64 --es msg '{b64_text}'"
await asyncio.to_thread(self.device.shell, broadcast_cmd)
return True
except Exception:
pass
# 4. Fallback: ADB input text
lines = text.split("\n")
# 3. Tier 3: Universal Native ADB input text fallback
lines = norm_text.split("\n")
for i, line in enumerate(lines):
if i > 0:
# Send Enter key between lines
await asyncio.to_thread(self.device.shell, "input keyevent 66")
if line:
escaped = _escape_for_adb_text(line)
+48 -1
View File
@@ -22,18 +22,27 @@ import sys
from rich.console import Console
from rich.panel import Panel
from rich.table import Table
import typer
from artemis.config import (
parse_llm_config,
settings,
)
from artemis.config.paths import ROOT_DIR
from artemis.core.diagnostics.adb_keys import heal_adb_keys, inspect_adb_keys
from artemis.platform import OSType, platform
from artemis.toolchain import toolchain
from artemis.utils.ocr_api import is_ocr_configured
def doctor_command() -> None:
def doctor_command(
fix: bool = typer.Option(
False,
"--fix",
"-f",
help="Automatically repair fixable environment issues (such as corrupted ADB authentication keys).",
),
) -> None:
"""Run diagnostics to inspect system dependencies, device connectivity, and configuration."""
console = Console()
console.print()
@@ -103,6 +112,44 @@ def doctor_command() -> None:
f"adb not found in PATH or Android SDK. {hint}",
)
# 2b. ADB Authentication RSA Keys Check
key_status = inspect_adb_keys()
if key_status.is_corrupted:
if fix:
heal_result = heal_adb_keys(adb_path=adb_path)
if heal_result.get("success"):
table.add_row(
"ADB Auth Keys",
"[bold green]✔ Repaired[/bold green]",
"Corrupted RSA keys detected and automatically regenerated.",
)
else:
all_passed = False
table.add_row(
"ADB Auth Keys",
"[bold red]✖ Error[/bold red]",
f"Failed to auto-heal keys: {heal_result.get('message')}",
)
else:
all_passed = False
table.add_row(
"ADB Auth Keys",
"[bold red]✖ Corrupted[/bold red]",
f"{key_status.error_reason} Run [bold cyan]artemis doctor --fix[/bold cyan] to auto-repair.",
)
elif key_status.exists:
table.add_row(
"ADB Auth Keys",
"[bold green]✔ OK[/bold green]",
f"{key_status.key_path} ({key_status.key_size_bytes} bytes RSA)",
)
else:
table.add_row(
"ADB Auth Keys",
"[bold cyan]○ Ready[/bold cyan]",
"Keys will be generated automatically on first device connection.",
)
# 3. Connected Android Devices
devices_info: list[tuple[str, str, str]] = []
if adb_path: