feat: initial release of ARTEMIS Android Agent & Automation Platform
ARTEMIS is an autonomous multimodal Android agent and mobile UI automation platform engineered for physical devices and emulators. It combines state-of-the-art multimodal Vision-Language Models (VLMs) with hierarchical closed-loop planning, resilient self-healing, and deep Android subsystem integration. Key Features & Architectural Highlights: - Dual-Engine Orchestration: * Pro Engine: Multi-agent closed-loop (Planner -> Operator -> Validator/Failure Analyzer -> Summarizer) powered by LangGraph. * Flash Engine: Ultra-fast reactive loop for deterministic, low-latency UI execution (<35 steps). - Multimodal Perception & Grounding: * Hybrid visual screenshot grounding, real-time OCR extraction, and XML accessibility hierarchy parsing. * Dynamic-First, Coordinate-Fallback locating strategy for maximum resilience against UI drift. - Autonomous Self-Healing & Diagnostics: * Failure analyzer and safety net for popup dismissal, crash recovery, and auto-retry. * Diagnostic sub-agents for log analysis, video recording inspection, and object detection. - Unified CLI & Python SDK: * Rich CLI with commands: init, doctor, run, trace, batch, bench, server, and mcp. * Fluent Python SDK with Pydantic structured output models and package locking. - Multi-Provider LLM Architecture: * Support for Google Gemini, OpenAI, Anthropic Claude, Vertex AI, OpenRouter, xAI Grok, and local endpoints (Ollama/vLLM). * Centralized configuration in config/artemis.jsonc. - Native MCP & Observability: * Model Context Protocol (MCP) server endpoints for AI IDEs. * Full trace recording with before/after screenshots, action overlays, and execution replays. Change-Id: I430b93ee2b76b54cc5a3c6eeccd15d2cf29830a1
This commit is contained in:
somew1nd committed
commit
206a9ce921
229 files changed
+49653
No files matched your search
@@ -0,0 +1,335 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
from collections.abc import Awaitable, Callable
|
||||
import logging
|
||||
import math
|
||||
import time
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def check_ui_change(
|
||||
img_before_bytes,
|
||||
img_after_bytes,
|
||||
action_item,
|
||||
full_screen_threshold=0.0001,
|
||||
roi_threshold=0.05,
|
||||
):
|
||||
"""Checks if the UI has changed after an action using image diff.
|
||||
|
||||
Args:
|
||||
img_before_bytes: Image bytes before action.
|
||||
img_after_bytes: Image bytes after action.
|
||||
action_item: Dict containing 'action' and optionally 'coordinates'.
|
||||
full_screen_threshold: Threshold for full screen change (fraction of
|
||||
changed pixels).
|
||||
roi_threshold: Threshold for ROI change (fraction of changed pixels in
|
||||
ROI).
|
||||
|
||||
Returns:
|
||||
bool: True if UI changed, False otherwise.
|
||||
"""
|
||||
try:
|
||||
# 1. Decode images
|
||||
nparr_b = np.frombuffer(img_before_bytes, np.uint8)
|
||||
nparr_a = np.frombuffer(img_after_bytes, np.uint8)
|
||||
img_b = cv2.imdecode(nparr_b, cv2.IMREAD_GRAYSCALE)
|
||||
img_a = cv2.imdecode(nparr_a, cv2.IMREAD_GRAYSCALE)
|
||||
|
||||
if img_b is None or img_a is None:
|
||||
logger.error("Failed to decode images.")
|
||||
return True # Fallback to assuming change if we can't decode
|
||||
|
||||
h, w = img_b.shape
|
||||
action_name = action_item.get("action")
|
||||
coordinates = action_item.get("coordinates")
|
||||
|
||||
# 2. Masking: Ignore status bar (top 10%) and nav bar (bottom 5%)
|
||||
top_crop = int(h * 0.10)
|
||||
bottom_crop = int(h * 0.05)
|
||||
|
||||
content_b = img_b[top_crop : h - bottom_crop, :]
|
||||
content_a = img_a[top_crop : h - bottom_crop, :]
|
||||
|
||||
# 3. Calculate absolute difference and threshold
|
||||
diff = cv2.absdiff(content_b, content_a)
|
||||
blurred = cv2.GaussianBlur(diff, (5, 5), 0)
|
||||
_, thresh = cv2.threshold(blurred, 25, 255, cv2.THRESH_BINARY)
|
||||
|
||||
non_zero_count = cv2.countNonZero(thresh)
|
||||
total_pixels = content_b.shape[0] * content_b.shape[1]
|
||||
global_change_ratio = non_zero_count / total_pixels
|
||||
|
||||
logger.info(f"Action: {action_name} | Global UI change ratio: {global_change_ratio:.4f}")
|
||||
|
||||
# 4. Strategy based on action type
|
||||
if action_name == "swipe":
|
||||
# Swipe expects large scale movement
|
||||
return global_change_ratio > full_screen_threshold
|
||||
|
||||
# For tap and others, check ROI if coordinates are available
|
||||
if (
|
||||
action_name
|
||||
in [
|
||||
"tap",
|
||||
"long_press_on",
|
||||
"focus_and_input_text",
|
||||
"focus_and_clear_text",
|
||||
]
|
||||
and coordinates
|
||||
and len(coordinates) == 2
|
||||
):
|
||||
cx, cy = coordinates
|
||||
# Scale ROI size dynamically based on screen diagonal (reference is 1080x2400)
|
||||
# and enforce a minimum of 15px to prevent collapsing on low-res screens
|
||||
ref_diagonal = math.sqrt(1080**2 + 2400**2)
|
||||
current_diagonal = math.sqrt(w**2 + h**2)
|
||||
scale_factor = current_diagonal / ref_diagonal
|
||||
roi_size = max(15, int(50 * scale_factor))
|
||||
|
||||
# Crop ROI from original images to avoid shifting issues with top_crop
|
||||
y1 = max(0, cy - roi_size)
|
||||
y2 = min(h, cy + roi_size)
|
||||
x1 = max(0, cx - roi_size)
|
||||
x2 = min(w, cx + roi_size)
|
||||
|
||||
roi_b = img_b[y1:y2, x1:x2]
|
||||
roi_a = img_a[y1:y2, x1:x2]
|
||||
|
||||
roi_diff = cv2.absdiff(roi_b, roi_a)
|
||||
roi_blurred = cv2.GaussianBlur(roi_diff, (3, 3), 0)
|
||||
_, roi_thresh = cv2.threshold(roi_blurred, 20, 255, cv2.THRESH_BINARY)
|
||||
|
||||
roi_non_zero = cv2.countNonZero(roi_thresh)
|
||||
roi_total = roi_b.shape[0] * roi_b.shape[1]
|
||||
roi_change_ratio = roi_non_zero / roi_total
|
||||
|
||||
logger.info(f"ROI change ratio at [{cx}, {cy}]: {roi_change_ratio:.4f}")
|
||||
|
||||
# If ROI changed significantly, we assume action worked
|
||||
if roi_change_ratio > roi_threshold:
|
||||
return True
|
||||
|
||||
# Fallback to global check if no ROI check was triggered or passed
|
||||
return global_change_ratio > full_screen_threshold
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error during image diff: {e}")
|
||||
return True # Fallback to assuming change on error
|
||||
|
||||
|
||||
async def wait_for_screen_stability(
|
||||
take_screenshot_fn: Callable[[], Awaitable[str]],
|
||||
max_timeout: float = 1.5,
|
||||
interval: float = 0.2,
|
||||
stability_threshold: float = 0.001,
|
||||
) -> str:
|
||||
"""Wait for the screen to stabilize (nearly zero pixel changes between consecutive frames)
|
||||
|
||||
before returning the latest base64 screenshot.
|
||||
|
||||
Args:
|
||||
take_screenshot_fn: Async callback function that captures a base64
|
||||
screenshot.
|
||||
max_timeout: Maximum time to wait for settling (in seconds) before
|
||||
forcing exit.
|
||||
interval: Polling time step between screenshot samples (in seconds).
|
||||
stability_threshold: Pixel change ratio below which the screen is
|
||||
considered settled (e.g. 0.001 for < 0.1%).
|
||||
|
||||
Returns:
|
||||
str: The final settled base64 screenshot.
|
||||
"""
|
||||
logger.info("Starting Dynamic Screen Settling...")
|
||||
|
||||
# 1. Capture the initial frame
|
||||
try:
|
||||
last_b64 = await take_screenshot_fn()
|
||||
last_bytes = base64.b64decode(last_b64)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to capture initial frame for settling: {e}")
|
||||
# Return fallback immediately on failure
|
||||
return await take_screenshot_fn()
|
||||
|
||||
start_time = time.time()
|
||||
while time.time() - start_time < max_timeout:
|
||||
await asyncio.sleep(interval)
|
||||
|
||||
try:
|
||||
curr_b64 = await take_screenshot_fn()
|
||||
curr_bytes = base64.b64decode(curr_b64)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to capture polling frame: {e}")
|
||||
continue
|
||||
|
||||
# 2. Decode and compare using OpenCV
|
||||
try:
|
||||
nparr_b = np.frombuffer(last_bytes, np.uint8)
|
||||
nparr_a = np.frombuffer(curr_bytes, np.uint8)
|
||||
img_b = cv2.imdecode(nparr_b, cv2.IMREAD_GRAYSCALE)
|
||||
img_a = cv2.imdecode(nparr_a, cv2.IMREAD_GRAYSCALE)
|
||||
|
||||
if img_b is None or img_a is None:
|
||||
last_b64, last_bytes = curr_b64, curr_bytes
|
||||
continue
|
||||
|
||||
h, w = img_b.shape
|
||||
|
||||
# Masking: crop status bar (top 10%) and navigation bar (bottom 5%)
|
||||
# to avoid false positives from notification icons, clocks, or blinking elements
|
||||
top_crop = int(h * 0.10)
|
||||
bottom_crop = int(h * 0.05)
|
||||
|
||||
content_b = img_b[top_crop : h - bottom_crop, :]
|
||||
content_a = img_a[top_crop : h - bottom_crop, :]
|
||||
|
||||
# Calculate absolute difference and filter noise
|
||||
diff = cv2.absdiff(content_b, content_a)
|
||||
blurred = cv2.GaussianBlur(diff, (5, 5), 0)
|
||||
_, thresh = cv2.threshold(blurred, 25, 255, cv2.THRESH_BINARY)
|
||||
|
||||
non_zero_count = cv2.countNonZero(thresh)
|
||||
total_pixels = content_b.shape[0] * content_b.shape[1]
|
||||
change_ratio = non_zero_count / total_pixels
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
logger.info(
|
||||
f"Screen settle poll: diff ratio = {change_ratio:.4f} (target <"
|
||||
f" {stability_threshold}) | Elapsed: {elapsed:.2f}s"
|
||||
)
|
||||
|
||||
# If the change ratio is below stability threshold, screen has stabilized!
|
||||
if change_ratio < stability_threshold:
|
||||
logger.info(f"Screen stabilized after {elapsed:.2f}s (diff: {change_ratio:.4f}).")
|
||||
return curr_b64
|
||||
|
||||
last_b64, last_bytes = curr_b64, curr_bytes
|
||||
|
||||
except Exception as ex:
|
||||
logger.error(f"Error during screen stability image diff calculation: {ex}")
|
||||
last_b64, last_bytes = curr_b64, curr_bytes
|
||||
continue
|
||||
|
||||
# 3. Timeout safety fallback (e.g., dynamic playing video/animated loaders)
|
||||
logger.warning(
|
||||
f"Screen settle timed out after {max_timeout}s without complete"
|
||||
" stability. Forcing return of latest frame."
|
||||
)
|
||||
return last_b64
|
||||
|
||||
|
||||
async def wait_for_screen_data_stability(
|
||||
get_screen_data_fn,
|
||||
max_timeout: float = 1.5,
|
||||
interval: float = 0.2,
|
||||
stability_threshold: float = 0.001,
|
||||
):
|
||||
"""Wait for the screen to stabilize (nearly zero pixel changes between consecutive frames)
|
||||
|
||||
while fetching both screenshot and XML hierarchy directly
|
||||
(ScreenDataResponse).
|
||||
|
||||
Args:
|
||||
get_screen_data_fn: Async callback function that captures a
|
||||
ScreenDataResponse.
|
||||
max_timeout: Maximum time to wait for settling (in seconds) before
|
||||
forcing exit.
|
||||
interval: Polling time step between screen data samples (in seconds).
|
||||
stability_threshold: Pixel change ratio below which the screen is
|
||||
considered settled.
|
||||
|
||||
Returns:
|
||||
ScreenDataResponse: The final settled ScreenDataResponse.
|
||||
"""
|
||||
logger.info("Starting Dynamic Screen Data Settling...")
|
||||
|
||||
# 1. Capture the initial frame
|
||||
try:
|
||||
last_data = await get_screen_data_fn()
|
||||
last_bytes = base64.b64decode(last_data.base64)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to capture initial frame for settling: {e}")
|
||||
# Return fallback immediately on failure
|
||||
return await get_screen_data_fn()
|
||||
|
||||
start_time = time.time()
|
||||
while time.time() - start_time < max_timeout:
|
||||
await asyncio.sleep(interval)
|
||||
|
||||
try:
|
||||
curr_data = await get_screen_data_fn()
|
||||
curr_bytes = base64.b64decode(curr_data.base64)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to capture polling frame: {e}")
|
||||
continue
|
||||
|
||||
# 2. Decode and compare using OpenCV
|
||||
try:
|
||||
nparr_b = np.frombuffer(last_bytes, np.uint8)
|
||||
nparr_a = np.frombuffer(curr_bytes, np.uint8)
|
||||
img_b = cv2.imdecode(nparr_b, cv2.IMREAD_GRAYSCALE)
|
||||
img_a = cv2.imdecode(nparr_a, cv2.IMREAD_GRAYSCALE)
|
||||
|
||||
if img_b is None or img_a is None:
|
||||
last_data, last_bytes = curr_data, curr_bytes
|
||||
continue
|
||||
|
||||
h, w = img_b.shape
|
||||
|
||||
# Masking: crop status bar (top 10%) and navigation bar (bottom 5%)
|
||||
top_crop = int(h * 0.10)
|
||||
bottom_crop = int(h * 0.05)
|
||||
|
||||
content_b = img_b[top_crop : h - bottom_crop, :]
|
||||
content_a = img_a[top_crop : h - bottom_crop, :]
|
||||
|
||||
# Calculate absolute difference and filter noise
|
||||
diff = cv2.absdiff(content_b, content_a)
|
||||
blurred = cv2.GaussianBlur(diff, (5, 5), 0)
|
||||
_, thresh = cv2.threshold(blurred, 25, 255, cv2.THRESH_BINARY)
|
||||
|
||||
non_zero_count = cv2.countNonZero(thresh)
|
||||
total_pixels = content_b.shape[0] * content_b.shape[1]
|
||||
change_ratio = non_zero_count / total_pixels
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
logger.info(
|
||||
f"Screen data settle poll: diff ratio = {change_ratio:.4f}"
|
||||
f" (target < {stability_threshold}) | Elapsed: {elapsed:.2f}s"
|
||||
)
|
||||
|
||||
# If the change ratio is below stability threshold, screen has stabilized!
|
||||
if change_ratio < stability_threshold:
|
||||
logger.info(f"Screen stabilized after {elapsed:.2f}s (diff: {change_ratio:.4f}).")
|
||||
return curr_data
|
||||
|
||||
last_data, last_bytes = curr_data, curr_bytes
|
||||
|
||||
except Exception as ex:
|
||||
logger.error(f"Error during screen stability image diff calculation: {ex}")
|
||||
last_data, last_bytes = curr_data, curr_bytes
|
||||
continue
|
||||
|
||||
# 3. Timeout safety fallback
|
||||
logger.warning(
|
||||
f"Screen data settle timed out after {max_timeout}s without complete"
|
||||
" stability. Forcing return of latest frame."
|
||||
)
|
||||
return last_data
|
||||
Reference in new issue
Block a user