feat: initial release of ARTEMIS Android Agent & Automation Platform
ARTEMIS is an autonomous multimodal Android agent and mobile UI automation platform engineered for physical devices and emulators. It combines state-of-the-art multimodal Vision-Language Models (VLMs) with hierarchical closed-loop planning, resilient self-healing, and deep Android subsystem integration. Key Features & Architectural Highlights: - Dual-Engine Orchestration: * Pro Engine: Multi-agent closed-loop (Planner -> Operator -> Validator/Failure Analyzer -> Summarizer) powered by LangGraph. * Flash Engine: Ultra-fast reactive loop for deterministic, low-latency UI execution (<35 steps). - Multimodal Perception & Grounding: * Hybrid visual screenshot grounding, real-time OCR extraction, and XML accessibility hierarchy parsing. * Dynamic-First, Coordinate-Fallback locating strategy for maximum resilience against UI drift. - Autonomous Self-Healing & Diagnostics: * Failure analyzer and safety net for popup dismissal, crash recovery, and auto-retry. * Diagnostic sub-agents for log analysis, video recording inspection, and object detection. - Unified CLI & Python SDK: * Rich CLI with commands: init, doctor, run, trace, batch, bench, server, and mcp. * Fluent Python SDK with Pydantic structured output models and package locking. - Multi-Provider LLM Architecture: * Support for Google Gemini, OpenAI, Anthropic Claude, Vertex AI, OpenRouter, xAI Grok, and local endpoints (Ollama/vLLM). * Centralized configuration in config/artemis.jsonc. - Native MCP & Observability: * Model Context Protocol (MCP) server endpoints for AI IDEs. * Full trace recording with before/after screenshots, action overlays, and execution replays. Change-Id: I430b93ee2b76b54cc5a3c6eeccd15d2cf29830a1
This commit is contained in:
somew1nd committed
commit
206a9ce921
229 files changed
+49653
No files matched your search
@@ -0,0 +1,389 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.controllers.types import (
|
||||
CoordinatesSelectorRequest,
|
||||
PercentagesSelectorRequest,
|
||||
)
|
||||
from artemis.controllers.unified_controller import UnifiedMobileController
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.types import Target
|
||||
from artemis.utils.logger import get_logger
|
||||
from artemis.utils.ui_hierarchy import (
|
||||
ElementBounds,
|
||||
Point,
|
||||
find_element_by_resource_id,
|
||||
get_bounds_for_element,
|
||||
get_element_text,
|
||||
is_element_focused,
|
||||
)
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
def find_element_by_text(
|
||||
ui_hierarchy: list[dict], text: str, index: int | None = None
|
||||
) -> dict | None:
|
||||
"""Find a UI element by its text content (adapted to both flat and rich hierarchy)
|
||||
|
||||
This function performs a recursive, case-insensitive search.
|
||||
|
||||
Args:
|
||||
ui_hierarchy: List of UI element dictionaries.
|
||||
text: The text content to search for.
|
||||
index: Optional index to select nth matching element.
|
||||
|
||||
Returns:
|
||||
The complete UI element dictionary if found, None otherwise.
|
||||
"""
|
||||
matches: list[dict] = []
|
||||
|
||||
def search_recursive(elements: list[dict]) -> None:
|
||||
for element in elements:
|
||||
if isinstance(element, dict):
|
||||
src = element.get("attributes", element)
|
||||
element_text = src.get("text", "")
|
||||
|
||||
# Guard against non-string values
|
||||
if not isinstance(element_text, str):
|
||||
element_text = ""
|
||||
|
||||
text_lower = text.lower()
|
||||
if element_text and text_lower == element_text.lower():
|
||||
matches.append(element)
|
||||
|
||||
if children := element.get("children", []):
|
||||
search_recursive(children)
|
||||
|
||||
search_recursive(ui_hierarchy)
|
||||
|
||||
if not matches:
|
||||
return None
|
||||
|
||||
if index is None:
|
||||
idx = 0
|
||||
elif index < 0:
|
||||
return None
|
||||
else:
|
||||
idx = index
|
||||
if idx < len(matches):
|
||||
return matches[idx]
|
||||
return None
|
||||
|
||||
|
||||
async def tap_bottom_right_of_element(bounds: ElementBounds, ctx: ArtemisContext):
|
||||
"""Tap the bottom-right corner of the specified element bounds."""
|
||||
bottom_right: Point = bounds.get_relative_point(x_percent=0.99, y_percent=0.99)
|
||||
await tap(
|
||||
ctx=ctx,
|
||||
selector_request=SelectorRequestWithCoordinates(
|
||||
coordinates=CoordinatesSelectorRequest(
|
||||
x=bottom_right.x,
|
||||
y=bottom_right.y,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# pylint: disable=too-many-return-statements
|
||||
async def move_cursor_to_end_if_bounds(
|
||||
ctx: ArtemisContext,
|
||||
state: State,
|
||||
target: Target,
|
||||
elt: dict | None = None,
|
||||
) -> dict | None:
|
||||
"""Best-effort move of the text cursor near the end of the input by tapping the
|
||||
|
||||
bottom-right area of the focused element (if bounds are available).
|
||||
"""
|
||||
if target.resource_id:
|
||||
if not elt:
|
||||
elt = find_element_by_resource_id(
|
||||
ui_hierarchy=state.latest_ui_hierarchy or [],
|
||||
resource_id=target.resource_id,
|
||||
index=target.resource_id_index,
|
||||
)
|
||||
if not elt:
|
||||
return None
|
||||
|
||||
bounds = get_bounds_for_element(elt)
|
||||
if not bounds:
|
||||
return elt
|
||||
|
||||
logger.debug("Tapping near the end of the input to move the cursor")
|
||||
await tap_bottom_right_of_element(bounds=bounds, ctx=ctx)
|
||||
logger.debug(f"Tapped end of input {target.resource_id}")
|
||||
return elt
|
||||
|
||||
if target.bounds:
|
||||
await tap_bottom_right_of_element(target.bounds, ctx=ctx)
|
||||
logger.debug("Tapped end of input by coordinates")
|
||||
return elt
|
||||
|
||||
if target.text:
|
||||
text_elt = find_element_by_text(
|
||||
state.latest_ui_hierarchy or [],
|
||||
target.text,
|
||||
index=target.text_index,
|
||||
)
|
||||
if text_elt:
|
||||
bounds = get_bounds_for_element(text_elt)
|
||||
if bounds:
|
||||
await tap_bottom_right_of_element(bounds=bounds, ctx=ctx)
|
||||
logger.debug(f"Tapped end of input that had text'{target.text}'")
|
||||
return text_elt
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
|
||||
async def focus_element_if_needed(
|
||||
ctx: ArtemisContext, target: Target
|
||||
) -> Literal["resource_id", "coordinates", "text", "already_focused"] | None:
|
||||
"""Ensures the element is focused, with a sanity check to prevent trusting misleading IDs.
|
||||
|
||||
If no target locator is provided (empty target), assumes the field is
|
||||
already focused
|
||||
and returns "already_focused" to allow typing without re-focusing.
|
||||
"""
|
||||
# If no locator is provided, assume already focused (e.g., keyboard is visible)
|
||||
if not target.resource_id and not target.bounds and not target.text:
|
||||
logger.debug("No target locator provided, assuming element is already focused")
|
||||
return "already_focused"
|
||||
|
||||
controller = UnifiedMobileController(ctx)
|
||||
rich_hierarchy = await controller.get_ui_elements()
|
||||
elt_from_id = None
|
||||
if target.resource_id:
|
||||
elt_from_id = find_element_by_resource_id(
|
||||
ui_hierarchy=rich_hierarchy,
|
||||
resource_id=target.resource_id,
|
||||
index=target.resource_id_index,
|
||||
is_rich_hierarchy=False,
|
||||
)
|
||||
|
||||
if elt_from_id and target.text:
|
||||
text_from_id_elt = get_element_text(elt_from_id)
|
||||
if not text_from_id_elt or target.text.lower() != text_from_id_elt.lower():
|
||||
logger.warning(
|
||||
f"ID '{target.resource_id}' and text '{target.text}' seem to be"
|
||||
" on different elements. Ignoring the resource_id and falling"
|
||||
" back to other locators."
|
||||
)
|
||||
elt_from_id = None
|
||||
|
||||
if elt_from_id:
|
||||
if not is_element_focused(elt_from_id):
|
||||
await tap(
|
||||
ctx=ctx,
|
||||
selector_request=IdSelectorRequest(id=target.resource_id), # type: ignore
|
||||
index=target.resource_id_index,
|
||||
)
|
||||
logger.debug(f"Focused (tap) on resource_id={target.resource_id}")
|
||||
rich_hierarchy = await controller.get_ui_elements()
|
||||
elt_from_id = find_element_by_resource_id(
|
||||
ui_hierarchy=rich_hierarchy,
|
||||
resource_id=target.resource_id, # type: ignore
|
||||
index=target.resource_id_index,
|
||||
is_rich_hierarchy=False,
|
||||
)
|
||||
if elt_from_id and is_element_focused(elt_from_id):
|
||||
logger.debug(f"Text input is focused: {target.resource_id}")
|
||||
return "resource_id"
|
||||
logger.warning(f"Failed to focus using resource_id='{target.resource_id}'. Fallback...")
|
||||
|
||||
if target.bounds:
|
||||
relative_point = target.bounds.get_center()
|
||||
await tap(
|
||||
ctx=ctx,
|
||||
selector_request=SelectorRequestWithCoordinates(
|
||||
coordinates=CoordinatesSelectorRequest(x=relative_point.x, y=relative_point.y)
|
||||
),
|
||||
)
|
||||
logger.debug(f"Tapped on coordinates ({relative_point.x}, {relative_point.y}) to focus.")
|
||||
return "coordinates"
|
||||
|
||||
if target.text:
|
||||
text_elt = find_element_by_text(rich_hierarchy, target.text, index=target.text_index)
|
||||
if text_elt:
|
||||
bounds = get_bounds_for_element(text_elt)
|
||||
if bounds:
|
||||
relative_point = bounds.get_center()
|
||||
await tap(
|
||||
ctx=ctx,
|
||||
selector_request=SelectorRequestWithCoordinates(
|
||||
coordinates=CoordinatesSelectorRequest(
|
||||
x=relative_point.x, y=relative_point.y
|
||||
)
|
||||
),
|
||||
)
|
||||
logger.debug(f"Tapped on text element '{target.text}' to focus.")
|
||||
return "text"
|
||||
|
||||
logger.error(
|
||||
"Failed to focus element. No valid locator (resource_id, coordinates, or text) succeeded."
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def validate_coordinates_bounds(
|
||||
target: Target, screen_width: int, screen_height: int
|
||||
) -> str | None:
|
||||
"""Validate that coordinates are within screen bounds.
|
||||
|
||||
Returns error message if invalid, None if valid.
|
||||
"""
|
||||
if not target.bounds:
|
||||
return None
|
||||
|
||||
center = target.bounds.get_center()
|
||||
errors = []
|
||||
|
||||
if center.x < 0 or center.x >= screen_width:
|
||||
errors.append(f"x={center.x} is outside screen width (0-{screen_width})")
|
||||
if center.y < 0 or center.y >= screen_height:
|
||||
errors.append(f"y={center.y} is outside screen height (0-{screen_height})")
|
||||
|
||||
return "; ".join(errors) if errors else None
|
||||
|
||||
|
||||
def has_valid_selectors(target: Target) -> bool:
|
||||
"""Check if target has at least one valid selector."""
|
||||
has_coordinates = target.bounds is not None
|
||||
has_resource_id = target.resource_id is not None and target.resource_id != ""
|
||||
has_text = target.text is not None and target.text != ""
|
||||
return has_coordinates or has_resource_id or has_text
|
||||
|
||||
|
||||
class IdSelectorRequest(BaseModel):
|
||||
"""Selector request using resource id."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True)
|
||||
id: str
|
||||
|
||||
def to_dict(self) -> dict[str, str | int]:
|
||||
"""Convert selector to dictionary."""
|
||||
return {"id": self.id}
|
||||
|
||||
|
||||
class TextSelectorRequest(BaseModel):
|
||||
"""Selector request using element text."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
text: str
|
||||
|
||||
def to_dict(self) -> dict[str, str | int]:
|
||||
"""Convert selector to dictionary."""
|
||||
return {"text": self.text}
|
||||
|
||||
|
||||
class SelectorRequestWithCoordinates(BaseModel):
|
||||
"""Selector request using absolute coordinates."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
coordinates: CoordinatesSelectorRequest
|
||||
|
||||
def to_dict(self) -> dict[str, str | int]:
|
||||
"""Convert selector to dictionary."""
|
||||
return {"point": self.coordinates.to_str()}
|
||||
|
||||
|
||||
class SelectorRequestWithPercentages(BaseModel):
|
||||
"""Selector request using percentage coordinates."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
percentages: PercentagesSelectorRequest
|
||||
|
||||
def to_dict(self) -> dict[str, str | int]:
|
||||
"""Convert selector to dictionary."""
|
||||
return {"point": self.percentages.to_str()}
|
||||
|
||||
|
||||
class IdWithTextSelectorRequest(BaseModel):
|
||||
"""Selector request using both resource id and text."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
id: str
|
||||
text: str
|
||||
|
||||
def to_dict(self) -> dict[str, str | int]:
|
||||
"""Convert selector to dictionary."""
|
||||
return {"id": self.id, "text": self.text}
|
||||
|
||||
|
||||
SelectorRequest = (
|
||||
IdSelectorRequest
|
||||
| SelectorRequestWithCoordinates
|
||||
| SelectorRequestWithPercentages
|
||||
| TextSelectorRequest
|
||||
| IdWithTextSelectorRequest
|
||||
)
|
||||
|
||||
|
||||
def _extract_resource_id_and_text_from_selector(
|
||||
selector: SelectorRequest,
|
||||
) -> tuple[str | None, str | None]:
|
||||
"""Extract resource_id and text from a selector."""
|
||||
resource_id = None
|
||||
text = None
|
||||
|
||||
if isinstance(selector, IdSelectorRequest):
|
||||
resource_id = selector.id
|
||||
elif isinstance(selector, TextSelectorRequest):
|
||||
text = selector.text
|
||||
elif isinstance(selector, IdWithTextSelectorRequest):
|
||||
resource_id = selector.id
|
||||
text = selector.text
|
||||
|
||||
return resource_id, text
|
||||
|
||||
|
||||
async def tap(
|
||||
ctx: ArtemisContext,
|
||||
selector_request: SelectorRequest,
|
||||
index: int | None = None,
|
||||
):
|
||||
"""Tap on a selector.
|
||||
|
||||
Index is optional and is used when you have multiple views matching the same
|
||||
selector. ui_hierarchy is optional and used for ADB taps to find elements.
|
||||
"""
|
||||
controller = UnifiedMobileController(ctx)
|
||||
if isinstance(selector_request, SelectorRequestWithCoordinates):
|
||||
result = await controller.tap_at(
|
||||
x=selector_request.coordinates.x, y=selector_request.coordinates.y
|
||||
)
|
||||
return result.error if result.error else None
|
||||
if isinstance(selector_request, SelectorRequestWithPercentages):
|
||||
coords = selector_request.percentages.to_coords(
|
||||
width=ctx.device.device_width,
|
||||
height=ctx.device.device_height,
|
||||
)
|
||||
return await controller.tap_at(coords.x, coords.y)
|
||||
|
||||
# For other selectors, we need the UI hierarchy
|
||||
resource_id, text = _extract_resource_id_and_text_from_selector(selector_request)
|
||||
|
||||
return await controller.tap_element(
|
||||
resource_id=resource_id,
|
||||
text=text,
|
||||
index=index if index is not None else 0,
|
||||
)
|
||||
Reference in new issue
Block a user