somew1nd
206a9ce921
ARTEMIS is an autonomous multimodal Android agent and mobile UI automation platform engineered for physical devices and emulators. It combines state-of-the-art multimodal Vision-Language Models (VLMs) with hierarchical closed-loop planning, resilient self-healing, and deep Android subsystem integration. Key Features & Architectural Highlights: - Dual-Engine Orchestration: * Pro Engine: Multi-agent closed-loop (Planner -> Operator -> Validator/Failure Analyzer -> Summarizer) powered by LangGraph. * Flash Engine: Ultra-fast reactive loop for deterministic, low-latency UI execution (<35 steps). - Multimodal Perception & Grounding: * Hybrid visual screenshot grounding, real-time OCR extraction, and XML accessibility hierarchy parsing. * Dynamic-First, Coordinate-Fallback locating strategy for maximum resilience against UI drift. - Autonomous Self-Healing & Diagnostics: * Failure analyzer and safety net for popup dismissal, crash recovery, and auto-retry. * Diagnostic sub-agents for log analysis, video recording inspection, and object detection. - Unified CLI & Python SDK: * Rich CLI with commands: init, doctor, run, trace, batch, bench, server, and mcp. * Fluent Python SDK with Pydantic structured output models and package locking. - Multi-Provider LLM Architecture: * Support for Google Gemini, OpenAI, Anthropic Claude, Vertex AI, OpenRouter, xAI Grok, and local endpoints (Ollama/vLLM). * Centralized configuration in config/artemis.jsonc. - Native MCP & Observability: * Model Context Protocol (MCP) server endpoints for AI IDEs. * Full trace recording with before/after screenshots, action overlays, and execution replays. Change-Id: I430b93ee2b76b54cc5a3c6eeccd15d2cf29830a1
91 lines
3.1 KiB
Python
91 lines
3.1 KiB
Python
# Copyright 2026 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import os
|
|
from typing import Any
|
|
|
|
import httpx
|
|
from artemis.config import settings
|
|
from artemis.utils.logger import get_logger
|
|
|
|
logger = get_logger(__name__)
|
|
_HTTP_CLIENT: httpx.AsyncClient | None = None
|
|
|
|
|
|
def get_http_client() -> httpx.AsyncClient:
|
|
global _HTTP_CLIENT
|
|
if _HTTP_CLIENT is None or _HTTP_CLIENT.is_closed:
|
|
_HTTP_CLIENT = httpx.AsyncClient(timeout=30.0)
|
|
return _HTTP_CLIENT
|
|
|
|
|
|
async def perform_ocr(
|
|
screenshot_b64: str, client: httpx.AsyncClient = None
|
|
) -> list[dict[str, Any]]:
|
|
"""Calls Google Vision API for OCR using a persistent client to reuse TLS handshakes.
|
|
|
|
Args:
|
|
screenshot_b64: Base64 encoded screenshot image.
|
|
|
|
Returns:
|
|
A list of dictionaries containing detected text and position vertices.
|
|
"""
|
|
ocr_secret = settings.get_api_key("ocr")
|
|
api_key = (
|
|
(ocr_secret.get_secret_value() if ocr_secret else None)
|
|
or os.environ.get("OCR_API_KEY")
|
|
or os.environ.get("VISION_API_KEY")
|
|
or os.environ.get("API_KEY")
|
|
or os.environ.get("GOOGLE_API_KEY")
|
|
or "API_KEY"
|
|
)
|
|
|
|
url = f"https://vision.googleapis.com/v1/images:annotate?key={api_key}"
|
|
headers = {"Content-Type": "application/json"}
|
|
data = {
|
|
"requests": [
|
|
{
|
|
"image": {"content": screenshot_b64},
|
|
"features": [{"type": "TEXT_DETECTION"}],
|
|
}
|
|
]
|
|
}
|
|
|
|
try:
|
|
active_client = client if client is not None else get_http_client()
|
|
response = await active_client.post(url, json=data, headers=headers, timeout=30.0)
|
|
return _parse_ocr_response(response)
|
|
except Exception as e:
|
|
logger.error(f"Failed to call Vision API for OCR: {e}")
|
|
raise e
|
|
|
|
|
|
def _parse_ocr_response(response: httpx.Response) -> list[dict[str, Any]]:
|
|
if response.status_code == 200:
|
|
res_json = response.json()
|
|
responses = res_json.get("responses", [])
|
|
if responses and "textAnnotations" in responses[0]:
|
|
annotations = responses[0]["textAnnotations"]
|
|
results = []
|
|
# Skip the first annotation (index 0) as it is the full-screen combined text
|
|
for ann in annotations[1:]:
|
|
desc = ann.get("description", "")
|
|
vertices = ann.get("boundingPoly", {}).get("vertices", [])
|
|
results.append({"text": desc, "position": vertices})
|
|
return results
|
|
else:
|
|
return []
|
|
else:
|
|
raise Exception(f"Vision API returned status {response.status_code}: {response.text}")
|