feat: initial release of ARTEMIS Android Agent & Automation Platform
ARTEMIS is an autonomous multimodal Android agent and mobile UI automation platform engineered for physical devices and emulators. It combines state-of-the-art multimodal Vision-Language Models (VLMs) with hierarchical closed-loop planning, resilient self-healing, and deep Android subsystem integration. Key Features & Architectural Highlights: - Dual-Engine Orchestration: * Pro Engine: Multi-agent closed-loop (Planner -> Operator -> Validator/Failure Analyzer -> Summarizer) powered by LangGraph. * Flash Engine: Ultra-fast reactive loop for deterministic, low-latency UI execution (<35 steps). - Multimodal Perception & Grounding: * Hybrid visual screenshot grounding, real-time OCR extraction, and XML accessibility hierarchy parsing. * Dynamic-First, Coordinate-Fallback locating strategy for maximum resilience against UI drift. - Autonomous Self-Healing & Diagnostics: * Failure analyzer and safety net for popup dismissal, crash recovery, and auto-retry. * Diagnostic sub-agents for log analysis, video recording inspection, and object detection. - Unified CLI & Python SDK: * Rich CLI with commands: init, doctor, run, trace, batch, bench, server, and mcp. * Fluent Python SDK with Pydantic structured output models and package locking. - Multi-Provider LLM Architecture: * Support for Google Gemini, OpenAI, Anthropic Claude, Vertex AI, OpenRouter, xAI Grok, and local endpoints (Ollama/vLLM). * Centralized configuration in config/artemis.jsonc. - Native MCP & Observability: * Model Context Protocol (MCP) server endpoints for AI IDEs. * Full trace recording with before/after screenshots, action overlays, and execution replays. Change-Id: I430b93ee2b76b54cc5a3c6eeccd15d2cf29830a1
This commit is contained in:
somew1nd committed
commit
206a9ce921
229 files changed
+49653
No files matched your search
@@ -0,0 +1,524 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Universal Multi-Model FlashRunner for Artemis.
|
||||
|
||||
Executes autonomous reactive mobile workflows across Google Gemini,
|
||||
OpenAI GPT-4o/o3, Anthropic Claude 3.5/3.7, and OpenRouter endpoints.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
import uuid
|
||||
|
||||
from jinja2 import Template
|
||||
from langchain_core.messages import (
|
||||
BaseMessage,
|
||||
HumanMessage,
|
||||
SystemMessage,
|
||||
ToolMessage,
|
||||
)
|
||||
|
||||
from artemis.agents.flash.context_compressor import compress_flash_messages
|
||||
from artemis.agents.flash.summarizer import VisualStepSummarizer
|
||||
from artemis.agents.validator.tool_declarations import (
|
||||
ACTION_TOOL_NAMES,
|
||||
ASK_EXPLORER_TOOL,
|
||||
CLICK_SEQUENCE_TOOL,
|
||||
MobileActionExecutor,
|
||||
REPORT_TASK_STATUS_TOOL,
|
||||
VALIDATOR_TOOLS_DECLARATION,
|
||||
capture_screenshot_and_parse_ui,
|
||||
prune_intermediate_screenshots,
|
||||
)
|
||||
from artemis.config import StepSummarizerConfig, load_agent_config
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.controllers.unified_controller import UnifiedMobileController
|
||||
from artemis.data_engine.trace import trace
|
||||
from artemis.graph.perception import _check_injected_instruction_file
|
||||
from artemis.graph.state import State
|
||||
from artemis.services.llm import (
|
||||
RobustChatModelWrapper,
|
||||
get_google_llm,
|
||||
get_llm,
|
||||
invoke_llm_with_timeout_message,
|
||||
)
|
||||
from artemis.utils.coordinates import parse_swipe_parameters
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class FlashRunner:
|
||||
"""Reactive, ultra-fast agent execution loop supporting all VLM providers."""
|
||||
|
||||
def __init__(self, ctx: ArtemisContext, goal: str, max_turns: int | None = None):
|
||||
self.ctx = ctx
|
||||
self.goal = goal
|
||||
try:
|
||||
cfg = load_agent_config()
|
||||
self.max_turns = max_turns if max_turns is not None else cfg.flash.max_turns
|
||||
self.step_summarizer_cfg = cfg.flash.step_summarizer
|
||||
except Exception:
|
||||
self.max_turns = max_turns if max_turns is not None else 30
|
||||
self.step_summarizer_cfg = StepSummarizerConfig()
|
||||
|
||||
self.controller = UnifiedMobileController(ctx)
|
||||
self.executor = MobileActionExecutor(ctx, self.controller)
|
||||
self.summarizer = (
|
||||
VisualStepSummarizer(ctx, model_name=self.step_summarizer_cfg.model)
|
||||
if self.step_summarizer_cfg.enabled
|
||||
else None
|
||||
)
|
||||
|
||||
def _get_tools(self) -> list:
|
||||
tools = [t for t in VALIDATOR_TOOLS_DECLARATION if t.name != "report_failure_analysis"]
|
||||
tools.insert(1, CLICK_SEQUENCE_TOOL)
|
||||
tools.append(ASK_EXPLORER_TOOL)
|
||||
tools.append(REPORT_TASK_STATUS_TOOL)
|
||||
return tools
|
||||
|
||||
def _prune_intermediate_screenshots(self, messages: list[BaseMessage]) -> None:
|
||||
"""Prunes binary screenshot blocks from intermediate observation messages."""
|
||||
prune_intermediate_screenshots(messages)
|
||||
|
||||
@trace(type="agent", name="FlashRunner")
|
||||
async def run(self, state: State) -> dict:
|
||||
logger.info(f"Starting Artemis Flash reactive loop for goal: {self.goal}")
|
||||
|
||||
# 1. Initialize Universal LLM via Service Layer
|
||||
try:
|
||||
llm = get_llm(self.ctx, name="operator")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to get operator LLM from config, using default: {e}")
|
||||
|
||||
llm = RobustChatModelWrapper(get_google_llm(model_name="gemini-2.5-flash"), self.ctx)
|
||||
|
||||
tools_declaration = self._get_tools()
|
||||
|
||||
# 2. Capture Initial State (Screenshot + UI Tree)
|
||||
shot_path, img_bytes, xml_list = await capture_screenshot_and_parse_ui(
|
||||
self.ctx, state, self.controller, skip_settling=False
|
||||
)
|
||||
state.latest_screenshot = shot_path
|
||||
|
||||
user_content: list[dict] = [
|
||||
{"type": "text", "text": f"Your objective is: {self.goal}"},
|
||||
]
|
||||
if img_bytes:
|
||||
img_b64 = base64.b64encode(img_bytes).decode("utf-8")
|
||||
user_content.append(
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": f"data:image/jpeg;base64,{img_b64}"},
|
||||
}
|
||||
)
|
||||
if xml_list:
|
||||
user_content.append(
|
||||
{
|
||||
"type": "text",
|
||||
"text": f"--- UI Element List ---\n{xml_list}",
|
||||
}
|
||||
)
|
||||
user_content.append(
|
||||
{
|
||||
"type": "text",
|
||||
"text": (
|
||||
"CRITICAL RULE: In every single turn, you MUST FIRST output a natural"
|
||||
" language reasoning/explanation paragraph BEFORE invoking any tool call."
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
# Render System Prompt
|
||||
prompt_path = Path(__file__).parent / "flash_runner.md"
|
||||
prompt_template = prompt_path.read_text(encoding="utf-8")
|
||||
system_prompt = Template(prompt_template).render(goal=self.goal)
|
||||
|
||||
messages: list[BaseMessage] = [
|
||||
SystemMessage(content=system_prompt),
|
||||
HumanMessage(content=user_content),
|
||||
]
|
||||
|
||||
turns = 0
|
||||
final_report = None
|
||||
current_pre_screenshot_bytes = img_bytes
|
||||
current_xml_list = xml_list
|
||||
|
||||
while turns < self.max_turns:
|
||||
turns += 1
|
||||
logger.info(f"--- Artemis Flash Turn {turns}/{self.max_turns} ---")
|
||||
|
||||
if self.ctx.data_engine:
|
||||
self.ctx.data_engine.allocate_step_id()
|
||||
|
||||
# Check for real-time injected instructions
|
||||
if self.ctx.data_engine and self.ctx.data_engine.base_dir:
|
||||
try:
|
||||
injected_instruction = await asyncio.to_thread(
|
||||
_check_injected_instruction_file,
|
||||
str(self.ctx.data_engine.base_dir),
|
||||
)
|
||||
if injected_instruction:
|
||||
messages.append(
|
||||
HumanMessage(
|
||||
content=(
|
||||
"[REAL-TIME INJECTED INSTRUCTION from user]:"
|
||||
f" {injected_instruction}\nYou MUST immediately follow"
|
||||
" this instruction and adjust your plan/actions."
|
||||
)
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to check injected instruction in FlashRunner: {e}")
|
||||
|
||||
# Compress previous history before LLM call to save context and inject objective summaries
|
||||
if self.summarizer:
|
||||
compress_flash_messages(
|
||||
messages,
|
||||
summarizer=self.summarizer,
|
||||
prune_history_xml=self.step_summarizer_cfg.prune_history_xml,
|
||||
)
|
||||
else:
|
||||
prune_intermediate_screenshots(messages)
|
||||
|
||||
# Tool restriction on the final turn
|
||||
if turns == self.max_turns:
|
||||
messages.append(
|
||||
HumanMessage(
|
||||
content=(
|
||||
"[WARNING] This is your final turn; only"
|
||||
" 'report_task_status' is available."
|
||||
)
|
||||
)
|
||||
)
|
||||
current_tools = [t for t in tools_declaration if t.name == "report_task_status"]
|
||||
else:
|
||||
current_tools = tools_declaration
|
||||
|
||||
# Bind active tools
|
||||
bound_llm = llm.bind_tools(current_tools)
|
||||
|
||||
# Invoke Model with Streaming & Retry
|
||||
response = None
|
||||
max_retries = 3
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
|
||||
async def run_stream():
|
||||
full_response = None
|
||||
async for chunk in bound_llm.astream(messages):
|
||||
if full_response is None:
|
||||
full_response = chunk
|
||||
else:
|
||||
full_response += chunk
|
||||
return full_response
|
||||
|
||||
response = await invoke_llm_with_timeout_message(
|
||||
run_stream(), timeout_seconds=10, hard_timeout=180
|
||||
)
|
||||
if response:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.warning(f"FlashRunner LLM turn failed on attempt {attempt + 1}: {e}")
|
||||
if attempt == max_retries - 1:
|
||||
raise e
|
||||
await asyncio.sleep(2.0**attempt)
|
||||
|
||||
if response is None:
|
||||
break
|
||||
|
||||
messages.append(response)
|
||||
|
||||
# Extract thought and tool calls
|
||||
raw_text = response.content if isinstance(response.content, str) else ""
|
||||
if isinstance(response.content, list):
|
||||
raw_text = "".join(
|
||||
b.get("text", "")
|
||||
for b in response.content
|
||||
if isinstance(b, dict) and "text" in b
|
||||
)
|
||||
|
||||
# Extract token usage metadata from response
|
||||
step_token_usage = None
|
||||
if hasattr(response, "usage_metadata") and response.usage_metadata:
|
||||
u = response.usage_metadata
|
||||
pr = u.get("input_tokens") or u.get("prompt_tokens") or 0
|
||||
co = u.get("output_tokens") or u.get("completion_tokens") or 0
|
||||
to = u.get("total_tokens") or (pr + co)
|
||||
if to > 0:
|
||||
step_token_usage = {
|
||||
"prompt_tokens": int(pr),
|
||||
"completion_tokens": int(co),
|
||||
"total_tokens": int(to),
|
||||
}
|
||||
elif hasattr(response, "response_metadata") and isinstance(
|
||||
response.response_metadata, dict
|
||||
):
|
||||
u = response.response_metadata.get(
|
||||
"usage_metadata"
|
||||
) or response.response_metadata.get("token_usage")
|
||||
if isinstance(u, dict):
|
||||
pr = (
|
||||
u.get("input_tokens")
|
||||
or u.get("prompt_tokens")
|
||||
or u.get("prompt_token_count")
|
||||
or 0
|
||||
)
|
||||
co = (
|
||||
u.get("output_tokens")
|
||||
or u.get("completion_tokens")
|
||||
or u.get("candidates_token_count")
|
||||
or 0
|
||||
)
|
||||
to = u.get("total_tokens") or u.get("total_token_count") or (pr + co)
|
||||
if to > 0:
|
||||
step_token_usage = {
|
||||
"prompt_tokens": int(pr),
|
||||
"completion_tokens": int(co),
|
||||
"total_tokens": int(to),
|
||||
}
|
||||
|
||||
if not step_token_usage or step_token_usage.get("total_tokens", 0) <= 0:
|
||||
calc_prompt_chars = 0
|
||||
calc_images = 0
|
||||
for msg in messages:
|
||||
c = getattr(msg, "content", "")
|
||||
if isinstance(c, str):
|
||||
calc_prompt_chars += len(c)
|
||||
elif isinstance(c, list):
|
||||
for block in c:
|
||||
if isinstance(block, dict):
|
||||
if block.get("type") == "image_url" or "image_url" in block:
|
||||
calc_images += 1
|
||||
else:
|
||||
calc_prompt_chars += len(str(block.get("text", "")))
|
||||
else:
|
||||
calc_prompt_chars += len(str(block))
|
||||
prompt_tokens = (calc_prompt_chars // 4) + (calc_images * 258)
|
||||
completion_tokens = len(raw_text) // 4
|
||||
step_token_usage = {
|
||||
"prompt_tokens": max(1, prompt_tokens),
|
||||
"completion_tokens": max(1, completion_tokens),
|
||||
"total_tokens": max(1, prompt_tokens + completion_tokens),
|
||||
}
|
||||
|
||||
if self.ctx.data_engine:
|
||||
current_step_id = getattr(self.ctx.data_engine, "current_step_id", None)
|
||||
self.ctx.data_engine.record_trace(
|
||||
type="llm_call",
|
||||
name="FlashRunner",
|
||||
payload={"token_usage": step_token_usage, "response": raw_text},
|
||||
step_id=current_step_id,
|
||||
status="success",
|
||||
)
|
||||
|
||||
tool_calls = response.tool_calls or []
|
||||
|
||||
# Fallback text parsing if tool_calls not parsed natively
|
||||
if not tool_calls and "```json" in raw_text:
|
||||
try:
|
||||
snippet = raw_text.split("```json")[1].split("```")[0].strip()
|
||||
parsed = json.loads(snippet)
|
||||
if isinstance(parsed, dict) and "name" in parsed:
|
||||
tool_calls = [
|
||||
{
|
||||
"name": parsed["name"],
|
||||
"args": parsed.get("args", {}),
|
||||
"id": str(uuid.uuid4()),
|
||||
}
|
||||
]
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tool_calls:
|
||||
logger.info(
|
||||
f"FlashRunner received response without tool calls at turn {turns}:"
|
||||
f" {raw_text[:100]}..."
|
||||
)
|
||||
if turns == self.max_turns:
|
||||
final_report = {"status": "failed", "explanation": raw_text}
|
||||
break
|
||||
messages.append(
|
||||
HumanMessage(
|
||||
content=(
|
||||
"You did not call any tools. Please make progress"
|
||||
" by calling an action tool or 'report_task_status'."
|
||||
)
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
# Process tool calls
|
||||
for tc in tool_calls:
|
||||
name = tc["name"].split(":")[-1] if ":" in tc["name"] else tc["name"]
|
||||
args = tc.get("args") or {}
|
||||
tc_id = tc.get("id") or str(uuid.uuid4())
|
||||
logger.info(f"Executing Flash tool: {name}({args})")
|
||||
|
||||
if name == "report_task_status":
|
||||
final_report = args
|
||||
if self.ctx.data_engine:
|
||||
try:
|
||||
if self.ctx.data_engine.current_step_id is None:
|
||||
self.ctx.data_engine.allocate_step_id()
|
||||
self.ctx.data_engine.record_step(
|
||||
pre_screenshot_bytes=current_pre_screenshot_bytes,
|
||||
ui_tree=current_xml_list,
|
||||
action_taken={"action": "report_task_status", "args": args},
|
||||
operator_raw_thinking=raw_text,
|
||||
last_execution_result={
|
||||
"result": "Task completed with final report."
|
||||
},
|
||||
extra_metadata={"token_usage": step_token_usage},
|
||||
)
|
||||
except Exception as step_err:
|
||||
logger.warning(f"Error recording final step in FlashRunner: {step_err}")
|
||||
messages.append(
|
||||
ToolMessage(
|
||||
tool_call_id=tc_id,
|
||||
name=name,
|
||||
content=json.dumps({"status": "acknowledged"}),
|
||||
status="success",
|
||||
)
|
||||
)
|
||||
if self.summarizer:
|
||||
await self.summarizer.flush()
|
||||
return final_report
|
||||
|
||||
try:
|
||||
exec_result = await self.executor.execute(name, args, tc_id, state)
|
||||
|
||||
post_img_bytes = exec_result.screenshot_bytes
|
||||
if not post_img_bytes and name in ACTION_TOOL_NAMES:
|
||||
try:
|
||||
controller = UnifiedMobileController(self.ctx)
|
||||
screen_data = await controller.get_screen_data()
|
||||
post_img_bytes = base64.b64decode(screen_data.base64)
|
||||
if not exec_result.ui_elements_text:
|
||||
exec_result.ui_elements_text = screen_data.elements
|
||||
except Exception as shot_err:
|
||||
logger.warning(
|
||||
f"Failed to capture fallback screenshot in FlashRunner: {shot_err}"
|
||||
)
|
||||
|
||||
# Record telemetry / step in DataEngine
|
||||
if self.ctx.data_engine and name in ACTION_TOOL_NAMES:
|
||||
try:
|
||||
if self.ctx.data_engine.current_step_id is None:
|
||||
self.ctx.data_engine.allocate_step_id()
|
||||
|
||||
# Extract and enrich coordinate metadata
|
||||
norm_coords = None
|
||||
norm_start = None
|
||||
norm_end = None
|
||||
if name == "swipe":
|
||||
kind, target_val, _ = parse_swipe_parameters(args)
|
||||
if (
|
||||
kind == "coords"
|
||||
and isinstance(target_val, list)
|
||||
and len(target_val) == 4
|
||||
):
|
||||
norm_coords = target_val
|
||||
norm_start = target_val[:2]
|
||||
norm_end = target_val[2:]
|
||||
elif kind == "direction" and isinstance(target_val, str):
|
||||
g_lower = target_val.lower()
|
||||
if "up" in g_lower:
|
||||
norm_coords = [600, 800, 600, 200]
|
||||
elif "down" in g_lower:
|
||||
norm_coords = [600, 200, 600, 800]
|
||||
elif "left" in g_lower:
|
||||
norm_coords = [800, 500, 200, 500]
|
||||
elif "right" in g_lower:
|
||||
norm_coords = [200, 500, 800, 500]
|
||||
if norm_coords:
|
||||
norm_start = norm_coords[:2]
|
||||
norm_end = norm_coords[2:]
|
||||
elif name in ("click", "tap", "long_press", "input_text"):
|
||||
target = args.get("target") or args.get("coordinates")
|
||||
if isinstance(target, (list, tuple)) and len(target) == 2:
|
||||
norm_coords = list(target)
|
||||
elif isinstance(target, str):
|
||||
nums = re.findall(r"-?\d+(?:\.\d+)?", target)
|
||||
if len(nums) == 2:
|
||||
norm_coords = [int(float(nums[0])), int(float(nums[1]))]
|
||||
|
||||
action_dict = {
|
||||
"action": name,
|
||||
"coordinates": (
|
||||
args.get("target")
|
||||
or args.get("coordinates")
|
||||
or args.get("sequence")
|
||||
or norm_coords
|
||||
),
|
||||
"args": args,
|
||||
}
|
||||
if norm_coords:
|
||||
action_dict["normalized_coordinates"] = norm_coords
|
||||
if norm_start and norm_end:
|
||||
action_dict["normalized_start_coordinates"] = norm_start
|
||||
action_dict["normalized_end_coordinates"] = norm_end
|
||||
|
||||
self.ctx.data_engine.record_step(
|
||||
pre_screenshot_bytes=current_pre_screenshot_bytes,
|
||||
post_screenshot_bytes=post_img_bytes,
|
||||
ui_tree=(exec_result.ui_elements_text or current_xml_list),
|
||||
action_taken=action_dict,
|
||||
operator_raw_thinking=raw_text,
|
||||
last_execution_result={"result": exec_result.text_summary},
|
||||
extra_metadata={"token_usage": step_token_usage},
|
||||
)
|
||||
except Exception as step_err:
|
||||
logger.warning(f"Error recording step in FlashRunner: {step_err}")
|
||||
|
||||
# ⚡ Non-blocking dispatch of objective visual transition summarizer
|
||||
if self.summarizer and name in ACTION_TOOL_NAMES:
|
||||
self.summarizer.dispatch(
|
||||
step_number=turns,
|
||||
action_name=name,
|
||||
action_args=args,
|
||||
pre_img_bytes=current_pre_screenshot_bytes,
|
||||
post_img_bytes=post_img_bytes,
|
||||
exec_outcome=exec_result.text_summary,
|
||||
)
|
||||
|
||||
if post_img_bytes:
|
||||
current_pre_screenshot_bytes = post_img_bytes
|
||||
if exec_result.ui_elements_text:
|
||||
current_xml_list = exec_result.ui_elements_text
|
||||
|
||||
messages.append(exec_result.to_langchain_tool_message())
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error executing tool {name}: {e}")
|
||||
messages.append(
|
||||
ToolMessage(
|
||||
tool_call_id=tc_id,
|
||||
name=name,
|
||||
content=f"Error executing tool {name}: {e}",
|
||||
status="error",
|
||||
)
|
||||
)
|
||||
|
||||
if self.summarizer:
|
||||
await self.summarizer.flush()
|
||||
|
||||
return final_report or {
|
||||
"status": "failed",
|
||||
"explanation": "Max turns reached without final status report.",
|
||||
}
|
||||
Reference in new issue
Block a user