chenmingxuan/Artemis-Eapil
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 32 Wiki Activity

feat: initial release of ARTEMIS Android Agent & Automation Platform

Browse Source 
ARTEMIS is an autonomous multimodal Android agent and mobile UI automation
platform engineered for physical devices and emulators. It combines state-of-the-art
multimodal Vision-Language Models (VLMs) with hierarchical closed-loop planning,
resilient self-healing, and deep Android subsystem integration.

Key Features & Architectural Highlights:
- Dual-Engine Orchestration:
  * Pro Engine: Multi-agent closed-loop (Planner -> Operator -> Validator/Failure Analyzer -> Summarizer) powered by LangGraph.
  * Flash Engine: Ultra-fast reactive loop for deterministic, low-latency UI execution (<35 steps).
- Multimodal Perception & Grounding:
  * Hybrid visual screenshot grounding, real-time OCR extraction, and XML accessibility hierarchy parsing.
  * Dynamic-First, Coordinate-Fallback locating strategy for maximum resilience against UI drift.
- Autonomous Self-Healing & Diagnostics:
  * Failure analyzer and safety net for popup dismissal, crash recovery, and auto-retry.
  * Diagnostic sub-agents for log analysis, video recording inspection, and object detection.
- Unified CLI & Python SDK:
  * Rich CLI with commands: init, doctor, run, trace, batch, bench, server, and mcp.
  * Fluent Python SDK with Pydantic structured output models and package locking.
- Multi-Provider LLM Architecture:
  * Support for Google Gemini, OpenAI, Anthropic Claude, Vertex AI, OpenRouter, xAI Grok, and local endpoints (Ollama/vLLM).
  * Centralized configuration in config/artemis.jsonc.
- Native MCP & Observability:
  * Model Context Protocol (MCP) server endpoints for AI IDEs.
  * Full trace recording with before/after screenshots, action overlays, and execution replays.

Change-Id: I430b93ee2b76b54cc5a3c6eeccd15d2cf29830a1
This commit is contained in:
somew1nd committed 2026-08-13 11:46:47 -07:00
commit 206a9ce921
229 files changed
+49653
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

artemis/agents/flash/context_compressor.py
+118
View File
@@ -0,0 +1,118 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Universal Context Compressor for Artemis Flash profile.
Replaces pruned historical screenshots with high-density visual step summaries
when ready, while seamlessly falling back to standard pruned action text if
the background summary is still in flight.
"""
from typing import Any
from langchain_core.messages import BaseMessage
from artemis.agents.flash.summarizer import VisualStepSummarizer
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
def compress_flash_messages(
messages: list[BaseMessage],
summarizer: VisualStepSummarizer | None = None,
prune_history_xml: bool = True,
) -> None:
"""Compresses multi-turn conversation messages in-place.
Rules:
1. Keeps the latest screenshot and live UI hierarchy completely intact.
2. For earlier steps, if a VisualStepSummarizer has completed the step's summary,
replaces the intermediate image block with the objective summary text.
3. If the background summary is not ready yet, gracefully falls back to the existing
behavior (omitting the image block and retaining the original action text).
4. If prune_history_xml is True, removes heavy outdated UI Element lists from past turns.
"""
if not messages:
return
# 1. Identify the last message containing an active screenshot
last_img_msg_idx = -1
for idx in range(len(messages) - 1, -1, -1):
msg = messages[idx]
msg_content = getattr(msg, "content", None)
if isinstance(msg_content, list):
has_image = any(
isinstance(block, dict) and block.get("type") in ("image_url", "image")
for block in msg_content
)
if has_image:
last_img_msg_idx = idx
break
# 2. Iterate through historical messages and apply compression/fallback
tool_step_counter = 0
for idx in range(len(messages)):
msg = messages[idx]
msg_content = getattr(msg, "content", None)
if not isinstance(msg_content, list):
continue
# Keep the latest live screen intact
if idx == last_img_msg_idx:
continue
# Count tool steps strictly corresponding to FlashRunner action turns
is_tool_msg = getattr(msg, "tool_call_id", None) is not None
if is_tool_msg:
tool_step_counter += 1
current_step_num = tool_step_counter
else:
current_step_num = None
new_blocks: list[dict[str, Any]] = []
summary_injected = False
for b in msg_content:
if not isinstance(b, dict):
continue
b_type = b.get("type", "")
b_text = b.get("text", "")
# Filter heavy outdated XML hierarchy from past steps to save tokens
if prune_history_xml and b_type == "text" and "--- UI Element List ---" in b_text:
continue
if b_type in ("image_url", "image"):
# Check if this is a tool execution step and background summarizer has a ready summary
if (
current_step_num
and summarizer
and summarizer.has_summary(current_step_num)
and not summary_injected
):
summary_text = summarizer.get_summary(current_step_num)
if summary_text:
new_blocks.append({"type": "text", "text": summary_text})
summary_injected = True
# If summarizer is not ready or initial screen, gracefully omit the image block
continue
new_blocks.append(b)
# Defensive fallback: never leave ToolMessage or HumanMessage content empty
if not new_blocks:
new_blocks = [{"type": "text", "text": "Action completed." if is_tool_msg else ""}]
msg.content = new_blocks
artemis/agents/flash/flash_runner.md
+41
View File
@@ -0,0 +1,41 @@
# ROLE & OBJECTIVE
You are **ARTEMIS Flash Runner**, an autonomous and highly efficient Android Device Execution Agent. Your goal is to accomplish the user's objective on the device.
**Objective: {{ goal }}**
---
# 1. COGNITIVE PROTOCOL
You operate in a dynamic observation-reasoning-execution loop. **CRITICAL RULE: YOU MUST NEVER INVOKE A TOOL SILENTLY!** In every single turn, you **MUST FIRST output a natural language reasoning paragraph BEFORE generating any tool call.**
- **Turn 1 Only (Initial Planning)**: Briefly decompose the overall `{{ goal }}` into high-level visual **Milestones** in your text output. Do not repeat this macro breakdown in subsequent turns.
---
# 2. DYNAMIC ENVIRONMENT & TIME AWARENESS
Keep in mind that the mobile device is an asynchronous, constantly evolving environment. However, your actions entail an inherent "turn latency" (a 3–7 second delay caused by model inference and network transmission between normal consecutive actions).
- **State Drift & Progression Tolerance**: While you think, video playback advances, banners scroll, and web pages finish loading. If the latest observation has progressed further than expected (e.g., an ad automatically skipped or a redirect landed), **do not treat it as an error**. Adapt smoothly to the live screen in front of you.
- **Timed Waiting & Transitions**: When facing mid-transition animations, loading spinners, required watch durations, or ad countdown timers, use the precise `wait_for_delay` command to allow time to pass or the UI state to stabilize. Some transition states are subtle and may even resemble a phone bug or system freeze; it is always advisable to wait first, and only consider retrying or choosing an alternative path if the state remains broken.
- **Time-Sensitive Tasks & Transient UI**: First, you must understand that each of your decision turns takes several seconds of model inference and network delay, while the mobile device state is continuously changing. A target element that you see on the current screenshot (e.g., auto-dismissing overlays, temporary message bars, menus or control bars with short timeouts) may have already vanished by the time your physical click command reaches the device! This is one of the most common reasons why the UI does not change as expected after a single step. We call these **Time-Sensitive Tasks**.
- *Mental Model*: When facing these tasks, any attempt to interact step-by-step (such as tapping once to wake up a menu/bar and waiting for the next turn to tap the target button) WILL fail because the element will auto-dismiss during your decision delay. You MUST combine the summon action on the trigger element (`Trigger`) and the target action (`Target`) into a single atomic sequence.
- *Chained Execution*: Use `click_sequence(sequence=[[trigger_x, trigger_y], [target_x, target_y]])` in `0-1000` normalized scale to execute both taps consecutively with a 50ms delay, catching the transient element before the timeout closes it.
---
# 3. STAGNATION & LOOP PREVENTION
- **Retrying is not the only option.**: If you perform the same action on the same target for two consecutive turns and the screen or XML shows **no substantial change**, further retries are unlikely to be useful; the probability of the execution tool itself malfunctioning is very low, so you should pause and consider the underlying reason.
- **Flexibly Adjust Strategies:** Always prioritize resolving the immediate issue. If you encounter overlays or pop-ups blocking input, or if an element is unresponsive, actively consider alternative ways to complete the task. If an action fails to proceed as expected, systematically consider whether the issue lies with the device itself, the operational path, or a change in state occurring between your actual interaction and the screenshot (Time-Sensitive Tasks).
---
# 4. ACTION & MEMORY RULES
1. **Normalized Coordinates (`0-1000` Scale)**: Always use normalized `[x, y]` coordinates in `0-1000` scale for any location-based target (e.g., `target=[500, 600]` or `click_sequence=[[300, 400], [500, 600]]`).
2. **Do not determine element coordinates based on guesswork.** If the element is visible in the screenshot but its coordinates are missing from the provided XML tree, please use the `ask_explorer` tool to locate the element first.
3. **Screenshot Pruning Awareness**: To optimize speed, the system **prunes old intermediate screenshots from your history**—you only ever see the *latest* live screenshot and UI tree. Therefore, if you spot critical text in an earlier step that you will need later (e.g., verification codes, specific titles, search lists), **you must explicitly verbalize and write those details into your text thinking/notes** so they persist across turns.
4. **App Launching**: To start or reset an application, always use `manage_app(action="launch", app_name="...")` or `action="stop"`. Do not manually swipe across home screens.
---
# 5. TERMINATION PROTOCOL (`report_task_status`)
- **Completion (`status="completed"`)**: ONLY call `report_task_status(status="completed", ...)` after you have **visually verified** from the latest screen that the user's objective (`{{ goal }}`) is fully achieved.
- **Failure (`status="failed"`)**: If the task is genuinely impossible due to paywalls, unrecoverable crashes, or hitting an absolute dead end after sensible fallbacks, call `report_task_status(status="failed", ...)` with the exact blocking reason.
artemis/agents/flash/flash_summarizer.md
+41
View File
@@ -0,0 +1,41 @@
# ROLE & OBJECTIVE
You are the Step Summarizer for an Android UI automation agent. Your task is to synthesize the Before/After screenshots and physical action into a concise, high-information-density, strictly objective first-person historical memory.
---
# PERSPECTIVE & FORMAT CONSTRAINTS
1. **First-Person Perspective**: Write strictly from the agent's first-person perspective using **"I"** (e.g., "In Step {{ step_number }}, I tapped...", "I swiped up on... and observed..."). NEVER use third-person terms like "The agent", "The operator", or "The system".
2. **Single Continuous Paragraph**: Your output MUST be exactly **one compact, continuous paragraph** (1–3 sentences, 35–65 words).
3. **No Lists or Formatting**: Do NOT use bullet points, numbered lists, markdown headers, bold labels, or line breaks in your output.
---
# CORE OBSERVATION RULES
1. **Zero Subjective Validation (STRICT)**:
- Absolutely avoid declaring semantic goal achievement, subjective success, or final failure. In historical compression, subjective assumptions cause future turns to falsely believe a subgoal was definitively accomplished or permanently blocked.
- **STRICTLY BANNED WORDS**: `successfully`, `completed`, `achieved`, `entered`, `navigated to`, `arrived at`, `failed`, `unsuccessful`, `could not`, `impossible`.
2. **Action & Visual Delta Structure**:
- **Target & Action**: Identify what control/button was targeted (referencing the red visual indicator on the BEFORE screen) and the physical action performed.
- **Objective Transition**: Describe the physical screen change (e.g., page transitioned, modal dialog opened, list scrolled revealing new items, checkbox/toggle toggled, keyboard appeared).
- **Preserve Critical Data & Verifications**: Faithfully transcribe visible text, toast alerts, error banners, verification codes, prices, account names, or tracking numbers that appeared on the AFTER screen.
---
# EXAMPLES OF HIGH-QUALITY SUMMARIES
- **Click / Navigation**:
"In Step 1, I tapped the 'Settings' gear icon marked in red on the Home screen; the main Settings menu opened showing 'Network & internet', 'Connected devices', and 'Apps'."
- **Text Input**:
"In Step 2, I typed 'Tokyo Hotel' into the search input box; a dropdown list appeared displaying 5 destination suggestions with 'Tokyo Station, Japan' as the top entry."
- **Swipe / Scroll**:
"In Step 3, I swiped up along the flight results list; the screen scrolled downward approximately one page, revealing 3 additional evening flights starting from '$420'."
- **Modal / Alert State**:
"In Step 4, I tapped the 'Confirm Booking' button; an alert dialog titled 'Payment Method Required' appeared over the view with an 'Add Card' option."
---
# TASK
Generate the single-paragraph first-person summary for Step {{ step_number }} following all constraints above:
artemis/agents/flash/runner.py
+524
View File
@@ -0,0 +1,524 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Universal Multi-Model FlashRunner for Artemis.
Executes autonomous reactive mobile workflows across Google Gemini,
OpenAI GPT-4o/o3, Anthropic Claude 3.5/3.7, and OpenRouter endpoints.
"""
import asyncio
import base64
import json
from pathlib import Path
import uuid
from jinja2 import Template
from langchain_core.messages import (
BaseMessage,
HumanMessage,
SystemMessage,
ToolMessage,
)
from artemis.agents.flash.context_compressor import compress_flash_messages
from artemis.agents.flash.summarizer import VisualStepSummarizer
from artemis.agents.validator.tool_declarations import (
ACTION_TOOL_NAMES,
ASK_EXPLORER_TOOL,
CLICK_SEQUENCE_TOOL,
MobileActionExecutor,
REPORT_TASK_STATUS_TOOL,
VALIDATOR_TOOLS_DECLARATION,
capture_screenshot_and_parse_ui,
prune_intermediate_screenshots,
)
from artemis.config import StepSummarizerConfig, load_agent_config
from artemis.context import ArtemisContext
from artemis.controllers.unified_controller import UnifiedMobileController
from artemis.data_engine.trace import trace
from artemis.graph.perception import _check_injected_instruction_file
from artemis.graph.state import State
from artemis.services.llm import (
RobustChatModelWrapper,
get_google_llm,
get_llm,
invoke_llm_with_timeout_message,
)
from artemis.utils.coordinates import parse_swipe_parameters
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
class FlashRunner:
"""Reactive, ultra-fast agent execution loop supporting all VLM providers."""
def __init__(self, ctx: ArtemisContext, goal: str, max_turns: int | None = None):
self.ctx = ctx
self.goal = goal
try:
cfg = load_agent_config()
self.max_turns = max_turns if max_turns is not None else cfg.flash.max_turns
self.step_summarizer_cfg = cfg.flash.step_summarizer
except Exception:
self.max_turns = max_turns if max_turns is not None else 30
self.step_summarizer_cfg = StepSummarizerConfig()
self.controller = UnifiedMobileController(ctx)
self.executor = MobileActionExecutor(ctx, self.controller)
self.summarizer = (
VisualStepSummarizer(ctx, model_name=self.step_summarizer_cfg.model)
if self.step_summarizer_cfg.enabled
else None
)
def _get_tools(self) -> list:
tools = [t for t in VALIDATOR_TOOLS_DECLARATION if t.name != "report_failure_analysis"]
tools.insert(1, CLICK_SEQUENCE_TOOL)
tools.append(ASK_EXPLORER_TOOL)
tools.append(REPORT_TASK_STATUS_TOOL)
return tools
def _prune_intermediate_screenshots(self, messages: list[BaseMessage]) -> None:
"""Prunes binary screenshot blocks from intermediate observation messages."""
prune_intermediate_screenshots(messages)
@trace(type="agent", name="FlashRunner")
async def run(self, state: State) -> dict:
logger.info(f"Starting Artemis Flash reactive loop for goal: {self.goal}")
# 1. Initialize Universal LLM via Service Layer
try:
llm = get_llm(self.ctx, name="operator")
except Exception as e:
logger.warning(f"Failed to get operator LLM from config, using default: {e}")
llm = RobustChatModelWrapper(get_google_llm(model_name="gemini-2.5-flash"), self.ctx)
tools_declaration = self._get_tools()
# 2. Capture Initial State (Screenshot + UI Tree)
shot_path, img_bytes, xml_list = await capture_screenshot_and_parse_ui(
self.ctx, state, self.controller, skip_settling=False
)
state.latest_screenshot = shot_path
user_content: list[dict] = [
{"type": "text", "text": f"Your objective is: {self.goal}"},
]
if img_bytes:
img_b64 = base64.b64encode(img_bytes).decode("utf-8")
user_content.append(
{
"type": "image_url",
"image_url": {"url": f"data:image/jpeg;base64,{img_b64}"},
}
)
if xml_list:
user_content.append(
{
"type": "text",
"text": f"--- UI Element List ---\n{xml_list}",
}
)
user_content.append(
{
"type": "text",
"text": (
"CRITICAL RULE: In every single turn, you MUST FIRST output a natural"
" language reasoning/explanation paragraph BEFORE invoking any tool call."
),
}
)
# Render System Prompt
prompt_path = Path(__file__).parent / "flash_runner.md"
prompt_template = prompt_path.read_text(encoding="utf-8")
system_prompt = Template(prompt_template).render(goal=self.goal)
messages: list[BaseMessage] = [
SystemMessage(content=system_prompt),
HumanMessage(content=user_content),
]
turns = 0
final_report = None
current_pre_screenshot_bytes = img_bytes
current_xml_list = xml_list
while turns < self.max_turns:
turns += 1
logger.info(f"--- Artemis Flash Turn {turns}/{self.max_turns} ---")
if self.ctx.data_engine:
self.ctx.data_engine.allocate_step_id()
# Check for real-time injected instructions
if self.ctx.data_engine and self.ctx.data_engine.base_dir:
try:
injected_instruction = await asyncio.to_thread(
_check_injected_instruction_file,
str(self.ctx.data_engine.base_dir),
)
if injected_instruction:
messages.append(
HumanMessage(
content=(
"[REAL-TIME INJECTED INSTRUCTION from user]:"
f" {injected_instruction}\nYou MUST immediately follow"
" this instruction and adjust your plan/actions."
)
)
)
except Exception as e:
logger.warning(f"Failed to check injected instruction in FlashRunner: {e}")
# Compress previous history before LLM call to save context and inject objective summaries
if self.summarizer:
compress_flash_messages(
messages,
summarizer=self.summarizer,
prune_history_xml=self.step_summarizer_cfg.prune_history_xml,
)
else:
prune_intermediate_screenshots(messages)
# Tool restriction on the final turn
if turns == self.max_turns:
messages.append(
HumanMessage(
content=(
"[WARNING] This is your final turn; only"
" 'report_task_status' is available."
)
)
)
current_tools = [t for t in tools_declaration if t.name == "report_task_status"]
else:
current_tools = tools_declaration
# Bind active tools
bound_llm = llm.bind_tools(current_tools)
# Invoke Model with Streaming & Retry
response = None
max_retries = 3
for attempt in range(max_retries):
try:
async def run_stream():
full_response = None
async for chunk in bound_llm.astream(messages):
if full_response is None:
full_response = chunk
else:
full_response += chunk
return full_response
response = await invoke_llm_with_timeout_message(
run_stream(), timeout_seconds=10, hard_timeout=180
)
if response:
break
except Exception as e:
logger.warning(f"FlashRunner LLM turn failed on attempt {attempt + 1}: {e}")
if attempt == max_retries - 1:
raise e
await asyncio.sleep(2.0**attempt)
if response is None:
break
messages.append(response)
# Extract thought and tool calls
raw_text = response.content if isinstance(response.content, str) else ""
if isinstance(response.content, list):
raw_text = "".join(
b.get("text", "")
for b in response.content
if isinstance(b, dict) and "text" in b
)
# Extract token usage metadata from response
step_token_usage = None
if hasattr(response, "usage_metadata") and response.usage_metadata:
u = response.usage_metadata
pr = u.get("input_tokens") or u.get("prompt_tokens") or 0
co = u.get("output_tokens") or u.get("completion_tokens") or 0
to = u.get("total_tokens") or (pr + co)
if to > 0:
step_token_usage = {
"prompt_tokens": int(pr),
"completion_tokens": int(co),
"total_tokens": int(to),
}
elif hasattr(response, "response_metadata") and isinstance(
response.response_metadata, dict
):
u = response.response_metadata.get(
"usage_metadata"
) or response.response_metadata.get("token_usage")
if isinstance(u, dict):
pr = (
u.get("input_tokens")
or u.get("prompt_tokens")
or u.get("prompt_token_count")
or 0
)
co = (
u.get("output_tokens")
or u.get("completion_tokens")
or u.get("candidates_token_count")
or 0
)
to = u.get("total_tokens") or u.get("total_token_count") or (pr + co)
if to > 0:
step_token_usage = {
"prompt_tokens": int(pr),
"completion_tokens": int(co),
"total_tokens": int(to),
}
if not step_token_usage or step_token_usage.get("total_tokens", 0) <= 0:
calc_prompt_chars = 0
calc_images = 0
for msg in messages:
c = getattr(msg, "content", "")
if isinstance(c, str):
calc_prompt_chars += len(c)
elif isinstance(c, list):
for block in c:
if isinstance(block, dict):
if block.get("type") == "image_url" or "image_url" in block:
calc_images += 1
else:
calc_prompt_chars += len(str(block.get("text", "")))
else:
calc_prompt_chars += len(str(block))
prompt_tokens = (calc_prompt_chars // 4) + (calc_images * 258)
completion_tokens = len(raw_text) // 4
step_token_usage = {
"prompt_tokens": max(1, prompt_tokens),
"completion_tokens": max(1, completion_tokens),
"total_tokens": max(1, prompt_tokens + completion_tokens),
}
if self.ctx.data_engine:
current_step_id = getattr(self.ctx.data_engine, "current_step_id", None)
self.ctx.data_engine.record_trace(
type="llm_call",
name="FlashRunner",
payload={"token_usage": step_token_usage, "response": raw_text},
step_id=current_step_id,
status="success",
)
tool_calls = response.tool_calls or []
# Fallback text parsing if tool_calls not parsed natively
if not tool_calls and "```json" in raw_text:
try:
snippet = raw_text.split("```json")[1].split("```")[0].strip()
parsed = json.loads(snippet)
if isinstance(parsed, dict) and "name" in parsed:
tool_calls = [
{
"name": parsed["name"],
"args": parsed.get("args", {}),
"id": str(uuid.uuid4()),
}
]
except Exception:
pass
if not tool_calls:
logger.info(
f"FlashRunner received response without tool calls at turn {turns}:"
f" {raw_text[:100]}..."
)
if turns == self.max_turns:
final_report = {"status": "failed", "explanation": raw_text}
break
messages.append(
HumanMessage(
content=(
"You did not call any tools. Please make progress"
" by calling an action tool or 'report_task_status'."
)
)
)
continue
# Process tool calls
for tc in tool_calls:
name = tc["name"].split(":")[-1] if ":" in tc["name"] else tc["name"]
args = tc.get("args") or {}
tc_id = tc.get("id") or str(uuid.uuid4())
logger.info(f"Executing Flash tool: {name}({args})")
if name == "report_task_status":
final_report = args
if self.ctx.data_engine:
try:
if self.ctx.data_engine.current_step_id is None:
self.ctx.data_engine.allocate_step_id()
self.ctx.data_engine.record_step(
pre_screenshot_bytes=current_pre_screenshot_bytes,
ui_tree=current_xml_list,
action_taken={"action": "report_task_status", "args": args},
operator_raw_thinking=raw_text,
last_execution_result={
"result": "Task completed with final report."
},
extra_metadata={"token_usage": step_token_usage},
)
except Exception as step_err:
logger.warning(f"Error recording final step in FlashRunner: {step_err}")
messages.append(
ToolMessage(
tool_call_id=tc_id,
name=name,
content=json.dumps({"status": "acknowledged"}),
status="success",
)
)
if self.summarizer:
await self.summarizer.flush()
return final_report
try:
exec_result = await self.executor.execute(name, args, tc_id, state)
post_img_bytes = exec_result.screenshot_bytes
if not post_img_bytes and name in ACTION_TOOL_NAMES:
try:
controller = UnifiedMobileController(self.ctx)
screen_data = await controller.get_screen_data()
post_img_bytes = base64.b64decode(screen_data.base64)
if not exec_result.ui_elements_text:
exec_result.ui_elements_text = screen_data.elements
except Exception as shot_err:
logger.warning(
f"Failed to capture fallback screenshot in FlashRunner: {shot_err}"
)
# Record telemetry / step in DataEngine
if self.ctx.data_engine and name in ACTION_TOOL_NAMES:
try:
if self.ctx.data_engine.current_step_id is None:
self.ctx.data_engine.allocate_step_id()
# Extract and enrich coordinate metadata
norm_coords = None
norm_start = None
norm_end = None
if name == "swipe":
kind, target_val, _ = parse_swipe_parameters(args)
if (
kind == "coords"
and isinstance(target_val, list)
and len(target_val) == 4
):
norm_coords = target_val
norm_start = target_val[:2]
norm_end = target_val[2:]
elif kind == "direction" and isinstance(target_val, str):
g_lower = target_val.lower()
if "up" in g_lower:
norm_coords = [600, 800, 600, 200]
elif "down" in g_lower:
norm_coords = [600, 200, 600, 800]
elif "left" in g_lower:
norm_coords = [800, 500, 200, 500]
elif "right" in g_lower:
norm_coords = [200, 500, 800, 500]
if norm_coords:
norm_start = norm_coords[:2]
norm_end = norm_coords[2:]
elif name in ("click", "tap", "long_press", "input_text"):
target = args.get("target") or args.get("coordinates")
if isinstance(target, (list, tuple)) and len(target) == 2:
norm_coords = list(target)
elif isinstance(target, str):
nums = re.findall(r"-?\d+(?:\.\d+)?", target)
if len(nums) == 2:
norm_coords = [int(float(nums[0])), int(float(nums[1]))]
action_dict = {
"action": name,
"coordinates": (
args.get("target")
or args.get("coordinates")
or args.get("sequence")
or norm_coords
),
"args": args,
}
if norm_coords:
action_dict["normalized_coordinates"] = norm_coords
if norm_start and norm_end:
action_dict["normalized_start_coordinates"] = norm_start
action_dict["normalized_end_coordinates"] = norm_end
self.ctx.data_engine.record_step(
pre_screenshot_bytes=current_pre_screenshot_bytes,
post_screenshot_bytes=post_img_bytes,
ui_tree=(exec_result.ui_elements_text or current_xml_list),
action_taken=action_dict,
operator_raw_thinking=raw_text,
last_execution_result={"result": exec_result.text_summary},
extra_metadata={"token_usage": step_token_usage},
)
except Exception as step_err:
logger.warning(f"Error recording step in FlashRunner: {step_err}")
# ⚡ Non-blocking dispatch of objective visual transition summarizer
if self.summarizer and name in ACTION_TOOL_NAMES:
self.summarizer.dispatch(
step_number=turns,
action_name=name,
action_args=args,
pre_img_bytes=current_pre_screenshot_bytes,
post_img_bytes=post_img_bytes,
exec_outcome=exec_result.text_summary,
)
if post_img_bytes:
current_pre_screenshot_bytes = post_img_bytes
if exec_result.ui_elements_text:
current_xml_list = exec_result.ui_elements_text
messages.append(exec_result.to_langchain_tool_message())
except Exception as e:
logger.error(f"Error executing tool {name}: {e}")
messages.append(
ToolMessage(
tool_call_id=tc_id,
name=name,
content=f"Error executing tool {name}: {e}",
status="error",
)
)
if self.summarizer:
await self.summarizer.flush()
return final_report or {
"status": "failed",
"explanation": "Max turns reached without final status report.",
}
artemis/agents/flash/summarizer.py
+226
View File
@@ -0,0 +1,226 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Asynchronous Visual Step Summarizer for Artemis Flash profile.
Processes Before/After screenshot pairs and action metadata in the background
to generate high-density, strictly objective visual state transition summaries.
Ensures zero blocking latency for the main runner and includes out-of-order self-healing retries.
"""
import asyncio
import base64
from pathlib import Path
from typing import Any
from jinja2 import Template
from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage
from artemis.context import ArtemisContext
from artemis.services.llm import get_google_llm, get_llm
from artemis.utils.logger import get_logger
from artemis.utils.visualization import draw_action_overlay_on_image
logger = get_logger(__name__)
class VisualStepSummarizer:
"""Non-blocking background worker generating objective visual step summaries.
Key design properties:
1. Zero-blocking dispatch: The main Flash runner dispatches and proceeds immediately.
2. Graceful fallback: If a step summary is not ready, it cleanly falls back to the original action text.
3. Self-healing re-trigger: If a previous step's summary is stalled or lost while subsequent steps
complete, it automatically re-dispatches the stalled step.
"""
def __init__(self, ctx: ArtemisContext, model_name: str | None = None):
self.ctx = ctx
self._summaries: dict[int, str] = {}
self._pending_tasks: dict[int, asyncio.Task] = {}
self._step_inputs: dict[int, dict[str, Any]] = {}
self._retry_counts: dict[int, int] = {}
self._max_retries = 2
# Initialize lightweight VLM: prioritize explicit model_name
target_model = model_name or "gemini-2.5-flash-lite"
try:
if model_name:
self._llm = get_google_llm(model_name=target_model, temperature=0.0)
else:
self._llm = get_llm(ctx, name="summarizer", is_utils=True)
except Exception:
self._llm = get_google_llm(model_name=target_model, temperature=0.0)
# Load system prompt template
prompt_path = Path(__file__).parent / "flash_summarizer.md"
if prompt_path.exists():
self._prompt_template = prompt_path.read_text(encoding="utf-8")
else:
self._prompt_template = (
"You are the Step Summarizer for an Android UI automation agent.\n"
"Synthesize the physical action and visual delta between BEFORE and AFTER screens in exactly ONE "
"continuous first-person paragraph using 'I' (e.g., 'In Step {{ step_number }}, I tapped... and observed...').\n"
"Strictly avoid subjective validation words: successfully, completed, failed, achieved, navigated to."
)
def dispatch(
self,
step_number: int,
action_name: str,
action_args: dict[str, Any],
pre_img_bytes: bytes | None,
post_img_bytes: bytes | None,
exec_outcome: str,
) -> None:
"""Dispatches an asynchronous summarization task without blocking the caller."""
self._step_inputs[step_number] = {
"action_name": action_name,
"action_args": action_args,
"pre_img_bytes": pre_img_bytes,
"post_img_bytes": post_img_bytes,
"exec_outcome": exec_outcome,
}
task = asyncio.create_task(self._run_summary(step_number))
self._pending_tasks[step_number] = task
# Check for and re-trigger any previous stalled steps
self._check_and_retrigger_stalled_steps(current_step=step_number)
def _check_and_retrigger_stalled_steps(self, current_step: int) -> None:
"""Re-dispatches any previous steps that remain unsummarized while later steps arrive."""
for past_step in range(1, current_step):
if past_step not in self._summaries and past_step in self._step_inputs:
task = self._pending_tasks.get(past_step)
retries = self._retry_counts.get(past_step, 0)
if (task is None or task.done()) and retries < self._max_retries:
self._retry_counts[past_step] = retries + 1
logger.info(
f"VisualStepSummarizer: Retriggering stalled summary for Step {past_step} "
f"(attempt {self._retry_counts[past_step] + 1})"
)
self._pending_tasks[past_step] = asyncio.create_task(
self._run_summary(past_step)
)
async def _run_summary(self, step_number: int) -> None:
"""Executes the lightweight VLM call to summarize the step transition."""
input_data = self._step_inputs.get(step_number)
if not input_data:
return
action_name = input_data["action_name"]
action_args = input_data["action_args"]
pre_bytes = input_data["pre_img_bytes"]
post_bytes = input_data["post_img_bytes"]
exec_outcome = input_data["exec_outcome"]
try:
rendered_prompt = Template(self._prompt_template).render(step_number=step_number)
content_blocks: list[dict[str, Any]] = [
{
"type": "text",
"text": (
f"Step {step_number} Physical Action: {action_name}({action_args})\n"
f"Controller Outcome: {exec_outcome}"
),
}
]
if pre_bytes:
# 🎨 Visually mark the exact action (tap ripple, sequence numbers, swipe arrow) on the BEFORE screenshot
annotated_pre_bytes = draw_action_overlay_on_image(
image_bytes=pre_bytes,
action_name=action_name,
action_args=action_args,
)
b64_pre = base64.b64encode(annotated_pre_bytes).decode("utf-8")
content_blocks.append(
{
"type": "text",
"text": "--- [1] BEFORE ACTION SCREEN (Action Marked Visually in Red) ---",
}
)
content_blocks.append(
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64_pre}"}}
)
if post_bytes:
b64_post = base64.b64encode(post_bytes).decode("utf-8")
content_blocks.append({"type": "text", "text": "--- [2] AFTER ACTION SCREEN ---"})
content_blocks.append(
{
"type": "image_url",
"image_url": {"url": f"data:image/jpeg;base64,{b64_post}"},
}
)
messages: list[BaseMessage] = [
SystemMessage(content=rendered_prompt),
HumanMessage(content=content_blocks),
]
response = await asyncio.wait_for(self._llm.ainvoke(messages), timeout=25.0)
summary_raw = response.content if isinstance(response.content, str) else ""
if isinstance(response.content, list):
summary_raw = "".join(
b.get("text", "")
for b in response.content
if isinstance(b, dict) and "text" in b
)
summary_text = summary_raw.strip()
if summary_text:
self._summaries[step_number] = summary_text
logger.info(
f"VisualStepSummarizer: Generated summary for Step {step_number}: {summary_text[:80]}..."
)
# Free binary image buffers from memory once summary is secured
if step_number in self._step_inputs:
self._step_inputs[step_number]["pre_img_bytes"] = None
self._step_inputs[step_number]["post_img_bytes"] = None
# Update DataEngine telemetry if active
if self.ctx.data_engine:
try:
self.ctx.data_engine.update_step_summary(step_number, summary_text)
except Exception as de_err:
logger.debug(f"DataEngine step summary update skipped: {de_err}")
except Exception as e:
logger.warning(
f"VisualStepSummarizer: Error generating summary for step {step_number}: {e}"
)
if self._retry_counts.get(step_number, 0) >= self._max_retries:
# Max retries reached, free memory buffer
if step_number in self._step_inputs:
self._step_inputs[step_number]["pre_img_bytes"] = None
self._step_inputs[step_number]["post_img_bytes"] = None
def get_summary(self, step_number: int, fallback_text: str | None = None) -> str | None:
"""Retrieves summarized state for step_number, or fallback_text if not ready yet."""
return self._summaries.get(step_number, fallback_text)
def has_summary(self, step_number: int) -> bool:
"""Checks if a step summary has completed."""
return step_number in self._summaries
async def flush(self) -> None:
"""Flushes and awaits all pending background summary tasks upon completion."""
tasks = [t for t in self._pending_tasks.values() if not t.done()]
if tasks:
logger.info(f"VisualStepSummarizer: Flushing {len(tasks)} pending summary tasks...")
await asyncio.gather(*tasks, return_exceptions=True)