somew1nd
206a9ce921
ARTEMIS is an autonomous multimodal Android agent and mobile UI automation platform engineered for physical devices and emulators. It combines state-of-the-art multimodal Vision-Language Models (VLMs) with hierarchical closed-loop planning, resilient self-healing, and deep Android subsystem integration. Key Features & Architectural Highlights: - Dual-Engine Orchestration: * Pro Engine: Multi-agent closed-loop (Planner -> Operator -> Validator/Failure Analyzer -> Summarizer) powered by LangGraph. * Flash Engine: Ultra-fast reactive loop for deterministic, low-latency UI execution (<35 steps). - Multimodal Perception & Grounding: * Hybrid visual screenshot grounding, real-time OCR extraction, and XML accessibility hierarchy parsing. * Dynamic-First, Coordinate-Fallback locating strategy for maximum resilience against UI drift. - Autonomous Self-Healing & Diagnostics: * Failure analyzer and safety net for popup dismissal, crash recovery, and auto-retry. * Diagnostic sub-agents for log analysis, video recording inspection, and object detection. - Unified CLI & Python SDK: * Rich CLI with commands: init, doctor, run, trace, batch, bench, server, and mcp. * Fluent Python SDK with Pydantic structured output models and package locking. - Multi-Provider LLM Architecture: * Support for Google Gemini, OpenAI, Anthropic Claude, Vertex AI, OpenRouter, xAI Grok, and local endpoints (Ollama/vLLM). * Centralized configuration in config/artemis.jsonc. - Native MCP & Observability: * Model Context Protocol (MCP) server endpoints for AI IDEs. * Full trace recording with before/after screenshots, action overlays, and execution replays. Change-Id: I430b93ee2b76b54cc5a3c6eeccd15d2cf29830a1
461 lines
18 KiB
Python
461 lines
18 KiB
Python
# Copyright 2026 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
from typing import Any
|
|
|
|
from jinja2 import Template
|
|
from langchain_core.messages import HumanMessage, SystemMessage, ToolMessage
|
|
from langchain_core.tools import tool
|
|
|
|
from pydantic import BaseModel, Field
|
|
|
|
from artemis.constants import CHECKER_MAX_ITERATIONS
|
|
from artemis.context import ArtemisContext
|
|
from artemis.controllers.unified_controller import UnifiedMobileController
|
|
from artemis.data_engine.trace import (
|
|
TraceSpan,
|
|
trace,
|
|
trace_langchain_tool,
|
|
)
|
|
from artemis.services.llm import get_llm, invoke_llm_with_timeout_message
|
|
from artemis.tools.index import get_tool_by_name
|
|
from artemis.tools.scratchpad import get_read_note_tool_pure
|
|
from artemis.tools.tool_wrapper import (
|
|
get_tool_result_content,
|
|
invoke_tool_with_injection,
|
|
)
|
|
from artemis.utils.logger import get_logger
|
|
from artemis.utils.notes import get_note_file_path, get_notes_dir
|
|
from artemis.utils.ocr_api import perform_ocr
|
|
from artemis.utils.ocr_xml_fusion import (
|
|
_crop_image_remove_status_bar,
|
|
_detect_status_bar_height,
|
|
_map_coordinates_back,
|
|
fuse_ocr_with_xml,
|
|
)
|
|
from artemis.utils.task_tree import (
|
|
build_plan_and_history,
|
|
get_recent_subgoal_hashes,
|
|
)
|
|
from artemis.utils.verification import (
|
|
append_verification_chat,
|
|
get_verification_chat_path,
|
|
get_verification_chat_rounds,
|
|
read_verification_chat,
|
|
)
|
|
from artemis.utils.text import safe_extract_text
|
|
from artemis.utils.visualization import format_minimal_list_with_elements
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
|
|
class CheckerResult(BaseModel):
|
|
"""Structured output contract for Checker verification."""
|
|
|
|
success: bool = Field(
|
|
description="Whether the active subgoal was successfully completed based on visual and note evidence."
|
|
)
|
|
reason: str = Field(
|
|
description="Detailed explanation of the result. If failed, specify what is missing or wrong and give actionable hints for the Operator."
|
|
)
|
|
|
|
|
|
@trace(type="agent", name="checker")
|
|
async def run_async_check(
|
|
ctx: ArtemisContext,
|
|
subgoal_text: str,
|
|
subgoal_hash: str | None = None,
|
|
raw_perception_data: dict[str, Any] | None = None,
|
|
latest_ui_hierarchy: list[Any] | None = None,
|
|
):
|
|
"""Runs the checker verification as a background task."""
|
|
logger.info(f"Starting async Checker for: {subgoal_text}")
|
|
|
|
if not ctx.data_engine:
|
|
logger.warning("DataEngine not available, skipping checker.")
|
|
return {"status": "success", "reason": "DataEngine unavailable."}
|
|
|
|
notes_dir = get_notes_dir(ctx.data_engine.base_dir)
|
|
status_path = notes_dir / "checker_status.json"
|
|
|
|
if ctx.execution_setup and ctx.execution_setup.disable_checker:
|
|
logger.info("Checker is disabled via flag. Skipping.")
|
|
try:
|
|
status_path.write_text(
|
|
json.dumps({"status": "success", "reason": "Checker disabled."}),
|
|
encoding="utf-8",
|
|
)
|
|
except Exception:
|
|
pass
|
|
return {"status": "success", "reason": "Checker disabled."}
|
|
|
|
# 1. Set status to running
|
|
try:
|
|
status_path.write_text(
|
|
json.dumps({"status": "running", "subgoal": subgoal_text}),
|
|
encoding="utf-8",
|
|
)
|
|
logger.info(f"Wrote checker status 'running' to {status_path}")
|
|
except Exception as e:
|
|
logger.error(f"Failed to write checker status: {e}")
|
|
return {"status": "success", "reason": f"Status write error: {e}"}
|
|
|
|
# 2. Run Checker logic
|
|
try:
|
|
# Retrieve perception data: directly reuse upstream perception if available
|
|
if isinstance(raw_perception_data, dict) and raw_perception_data.get("screenshot_b64"):
|
|
screenshot_b64 = raw_perception_data["screenshot_b64"]
|
|
width = raw_perception_data.get("width", 1080)
|
|
height = raw_perception_data.get("height", 2400)
|
|
fused_xml = latest_ui_hierarchy or raw_perception_data.get("xml_hierarchy") or []
|
|
elif (
|
|
hasattr(ctx, "latest_perception_data")
|
|
and isinstance(ctx.latest_perception_data, dict)
|
|
and ctx.latest_perception_data
|
|
):
|
|
screenshot_b64 = ctx.latest_perception_data.get("screenshot_b64")
|
|
width = ctx.latest_perception_data.get("width", 1080)
|
|
height = ctx.latest_perception_data.get("height", 2400)
|
|
fused_xml = ctx.latest_perception_data.get("latest_ui_hierarchy") or []
|
|
else:
|
|
# Fallback to direct controller query
|
|
controller = UnifiedMobileController(ctx)
|
|
device_data = await controller.get_screen_data()
|
|
screenshot_b64 = device_data.base64
|
|
xml_hierarchy = device_data.elements
|
|
width = device_data.width
|
|
height = device_data.height
|
|
|
|
ocr_results = []
|
|
try:
|
|
status_bar_height = _detect_status_bar_height(xml_hierarchy, height)
|
|
if status_bar_height > 0:
|
|
cropped_b64, _, _ = _crop_image_remove_status_bar(
|
|
screenshot_b64, status_bar_height
|
|
)
|
|
raw_ocr_results = await perform_ocr(cropped_b64)
|
|
ocr_results = _map_coordinates_back(raw_ocr_results, status_bar_height)
|
|
else:
|
|
ocr_results = await perform_ocr(screenshot_b64)
|
|
except Exception as e:
|
|
logger.error(f"Checker OCR failed, proceeding without OCR: {e}")
|
|
|
|
fused_xml = fuse_ocr_with_xml(xml_hierarchy, ocr_results)
|
|
|
|
minimal_list, elements, labels = format_minimal_list_with_elements(fused_xml, width, height)
|
|
|
|
# Build task tree
|
|
task_plan = "No task plan yet."
|
|
task_plan_path = get_note_file_path(ctx.data_engine.base_dir, "task_plan")
|
|
if task_plan_path.exists():
|
|
try:
|
|
task_plan = task_plan_path.read_text(encoding="utf-8")
|
|
except Exception as e:
|
|
logger.error(f"Failed to read task plan for Checker: {e}")
|
|
|
|
steps = ctx.data_engine.get_agent_friendly_steps() if ctx.data_engine else []
|
|
|
|
# Calculate exact subgoal hash from subgoal_text directly to avoid 'default' placeholder
|
|
if not subgoal_hash or subgoal_hash == "default":
|
|
clean_subgoal = safe_extract_text(subgoal_text).strip()
|
|
subgoal_hash = hashlib.md5(clean_subgoal.encode("utf-8")).hexdigest()
|
|
|
|
keep_hashes = get_recent_subgoal_hashes(steps, subgoal_hash, ctx.data_engine.base_dir)
|
|
|
|
plan_and_history = build_plan_and_history(
|
|
task_plan, steps, subgoal_hash, keep_subgoal_hashes=keep_hashes
|
|
)
|
|
|
|
chat_path = get_verification_chat_path(ctx.data_engine.base_dir, subgoal_hash)
|
|
turns = read_verification_chat(chat_path)
|
|
|
|
# Calculate round number
|
|
max_op, max_chk = get_verification_chat_rounds(turns)
|
|
round_num = max(max_op, max_chk + 1)
|
|
if round_num == 0:
|
|
round_num = 1
|
|
|
|
# Render dialogue history for prompt
|
|
dialogue_lines = []
|
|
for t in turns:
|
|
role = "Operator" if t["role"] == "operator" else "Checker"
|
|
dialogue_lines.append(f"**{role} (Round {t['round']})**:\n{t['content']}")
|
|
dialogue_history = (
|
|
"\n\n".join(dialogue_lines) if dialogue_lines else "No previous dialogue."
|
|
)
|
|
|
|
prompts = {}
|
|
prompts_path = Path(__file__).parent / "checker.json"
|
|
if prompts_path.exists():
|
|
try:
|
|
prompts = json.loads(prompts_path.read_text(encoding="utf-8"))
|
|
except Exception as e:
|
|
logger.error(f"Failed to load checker prompts: {e}")
|
|
|
|
prompt_template = prompts.get(
|
|
"checker_prompt",
|
|
"You are the Checker. Your job is to verify if the current subtask"
|
|
" was successfully completed.",
|
|
)
|
|
|
|
full_prompt = Template(prompt_template).render(
|
|
subgoal=subgoal_text,
|
|
plan_and_history=plan_and_history,
|
|
dialogue_history=dialogue_history if dialogue_history else "No previous dialogue.",
|
|
minimal_list=minimal_list,
|
|
)
|
|
|
|
llm = get_llm(ctx=ctx, name="checker")
|
|
read_note = get_read_note_tool_pure(ctx)
|
|
|
|
@tool
|
|
def submit_verification(
|
|
success: bool,
|
|
reason: str,
|
|
) -> str:
|
|
"""[TERMINAL] Submit the final verification outcome (success or failure with reason). This completes verification."""
|
|
return f"Verification decision: success={success}, reason={reason}"
|
|
|
|
tools_to_bind = [
|
|
trace_langchain_tool(read_note, ctx),
|
|
submit_verification,
|
|
]
|
|
|
|
content = [
|
|
{"type": "text", "text": full_prompt},
|
|
{"type": "text", "text": "--- Current Screenshot ---"},
|
|
{
|
|
"type": "image_url",
|
|
"image_url": {"url": f"data:image/jpeg;base64,{screenshot_b64}"},
|
|
},
|
|
]
|
|
|
|
messages = [
|
|
SystemMessage(
|
|
content=prompts.get(
|
|
"checker_system",
|
|
"You are the Checker agent. Your objective is to accurately"
|
|
" assess whether the active subgoal was successfully"
|
|
" completed.",
|
|
)
|
|
),
|
|
HumanMessage(content=content),
|
|
]
|
|
|
|
max_iterations = (
|
|
getattr(
|
|
ctx.execution_setup,
|
|
"checker_max_iterations",
|
|
CHECKER_MAX_ITERATIONS,
|
|
)
|
|
if ctx.execution_setup
|
|
else CHECKER_MAX_ITERATIONS
|
|
)
|
|
|
|
success = True
|
|
reason = "Checker completed."
|
|
|
|
for i in range(max_iterations):
|
|
is_final_round = i == max_iterations - 1
|
|
|
|
if is_final_round:
|
|
messages.append(
|
|
HumanMessage(
|
|
content=(
|
|
"This is your final iteration; all tools are"
|
|
" stripped, and you must provide your final structured verification result."
|
|
)
|
|
)
|
|
)
|
|
structured_llm = llm.with_structured_output(CheckerResult)
|
|
structured_res: CheckerResult = await invoke_llm_with_timeout_message(
|
|
structured_llm.ainvoke(messages)
|
|
)
|
|
if isinstance(structured_res, CheckerResult):
|
|
success = structured_res.success
|
|
reason = structured_res.reason
|
|
elif isinstance(structured_res, dict):
|
|
success = structured_res.get("success", True)
|
|
reason = structured_res.get("reason", "Verification completed.")
|
|
break
|
|
else:
|
|
if i > 0:
|
|
messages.append(
|
|
HumanMessage(
|
|
content=(
|
|
"You have not completed verification yet"
|
|
f" (iteration {i + 1} of {max_iterations})."
|
|
)
|
|
)
|
|
)
|
|
active_llm = llm.bind_tools(tools=tools_to_bind)
|
|
|
|
async def run_stream():
|
|
full_response = None
|
|
async for chunk in active_llm.astream(messages):
|
|
if full_response is None:
|
|
full_response = chunk
|
|
else:
|
|
full_response += chunk
|
|
return full_response
|
|
|
|
response = await invoke_llm_with_timeout_message(run_stream())
|
|
|
|
# Check if submit_verification was invoked
|
|
submit_tc = next(
|
|
(
|
|
tc
|
|
for tc in response.tool_calls
|
|
if tc["name"].endswith("submit_verification")
|
|
),
|
|
None,
|
|
)
|
|
if submit_tc:
|
|
tc_args = dict(submit_tc["args"])
|
|
success = bool(tc_args.get("success", True))
|
|
reason = str(tc_args.get("reason", "Verification submitted."))
|
|
break
|
|
|
|
if not response.tool_calls:
|
|
# Model answered in plain text; use structured_llm directly
|
|
structured_llm = llm.with_structured_output(CheckerResult)
|
|
structured_res: CheckerResult = await invoke_llm_with_timeout_message(
|
|
structured_llm.ainvoke(messages + [response])
|
|
)
|
|
if isinstance(structured_res, CheckerResult):
|
|
success = structured_res.success
|
|
reason = structured_res.reason
|
|
elif isinstance(structured_res, dict):
|
|
success = structured_res.get("success", True)
|
|
reason = structured_res.get("reason", "Verification completed.")
|
|
break
|
|
|
|
messages.append(response)
|
|
|
|
async def run_tool(tc):
|
|
tool_name = tc["name"].split(":")[-1] if ":" in tc["name"] else tc["name"]
|
|
logger.info(f"Checker requested tool: {tool_name}")
|
|
|
|
result_str = ""
|
|
tool_status = "success"
|
|
try:
|
|
if ":" in tool_name:
|
|
tool_to_run = get_tool_by_name(tool_name, traced_tools)
|
|
else:
|
|
tool_to_run = next(
|
|
(t for t in traced_tools if t.name == tool_name),
|
|
None,
|
|
)
|
|
if tool_to_run:
|
|
args = dict(tc["args"])
|
|
with TraceSpan(name=tool_name, ctx=ctx) as span:
|
|
span.payload = {"args": args}
|
|
try:
|
|
result_obj = await invoke_tool_with_injection(
|
|
tool=tool_to_run,
|
|
args=args,
|
|
tool_call_id=tc["id"],
|
|
)
|
|
result_str = get_tool_result_content(result_obj)
|
|
span.result = result_str
|
|
if result_str.startswith("Error"):
|
|
tool_status = "error"
|
|
span.status = "failed"
|
|
span.error = result_str
|
|
except Exception as err:
|
|
tool_status = "error"
|
|
span.status = "failed"
|
|
span.error = str(err)
|
|
raise err
|
|
else:
|
|
result_str = f"Error: Tool {tool_name} not supported"
|
|
tool_status = "error"
|
|
except Exception as err:
|
|
logger.error(f"Error running tool {tool_name}: {err}")
|
|
result_str = f"Error running tool {tool_name}: {err}"
|
|
tool_status = "error"
|
|
|
|
return ToolMessage(
|
|
tool_call_id=tc["id"],
|
|
content=result_str,
|
|
status=tool_status,
|
|
)
|
|
|
|
tool_outputs = await asyncio.gather(*(run_tool(tc) for tc in response.tool_calls))
|
|
for tm in tool_outputs:
|
|
messages.append(tm)
|
|
|
|
logger.info(
|
|
f"Async Checker result for '{subgoal_text}': Success={success}, Reason={reason}"
|
|
)
|
|
|
|
# Update status file
|
|
status_path.write_text(
|
|
json.dumps({"status": "success" if success else "failed", "reason": reason}),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
# Clean up dialogue file if successful
|
|
if success:
|
|
if chat_path.exists():
|
|
logger.info(f"Checker succeeded. Cleaning up dialogue file: {chat_path}")
|
|
chat_path.unlink()
|
|
|
|
# If failed, revert status in task_plan.md and write feedback
|
|
if not success:
|
|
task_plan_path = get_note_file_path(ctx.data_engine.base_dir, "task_plan")
|
|
if task_plan_path.exists():
|
|
task_plan = task_plan_path.read_text(encoding="utf-8")
|
|
lines = task_plan.split("\n")
|
|
|
|
for i in range(len(lines) - 1, -1, -1):
|
|
if lines[i].startswith("- ["):
|
|
task_content = safe_extract_text(lines[i][5:]).strip()
|
|
safe_subgoal = safe_extract_text(subgoal_text).strip()
|
|
if task_content == safe_subgoal:
|
|
lines[i] = re.sub(r"^(- \[)[x/ ](\])", r"\g<1>/\g<2>", lines[i])
|
|
logger.info(
|
|
"Matched top-level subgoal by content."
|
|
f" Reverted status to [/]: {subgoal_text}"
|
|
)
|
|
break
|
|
task_plan_path.write_text("\n".join(lines), encoding="utf-8")
|
|
|
|
append_verification_chat(
|
|
chat_path,
|
|
"checker",
|
|
f"Status: Failed.\nReason: {reason}",
|
|
round_num,
|
|
)
|
|
|
|
return {"status": "success" if success else "failed", "reason": reason}
|
|
|
|
except Exception as e:
|
|
logger.error(f"Async Checker encountered exception (fail-open): {e}")
|
|
status_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"status": "success",
|
|
"reason": f"Checker Fail-Open: {e}",
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
return {"status": "success", "reason": f"Checker Fail-Open: {e}"}
|