ARTEMIS is an autonomous multimodal Android agent and mobile UI automation platform engineered for physical devices and emulators. It combines state-of-the-art multimodal Vision-Language Models (VLMs) with hierarchical closed-loop planning, resilient self-healing, and deep Android subsystem integration. Key Features & Architectural Highlights: - Dual-Engine Orchestration: * Pro Engine: Multi-agent closed-loop (Planner -> Operator -> Validator/Failure Analyzer -> Summarizer) powered by LangGraph. * Flash Engine: Ultra-fast reactive loop for deterministic, low-latency UI execution (<35 steps). - Multimodal Perception & Grounding: * Hybrid visual screenshot grounding, real-time OCR extraction, and XML accessibility hierarchy parsing. * Dynamic-First, Coordinate-Fallback locating strategy for maximum resilience against UI drift. - Autonomous Self-Healing & Diagnostics: * Failure analyzer and safety net for popup dismissal, crash recovery, and auto-retry. * Diagnostic sub-agents for log analysis, video recording inspection, and object detection. - Unified CLI & Python SDK: * Rich CLI with commands: init, doctor, run, trace, batch, bench, server, and mcp. * Fluent Python SDK with Pydantic structured output models and package locking. - Multi-Provider LLM Architecture: * Support for Google Gemini, OpenAI, Anthropic Claude, Vertex AI, OpenRouter, xAI Grok, and local endpoints (Ollama/vLLM). * Centralized configuration in config/artemis.jsonc. - Native MCP & Observability: * Model Context Protocol (MCP) server endpoints for AI IDEs. * Full trace recording with before/after screenshots, action overlays, and execution replays. Change-Id: I430b93ee2b76b54cc5a3c6eeccd15d2cf29830a1
163 lines
6.1 KiB
Python
163 lines
6.1 KiB
Python
# Copyright 2026 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
from pathlib import Path
|
|
from typing import Annotated
|
|
from langchain_core.messages import HumanMessage, SystemMessage, ToolMessage
|
|
from langchain_core.tools import tool
|
|
from artemis.context import ArtemisContext
|
|
from artemis.services.llm import get_llm, invoke_llm_with_timeout_message
|
|
from artemis.tools.mobile.search_logs import search_and_merge_logs
|
|
from artemis.utils.logger import get_logger
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
|
|
class TaskOutputAnalyzerNode:
|
|
"""Agent that analyzes large command/task execution outputs using LLM and tools."""
|
|
|
|
def __init__(self, ctx: ArtemisContext):
|
|
self.ctx = ctx
|
|
|
|
async def run(
|
|
self,
|
|
command: str,
|
|
output_text: str,
|
|
query: str,
|
|
) -> str:
|
|
prompt_path = Path(__file__).parent.joinpath("output_analyzer.md")
|
|
if not prompt_path.exists():
|
|
raise FileNotFoundError(f"System prompt file not found at {prompt_path}")
|
|
|
|
system_prompt = prompt_path.read_text(encoding="utf-8")
|
|
|
|
# Get the LLM for output_analyzer (falls back to log_analyzer)
|
|
llm = get_llm(ctx=self.ctx, name="output_analyzer")
|
|
|
|
# Local tools created with closure on output_text
|
|
@tool
|
|
def read_task_output(
|
|
start_line: Annotated[int, "The 1-indexed line number to start reading from."],
|
|
end_line: Annotated[int, "The 1-indexed line number to stop reading at (inclusive)."],
|
|
) -> str:
|
|
"""Reads a specific range of lines from the task output.
|
|
|
|
Use this tool when you know the line number range to read a specific
|
|
portion of the output.
|
|
"""
|
|
lines = output_text.splitlines()
|
|
if not lines:
|
|
return "Task output is empty."
|
|
start = max(1, start_line)
|
|
end = min(len(lines), end_line)
|
|
if start > end:
|
|
return (
|
|
f"Error: start_line {start_line} is greater than end_line"
|
|
f" {end_line} or task output length {len(lines)}."
|
|
)
|
|
segment = lines[start - 1 : end]
|
|
return "\n".join(segment)
|
|
|
|
@tool
|
|
def search_task_output(
|
|
keyword: Annotated[
|
|
str,
|
|
"The keyword or regular expression to search for in the output.",
|
|
],
|
|
is_regex: Annotated[
|
|
bool,
|
|
"Whether to treat the keyword as a regular expression. Default is False.",
|
|
] = False,
|
|
context_lines: Annotated[
|
|
int,
|
|
"Number of context lines to include before and after the"
|
|
" matched line. Default is 5.",
|
|
] = 5,
|
|
) -> str:
|
|
"""Searches the task output for a keyword or regular expression, returning matching lines and context.
|
|
|
|
Use this tool to quickly locate error messages, warnings, or
|
|
specific key-value pairs in large outputs.
|
|
"""
|
|
return search_and_merge_logs(
|
|
logs=output_text,
|
|
keyword=keyword,
|
|
context_lines=context_lines,
|
|
is_regex=is_regex,
|
|
)
|
|
|
|
tools = [read_task_output, search_task_output]
|
|
llm = llm.bind_tools(tools=tools)
|
|
|
|
# Optimize prompt context with preview if output is large
|
|
lines = output_text.splitlines()
|
|
if len(lines) <= 400:
|
|
log_preview = f"Full Output:\n{output_text}"
|
|
else:
|
|
first_part = "\n".join(lines[:200])
|
|
last_part = "\n".join(lines[-200:])
|
|
log_preview = (
|
|
f"Note: Output is very large ({len(lines)} lines). Here is a"
|
|
" preview of the start and end of the output:\n\n--- START OF"
|
|
f" OUTPUT ---\n{first_part}\n--- TRUNCATED {len(lines) - 400}"
|
|
f" LINES ---\n--- END OF OUTPUT ---\n{last_part}\n\nYou can use"
|
|
" `read_task_output` or `search_task_output` to fetch any"
|
|
" lines in the middle if needed."
|
|
)
|
|
|
|
user_content = f"Command Executed: {command}\n\n{log_preview}\n\nQuestion: {query}"
|
|
|
|
current_messages = [
|
|
SystemMessage(content=system_prompt),
|
|
HumanMessage(content=user_content),
|
|
]
|
|
|
|
max_iterations = 5
|
|
outcome = "Failed to analyze output."
|
|
|
|
for _ in range(max_iterations):
|
|
response = await invoke_llm_with_timeout_message(llm.ainvoke(current_messages))
|
|
|
|
if response.content:
|
|
outcome = response.content
|
|
|
|
if not response.tool_calls:
|
|
break
|
|
|
|
current_messages.append(response)
|
|
|
|
for tc in response.tool_calls:
|
|
tool_name = tc["name"]
|
|
args = tc["args"]
|
|
logger.info(f"Output Analyzer requested tool: {tool_name}")
|
|
|
|
tool_to_run = next((t for t in tools if t.name == tool_name), None)
|
|
if tool_to_run:
|
|
try:
|
|
result = tool_to_run.invoke(args)
|
|
except Exception as e:
|
|
result = f"Failed to run tool {tool_name}: {e}"
|
|
else:
|
|
result = f"Error: Tool {tool_name} not found."
|
|
|
|
current_messages.append(
|
|
ToolMessage(
|
|
tool_call_id=tc["id"],
|
|
content=str(result),
|
|
status="success" if not str(result).startswith("Error") else "error",
|
|
)
|
|
)
|
|
|
|
return outcome
|