feat: initial release of ARTEMIS Android Agent & Automation Platform
ARTEMIS is an autonomous multimodal Android agent and mobile UI automation platform engineered for physical devices and emulators. It combines state-of-the-art multimodal Vision-Language Models (VLMs) with hierarchical closed-loop planning, resilient self-healing, and deep Android subsystem integration. Key Features & Architectural Highlights: - Dual-Engine Orchestration: * Pro Engine: Multi-agent closed-loop (Planner -> Operator -> Validator/Failure Analyzer -> Summarizer) powered by LangGraph. * Flash Engine: Ultra-fast reactive loop for deterministic, low-latency UI execution (<35 steps). - Multimodal Perception & Grounding: * Hybrid visual screenshot grounding, real-time OCR extraction, and XML accessibility hierarchy parsing. * Dynamic-First, Coordinate-Fallback locating strategy for maximum resilience against UI drift. - Autonomous Self-Healing & Diagnostics: * Failure analyzer and safety net for popup dismissal, crash recovery, and auto-retry. * Diagnostic sub-agents for log analysis, video recording inspection, and object detection. - Unified CLI & Python SDK: * Rich CLI with commands: init, doctor, run, trace, batch, bench, server, and mcp. * Fluent Python SDK with Pydantic structured output models and package locking. - Multi-Provider LLM Architecture: * Support for Google Gemini, OpenAI, Anthropic Claude, Vertex AI, OpenRouter, xAI Grok, and local endpoints (Ollama/vLLM). * Centralized configuration in config/artemis.jsonc. - Native MCP & Observability: * Model Context Protocol (MCP) server endpoints for AI IDEs. * Full trace recording with before/after screenshots, action overlays, and execution replays. Change-Id: I430b93ee2b76b54cc5a3c6eeccd15d2cf29830a1
This commit is contained in:
somew1nd committed
commit
206a9ce921
229 files changed
+49653
No files matched your search
@@ -0,0 +1,823 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import difflib
|
||||
import functools
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
from typing import Annotated, Literal
|
||||
|
||||
from langchain_core.messages import SystemMessage
|
||||
from langchain_core.tools import StructuredTool
|
||||
from langchain_core.tools.base import InjectedToolCallId
|
||||
from langgraph.constants import END, START
|
||||
from langgraph.graph import StateGraph
|
||||
from langgraph.graph.state import CompiledStateGraph
|
||||
from langgraph.prebuilt import InjectedState
|
||||
from langgraph.types import Command
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from artemis.agents.checker.checker import run_async_check
|
||||
from artemis.agents.operator.operator import OperatorNode
|
||||
from artemis.agents.planner.planner import (
|
||||
PlannerNode,
|
||||
run_async_planner_validation,
|
||||
)
|
||||
from artemis.agents.summarizer.summarizer import SummarizerNode
|
||||
from artemis.agents.validator.validator import ValidatorNode
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.perception import perception_node
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.command_tool import (
|
||||
manage_task_wrapper,
|
||||
run_adb_command_wrapper,
|
||||
)
|
||||
from artemis.tools.committee_tool import ask_committee_wrapper
|
||||
from artemis.tools.diagnostic_tool import ask_diagnoser_wrapper
|
||||
from artemis.tools.explorer_tool import ask_explorer_wrapper
|
||||
from artemis.tools.index import get_tools_from_wrappers
|
||||
from artemis.tools.scratchpad import (
|
||||
append_note_wrapper,
|
||||
list_notes_wrapper,
|
||||
read_note_wrapper,
|
||||
save_note_wrapper,
|
||||
update_note_wrapper,
|
||||
)
|
||||
from artemis.tools.tool_wrapper import invoke_tool_with_injection
|
||||
from artemis.tools.video_tool import get_video_analyzer_tool
|
||||
from artemis.utils.file import restore_snapshot
|
||||
from artemis.utils.logger import get_logger
|
||||
from artemis.utils.notes import (
|
||||
SAVE_NOTE_ARG_CONTENT_DESC,
|
||||
SAVE_NOTE_ARG_KEY_DESC,
|
||||
UPDATE_NOTE_ARG_KEY_DESC,
|
||||
UPDATE_NOTE_ARG_REPLACEMENT_DESC,
|
||||
UPDATE_NOTE_ARG_TARGET_DESC,
|
||||
get_note_file_path,
|
||||
get_notes_dir,
|
||||
)
|
||||
from artemis.utils.task_tree import get_active_subgoal_hashes
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
def convergence_node(state: State):
|
||||
"""Convergence point for parallel execution paths."""
|
||||
return {}
|
||||
|
||||
|
||||
async def execution_check_node(state: State, ctx: ArtemisContext):
|
||||
logger.info("Starting execution_check_node")
|
||||
|
||||
if not ctx.data_engine:
|
||||
return {"checker_success": True}
|
||||
|
||||
checker_success = True
|
||||
disable_checker = ctx.execution_setup and ctx.execution_setup.disable_checker
|
||||
|
||||
if disable_checker:
|
||||
logger.info("Checker is disabled via flag. Skipping checker invocation.")
|
||||
else:
|
||||
if (
|
||||
(not hasattr(ctx, "checker_task") or not ctx.checker_task)
|
||||
and ctx.data_engine
|
||||
and getattr(state, "operator_replied", False)
|
||||
):
|
||||
subgoal_hash, _ = _get_active_subgoal_hashes(ctx)
|
||||
notes_dir = get_notes_dir(ctx.data_engine.base_dir)
|
||||
chat_path = notes_dir / f"verification_chat_{subgoal_hash}.json"
|
||||
|
||||
if chat_path.exists():
|
||||
subgoal_text = "Unknown subgoal"
|
||||
task_plan_path = get_note_file_path(ctx.data_engine.base_dir, "task_plan")
|
||||
if task_plan_path.exists():
|
||||
try:
|
||||
content = task_plan_path.read_text(encoding="utf-8")
|
||||
lines = content.split("\n")
|
||||
for line in lines:
|
||||
if line.strip().startswith("- [/]"):
|
||||
subgoal_text = line.strip()[5:].strip()
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to parse active subgoal text: {e}")
|
||||
|
||||
raw_perception = getattr(state, "operator_raw_data", None)
|
||||
ui_hierarchy = getattr(state, "latest_ui_hierarchy", None)
|
||||
|
||||
ctx.checker_task = asyncio.create_task(
|
||||
run_async_check(
|
||||
ctx,
|
||||
subgoal_text,
|
||||
subgoal_hash=subgoal_hash,
|
||||
raw_perception_data=raw_perception,
|
||||
latest_ui_hierarchy=ui_hierarchy,
|
||||
)
|
||||
)
|
||||
|
||||
if hasattr(ctx, "checker_task") and ctx.checker_task:
|
||||
logger.info("Waiting for checker task to complete...")
|
||||
try:
|
||||
result = await ctx.checker_task
|
||||
ctx.checker_task = None # Clear it
|
||||
|
||||
if result:
|
||||
status = result.get("status")
|
||||
if status == "success":
|
||||
logger.info("Checker succeeded.")
|
||||
checker_success = True
|
||||
elif status == "failed":
|
||||
logger.warning("Checker failed.")
|
||||
checker_success = False
|
||||
else:
|
||||
logger.warning("Checker task returned None.")
|
||||
checker_success = False
|
||||
except Exception as e:
|
||||
logger.error(f"Checker task failed: {e}")
|
||||
checker_success = False
|
||||
|
||||
if hasattr(ctx, "planner_task") and ctx.planner_task:
|
||||
logger.info("Waiting for planner validation task to complete...")
|
||||
try:
|
||||
planner_result = await ctx.planner_task
|
||||
ctx.planner_task = None
|
||||
|
||||
if planner_result and planner_result.get("status") == "failed":
|
||||
logger.warning("Planner rejected the task plan changes.")
|
||||
|
||||
# Targeted rollback: restore task_plan.md only
|
||||
if hasattr(ctx, "task_plan_content_before"):
|
||||
task_plan_path = get_note_file_path(ctx.data_engine.base_dir, "task_plan")
|
||||
if task_plan_path.exists():
|
||||
task_plan_path.write_text(ctx.task_plan_content_before, encoding="utf-8")
|
||||
logger.info("Rolled back task_plan.md to its original state.")
|
||||
|
||||
# Intercept action and inject message
|
||||
feedback = planner_result.get("feedback", "Your task plan changes were rejected.")
|
||||
|
||||
system_msg = SystemMessage(
|
||||
content=(
|
||||
"Your recent modification to the top-level task plan"
|
||||
" was rejected by the Planner. You should view this"
|
||||
f" feedback critically.\n\nReason: {feedback}\n\nThe"
|
||||
" task plan has been rolled back to its previous"
|
||||
" state. Any terminal actions you outputted were not"
|
||||
" executed. Please review the task goal again and"
|
||||
" consider what the optimal strategy is now."
|
||||
)
|
||||
)
|
||||
|
||||
return {
|
||||
"checker_success": False,
|
||||
"operator_replied": False,
|
||||
VALIDATOR_MESSAGES_KEY: [system_msg],
|
||||
"structured_decisions": "",
|
||||
}
|
||||
|
||||
else:
|
||||
logger.info("Planner approved the task plan changes.")
|
||||
except Exception as e:
|
||||
logger.error(f"Planner validation task failed: {e}")
|
||||
|
||||
current_step_id = None
|
||||
if checker_success:
|
||||
raw_data = state.operator_raw_data
|
||||
if raw_data:
|
||||
try:
|
||||
screenshot_b64 = raw_data.get("screenshot_b64")
|
||||
xml_hierarchy = raw_data.get("xml_hierarchy")
|
||||
ocr_results = raw_data.get("ocr_results")
|
||||
|
||||
screenshot_bytes = base64.b64decode(screenshot_b64)
|
||||
|
||||
action_taken = None
|
||||
if state.structured_decisions:
|
||||
try:
|
||||
action_taken = json.loads(state.structured_decisions)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to parse structured decisions: {e}")
|
||||
|
||||
subgoal_hash, sub_subgoal_hash = _get_active_subgoal_hashes(ctx)
|
||||
step_id = ctx.data_engine.record_step(
|
||||
pre_screenshot_bytes=screenshot_bytes,
|
||||
ui_tree=xml_hierarchy,
|
||||
ocr_result=ocr_results,
|
||||
action_taken=action_taken,
|
||||
operator_raw_thinking=getattr(state, "operator_raw_thinking", None),
|
||||
operator_native_thinking=getattr(state, "operator_native_thinking", None),
|
||||
extra_metadata={
|
||||
"subgoal_hash": subgoal_hash,
|
||||
"sub_subgoal_hash": sub_subgoal_hash,
|
||||
"width": raw_data.get("width"),
|
||||
"height": raw_data.get("height"),
|
||||
},
|
||||
)
|
||||
current_step_id = str(step_id)
|
||||
logger.info(f"Recorded step in DataEngine: {current_step_id}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to record step in DataEngine: {e}")
|
||||
|
||||
# Handle optimistic execution rollback/commit
|
||||
if ctx.data_engine and getattr(ctx, "task_plan_snapshot", None):
|
||||
snapshot_dir = ctx.task_plan_snapshot
|
||||
notes_dir = get_notes_dir(ctx.data_engine.base_dir)
|
||||
|
||||
if checker_success:
|
||||
logger.info("Checker succeeded. Committing optimistic changes (cleaning up snapshot).")
|
||||
if snapshot_dir.exists():
|
||||
await asyncio.to_thread(shutil.rmtree, snapshot_dir)
|
||||
logger.info(f"Offloaded snapshot deletion at {snapshot_dir} via asyncio.to_thread")
|
||||
else:
|
||||
logger.warning("Checker failed. Rolling back optimistic changes.")
|
||||
if snapshot_dir.exists():
|
||||
|
||||
def _restore_and_preserve():
|
||||
chat_contents = {}
|
||||
for p in notes_dir.glob("verification_chat_*.json"):
|
||||
chat_contents[p.name] = p.read_text(encoding="utf-8")
|
||||
restore_snapshot(snapshot_dir, notes_dir)
|
||||
for name, content in chat_contents.items():
|
||||
(notes_dir / name).write_text(content, encoding="utf-8")
|
||||
logger.info(f"Restored verification chat file: {name}")
|
||||
|
||||
await asyncio.to_thread(_restore_and_preserve)
|
||||
logger.info(f"Restored notes from snapshot {snapshot_dir}")
|
||||
else:
|
||||
raise FileNotFoundError(
|
||||
f"Snapshot directory {snapshot_dir} not found for rollback!"
|
||||
)
|
||||
|
||||
update = {
|
||||
"checker_success": checker_success,
|
||||
"current_step_id": current_step_id,
|
||||
"operator_replied": False,
|
||||
}
|
||||
if checker_success:
|
||||
update["subagent_calls"] = []
|
||||
return update
|
||||
|
||||
|
||||
def execution_check_edge(
|
||||
state: State,
|
||||
) -> Literal["execute_decisions", "review_subgoals"]:
|
||||
if state.checker_success:
|
||||
if state.structured_decisions:
|
||||
return "execute_decisions"
|
||||
return "review_subgoals"
|
||||
return "review_subgoals"
|
||||
|
||||
|
||||
def _get_active_subgoal_hashes(ctx: ArtemisContext) -> tuple[str, str | None]:
|
||||
"""Parses task_plan.md to find the active subgoal and sub-subgoal hashes."""
|
||||
|
||||
if not ctx.data_engine:
|
||||
return "default", None
|
||||
|
||||
task_plan_path = get_note_file_path(ctx.data_engine.base_dir, "task_plan")
|
||||
|
||||
if not task_plan_path.exists():
|
||||
return "default", None
|
||||
|
||||
try:
|
||||
content = task_plan_path.read_text(encoding="utf-8")
|
||||
|
||||
return get_active_subgoal_hashes(content)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to parse active subgoal: {e}")
|
||||
|
||||
return "default", None
|
||||
|
||||
|
||||
def validate_milestones(
|
||||
content_before: str, content_after: str, similarity_threshold: float = 0.85
|
||||
) -> bool:
|
||||
"""Validates top-level milestones to see if async validation is needed.
|
||||
|
||||
Returns True if structural changes or major wording changes (<similarity_threshold ratio)
|
||||
are detected.
|
||||
"""
|
||||
|
||||
milestone_pattern = re.compile(r"^-\s*\[([\sx!/])\]\s*(.*)$", re.MULTILINE)
|
||||
before_milestones = [text.strip() for status, text in milestone_pattern.findall(content_before)]
|
||||
after_milestones = [text.strip() for status, text in milestone_pattern.findall(content_after)]
|
||||
|
||||
if len(before_milestones) != len(after_milestones):
|
||||
return True
|
||||
|
||||
if before_milestones == after_milestones:
|
||||
return False
|
||||
|
||||
def normalize(s: str) -> str:
|
||||
cleaned = re.sub(r"\W+", " ", s.lower()).strip()
|
||||
return re.sub(r"\s+", " ", cleaned)
|
||||
|
||||
for b, a in zip(before_milestones, after_milestones):
|
||||
if b == a:
|
||||
continue
|
||||
|
||||
norm_b = normalize(b)
|
||||
norm_a = normalize(a)
|
||||
|
||||
ratio = difflib.SequenceMatcher(None, norm_b, norm_a).ratio()
|
||||
if ratio < similarity_threshold:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
class _CyFunctionDetectorMeta(type):
|
||||
def __instancecheck__(self, instance):
|
||||
name = type(instance).__name__
|
||||
return (
|
||||
name
|
||||
in (
|
||||
"cyfunction",
|
||||
"cython_function_or_method",
|
||||
"builtin_function_or_method",
|
||||
)
|
||||
or "cyfunction" in name.lower()
|
||||
)
|
||||
|
||||
|
||||
class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta):
|
||||
pass
|
||||
|
||||
|
||||
class NoteArgs(BaseModel):
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
key: str = Field(..., description=SAVE_NOTE_ARG_KEY_DESC)
|
||||
content: str = Field(..., description=SAVE_NOTE_ARG_CONTENT_DESC)
|
||||
|
||||
|
||||
def wrap_note_tool(ctx: ArtemisContext, original_tool):
|
||||
|
||||
async def wrapped_note_tool(
|
||||
key: str,
|
||||
content: str,
|
||||
tool_call_id: Annotated[str, InjectedToolCallId] = None,
|
||||
state: Annotated[State, InjectedState] = None,
|
||||
):
|
||||
"""Wrapped tool to intercept task plan updates."""
|
||||
if key == "task_plan" and ctx.data_engine:
|
||||
task_plan_path = get_note_file_path(ctx.data_engine.base_dir, "task_plan")
|
||||
|
||||
content_before = ""
|
||||
if task_plan_path.exists():
|
||||
content_before = task_plan_path.read_text(encoding="utf-8")
|
||||
|
||||
result = await invoke_tool_with_injection(
|
||||
original_tool,
|
||||
{"key": key, "content": content},
|
||||
tool_call_id,
|
||||
state,
|
||||
record_trace=False,
|
||||
)
|
||||
|
||||
content_after = content
|
||||
if task_plan_path.exists():
|
||||
content_after = task_plan_path.read_text(encoding="utf-8")
|
||||
|
||||
lines = content_after.splitlines()
|
||||
top_level_lines = [line for line in lines if line.startswith("- [")]
|
||||
if top_level_lines and all(line.startswith("- [x]") for line in top_level_lines):
|
||||
has_incomplete = any(re.match(r"^\s*-\s*\[[ /]\]", line) for line in lines)
|
||||
if has_incomplete:
|
||||
logger.info(
|
||||
"Rejected: Top-level subgoals are complete but"
|
||||
" incomplete nested subgoals remain."
|
||||
)
|
||||
if task_plan_path.exists():
|
||||
task_plan_path.write_text(content_before, encoding="utf-8")
|
||||
|
||||
return Command(
|
||||
update={
|
||||
VALIDATOR_MESSAGES_KEY: [
|
||||
SystemMessage(
|
||||
content=(
|
||||
"Your changes to the task plan were not"
|
||||
" applied. Your changes would cause the"
|
||||
" task to end completely, but there"
|
||||
" seem to be goals in the task plan"
|
||||
" that have not been marked as"
|
||||
" completed. If you believe these goals"
|
||||
" are already completed, please update"
|
||||
" their statuses."
|
||||
)
|
||||
)
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
threshold = (
|
||||
ctx.execution_setup.planner_validation_threshold
|
||||
if ctx.execution_setup
|
||||
and hasattr(ctx.execution_setup, "planner_validation_threshold")
|
||||
else 0.85
|
||||
)
|
||||
needs_validation = validate_milestones(
|
||||
content_before, content_after, similarity_threshold=threshold
|
||||
)
|
||||
disable_planner_validation = (
|
||||
ctx.execution_setup and ctx.execution_setup.disable_planner_validation
|
||||
)
|
||||
|
||||
if needs_validation and not disable_planner_validation:
|
||||
logger.info(
|
||||
"Detected significant top-level task plan modification."
|
||||
" Triggering async Planner Validation."
|
||||
)
|
||||
|
||||
initial_goal = (
|
||||
state.initial_goal
|
||||
if state and hasattr(state, "initial_goal")
|
||||
else "Unknown Goal"
|
||||
)
|
||||
ctx.task_plan_content_before = content_before
|
||||
|
||||
ctx.planner_task = asyncio.create_task(
|
||||
run_async_planner_validation(
|
||||
ctx,
|
||||
initial_goal,
|
||||
content_before,
|
||||
content_after,
|
||||
getattr(state, "operator_raw_thinking", None),
|
||||
getattr(state, "operator_native_thinking", None),
|
||||
)
|
||||
)
|
||||
|
||||
if isinstance(result, Command) and VALIDATOR_MESSAGES_KEY in result.update:
|
||||
msgs = result.update[VALIDATOR_MESSAGES_KEY]
|
||||
if msgs and hasattr(msgs[0], "content"):
|
||||
msgs[0].content = (
|
||||
f"{msgs[0].content}\n\nWe have applied your changes"
|
||||
" to the task plan. A background verification task"
|
||||
" is currently reviewing these changes. If there"
|
||||
" are any issues with your modifications, you will"
|
||||
" be notified shortly."
|
||||
)
|
||||
|
||||
# Only match top-level subgoals, intentionally ignoring indented sub-subgoals
|
||||
disable_checker = ctx.execution_setup and ctx.execution_setup.disable_checker
|
||||
pattern = re.compile(r"^-\s*\[x\]\s*(.*)$", re.MULTILINE)
|
||||
|
||||
before_matches = set(pattern.findall(content_before))
|
||||
after_matches = set(pattern.findall(content_after))
|
||||
|
||||
new_completions = after_matches - before_matches
|
||||
|
||||
if new_completions and not disable_checker:
|
||||
logger.info(f"Detected new top-level completions: {new_completions}.")
|
||||
|
||||
all_matches_in_order = pattern.findall(content_after)
|
||||
latest_completion = None
|
||||
for match in reversed(all_matches_in_order):
|
||||
if match in new_completions:
|
||||
latest_completion = match
|
||||
break
|
||||
|
||||
if latest_completion:
|
||||
logger.info(f"Triggering Checker for latest completion: {latest_completion}")
|
||||
raw_perception = getattr(state, "operator_raw_data", None)
|
||||
ui_hierarchy = getattr(state, "latest_ui_hierarchy", None)
|
||||
ctx.checker_task = asyncio.create_task(
|
||||
run_async_check(
|
||||
ctx,
|
||||
latest_completion,
|
||||
raw_perception_data=raw_perception,
|
||||
latest_ui_hierarchy=ui_hierarchy,
|
||||
)
|
||||
)
|
||||
|
||||
return result
|
||||
else:
|
||||
return await invoke_tool_with_injection(
|
||||
original_tool,
|
||||
{"key": key, "content": content},
|
||||
tool_call_id,
|
||||
state,
|
||||
record_trace=False,
|
||||
)
|
||||
|
||||
return StructuredTool.from_function(
|
||||
func=lambda *a, **kw: None,
|
||||
coroutine=wrapped_note_tool,
|
||||
name=original_tool.name,
|
||||
description=original_tool.description,
|
||||
args_schema=NoteArgs,
|
||||
)
|
||||
|
||||
|
||||
class UpdateNoteArgs(BaseModel):
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
key: str = Field(..., description=UPDATE_NOTE_ARG_KEY_DESC)
|
||||
target: str = Field(..., description=UPDATE_NOTE_ARG_TARGET_DESC)
|
||||
replacement: str = Field(..., description=UPDATE_NOTE_ARG_REPLACEMENT_DESC)
|
||||
|
||||
|
||||
def wrap_update_note_tool(ctx: ArtemisContext, original_tool):
|
||||
|
||||
async def wrapped_update_note(
|
||||
key: str,
|
||||
target: str,
|
||||
replacement: str,
|
||||
tool_call_id: Annotated[str, InjectedToolCallId] = None,
|
||||
state: Annotated[State, InjectedState] = None,
|
||||
):
|
||||
"""Wrapped tool to intercept task plan updates via update_note."""
|
||||
if key == "task_plan" and ctx.data_engine:
|
||||
task_plan_path = get_note_file_path(ctx.data_engine.base_dir, "task_plan")
|
||||
|
||||
content_before = ""
|
||||
if task_plan_path.exists():
|
||||
content_before = task_plan_path.read_text(encoding="utf-8")
|
||||
|
||||
result = await invoke_tool_with_injection(
|
||||
original_tool,
|
||||
{"key": key, "target": target, "replacement": replacement},
|
||||
tool_call_id,
|
||||
state,
|
||||
record_trace=False,
|
||||
)
|
||||
|
||||
content_after = ""
|
||||
if task_plan_path.exists():
|
||||
content_after = task_plan_path.read_text(encoding="utf-8")
|
||||
|
||||
lines = content_after.splitlines()
|
||||
top_level_lines = [line for line in lines if line.startswith("- [")]
|
||||
if top_level_lines and all(line.startswith("- [x]") for line in top_level_lines):
|
||||
has_incomplete = any(re.match(r"^\s*-\s*\[[ /]\]", line) for line in lines)
|
||||
if has_incomplete:
|
||||
logger.info(
|
||||
"Rejected: Top-level subgoals are complete but"
|
||||
" incomplete nested subgoals remain."
|
||||
)
|
||||
if task_plan_path.exists():
|
||||
task_plan_path.write_text(content_before, encoding="utf-8")
|
||||
|
||||
return Command(
|
||||
update={
|
||||
VALIDATOR_MESSAGES_KEY: [
|
||||
SystemMessage(
|
||||
content=(
|
||||
"Your changes to the task plan were not"
|
||||
" applied. Your changes would cause the"
|
||||
" task to end completely, but there"
|
||||
" seem to be goals in the task plan"
|
||||
" that have not been marked as"
|
||||
" completed. If you believe these goals"
|
||||
" are already completed, please update"
|
||||
" their statuses."
|
||||
)
|
||||
)
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
threshold = (
|
||||
ctx.execution_setup.planner_validation_threshold
|
||||
if ctx.execution_setup
|
||||
and hasattr(ctx.execution_setup, "planner_validation_threshold")
|
||||
else 0.85
|
||||
)
|
||||
needs_validation = validate_milestones(
|
||||
content_before, content_after, similarity_threshold=threshold
|
||||
)
|
||||
disable_planner_validation = (
|
||||
ctx.execution_setup and ctx.execution_setup.disable_planner_validation
|
||||
)
|
||||
|
||||
if needs_validation and not disable_planner_validation:
|
||||
logger.info(
|
||||
"Detected significant top-level task plan modification."
|
||||
" Triggering async Planner Validation."
|
||||
)
|
||||
|
||||
initial_goal = (
|
||||
state.initial_goal
|
||||
if state and hasattr(state, "initial_goal")
|
||||
else "Unknown Goal"
|
||||
)
|
||||
ctx.task_plan_content_before = content_before
|
||||
|
||||
ctx.planner_task = asyncio.create_task(
|
||||
run_async_planner_validation(
|
||||
ctx,
|
||||
initial_goal,
|
||||
content_before,
|
||||
content_after,
|
||||
getattr(state, "operator_raw_thinking", None),
|
||||
getattr(state, "operator_native_thinking", None),
|
||||
)
|
||||
)
|
||||
|
||||
if isinstance(result, Command) and VALIDATOR_MESSAGES_KEY in result.update:
|
||||
msgs = result.update[VALIDATOR_MESSAGES_KEY]
|
||||
if msgs and hasattr(msgs[0], "content"):
|
||||
msgs[0].content = (
|
||||
f"{msgs[0].content}\n\nWe have applied your changes"
|
||||
" to the task plan. A background verification task"
|
||||
" is currently reviewing these changes. If there"
|
||||
" are any issues with your modifications, you will"
|
||||
" be notified shortly."
|
||||
)
|
||||
|
||||
disable_checker = ctx.execution_setup and ctx.execution_setup.disable_checker
|
||||
pattern = re.compile(r"^-\s*\[x\]\s*(.*)$", re.MULTILINE)
|
||||
before_matches = set(pattern.findall(content_before))
|
||||
after_matches = set(pattern.findall(content_after))
|
||||
new_completions = after_matches - before_matches
|
||||
|
||||
if new_completions and not disable_checker:
|
||||
logger.info(f"Detected new top-level completions: {new_completions}.")
|
||||
all_matches_in_order = pattern.findall(content_after)
|
||||
latest_completion = None
|
||||
for match in reversed(all_matches_in_order):
|
||||
if match in new_completions:
|
||||
latest_completion = match
|
||||
break
|
||||
|
||||
if latest_completion:
|
||||
logger.info(f"Triggering Checker for latest completion: {latest_completion}")
|
||||
raw_perception = getattr(state, "operator_raw_data", None)
|
||||
ui_hierarchy = getattr(state, "latest_ui_hierarchy", None)
|
||||
ctx.checker_task = asyncio.create_task(
|
||||
run_async_check(
|
||||
ctx,
|
||||
latest_completion,
|
||||
raw_perception_data=raw_perception,
|
||||
latest_ui_hierarchy=ui_hierarchy,
|
||||
)
|
||||
)
|
||||
|
||||
return result
|
||||
else:
|
||||
return await invoke_tool_with_injection(
|
||||
original_tool,
|
||||
{"key": key, "target": target, "replacement": replacement},
|
||||
tool_call_id,
|
||||
state,
|
||||
record_trace=False,
|
||||
)
|
||||
|
||||
return StructuredTool.from_function(
|
||||
func=lambda *a, **kw: None,
|
||||
coroutine=wrapped_update_note,
|
||||
name=original_tool.name,
|
||||
description=original_tool.description,
|
||||
args_schema=UpdateNoteArgs,
|
||||
)
|
||||
|
||||
|
||||
async def get_graph(ctx: ArtemisContext) -> CompiledStateGraph:
|
||||
graph_builder = StateGraph(State)
|
||||
|
||||
def convergence_gate(state: State) -> Literal["continue", "end"]:
|
||||
logger.info("Starting convergence_gate")
|
||||
|
||||
# Check checker result
|
||||
if hasattr(state, "checker_success") and not state.checker_success:
|
||||
logger.info("Checker failed, returning to operator for retry.")
|
||||
return "continue"
|
||||
|
||||
task_plan_content = ""
|
||||
if ctx.data_engine:
|
||||
file_path = get_note_file_path(ctx.data_engine.base_dir, "task_plan")
|
||||
if file_path.exists():
|
||||
try:
|
||||
task_plan_content = file_path.read_text(encoding="utf-8")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to read task plan: {e}")
|
||||
return "continue"
|
||||
else:
|
||||
logger.warning(f"Task plan file not found at {file_path}")
|
||||
return "continue"
|
||||
|
||||
lines = task_plan_content.splitlines()
|
||||
|
||||
# Check for all completed
|
||||
top_level_lines = [line for line in lines if line.startswith("- [")]
|
||||
|
||||
if not top_level_lines:
|
||||
logger.info("No subgoals found in file, ending")
|
||||
return "end"
|
||||
|
||||
all_done = all(line.startswith("- [x]") for line in top_level_lines)
|
||||
if all_done:
|
||||
logger.info("All subgoals are completed, ending the goal")
|
||||
return "end"
|
||||
|
||||
return "continue"
|
||||
|
||||
# Get native tools
|
||||
native_wrappers = [
|
||||
save_note_wrapper,
|
||||
read_note_wrapper,
|
||||
list_notes_wrapper,
|
||||
update_note_wrapper,
|
||||
append_note_wrapper,
|
||||
]
|
||||
|
||||
native_tools = get_tools_from_wrappers(ctx, native_wrappers)
|
||||
|
||||
## Define nodes
|
||||
graph_builder.add_node("planner", PlannerNode(ctx, tools=native_tools))
|
||||
|
||||
# Wrap native tools for Operator
|
||||
operator_native_tools = []
|
||||
for t in native_tools:
|
||||
if t.name == "save_note":
|
||||
operator_native_tools.append(wrap_note_tool(ctx, t))
|
||||
elif t.name == "update_note":
|
||||
operator_native_tools.append(wrap_update_note_tool(ctx, t))
|
||||
elif t.name == "append_note":
|
||||
operator_native_tools.append(wrap_note_tool(ctx, t))
|
||||
else:
|
||||
operator_native_tools.append(t)
|
||||
|
||||
# Get specialized tools for Operator
|
||||
operator_specialized_wrappers = [
|
||||
ask_diagnoser_wrapper,
|
||||
run_adb_command_wrapper,
|
||||
manage_task_wrapper,
|
||||
ask_explorer_wrapper,
|
||||
]
|
||||
if ctx.execution_setup and ctx.execution_setup.enable_committee:
|
||||
operator_specialized_wrappers.append(ask_committee_wrapper)
|
||||
|
||||
operator_specialized_tools = get_tools_from_wrappers(ctx, operator_specialized_wrappers)
|
||||
|
||||
# Customize descriptions for run_adb_command and manage_task to match exploratory role
|
||||
for t in operator_specialized_tools:
|
||||
if t.name == "run_adb_command":
|
||||
t.description = (
|
||||
"[EXPLORER] Executes a shell command directly on the Android"
|
||||
" mobile device via ADB shell. Use this tool when normal"
|
||||
" operating tools cannot achieve the goal, do not rely on it."
|
||||
" You must not use this tool to bypass goals that could be"
|
||||
" achieved by action tools.\nCan run synchronously or"
|
||||
" transition to a background task if execution takes longer"
|
||||
" than WaitMsBeforeAsync.\nSupports persistent environments"
|
||||
" (environment variables) on the phone across invocations."
|
||||
)
|
||||
elif t.name == "manage_task":
|
||||
t.description = (
|
||||
"[EXPLORER] Manage background ADB shell tasks launched via"
|
||||
" run_adb_command to monitor or terminate ongoing background"
|
||||
" commands."
|
||||
)
|
||||
|
||||
operator_tools = operator_native_tools + operator_specialized_tools
|
||||
|
||||
if ctx.execution_setup and ctx.execution_setup.video_recording_tools_enabled:
|
||||
operator_tools.append(get_video_analyzer_tool(ctx, role="operator"))
|
||||
|
||||
graph_builder.add_node("operator", OperatorNode(ctx, tools=operator_tools))
|
||||
graph_builder.add_node("validator", ValidatorNode(ctx))
|
||||
|
||||
graph_builder.add_node("summarizer", SummarizerNode(ctx))
|
||||
graph_builder.add_node("execution_check", functools.partial(execution_check_node, ctx=ctx))
|
||||
graph_builder.add_node("perception", functools.partial(perception_node, ctx=ctx))
|
||||
graph_builder.add_node(node="convergence", action=convergence_node, defer=True)
|
||||
|
||||
## Linking nodes
|
||||
graph_builder.add_edge(START, "planner")
|
||||
graph_builder.add_edge("planner", "convergence")
|
||||
graph_builder.add_edge("operator", "execution_check")
|
||||
graph_builder.add_conditional_edges(
|
||||
"execution_check",
|
||||
execution_check_edge,
|
||||
{
|
||||
"review_subgoals": "convergence",
|
||||
"execute_decisions": "validator",
|
||||
},
|
||||
)
|
||||
graph_builder.add_edge("validator", "summarizer")
|
||||
graph_builder.add_edge("summarizer", "convergence")
|
||||
|
||||
graph_builder.add_conditional_edges(
|
||||
source="convergence",
|
||||
path=convergence_gate,
|
||||
path_map={
|
||||
"continue": "perception",
|
||||
"end": END,
|
||||
},
|
||||
)
|
||||
graph_builder.add_edge("perception", "operator")
|
||||
|
||||
return graph_builder.compile()
|
||||
Reference in new issue
Block a user