# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json from typing import Any from uuid import uuid4 from langchain_core.messages import SystemMessage, ToolMessage from langchain_core.tools import BaseTool from artemis.context import ArtemisContext from artemis.data_engine.trace import ( TraceSpan, trace, trace_langchain_tool, ) from artemis.graph.state import State from artemis.graph.visibility import strict_state from artemis.mcp.action_specs import OPERATOR_SHELL_ORDER, operator_shell_tool from artemis.services.llm import acomplete, get_llm, invoke_llm_with_timeout_message from artemis.tools.command_tool import ( analyze_task_output_wrapper, get_adb_task_registry, ) from artemis.tools.index import get_tool_by_name from artemis.tools.tool_wrapper import ( get_tool_result_content, invoke_tool_with_injection, tool_result_messages, ) from artemis.utils.coordinates import ( COORDINATE_SPACE_KEY, COORDINATE_SPACE_PIXEL, compute_smart_swipe_coordinates, parse_swipe_parameters, ) from artemis.utils.decorators import wrap_with_callbacks from artemis.utils.element_hit_test import find_element_at_point from artemis.utils.logger import get_logger from artemis.utils.notes import get_note_file_path from artemis.utils.plan_grammar import parse_plan from artemis.memory.context_policy import build_history_for from artemis.utils.task_tree import get_active_subgoal_hashes from artemis.utils.visualization import format_minimal_list_with_elements logger = get_logger(__name__) DEFERRING_TOOLS = { "ask_diagnoser", "video_analyzer", "read_note", "list_notes", "search_history", "replay_steps", "get_step_screenshot", "run_adb_command", "manage_task", "analyze_task_output", "ask_explorer", } from artemis.agents.operator.prompts import ( OPERATOR_MAX_TOOL_ITERATIONS, PromptBuilder, PromptComponent, TemplatePromptComponent, ObservationPromptComponent, PlanRecitationPromptComponent, ScreenshotSimilarityPromptComponent, HistoricalStateHintPromptComponent, FeedbackPromptComponent, ExecutionIncidentPromptComponent, CheckItemsExplainerPromptComponent, BackgroundTasksPromptComponent, render_plan_ledger_bounce, unwritten_action_streak, ToolLimitWarningPromptComponent, InjectedInstructionPromptComponent, render_transcript_static_system, ) from artemis.agents.operator.prompts import load_operator_prompts class OperatorNode: def __init__( self, ctx: ArtemisContext, tools: list[BaseTool] | None = None, prompt_components: list[PromptComponent] | None = None, last_n_detailed: int = 1, transcript_config=None, ): self.ctx = ctx self.tools = tools or [] self.prompt_components = prompt_components or [] self.last_n_detailed = last_n_detailed try: self.prompts = load_operator_prompts() except (OSError, ValueError, json.JSONDecodeError) as e: raise RuntimeError(f"Failed to load operator prompts: {e}") from e # M2 transcript flag (agent.memory.transcript.enabled). Off keeps the # legacy 2-message prompt path byte-for-byte; an explicit # ``transcript_config`` overrides the loaded configuration (tests). if transcript_config is not None: self._transcript_cfg = transcript_config else: try: from artemis.config import load_agent_config self._transcript_cfg = load_agent_config().memory.transcript except Exception: from artemis.config.agent import MemoryTranscriptConfig self._transcript_cfg = MemoryTranscriptConfig() # Index into the rendered message list where this turn's tail begins; # set by the transcript build and consumed after the tool loop. self._transcript_turn_base: int | None = None self._turn_plan_baseline: str | None = None self._turn_ledger_streak: int = 0 def _available_device_actions(self) -> frozenset[str]: """Device actions the installed actuator backend provides. Without an explicit ``ctx.actuator`` the full manifest set applies (the default AdbActuator implements everything), so behavior is unchanged until a partial backend is actually installed. """ from artemis.mcp.action_manifest import ( OPTIONAL_ACTIONS, REQUIRED_ACTIONS, available_device_actions, ) actuator = getattr(self.ctx, "actuator", None) if actuator is not None and callable(getattr(actuator, "capabilities", None)): try: extension_names = frozenset( e.name for e in actuator.extensions() if "operator" in e.targets ) return available_device_actions(actuator) | extension_names except Exception as e: logger.warning(f"Failed to read actuator capabilities: {e}") return REQUIRED_ACTIONS | OPTIONAL_ACTIONS def _get_history_and_plan(self) -> tuple[list[dict], str]: steps = [] if self.ctx.data_engine: steps = self.ctx.data_engine.get_agent_friendly_steps() task_plan = "No task plan yet." if self.ctx.data_engine: task_plan_path = get_note_file_path(self.ctx.data_engine.base_dir, "task_plan") if task_plan_path.exists(): try: task_plan = task_plan_path.read_text(encoding="utf-8") except Exception as e: task_plan = f"Error reading task plan: {e}" return steps, task_plan async def _build_prompt( self, state: State, latest_screenshot_b64: str, fused_xml: list[dict], minimal_list: str, current_step_num: int, steps: list[dict], task_plan: str, active_background_tasks: list[dict] = None, newly_finished_tasks: list[dict] = None, ) -> list: self._transcript_turn_base = None if getattr(self._transcript_cfg, "enabled", False): try: return await self._build_prompt_transcript( state, latest_screenshot_b64, minimal_list, current_step_num, steps, task_plan, active_background_tasks=active_background_tasks, newly_finished_tasks=newly_finished_tasks, ) except Exception as e: logger.error( f"Transcript prompt path failed; falling back to the legacy" f" 2-message build for this turn: {e}" ) self._transcript_turn_base = None plan_and_history = self._build_plan_and_history(steps, task_plan) builder = PromptBuilder() # One constant component list: the prompt template is never switched by # verification results. Check-related components are append-only and # render nothing when there is nothing to say. components = self.prompt_components if not components: components = [ ( TemplatePromptComponent(), {"template_name": "main_template"}, ), (CheckItemsExplainerPromptComponent(), {}), (ExecutionIncidentPromptComponent(), {}), (ObservationPromptComponent(), {}), (ScreenshotSimilarityPromptComponent(), {}), (HistoricalStateHintPromptComponent(), {}), (InjectedInstructionPromptComponent(), {}), (BackgroundTasksPromptComponent(), {}), (FeedbackPromptComponent(), {}), (ToolLimitWarningPromptComponent(), {}), ] else: components = [(c, {}) for c in components] + [(ToolLimitWarningPromptComponent(), {})] # Active and finished tasks are now passed as arguments active_tasks = active_background_tasks or [] newly_finished = newly_finished_tasks or [] for component, extra_kwargs in components: kwargs_to_pass = { "prompts": self.prompts, "plan_and_history": plan_and_history, "task_plan": task_plan, "latest_screenshot_b64": latest_screenshot_b64, "minimal_list": minimal_list, "current_step_num": current_step_num, "active_background_tasks": active_tasks, "newly_finished_tasks": newly_finished, "steps": steps, } kwargs_to_pass.update(extra_kwargs) await component(builder, state, self.ctx, **kwargs_to_pass) return builder.build() # ------------------------------------------------------------------ # Transcript prompt path (M2, flag agent.memory.transcript.enabled) # ------------------------------------------------------------------ def _ensure_transcript_ledger(self, state=None): """Session ledger on the composition root, created on first use.""" from artemis.memory import TranscriptLedger, ensure_step_memory ledger = getattr(self.ctx, "transcript_ledger", None) if isinstance(ledger, TranscriptLedger): return ledger cfg = self._transcript_cfg # Anchor the ``T+mm:ss`` clock to the DataEngine session start so the # tail offsets agree with the chunk ledger lines, the restored-history # block and the video analyzer's action timeline. session_start = getattr(self.ctx.data_engine, "session_start_time", None) ledger = TranscriptLedger( step_memory=ensure_step_memory(self.ctx), image_scrub_depth=getattr(cfg, "image_scrub_depth", 3), pending_grace_steps=getattr(cfg, "pending_grace_steps", 3), xml_scrub_depth=getattr(cfg, "xml_scrub_depth", 1), session_start=session_start if isinstance(session_start, (int, float)) else None, ) # L2/L3 chunk compression (M3) rides the same flag; without a # DataEngine there are no step records to chunk, so the ledger simply # runs scrub-edge-only. if self.ctx.data_engine is not None: try: from artemis.memory import HistoryChunkManager chunking_cfg = None try: from artemis.config import load_agent_config chunking_cfg = load_agent_config().memory.chunking except Exception as exc: logger.debug( "Chunking config unavailable; using chunker defaults: %s", exc, exc_info=True, ) ledger.attach_chunker( HistoryChunkManager( engine=self.ctx.data_engine, ctx=self.ctx, chunking_config=chunking_cfg, transcript_config=cfg, goal=getattr(state, "initial_goal", None) if state else None, ) ) except Exception as e: logger.error(f"History chunk manager unavailable: {e}") try: self.ctx.transcript_ledger = ledger except (AttributeError, TypeError, ValueError): pass return ledger async def _build_prompt_transcript( self, state: State, latest_screenshot_b64: str, minimal_list: str, current_step_num: int, steps: list[dict], task_plan: str, active_background_tasks: list[dict] = None, newly_finished_tasks: list[dict] = None, ) -> list: """Build ``S + F + A + tail`` from the session transcript ledger. The static system message renders once per session (byte-stable S region); the previous turn is committed into the append-only active region together with its step key and validator result; the fresh tail carries the current observation plus the task-plan recitation. """ from langchain_core.messages import HumanMessage ledger = self._ensure_transcript_ledger(state) # 1. S region: rendered exactly once per session. if not ledger.has_static_prefix: static_text = render_transcript_static_system(self.prompts, self.ctx, state) ledger.set_static_prefix([SystemMessage(content=static_text)]) # 2. F region cold start: an empty ledger over an existing step record # trail means the process restarted — freeze the compiled history once. # A staged (not yet committed) turn means this is turn 2+ of a live # session, not a cold start — step records already exist then, and # seeding would be rejected by the empty-ledger invariant. if ( ledger.turn_count == 0 and not ledger.has_staged_turn and not ledger.has_restored_history and steps ): restored = build_history_for( "operator_cold_start", task_plan, steps, self._get_active_subgoal_hash(), ) ledger.set_restored_history( "[Restored history] Rebuilt from the step records of this task" " after a process restart; step times below are relative to the" " original session start.\n" f"{restored}" ) # 3. Commit the previous turn (its step id and validator result only # exist now). A turn without a terminal action never ran the validator, # and ``structured_decisions`` is cleared on planner rejection — the # gate keeps stale reports out of the ledger. validator_result = ( getattr(state, "last_execution_result", None) if getattr(state, "structured_decisions", None) else None ) ledger.commit_staged( step_key=getattr(state, "current_step_id", None), validator_result=validator_result, ) # 4. Current tail: observation + plan recitation + injected components. builder = PromptBuilder() builder.add_human_content(f"# CURRENT OBSERVATION [{ledger.elapsed_label()}]") components = [ (PlanRecitationPromptComponent(), {}), (CheckItemsExplainerPromptComponent(), {}), (ExecutionIncidentPromptComponent(), {}), (ObservationPromptComponent(), {}), (ScreenshotSimilarityPromptComponent(), {}), (HistoricalStateHintPromptComponent(), {}), (InjectedInstructionPromptComponent(), {}), (BackgroundTasksPromptComponent(), {}), (FeedbackPromptComponent(), {}), (ToolLimitWarningPromptComponent(), {}), ] for component, extra_kwargs in components: kwargs_to_pass = { "prompts": self.prompts, "task_plan": task_plan, "latest_screenshot_b64": latest_screenshot_b64, "minimal_list": minimal_list, "current_step_num": current_step_num, "active_background_tasks": active_background_tasks or [], "newly_finished_tasks": newly_finished_tasks or [], "steps": steps, } kwargs_to_pass.update(extra_kwargs) await component(builder, state, self.ctx, **kwargs_to_pass) tail_content = [] for part in builder.human_parts: if isinstance(part, str): tail_content.append({"type": "text", "text": part}) else: tail_content.append(part) tail = HumanMessage(content=tail_content) messages = ledger.render([tail]) self._transcript_turn_base = len(messages) - 1 return messages def _stage_transcript_turn(self, messages: list) -> None: """Hand this turn's tail + tool-loop products to the ledger.""" base = self._transcript_turn_base self._transcript_turn_base = None if base is None: return try: ledger = getattr(self.ctx, "transcript_ledger", None) if ledger is not None: ledger.stage_turn(messages[base:]) except Exception as e: logger.error(f"Failed to stage transcript turn: {e}") async def _invoke_llm_loop( self, base_llm, current_messages: list, traced_tools: list, new_subagent_calls: list, state: State, ) -> tuple[list[dict] | None, str | None, str | None, bool]: max_iterations = OPERATOR_MAX_TOOL_ITERATIONS action_result = None raw_thoughts = [] native_thoughts = [] tool_limit_exceeded = False plan_gate_bounced = False for iteration in range(max_iterations): if iteration == max_iterations - 1: logger.warning( "Operator has reached the maximum number of tool calls. Adding reminder." ) current_messages.append( SystemMessage( content=( "You have reached the maximum number of tool calls" " for this turn. You must make a final action, or" " save your current research results via notes to" " continue later." ) ) ) bound_llm = base_llm.bind_tools(tools=traced_tools) response = await invoke_llm_with_timeout_message(acomplete(bound_llm, current_messages)) if hasattr(response, "response_metadata") and response.response_metadata: usage = ( response.response_metadata.get("usage_metadata") or response.response_metadata.get("token_usage") or {} ) prompt_tokens = usage.get("prompt_token_count") or usage.get("prompt_tokens") cached_tokens = usage.get("cached_content_token_count") or usage.get( "cached_tokens", 0 ) completion_tokens = usage.get("candidates_token_count") or usage.get( "completion_tokens" ) if prompt_tokens is not None: logger.info( f"LLM usage: prompt_tokens={prompt_tokens}" f" (cached={cached_tokens})," f" completion_tokens={completion_tokens}" ) if response.content: if isinstance(response.content, str): raw_thoughts.append(response.content) elif isinstance(response.content, list): for item in response.content: if isinstance(item, dict): if item.get("type") == "text": raw_thoughts.append(item["text"]) elif item.get("type") == "thinking": native_thoughts.append(item["thinking"]) additional_kwargs = getattr(response, "additional_kwargs", None) if ( not response.tool_calls and type(additional_kwargs) is dict and additional_kwargs.get("function_call") ): fc = additional_kwargs["function_call"] try: arguments_str = fc.get("arguments", "{}") if isinstance(arguments_str, str): arguments = json.loads(arguments_str) else: arguments = arguments_str tool_call = { "name": fc["name"], "args": arguments, "id": f"call_{uuid4().hex[:8]}", "type": "tool_call", } response.tool_calls = [tool_call] logger.info( "Fallback: Manually parsed function_call from" f" additional_kwargs: {tool_call}" ) except Exception as e: logger.error(f"Failed to parse function_call fallback: {e}") if not response.tool_calls: logger.warning("LLM stopped without calling any tool. Encouraging action.") break current_messages.append(response) # Manifest-driven, so a backend's extension actions are classified as # turn-ending device actions without touching this file. action_tool_names = sorted(self._available_device_actions()) def normalize_name(name: str) -> str: return name.split(":")[-1] if ":" in name else name action_calls = [ tc for tc in response.tool_calls if normalize_name(tc["name"]) in action_tool_names ] other_calls = [ tc for tc in response.tool_calls if normalize_name(tc["name"]) not in action_tool_names ] tool_outputs = [] validation_errors = False other_tool_failed = False other_tool_failure_msg = "" has_terminal_calls = bool(action_calls) if other_calls: results = [] for tc in other_calls: tool_name = tc["name"] logger.info(f"Operator requested tool: {tool_name}") if tool_name in ["ask_diagnoser", "video_analyzer"]: new_subagent_calls.append(tool_name) if ":" in tool_name: tool_to_run = get_tool_by_name(tool_name, traced_tools) else: tool_to_run = next( (t for t in traced_tools if t.name == tool_name), None, ) if tool_to_run: args = dict(tc["args"]) span = TraceSpan(name=tool_name, trace_type="tool", ctx=self.ctx) span.payload = {"args": args} with span: try: state.operator_raw_thinking = ( "\n".join(raw_thoughts) if raw_thoughts else None ) state.operator_native_thinking = ( "\n".join(native_thoughts) if native_thoughts else None ) result_obj = await invoke_tool_with_injection( tool=tool_to_run, args=args, tool_call_id=tc["id"], state=state, ) content = get_tool_result_content(result_obj) status = "success" if isinstance(result_obj, ToolMessage) and result_obj.status: status = result_obj.status is_err = status == "error" if not is_err: if isinstance(content, str): if content.startswith( "Failed" ) or content.lower().startswith("error"): is_err = True elif isinstance(content, list): for block in content: if ( isinstance(block, dict) and block.get("type") == "text" ): text = block.get("text", "") if text.startswith( "Failed" ) or text.lower().startswith("error"): is_err = True break if is_err: other_tool_failed = True other_tool_failure_msg = str(content) status = "error" span.status = "failed" span.error = str(content) else: span.result = content results.append((content, status)) except Exception as e: other_tool_failed = True other_tool_failure_msg = str(e) span.status = "failed" span.error = str(e) results.append( ( f"Error running tool {tool_name}: {e}", "error", ) ) else: other_tool_failed = True other_tool_failure_msg = f"Tool {tool_name} not supported" results.append((f"Error: Tool {tool_name} not supported", "error")) for tc, (content, status) in zip(other_calls, results): # A step screenshot (content blocks) travels in the carrier # this model's provider accepts; text stays a ToolMessage. tool_outputs.extend( tool_result_messages( tc["id"], content, name=normalize_name(tc["name"]), status=status, llm=base_llm, ) ) if other_tool_failed: if action_calls: logger.warning( "Scratchpad/helper tool call failed. Deferring" f" screen actions: {other_tool_failure_msg}" ) for tc in action_calls: tool_outputs.append( ToolMessage( tool_call_id=tc["id"], content=( "This screen action was rejected" " because an accompanying tool call" " failed. Error:" f" {other_tool_failure_msg}. Currently," " no screen actions have been" " executed. Please correct the error" " and try again." ), status="error", ) ) action_calls = [] elif has_terminal_calls: has_deferring_calls = any( normalize_name(tc["name"]) in DEFERRING_TOOLS for tc in other_calls ) if has_deferring_calls: logger.info( "Operator mixed pre-decision tools" " (memory/exploration) and terminal action calls." " Deferring terminal actions." ) for tc in action_calls: tool_outputs.append( ToolMessage( tool_call_id=tc["id"], content=( "Your pre-decision tools have been" " successfully processed. However, the" " screen action you attempted to" " execute was rejected because you are" " not allowed to simultaneously" " use result-dependent pre-decision" " tools and execute physical actions in" " the same turn." " Currently, no screen actions have" " been executed. Please review your" " updated context and re-output your" " intended screen action." ), status="success", ) ) action_calls = [] if action_calls: logger.info( f"Operator requested {len(action_calls)} action(s)." " Translating and processing all." ) actions_list = [] burst_cap = self._max_burst_actions() bounce = None if ( not plan_gate_bounced and iteration < max_iterations - 1 and len(action_calls) == 1 and not getattr(state, "open_incident", None) and self._plan_changed_this_turn() is False ): bounce = self._plan_ledger_bounce_reason() if bounce: plan_gate_bounced = True validation_errors = True logger.info(f"Plan ledger gate: bouncing action. {bounce}") for tc in action_calls: tool_outputs.append( ToolMessage(tool_call_id=tc["id"], content=bounce, status="error") ) action_calls_to_translate = [] elif len(action_calls) > burst_cap: # A multi-action turn is a fast-action burst that the Validator # fires without the safety net; cap its length before it runs. validation_errors = True for tc in action_calls: tool_outputs.append( ToolMessage( tool_call_id=tc["id"], content=( f"Error: {len(action_calls)} Turn-Ending Actions in one" " turn exceed the fast-action burst limit of" f" {burst_cap}. Nothing was executed. Re-issue at most" f" {burst_cap} actions (a burst is for sub-second" " sequences on transient UI, not for batching normal" " steps), or a single vetted action." ), status="error", ) ) action_calls_to_translate = [] else: if plan_gate_bounced and self._plan_changed_this_turn() is False: logger.warning( "Plan ledger gate: executing actions after one bounce without" " a task_plan change (ledger gap this turn)." ) action_calls_to_translate = action_calls for tc in action_calls_to_translate: translated_actions, error = self._translate_and_validate_tool(tc, state) if error: validation_errors = True tool_outputs.append( ToolMessage( tool_call_id=tc["id"], content=error, status="error", ) ) else: actions_list.extend(translated_actions) tool_outputs.append( ToolMessage( tool_call_id=tc["id"], content="Action Recorded", status="success", ) ) if not validation_errors: action_result = actions_list for tm in tool_outputs: current_messages.append(tm) if action_calls and not validation_errors: break if validation_errors: logger.warning("Some action calls failed validation. Feeding back to LLM.") continue else: tool_limit_exceeded = True raw_thinking = "\n".join(raw_thoughts) if raw_thoughts else None native_thinking = "\n".join(native_thoughts) if native_thoughts else None return ( action_result, raw_thinking, native_thinking, tool_limit_exceeded, ) async def _run_other_tool( self, tc, traced_tools, new_subagent_calls, state: State, raw_thoughts: list, native_thoughts: list, ) -> str: tool_name = tc["name"] logger.info(f"Operator requested tool: {tool_name}") if tool_name in ["ask_diagnoser", "video_analyzer"]: new_subagent_calls.append(tool_name) if ":" in tool_name: tool_to_run = get_tool_by_name(tool_name, traced_tools) else: tool_to_run = next((t for t in traced_tools if t.name == tool_name), None) if tool_to_run: try: args = dict(tc["args"]) state.operator_raw_thinking = "\n".join(raw_thoughts) if raw_thoughts else None state.operator_native_thinking = ( "\n".join(native_thoughts) if native_thoughts else None ) result_obj = await invoke_tool_with_injection( tool=tool_to_run, args=args, tool_call_id=tc["id"], state=state, ) return get_tool_result_content(result_obj) except Exception as e: return f"Error running tool {tool_name}: {e}" else: return f"Error: Tool {tool_name} not supported" @wrap_with_callbacks( before=lambda: logger.info("Starting Operator Agent..."), on_success=lambda _: logger.success("Operator Agent"), on_failure=lambda _: logger.error("Operator Agent"), ) @trace(type="agent", name="operator") async def __call__(self, state: State): state = strict_state(state, "operator") # 1. Get perception data from state (populated by Perception node) operator_raw_data = getattr(state, "operator_raw_data", {}) or {} latest_screenshot_b64 = operator_raw_data.get("screenshot_b64") xml_hierarchy = operator_raw_data.get("xml_hierarchy") operator_raw_data.get("ocr_results") fused_xml = state.latest_ui_hierarchy if hasattr(state, "latest_ui_hierarchy") else [] state.latest_screenshot if hasattr(state, "latest_screenshot") else None if not latest_screenshot_b64 or xml_hierarchy is None: logger.error( "Operator requires perception data in state (populated by Perception node)" ) raise ValueError("Operator requires perception data in state") # 3. Get history and plan steps, task_plan = self._get_history_and_plan() self._turn_plan_baseline = self._plan_ledger_baseline(task_plan) self._turn_ledger_streak = unwritten_action_streak(steps) # 4. Format minimal list width = 1080 height = 2400 w_raw = operator_raw_data.get("width") h_raw = operator_raw_data.get("height") if isinstance(w_raw, int) and isinstance(h_raw, int): width = w_raw height = h_raw else: try: if self.ctx and hasattr(self.ctx, "device") and self.ctx.device: w = getattr(self.ctx.device, "device_width", 1080) h = getattr(self.ctx.device, "device_height", 2400) if isinstance(w, int): width = w if isinstance(h, int): height = h except (AttributeError, TypeError): pass minimal_list, elements, labels = format_minimal_list_with_elements(fused_xml, width, height) # Save transient indexed_points and indexed_elements on the state instance state.indexed_points = [el["center"] for el in elements] state.indexed_elements = elements current_step_num = len(steps) + 1 # 5. Prepare Tools. Device-action shells come from the canonical manifest # (artemis/mcp/action_specs.py) and are assembled against the installed # actuator backend's capabilities: an action the backend does not implement # is simply never declared (and the prompt assembly drops its teaching # segments in lockstep). available_actions = self._available_device_actions() all_tools = [ operator_shell_tool(name) for name in OPERATOR_SHELL_ORDER if name in available_actions ] + self.tools # 5. Mount the task-output analyzer and collect background ADB task state. # The analyzer is always available: the "output truncated, use # analyze_task_output" hint arrives mid-turn, and tools are bound once per # turn, so a conditional mount would be one turn late. if not any(t.name == "analyze_task_output" for t in all_tools): all_tools.append(analyze_task_output_wrapper.tool_fn_getter(self.ctx)) traced_tools = [trace_langchain_tool(t, self.ctx) for t in all_tools] adb_registry = get_adb_task_registry(self.ctx) active_background_tasks = adb_registry.active_task_summaries() newly_finished_tasks = adb_registry.pop_unnotified_finished() # 6. Prepare LLM llm = get_llm(ctx=self.ctx, name="operator") # 7. Build Prompt messages = await self._build_prompt( state, latest_screenshot_b64, fused_xml, minimal_list, current_step_num, steps, task_plan, active_background_tasks=active_background_tasks, newly_finished_tasks=newly_finished_tasks, ) # 8. Invoke LLM Loop new_subagent_calls = [] ( action_result, raw_thinking, native_thinking, tool_limit_exceeded, ) = await self._invoke_llm_loop( base_llm=llm, current_messages=messages, traced_tools=traced_tools, new_subagent_calls=new_subagent_calls, state=state, ) # 8b. Transcript path: hold this turn's messages (observation tail + # tool-loop products) for commit at the next build, when the turn's # step id and validator result exist. self._stage_transcript_turn(messages) # 9. Update State structured_decisions = json.dumps(action_result) if action_result else None return { "structured_decisions": structured_decisions, "operator_raw_thinking": raw_thinking, "operator_native_thinking": native_thinking, "indexed_points": state.indexed_points, "indexed_elements": state.indexed_elements, "current_step_id": state.current_step_id, "subagent_calls": ((state.subagent_calls or []) + new_subagent_calls), "operator_tool_limit_exceeded": tool_limit_exceeded, } def _plan_ledger_gate_enabled(self) -> bool: try: from artemis.config import load_agent_config return bool(load_agent_config().pro.execution.plan_ledger_gate) except (AttributeError, OSError, TypeError, ValueError): return True def _read_task_plan_text(self) -> str | None: """Current ``task_plan`` note text, or ``None`` when it cannot be read.""" engine = getattr(self.ctx, "data_engine", None) if not engine: return None try: path = get_note_file_path(engine.base_dir, "task_plan") if not path.exists(): return None return path.read_text(encoding="utf-8") except (AttributeError, OSError, TypeError) as e: logger.warning(f"Plan ledger gate: failed to read task_plan: {e}") return None def _plan_ledger_baseline(self, task_plan: str) -> str | None: """Return the plan text used as the current turn's baseline.""" if not self._plan_ledger_gate_enabled() or not getattr(self.ctx, "data_engine", None): return None if not parse_plan(task_plan).has_top_level: return None return task_plan def _plan_ledger_stale_turns(self) -> int: try: from artemis.config import load_agent_config return int(load_agent_config().pro.execution.plan_ledger_stale_turns) except (AttributeError, OSError, TypeError, ValueError): return 4 def _plan_ledger_bounce_reason(self) -> str | None: """Bounce message when the (unchanged) ledger is broken or stale, else ``None``.""" current = self._read_task_plan_text() if current is None: return None return render_plan_ledger_bounce( current, self._turn_ledger_streak, self._plan_ledger_stale_turns() ) def _plan_changed_this_turn(self) -> bool | None: """Whether ``task_plan`` differs from the turn's baseline (``None``: gate inactive).""" baseline = self._turn_plan_baseline if baseline is None: return None current = self._read_task_plan_text() if current is None: return None return current != baseline def _get_active_subgoal_hash(self) -> str: """Parses task_plan.md to find the active top-level subgoal hash.""" if not self.ctx.data_engine: return "default" task_plan_path = get_note_file_path(self.ctx.data_engine.base_dir, "task_plan") if not task_plan_path.exists(): return "default" try: content = task_plan_path.read_text(encoding="utf-8") parent_hash, _ = get_active_subgoal_hashes(content) return parent_hash except Exception as e: logger.error(f"Failed to parse active subgoal: {e}") return "default" def _build_plan_and_history(self, steps: list[dict[str, Any]], task_plan: str) -> str: """Builds a plan checklist and separate execution history.""" active_subgoal_hash = self._get_active_subgoal_hash() return build_history_for( "operator", task_plan, steps, active_subgoal_hash, last_n_detailed=self.last_n_detailed, ) def _max_burst_actions(self) -> int: """Configured ceiling for actions in one fast-action burst (default 4).""" try: from artemis.config import load_agent_config return int(load_agent_config().pro.execution.max_burst_actions) except Exception: return 4 def _translate_and_validate_tool(self, tc: dict, state: State) -> tuple[list[dict], str | None]: actions, err = self._translate_and_validate_tool_inner(tc, state) if err: return [], err return actions, None def _translate_and_validate_tool_inner( self, tc: dict, state: State ) -> tuple[list[dict], str | None]: tool_name = tc["name"].split(":")[-1] if ":" in tc["name"] else tc["name"] args = tc["args"] # Get device resolution fallbacks width = 1080 height = 2400 operator_raw_data = getattr(state, "operator_raw_data", {}) or {} w_raw = operator_raw_data.get("width") h_raw = operator_raw_data.get("height") if isinstance(w_raw, int) and isinstance(h_raw, int): width = w_raw height = h_raw else: try: if self.ctx and hasattr(self.ctx, "device") and self.ctx.device: w = getattr(self.ctx.device, "device_width", 1080) h = getattr(self.ctx.device, "device_height", 2400) if isinstance(w, int): width = w if isinstance(h, int): height = h except (AttributeError, TypeError): pass def resolve_target_element( target: Any, ) -> tuple[dict | None, str | None]: if isinstance(target, (int, float)): try: target_int = int(target) except (ValueError, TypeError): return None, f"Error: Invalid target index {target}." indexed_elements = getattr(state, "indexed_elements", None) or [] if 1 <= target_int <= len(indexed_elements): return {**indexed_elements[target_int - 1], "label_source": "index"}, None return ( None, ( f"Error: Invalid target index {target_int}. Active" f" index range is 1 to {len(indexed_elements)}." ), ) elif isinstance(target, list) and len(target) == 2: try: nx, ny = map(float, target) x = int(max(0, min(width - 1, nx * width / 1000))) y = int(max(0, min(height - 1, ny * height / 1000))) # Record-time enrichment: a bare-coordinate target carries no # element semantics, so hit test the pre-action frame's indexed # elements to best-effort recover what sits under the point. hit_el, hit_source = find_element_at_point( getattr(state, "indexed_elements", None), x, y ) hit_el = hit_el or {} return { "center": [x, y], "text": hit_el.get("text"), "bounds": hit_el.get("bounds"), "class": hit_el.get("class"), "resource_id": hit_el.get("resource_id"), "is_ocr": bool(hit_el.get("is_ocr")), "label_source": hit_source, }, None except (ValueError, TypeError): return ( None, (f"Error: Invalid coordinates {target}. Must be list of 2 numbers."), ) return ( None, ( f"Error: Invalid target parameter {target}. Must be index" " (int/float) or coordinates [x, y]." ), ) if tool_name == "click": target = args.get("target") times = args.get("times", 1) delay_ms = args.get("delay_ms", 100) el, err = resolve_target_element(target) if err: return [], err norm_c = [ int(round(el["center"][0] * 1000.0 / width)), int(round(el["center"][1] * 1000.0 / height)), ] return [ { "action": "tap", "coordinates": el["center"], COORDINATE_SPACE_KEY: COORDINATE_SPACE_PIXEL, "normalized_coordinates": norm_c, "times": times, "delay_ms": delay_ms, "target_text": el.get("text"), "target_bounds": el.get("bounds"), "target_resource_id": el.get("resource_id"), "target_class": el.get("class"), "target_label_source": el.get("label_source", "none"), } ], None elif tool_name == "long_press": target = args.get("target") duration = args.get("duration", 1000) el, err = resolve_target_element(target) if err: return [], err norm_c = [ int(round(el["center"][0] * 1000.0 / width)), int(round(el["center"][1] * 1000.0 / height)), ] return [ { "action": "long_press_on", "coordinates": el["center"], COORDINATE_SPACE_KEY: COORDINATE_SPACE_PIXEL, "normalized_coordinates": norm_c, "duration": duration, "target_text": el.get("text"), "target_bounds": el.get("bounds"), "target_resource_id": el.get("resource_id"), "target_class": el.get("class"), "target_label_source": el.get("label_source", "none"), } ], None elif tool_name == "input_text": text = args.get("text") target = args.get("target") clear_exist = args.get("clear_exist", True) if text is None: return [], "Error: Missing required argument 'text'." if not isinstance(text, str): return ( [], (f"Error: 'text' must be a string, got {type(text).__name__}."), ) el, err = resolve_target_element(target) if err: return [], err norm_c = [ int(round(el["center"][0] * 1000.0 / width)), int(round(el["center"][1] * 1000.0 / height)), ] actions = [] actions.append( { "action": "focus_and_input_text", "coordinates": el["center"], COORDINATE_SPACE_KEY: COORDINATE_SPACE_PIXEL, "normalized_coordinates": norm_c, "text": text, "clear_before_input": clear_exist, "target_text": el.get("text"), "target_bounds": el.get("bounds"), "target_resource_id": el.get("resource_id"), "target_class": el.get("class"), "target_label_source": el.get("label_source", "none"), } ) return actions, None elif tool_name == "swipe": kind, target, parsed_duration = parse_swipe_parameters(args, default_duration=None) duration = args.get("duration") if duration is not None: if not isinstance(duration, (int, float)): try: duration = int(duration) except (ValueError, TypeError): return ( [], (f"Error: 'duration' must be a number, got {type(duration).__name__}."), ) else: duration = int(duration) else: duration = parsed_duration if kind == "direction": x1, y1, x2, y2, smart_dur = compute_smart_swipe_coordinates( direction=target, target=args.get("target"), indexed_elements=getattr(state, "indexed_elements", None), ui_hierarchy=getattr(state, "latest_ui_hierarchy", None), width=width, height=height, duration=duration, ) if duration is None: duration = smart_dur elif kind == "coords" and isinstance(target, list) and len(target) == 4: try: nx1, ny1, nx2, ny2 = map(float, target) x1 = int(max(0, min(width - 1, nx1 * width / 1000))) y1 = int(max(0, min(height - 1, ny1 * height / 1000))) x2 = int(max(0, min(width - 1, nx2 * width / 1000))) y2 = int(max(0, min(height - 1, ny2 * height / 1000))) except (ValueError, TypeError): return ( [], f"Error: Invalid custom swipe coordinates {target}.", ) else: return ( [], ( f"Error: Invalid swipe gesture parameters {args}. Must be" " direction string or custom coordinates." ), ) # Calculate duration automatically if not provided (safe range 350-900ms to eliminate Android inertial fling) if duration is None: dist = ((x2 - x1) ** 2 + (y2 - y1) ** 2) ** 0.5 if dist <= 300: duration = 350 elif dist <= 600: duration = 550 elif dist <= 1000: duration = 800 else: duration = 900 nx1, ny1 = int(round(x1 * 1000.0 / width)), int(round(y1 * 1000.0 / height)) nx2, ny2 = int(round(x2 * 1000.0 / width)), int(round(y2 * 1000.0 / height)) return [ { "action": "swipe", "coordinates": [x1, y1, x2, y2], COORDINATE_SPACE_KEY: COORDINATE_SPACE_PIXEL, "normalized_coordinates": [nx1, ny1, nx2, ny2], "normalized_start_coordinates": [nx1, ny1], "normalized_end_coordinates": [nx2, ny2], "duration": duration, } ], None elif tool_name == "press_key": key = args.get("key") or args.get("keycode") if not key: return [], "Error: Missing required argument 'key'." if not isinstance(key, str): return ( [], f"Error: 'key' must be a string, got {type(key).__name__}.", ) normalized_key = key[8:] if key.startswith("KEYCODE_") else key if normalized_key not in ["ENTER", "BACK", "HOME", "APP_SWITCH"]: return [], f"Error: Unsupported key '{key}'." return [{"action": "press_key", "keycode": f"KEYCODE_{normalized_key}"}], None elif tool_name == "manage_app": action = args.get("action") app_name = args.get("app_name") if action not in ["launch", "stop"]: return ( [], f"Error: Unsupported app management action '{action}'.", ) if not app_name or not isinstance(app_name, str): return [], "Error: 'app_name' must be a valid non-empty string." action_str = "launch_app" if action == "launch" else "stop_app" return [{"action": action_str, "app_name": app_name}], None elif tool_name == "wait_for_delay": time_in_ms = args.get("time_in_ms") if time_in_ms is None: return [], "Error: Missing required argument 'time_in_ms'." try: time_in_ms = int(time_in_ms) except (ValueError, TypeError): return [], f"Error: Invalid time_in_ms value {time_in_ms}." return [{"action": "wait_for_delay", "time_in_ms": time_in_ms}], None return [], f"Error: Unsupported tool name '{tool_name}'."