# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import asyncio import json from pathlib import Path import re from typing import Annotated, Any, Literal from uuid import uuid4 from langchain_core.messages import SystemMessage, ToolMessage from langchain_core.tools import BaseTool, tool from langgraph.types import Command from artemis.constants import ( OPERATOR_MAX_CHAT_ROUNDS, VALIDATOR_MESSAGES_KEY, ) from artemis.context import ArtemisContext from artemis.data_engine.trace import ( TraceSpan, trace, trace_langchain_tool, ) from artemis.graph.state import State from artemis.services.llm import get_llm, invoke_llm_with_timeout_message from artemis.tools.command_tool import ( _BACKGROUND_TASKS, _FINISHED_TASKS_LOGS, _is_output_long, analyze_task_output_wrapper, ) from artemis.tools.index import get_tool_by_name from artemis.tools.tool_wrapper import ( get_tool_result_content, invoke_tool_with_injection, ) from artemis.utils.coordinates import parse_swipe_parameters from artemis.utils.decorators import wrap_with_callbacks from artemis.utils.file import create_snapshot from artemis.utils.logger import get_logger from artemis.utils.notes import get_note_file_path, get_notes_dir from artemis.utils.task_tree import ( build_plan_and_history, get_active_subgoal_hashes, ) from artemis.utils.verification import ( append_verification_chat, get_verification_chat_path, get_verification_chat_rounds, read_verification_chat, ) from artemis.utils.visualization import format_minimal_list_with_elements logger = get_logger(__name__) DEFERRING_TOOLS = { "ask_diagnoser", "video_analyzer", "read_note", "list_notes", "save_note", "run_adb_command", "manage_task", "analyze_task_output", "ask_explorer", } from artemis.agents.operator.prompts import ( PromptBuilder, PromptComponent, TemplatePromptComponent, ObservationPromptComponent, ScreenshotSimilarityPromptComponent, CheckerFeedbackPromptComponent, BackgroundTasksPromptComponent, ShortTermMemoryPromptComponent, TaskPlanWarningPromptComponent, ToolLimitWarningPromptComponent, InjectedInstructionPromptComponent, ) class OperatorNode: def __init__( self, ctx: ArtemisContext, tools: list[BaseTool] | None = None, prompt_components: list[PromptComponent] | None = None, last_n_detailed: int = 1, ): self.ctx = ctx self.tools = tools or [] self.prompt_components = prompt_components or [] self.last_n_detailed = last_n_detailed self.prompts = {} prompts_path = Path(__file__).parent / "operator.json" if prompts_path.exists(): try: self.prompts = json.loads(prompts_path.read_text(encoding="utf-8")) except Exception as e: logger.error(f"Failed to load operator prompts: {e}") def _check_infinite_loop(self, state: State): subagent_calls = state.subagent_calls or [] if len(subagent_calls) >= 3: last_three = subagent_calls[-3:] if len(set(last_three)) == 1: logger.error( f"Infinite loop detected: Sub-agent {last_three[0]} called" " 3 times consecutively." ) raise RuntimeError( f"Infinite loop detected: Sub-agent {last_three[0]} called" " 3 times consecutively." ) def _get_history_and_plan(self) -> tuple[list[dict], str]: steps = [] if self.ctx.data_engine: steps = self.ctx.data_engine.get_agent_friendly_steps() task_plan = "No task plan yet." if self.ctx.data_engine: task_plan_path = get_note_file_path(self.ctx.data_engine.base_dir, "task_plan") if task_plan_path.exists(): try: task_plan = task_plan_path.read_text(encoding="utf-8") except Exception as e: task_plan = f"Error reading task plan: {e}" return steps, task_plan def _get_verification_chat_rounds(self) -> tuple[int, int]: if not self.ctx.data_engine: return 0, 0 subgoal_hash = self._get_active_subgoal_hash() chat_path = get_verification_chat_path(self.ctx.data_engine.base_dir, subgoal_hash) turns = read_verification_chat(chat_path) return get_verification_chat_rounds(turns) def _has_checker_feedback(self) -> bool: if not self.ctx.data_engine: return False subgoal_hash = self._get_active_subgoal_hash() chat_path = get_verification_chat_path(self.ctx.data_engine.base_dir, subgoal_hash) return chat_path.exists() def _get_reply_to_checker_tool(self) -> BaseTool: @tool def reply_to_checker( reasoning: Annotated[ str, "Your reasoning or observations regarding the verification feedback.", ], ): """[TERMINAL] Use this tool to reply to the verification feedback with your reasoning or observations. This ends your turn. """ if self.ctx.data_engine: subgoal_hash = self._get_active_subgoal_hash() chat_path = get_verification_chat_path(self.ctx.data_engine.base_dir, subgoal_hash) turns = read_verification_chat(chat_path) max_op, max_chk = get_verification_chat_rounds(turns) round_num = max(max_op + 1, max_chk) if round_num == 0: round_num = 1 if append_verification_chat(chat_path, "operator", reasoning, round_num): return "Successfully recorded your reply to the Checker." else: return "Failed to save reply." else: return "Error: DataEngine not available." return reply_to_checker async def _build_prompt( self, state: State, latest_screenshot_b64: str, fused_xml: list[dict], minimal_list: str, current_step_num: int, steps: list[dict], task_plan: str, active_background_tasks: list[dict] = None, newly_finished_tasks: list[dict] = None, ) -> list: plan_and_history = self._build_plan_and_history(steps, task_plan) builder = PromptBuilder() has_feedback = self._has_checker_feedback() components = self.prompt_components if not components: if has_feedback: components = [ ( TemplatePromptComponent(), {"template_name": "troubleshooter_template"}, ), (ObservationPromptComponent(), {}), (ScreenshotSimilarityPromptComponent(), {}), (InjectedInstructionPromptComponent(), {}), (ShortTermMemoryPromptComponent(), {}), (BackgroundTasksPromptComponent(), {}), (CheckerFeedbackPromptComponent(), {}), (TaskPlanWarningPromptComponent(), {}), (ToolLimitWarningPromptComponent(), {}), ] else: components = [ ( TemplatePromptComponent(), {"template_name": "main_template"}, ), (ObservationPromptComponent(), {}), (ScreenshotSimilarityPromptComponent(), {}), (InjectedInstructionPromptComponent(), {}), (ShortTermMemoryPromptComponent(), {}), (BackgroundTasksPromptComponent(), {}), (TaskPlanWarningPromptComponent(), {}), (ToolLimitWarningPromptComponent(), {}), ] else: components = [(c, {}) for c in components] + [(ToolLimitWarningPromptComponent(), {})] # Active and finished tasks are now passed as arguments active_tasks = active_background_tasks or [] newly_finished = newly_finished_tasks or [] for component, extra_kwargs in components: kwargs_to_pass = { "prompts": self.prompts, "plan_and_history": plan_and_history, "latest_screenshot_b64": latest_screenshot_b64, "minimal_list": minimal_list, "current_step_num": current_step_num, "active_background_tasks": active_tasks, "newly_finished_tasks": newly_finished, "steps": steps, } kwargs_to_pass.update(extra_kwargs) await component(builder, state, self.ctx, **kwargs_to_pass) return builder.build() async def _invoke_llm_loop( self, base_llm, current_messages: list, traced_tools: list, new_subagent_calls: list, state: State, ) -> tuple[list[dict] | None, bool, str | None, str | None, bool]: max_iterations = 20 action_result = None requested_argue = False raw_thoughts = [] native_thoughts = [] tool_limit_exceeded = False async def read_stream_and_accumulate(target_llm, msgs): try: full_response = None has_chunks = False async for chunk in target_llm.astream(msgs): has_chunks = True if full_response is None: full_response = chunk else: full_response += chunk if has_chunks and full_response is not None: return full_response except Exception as e: logger.debug(f"astream failed or not supported: {e}. Falling back to ainvoke.") return await target_llm.ainvoke(msgs) for iteration in range(max_iterations): if iteration == max_iterations - 1: logger.warning( "Operator has reached the maximum number of tool calls. Adding reminder." ) current_messages.append( SystemMessage( content=( "You have reached the maximum number of tool calls" " for this turn. You must make a final action, or" " save your current research results via notes to" " continue later." ) ) ) bound_llm = base_llm.bind_tools(tools=traced_tools) response = await invoke_llm_with_timeout_message( read_stream_and_accumulate(bound_llm, current_messages) ) if hasattr(response, "response_metadata") and response.response_metadata: usage = ( response.response_metadata.get("usage_metadata") or response.response_metadata.get("token_usage") or {} ) prompt_tokens = usage.get("prompt_token_count") or usage.get("prompt_tokens") cached_tokens = usage.get("cached_content_token_count") or usage.get( "cached_tokens", 0 ) completion_tokens = usage.get("candidates_token_count") or usage.get( "completion_tokens" ) if prompt_tokens is not None: logger.info( f"LLM usage: prompt_tokens={prompt_tokens}" f" (cached={cached_tokens})," f" completion_tokens={completion_tokens}" ) if response.content: if isinstance(response.content, str): raw_thoughts.append(response.content) elif isinstance(response.content, list): for item in response.content: if isinstance(item, dict): if item.get("type") == "text": raw_thoughts.append(item["text"]) elif item.get("type") == "thinking": native_thoughts.append(item["thinking"]) additional_kwargs = getattr(response, "additional_kwargs", None) if ( not response.tool_calls and isinstance(additional_kwargs, dict) and not hasattr(additional_kwargs, "assert_called_with") # Filter out mocks and additional_kwargs.get("function_call") ): fc = additional_kwargs["function_call"] try: arguments_str = fc.get("arguments", "{}") if isinstance(arguments_str, str): arguments = json.loads(arguments_str) else: arguments = arguments_str tool_call = { "name": fc["name"], "args": arguments, "id": f"call_{uuid4().hex[:8]}", "type": "tool_call", } response.tool_calls = [tool_call] logger.info( "Fallback: Manually parsed function_call from" f" additional_kwargs: {tool_call}" ) except Exception as e: logger.error(f"Failed to parse function_call fallback: {e}") if not response.tool_calls: logger.warning("LLM stopped without calling any tool. Encouraging action.") break current_messages.append(response) action_tool_names = [ "click", "input_text", "swipe", "press_key", "manage_app", "wait_for_delay", "long_press", ] def normalize_name(name: str) -> str: return name.split(":")[-1] if ":" in name else name action_calls = [ tc for tc in response.tool_calls if normalize_name(tc["name"]) in action_tool_names ] reply_to_checker_calls = [ tc for tc in response.tool_calls if normalize_name(tc["name"]) == "reply_to_checker" ] other_calls = [ tc for tc in response.tool_calls if normalize_name(tc["name"]) not in action_tool_names + ["reply_to_checker"] ] tool_outputs = [] validation_errors = False requested_argue = False other_tool_failed = False other_tool_failure_msg = "" has_terminal_calls = bool(action_calls or reply_to_checker_calls) if other_calls: results = [] for tc in other_calls: tool_name = tc["name"] logger.info(f"Operator requested tool: {tool_name}") if tool_name in ["ask_diagnoser", "video_analyzer"]: new_subagent_calls.append(tool_name) if ":" in tool_name: tool_to_run = get_tool_by_name(tool_name, traced_tools) else: tool_to_run = next( (t for t in traced_tools if t.name == tool_name), None, ) if tool_to_run: args = dict(tc["args"]) span = TraceSpan(name=tool_name, trace_type="tool", ctx=self.ctx) span.payload = {"args": args} with span: try: state.operator_raw_thinking = ( "\n".join(raw_thoughts) if raw_thoughts else None ) state.operator_native_thinking = ( "\n".join(native_thoughts) if native_thoughts else None ) result_obj = await invoke_tool_with_injection( tool=tool_to_run, args=args, tool_call_id=tc["id"], state=state, ) content = get_tool_result_content(result_obj) status = "success" if isinstance(result_obj, Command): updates = result_obj.update if VALIDATOR_MESSAGES_KEY in updates: msgs = updates[VALIDATOR_MESSAGES_KEY] if msgs and hasattr(msgs[0], "status"): status = msgs[0].status is_err = status == "error" if not is_err: if isinstance(content, str): if content.startswith( "Failed" ) or content.lower().startswith("error"): is_err = True elif isinstance(content, list): for block in content: if ( isinstance(block, dict) and block.get("type") == "text" ): text = block.get("text", "") if text.startswith( "Failed" ) or text.lower().startswith("error"): is_err = True break if is_err: other_tool_failed = True other_tool_failure_msg = str(content) status = "error" span.status = "failed" span.error = str(content) else: span.result = content results.append((content, status)) except Exception as e: other_tool_failed = True other_tool_failure_msg = str(e) span.status = "failed" span.error = str(e) results.append( ( f"Error running tool {tool_name}: {e}", "error", ) ) else: other_tool_failed = True other_tool_failure_msg = f"Tool {tool_name} not supported" results.append((f"Error: Tool {tool_name} not supported", "error")) for tc, (content, status) in zip(other_calls, results): tool_outputs.append( ToolMessage( tool_call_id=tc["id"], content=content, status=status, ) ) if other_tool_failed: if action_calls: logger.warning( "Scratchpad/helper tool call failed. Deferring" f" screen actions: {other_tool_failure_msg}" ) for tc in action_calls: tool_outputs.append( ToolMessage( tool_call_id=tc["id"], content=( "This screen action was rejected" " because an accompanying tool call" " failed. Error:" f" {other_tool_failure_msg}. Currently," " no screen actions have been" " executed. Please correct the error" " and try again." ), status="error", ) ) action_calls = [] if reply_to_checker_calls: logger.warning( "Scratchpad/helper tool call failed. Deferring" f" reply to checker: {other_tool_failure_msg}" ) for tc in reply_to_checker_calls: tool_outputs.append( ToolMessage( tool_call_id=tc["id"], content=( "Your reply to the checker was" " rejected because an accompanying" " tool call failed. Error:" f" {other_tool_failure_msg}. Currently," " no response has been sent to the" " checker. Please correct the error" " and try again." ), status="error", ) ) reply_to_checker_calls = [] elif has_terminal_calls: has_deferring_calls = any( normalize_name(tc["name"]) in DEFERRING_TOOLS for tc in other_calls ) if has_deferring_calls: logger.info( "Operator mixed pre-decision tools" " (memory/exploration) and terminal action calls." " Deferring terminal actions." ) for tc in action_calls: tool_outputs.append( ToolMessage( tool_call_id=tc["id"], content=( "Your pre-decision tools have been" " successfully processed. However, the" " screen action you attempted to" " execute was rejected because you are" " not allowed to simultaneously" " read/write memory and execute" " physical actions in the same turn." " Currently, no screen actions have" " been executed. Please review your" " updated context and re-output your" " intended screen action." ), status="success", ) ) for tc in reply_to_checker_calls: tool_outputs.append( ToolMessage( tool_call_id=tc["id"], content=( "Your pre-decision tools have been" " successfully processed. However, your" " response to the checker was rejected" " because you are not allowed to" " simultaneously read/write memory and" " reply to the checker in the same" " turn. Currently, no response has been" " sent to the checker. Please review" " your updated context and re-output" " your reply." ), status="success", ) ) action_calls = [] reply_to_checker_calls = [] if action_calls: logger.info( f"Operator requested {len(action_calls)} action(s)." " Translating and processing all." ) actions_list = [] for tc in action_calls: translated_actions, error = self._translate_and_validate_tool(tc, state) if error: validation_errors = True tool_outputs.append( ToolMessage( tool_call_id=tc["id"], content=error, status="error", ) ) else: actions_list.extend(translated_actions) tool_outputs.append( ToolMessage( tool_call_id=tc["id"], content="Action Recorded", status="success", ) ) if not validation_errors: action_result = actions_list if reply_to_checker_calls: logger.info("Operator replied to checker.") tc = reply_to_checker_calls[0] requested_argue = True # Execute it immediately result = await self._run_other_tool( tc, traced_tools, new_subagent_calls, state, raw_thoughts, native_thoughts, ) tool_outputs.append( ToolMessage( tool_call_id=tc["id"], content=result, status="success" if not result.startswith("Error") else "error", ) ) for tm in tool_outputs: current_messages.append(tm) if (action_calls and not validation_errors) or requested_argue: break if validation_errors: logger.warning("Some action calls failed validation. Feeding back to LLM.") continue else: tool_limit_exceeded = True raw_thinking = "\n".join(raw_thoughts) if raw_thoughts else None native_thinking = "\n".join(native_thoughts) if native_thoughts else None return ( action_result, requested_argue, raw_thinking, native_thinking, tool_limit_exceeded, ) async def _run_other_tool( self, tc, traced_tools, new_subagent_calls, state: State, raw_thoughts: list, native_thoughts: list, ) -> str: tool_name = tc["name"] logger.info(f"Operator requested tool: {tool_name}") if tool_name in ["ask_diagnoser", "video_analyzer"]: new_subagent_calls.append(tool_name) if ":" in tool_name: tool_to_run = get_tool_by_name(tool_name, traced_tools) else: tool_to_run = next((t for t in traced_tools if t.name == tool_name), None) if tool_to_run: try: args = dict(tc["args"]) state.operator_raw_thinking = "\n".join(raw_thoughts) if raw_thoughts else None state.operator_native_thinking = ( "\n".join(native_thoughts) if native_thoughts else None ) result_obj = await invoke_tool_with_injection( tool=tool_to_run, args=args, tool_call_id=tc["id"], state=state, ) return get_tool_result_content(result_obj) except Exception as e: return f"Error running tool {tool_name}: {e}" else: return f"Error: Tool {tool_name} not supported" @wrap_with_callbacks( before=lambda: logger.info("Starting Operator Agent..."), on_success=lambda _: logger.success("Operator Agent"), on_failure=lambda _: logger.error("Operator Agent"), ) @trace(type="agent", name="operator") async def __call__(self, state: State): # 1. Check for infinite loops self._check_infinite_loop(state) # Create snapshot for optimistic execution if self.ctx.data_engine: notes_dir = get_notes_dir(self.ctx.data_engine.base_dir) if notes_dir.exists(): snapshot_dir = notes_dir.with_name("notes_snapshot") await asyncio.to_thread(create_snapshot, notes_dir, snapshot_dir) self.ctx.task_plan_snapshot = snapshot_dir logger.info(f"Created snapshot of notes at {snapshot_dir}") # 2. Get perception data from state (populated by Perception node) operator_raw_data = getattr(state, "operator_raw_data", {}) or {} latest_screenshot_b64 = operator_raw_data.get("screenshot_b64") xml_hierarchy = operator_raw_data.get("xml_hierarchy") operator_raw_data.get("ocr_results") fused_xml = state.latest_ui_hierarchy if hasattr(state, "latest_ui_hierarchy") else [] state.latest_screenshot if hasattr(state, "latest_screenshot") else None if not latest_screenshot_b64 or xml_hierarchy is None: logger.error( "Operator requires perception data in state (populated by Perception node)" ) raise ValueError("Operator requires perception data in state") # 3. Get history and plan steps, task_plan = self._get_history_and_plan() # 4. Format minimal list width = 1080 height = 2400 w_raw = operator_raw_data.get("width") h_raw = operator_raw_data.get("height") if isinstance(w_raw, int) and isinstance(h_raw, int): width = w_raw height = h_raw else: try: if self.ctx and hasattr(self.ctx, "device") and self.ctx.device: w = getattr(self.ctx.device, "device_width", 1080) h = getattr(self.ctx.device, "device_height", 2400) if isinstance(w, int): width = w if isinstance(h, int): height = h except Exception: pass minimal_list, elements, labels = format_minimal_list_with_elements(fused_xml, width, height) # Save transient indexed_points and indexed_elements on the state instance state.indexed_points = [el["center"] for el in elements] state.indexed_elements = elements current_step_num = len(steps) + 1 # 5. Prepare Tools click_tool = self._get_click_tool() input_text_tool = self._get_input_text_tool() swipe_tool = self._get_swipe_tool() press_key_tool = self._get_press_key_tool() manage_app_tool = self._get_manage_app_tool() wait_for_delay_tool = self._get_wait_for_delay_tool() long_press_tool = self._get_long_press_tool() all_tools = [ click_tool, input_text_tool, swipe_tool, press_key_tool, manage_app_tool, wait_for_delay_tool, long_press_tool, ] + self.tools if self._has_checker_feedback(): max_op, max_chk = self._get_verification_chat_rounds() max_rounds = ( getattr( getattr(self.ctx, "execution_setup", None), "checker_max_chat_rounds", OPERATOR_MAX_CHAT_ROUNDS, ) if self.ctx and getattr(self.ctx, "execution_setup", None) else OPERATOR_MAX_CHAT_ROUNDS ) if max_op < max_rounds: reply_to_checker_tool = self._get_reply_to_checker_tool() all_tools.append(reply_to_checker_tool) else: logger.info( f"Maximum chat rounds ({max_rounds}) reached. Removing reply_to_checker tool." ) # 5. Evaluate dynamic tools and prepare background tasks has_long_output = False for tinfo in _FINISHED_TASKS_LOGS.values(): if _is_output_long(tinfo.get("output", "")): has_long_output = True break if not has_long_output: for t in _BACKGROUND_TASKS.values(): if _is_output_long("".join(t.stdout_log)): has_long_output = True break if has_long_output and not any(t.name == "analyze_task_output" for t in all_tools): analyze_tool_fn = analyze_task_output_wrapper.tool_fn_getter(self.ctx) all_tools.append(analyze_tool_fn) traced_tools = [trace_langchain_tool(t, self.ctx) for t in all_tools] active_background_tasks = [] for tid, t in _BACKGROUND_TASKS.items(): active_background_tasks.append( { "task_id": tid, "command": t.command, "cwd": t.cwd, "terminal_id": t.terminal_id, "output_line_count": len(t.stdout_log), } ) newly_finished_tasks = [] for tid, tinfo in _FINISHED_TASKS_LOGS.items(): if not tinfo.get("notified", False): newly_finished_tasks.append( { "task_id": tid, "command": tinfo.get("command", ""), "status": tinfo.get("status", "completed"), "output_text": tinfo.get("output", ""), } ) tinfo["notified"] = True # Mark as read # 6. Prepare LLM llm = get_llm(ctx=self.ctx, name="operator") # 7. Build Prompt messages = await self._build_prompt( state, latest_screenshot_b64, fused_xml, minimal_list, current_step_num, steps, task_plan, active_background_tasks=active_background_tasks, newly_finished_tasks=newly_finished_tasks, ) # 8. Invoke LLM Loop new_subagent_calls = [] ( action_result, requested_argue, raw_thinking, native_thinking, tool_limit_exceeded, ) = await self._invoke_llm_loop( base_llm=llm, current_messages=messages, traced_tools=traced_tools, new_subagent_calls=new_subagent_calls, state=state, ) # 9. Update State structured_decisions = json.dumps(action_result) if action_result else None short_term_memory = None combined_thinking = "" if raw_thinking: combined_thinking += raw_thinking + "\n" if native_thinking: combined_thinking += native_thinking if combined_thinking: stm_match = re.search( r"(.*?)", combined_thinking, re.DOTALL, ) if stm_match: short_term_memory = stm_match.group(1).strip() return await state.asanitize_update( ctx=self.ctx, update={ "structured_decisions": structured_decisions, "operator_raw_thinking": raw_thinking, "operator_native_thinking": native_thinking, "short_term_memory": short_term_memory, "indexed_points": state.indexed_points, "complete_subgoals_by_ids": [], "current_step_id": state.current_step_id, "subagent_calls": ((state.subagent_calls or []) + new_subagent_calls), "operator_replied": requested_argue, "operator_tool_limit_exceeded": tool_limit_exceeded, }, agent="operator", ) def _get_active_subgoal_hash(self) -> str: """Parses task_plan.md to find the active top-level subgoal hash.""" if not self.ctx.data_engine: return "default" task_plan_path = get_note_file_path(self.ctx.data_engine.base_dir, "task_plan") if not task_plan_path.exists(): return "default" try: content = task_plan_path.read_text(encoding="utf-8") parent_hash, _ = get_active_subgoal_hashes(content) return parent_hash except Exception as e: logger.error(f"Failed to parse active subgoal: {e}") return "default" def _build_plan_and_history(self, steps: list[dict[str, Any]], task_plan: str) -> str: """Builds a plan checklist and separate execution history.""" active_subgoal_hash = self._get_active_subgoal_hash() return build_plan_and_history( task_plan, steps, active_subgoal_hash, last_n_detailed=self.last_n_detailed, strict_milestone_pruning=True, recent_window_size=3, chronological_last_step=True, ) def _get_long_press_tool(self) -> BaseTool: @tool def long_press( target: Annotated[ int | list[int], "Long press target. Can be an element index number (int, e.g." " 3) OR normalized coordinates (list of 2 integers, e.g. [500," " 600]).", ], duration: Annotated[int, "Long press duration in milliseconds (default 1000)."] = 1000, ): """[ACTION] Long press on the target location on the screen (supports element index or absolute normalized coordinates).""" return "Action Recorded" return long_press def _get_click_tool(self) -> BaseTool: @tool def click( target: Annotated[ int | list[int], "Click target. Can be an element index number (int, e.g. 3) OR" " normalized coordinates (list of 2 integers, e.g. [500," " 600]).", ], times: Annotated[ int, "Number of consecutive clicks on this target. Use this for" " double-clicks or multi-clicks (e.g. 7 to enter developer" " mode). Default is 1.", ] = 1, delay_ms: Annotated[ int, "Delay in milliseconds between consecutive clicks. Default is 100.", ] = 100, ): """[ACTION] Click on the target location on the screen (supports element index or absolute normalized coordinates).""" return "Action Recorded" return click def _get_input_text_tool(self) -> BaseTool: @tool def input_text( text: Annotated[ str, "The text content to input. Supports multi-line content with '\\n'.", ], target: Annotated[ int | list[int], "Input target field. Can be an input box element index number" " (int, e.g. 3) OR normalized coordinates (list of 2 integers," " e.g. [500, 600]).", ], clear_exist: Annotated[ bool, "Whether to clear existing text before typing. True (default):" " clear/replace entire text. False: append at the end of" " existing content.", ] = True, ): """[ACTION] Type text into the target input field (supports replacing whole text or appending to the end, and multi-line strings with '\\n').""" return "Action Recorded" return input_text def _get_swipe_tool(self) -> BaseTool: @tool def swipe( direction: Annotated[ Literal["up", "down", "left", "right"] | None, "Smart directional swipe ('up', 'down', 'left', 'right') for general page" " scrolling/browsing. Automatically computes safe swipe vectors (locking axes" " to prevent sidebar/edge mis-clicks), retains a ~40% visual overlap anchor," " and prevents inertial flings: 'up' (drags bottom-to-top, scrolls down to reveal content below)," " 'down' (drags top-to-bottom, scrolls up to reveal content above)," " 'left' (drags right-to-left, scrolls right), 'right' (drags left-to-right, scrolls left).", ] = None, start: Annotated[ list[int] | None, "Start point normalized coordinates [start_x, start_y] in 0-1000 scale for precise," " local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop).", ] = None, end: Annotated[ list[int] | None, "End point normalized coordinates [end_x, end_y] in 0-1000 scale for precise," " local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop).", ] = None, gesture: Annotated[ Literal["up", "down", "left", "right"] | list[int] | None, "Backward-compatible swipe gesture: smart direction string ('up', 'down', 'left', 'right')" " OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale.", ] = None, duration: Annotated[ int | None, "Optional swipe/drag duration in milliseconds. If specified, the gesture will run at this exact speed." " For drag-and-drop, list reordering, or sliding/adjusting sliders (e.g., volume, brightness, SeekBars)," " set duration >= 1000 (e.g. 1500). If omitted for directional swipe, the system automatically" " calculates the best duration based on the swipe distance to prevent unintended inertial flings.", ] = None, ): """[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen. • Smart Directional Scrolling ('direction'): Best for general browsing and page scrolling. Automatically computes safe swipe vectors (locking axes to prevent edge/sidebar mis-clicks), retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. • Precise Coordinate Gestures ('start', 'end'): Best for local, fine-grained interactions such as adjusting sliders/SeekBars (e.g., volume, brightness, progress bars) or drag-and-drop / list reordering. Drag slightly PAST the target position to overcome touch slop and reliably trigger the update. When setting a slider to Maximum (100%) or Minimum (0%), always swipe fully to the extreme boundary to guarantee reaching the absolute limits. Args: direction: Smart directional scrolling ('up', 'down', 'left', 'right'). Automatically computes safe swipe vectors, retaining 40% of the previous screen's visible content as an overlapping anchor: 'up' (drags bottom-to-top, scrolling down to reveal content below), 'down' (drags top-to-bottom, scrolling up to reveal content above), 'left' (drags right-to-left), 'right' (drags left-to-right). start: Start normalized coordinates [start_x, start_y] in 0-1000 scale for precise/local interactions. end: End normalized coordinates [end_x, end_y] in 0-1000 scale for precise/local interactions. gesture: Backward-compatible parameter: direction string OR custom coordinates list [start_x, start_y, end_x, end_y] in 0-1000 scale. duration: Optional gesture duration in milliseconds. If omitted for directional swipe, the system automatically computes a reliable duration based on the swipe distance to prevent unintended inertial flings. """ return "Action Recorded" return swipe def _get_press_key_tool(self) -> BaseTool: @tool def press_key( key: Annotated[ Literal["ENTER", "BACK", "HOME", "APP_SWITCH"], "Standard Android system button name (ENTER, BACK, HOME, APP_SWITCH).", ], ): """[ACTION] Press a physical or virtual system button (e.g. ENTER, BACK, HOME, APP_SWITCH). """ return "Action Recorded" return press_key def _get_manage_app_tool(self) -> BaseTool: @tool def manage_app( action: Annotated[Literal["launch", "stop"], "The action type."], app_name: Annotated[str, "Display name or package name of the application."], ): """[ACTION] Launch or force stop a specified application.""" return "Action Recorded" return manage_app def _get_wait_for_delay_tool(self) -> BaseTool: @tool def wait_for_delay( time_in_ms: Annotated[ int, "The duration to wait in milliseconds (e.g., 2000 for 2" " seconds, 5000 for 5 seconds). Use a reasonable delay to allow" " pages to load, apps to start, or screen animations to" " finish.", ], ): """[ACTION] Pause execution and wait for a fixed duration. Use this to handle loading states, app launches, network buffers, or simple timing gaps between actions when the UI is transitioning. """ return "Action Recorded" return wait_for_delay def _get_wait_for_text_tool(self) -> BaseTool: @tool def wait_for_text( text: Annotated[ str, "A simple, distinct keyword (e.g., 'Success', 'Done'," " 'Loading') to watch on the screen. Try to pass a clean word" " without trailing punctuation (e.g. 'Loading' instead of" " 'Loading...') for optimal matching reliability.", ], state: Annotated[ Literal["visible", "hidden"], "Wait for the keyword to appear ('visible') or disappear" " ('hidden') from the screen.", ] = "visible", timeout_ms: Annotated[int, "Maximum timeout in milliseconds (default 5000)."] = 5000, ): """[ACTION] Pause execution and wait intelligently for a specific, distinct text or keyword to appear or disappear on the screen. Use this when waiting for a specific loading screen to finish or a specific success/confirmation state to appear. """ return "Action Recorded" return wait_for_text def _translate_and_validate_tool(self, tc: dict, state: State) -> tuple[list[dict], str | None]: actions, err = self._translate_and_validate_tool_inner(tc, state) if err: return [], err return actions, None def _translate_and_validate_tool_inner( self, tc: dict, state: State ) -> tuple[list[dict], str | None]: tool_name = tc["name"].split(":")[-1] if ":" in tc["name"] else tc["name"] args = tc["args"] # Get device resolution fallbacks width = 1080 height = 2400 operator_raw_data = getattr(state, "operator_raw_data", {}) or {} w_raw = operator_raw_data.get("width") h_raw = operator_raw_data.get("height") if isinstance(w_raw, int) and isinstance(h_raw, int): width = w_raw height = h_raw else: try: if self.ctx and hasattr(self.ctx, "device") and self.ctx.device: w = getattr(self.ctx.device, "device_width", 1080) h = getattr(self.ctx.device, "device_height", 2400) if isinstance(w, int): width = w if isinstance(h, int): height = h except Exception: pass def resolve_target_element( target: Any, ) -> tuple[dict | None, str | None]: if isinstance(target, (int, float)): try: target_int = int(target) except (ValueError, TypeError): return None, f"Error: Invalid target index {target}." indexed_elements = getattr(state, "indexed_elements", None) or [] if 1 <= target_int <= len(indexed_elements): return indexed_elements[target_int - 1], None return ( None, ( f"Error: Invalid target index {target_int}. Active" f" index range is 1 to {len(indexed_elements)}." ), ) elif isinstance(target, list) and len(target) == 2: try: nx, ny = map(float, target) x = int(max(0, min(width - 1, nx * width / 1000))) y = int(max(0, min(height - 1, ny * height / 1000))) return { "center": [x, y], "text": None, "bounds": None, "class": None, "resource_id": None, "is_ocr": False, }, None except (ValueError, TypeError): return ( None, (f"Error: Invalid coordinates {target}. Must be list of 2 numbers."), ) return ( None, ( f"Error: Invalid target parameter {target}. Must be index" " (int/float) or coordinates [x, y]." ), ) if tool_name == "click": target = args.get("target") times = args.get("times", 1) delay_ms = args.get("delay_ms", 100) el, err = resolve_target_element(target) if err: return [], err norm_c = [ int(round(el["center"][0] * 1000.0 / width)), int(round(el["center"][1] * 1000.0 / height)), ] return [ { "action": "tap", "coordinates": el["center"], "normalized_coordinates": norm_c, "times": times, "delay_ms": delay_ms, "target_text": el.get("text"), "target_bounds": el.get("bounds"), "target_resource_id": el.get("resource_id"), "target_class": el.get("class"), } ], None elif tool_name == "long_press": target = args.get("target") duration = args.get("duration", 1000) el, err = resolve_target_element(target) if err: return [], err norm_c = [ int(round(el["center"][0] * 1000.0 / width)), int(round(el["center"][1] * 1000.0 / height)), ] return [ { "action": "long_press_on", "coordinates": el["center"], "normalized_coordinates": norm_c, "duration": duration, "target_text": el.get("text"), "target_bounds": el.get("bounds"), "target_resource_id": el.get("resource_id"), "target_class": el.get("class"), } ], None elif tool_name == "input_text": text = args.get("text") target = args.get("target") clear_exist = args.get("clear_exist", True) if text is None: return [], "Error: Missing required argument 'text'." if not isinstance(text, str): return ( [], (f"Error: 'text' must be a string, got {type(text).__name__}."), ) el, err = resolve_target_element(target) if err: return [], err norm_c = [ int(round(el["center"][0] * 1000.0 / width)), int(round(el["center"][1] * 1000.0 / height)), ] actions = [] actions.append( { "action": "focus_and_input_text", "coordinates": el["center"], "normalized_coordinates": norm_c, "text": text, "clear_before_input": clear_exist, "target_text": el.get("text"), "target_bounds": el.get("bounds"), "target_resource_id": el.get("resource_id"), "target_class": el.get("class"), } ) return actions, None elif tool_name == "swipe": kind, target, parsed_duration = parse_swipe_parameters(args, default_duration=None) duration = args.get("duration") if duration is not None: if not isinstance(duration, (int, float)): try: duration = int(duration) except (ValueError, TypeError): return ( [], (f"Error: 'duration' must be a number, got {type(duration).__name__}."), ) else: duration = int(duration) else: duration = parsed_duration if kind == "direction": gesture = target # Align with FinalRun directional swipe vector rules: # Locks X axis to 60% of width for up/down to prevent mis-clicks on alphabet sidebar. # Horizontal Y axis is locked to 60% of height. if gesture == "up": coords = [ int(width * 0.6), int(height * 0.8), int(width * 0.6), int(height * 0.2), ] elif gesture == "down": coords = [ int(width * 0.6), int(height * 0.2), int(width * 0.6), int(height * 0.8), ] elif gesture == "left": coords = [ int(width * 0.8), int(height * 0.6), int(width * 0.2), int(height * 0.6), ] elif gesture == "right": coords = [ int(width * 0.2), int(height * 0.6), int(width * 0.8), int(height * 0.6), ] else: return [], f"Error: Invalid swipe direction '{gesture}'." x1, y1, x2, y2 = coords elif kind == "coords" and isinstance(target, list) and len(target) == 4: try: nx1, ny1, nx2, ny2 = map(float, target) x1 = int(max(0, min(width - 1, nx1 * width / 1000))) y1 = int(max(0, min(height - 1, ny1 * height / 1000))) x2 = int(max(0, min(width - 1, nx2 * width / 1000))) y2 = int(max(0, min(height - 1, ny2 * height / 1000))) except (ValueError, TypeError): return ( [], f"Error: Invalid custom swipe coordinates {target}.", ) else: return ( [], ( f"Error: Invalid swipe gesture parameters {args}. Must be" " direction string or custom coordinates." ), ) # Calculate duration automatically if not provided (safe range 280-400ms to avoid long-press and fling) if duration is None: dist = ((x2 - x1) ** 2 + (y2 - y1) ** 2) ** 0.5 if dist <= 400: duration = 280 elif dist <= 800: duration = 320 elif dist <= 1200: duration = 350 elif dist <= 1600: duration = 380 else: duration = 400 nx1, ny1 = int(round(x1 * 1000.0 / width)), int(round(y1 * 1000.0 / height)) nx2, ny2 = int(round(x2 * 1000.0 / width)), int(round(y2 * 1000.0 / height)) return [ { "action": "swipe", "coordinates": [x1, y1, x2, y2], "normalized_coordinates": [nx1, ny1, nx2, ny2], "normalized_start_coordinates": [nx1, ny1], "normalized_end_coordinates": [nx2, ny2], "duration": duration, } ], None elif tool_name == "press_key": key = args.get("key") or args.get("keycode") if not key: return [], "Error: Missing required argument 'key'." if not isinstance(key, str): return ( [], f"Error: 'key' must be a string, got {type(key).__name__}.", ) normalized_key = key[8:] if key.startswith("KEYCODE_") else key if normalized_key not in ["ENTER", "BACK", "HOME", "APP_SWITCH"]: return [], f"Error: Unsupported key '{key}'." return [{"action": "press_key", "keycode": f"KEYCODE_{normalized_key}"}], None elif tool_name == "manage_app": action = args.get("action") app_name = args.get("app_name") if action not in ["launch", "stop"]: return ( [], f"Error: Unsupported app management action '{action}'.", ) if not app_name or not isinstance(app_name, str): return [], "Error: 'app_name' must be a valid non-empty string." action_str = "launch_app" if action == "launch" else "stop_app" return [{"action": action_str, "app_name": app_name}], None elif tool_name == "wait_for_delay": time_in_ms = args.get("time_in_ms") if time_in_ms is None: return [], "Error: Missing required argument 'time_in_ms'." try: time_in_ms = int(time_in_ms) except (ValueError, TypeError): return [], f"Error: Invalid time_in_ms value {time_in_ms}." return [{"action": "wait_for_delay", "time_in_ms": time_in_ms}], None return [], f"Error: Unsupported tool name '{tool_name}'."