1441 lines
61 KiB
Python
1441 lines
61 KiB
Python
# Copyright 2026 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import json
|
|
from typing import Any
|
|
from uuid import uuid4
|
|
|
|
from langchain_core.messages import SystemMessage, ToolMessage
|
|
from langchain_core.tools import BaseTool
|
|
|
|
from artemis.agents.validator.tool_declarations import normalize_coordinate_target
|
|
from artemis.config.settings import settings
|
|
from artemis.context import ArtemisContext
|
|
from artemis.core.tool_failure import is_tool_failure
|
|
from artemis.data_engine.trace import (
|
|
TraceSpan,
|
|
trace,
|
|
trace_langchain_tool,
|
|
)
|
|
from artemis.graph.state import State
|
|
from artemis.graph.visibility import strict_state
|
|
from artemis.mcp.action_specs import OPERATOR_SHELL_ORDER, operator_shell_tool
|
|
from artemis.llm.usage import usage_from_message
|
|
from artemis.services.llm import acomplete, get_llm, invoke_llm_with_timeout_message
|
|
from artemis.tools.command_tool import (
|
|
analyze_task_output_wrapper,
|
|
get_adb_task_registry,
|
|
)
|
|
from artemis.tools.index import get_tool_by_name
|
|
from artemis.tools.tool_wrapper import (
|
|
get_tool_result_content,
|
|
invoke_tool_with_injection,
|
|
tool_result_messages,
|
|
)
|
|
from artemis.utils.coordinates import (
|
|
COORDINATE_SPACE_KEY,
|
|
COORDINATE_SPACE_PIXEL,
|
|
compute_smart_swipe_coordinates,
|
|
parse_swipe_parameters,
|
|
)
|
|
from artemis.utils.decorators import wrap_with_callbacks
|
|
from artemis.utils.logger import get_logger
|
|
from artemis.utils.notes import get_note_file_path
|
|
from artemis.utils.plan_grammar import parse_plan
|
|
from artemis.memory.context_policy import build_history_for
|
|
from artemis.utils.task_tree import get_active_subgoal_hashes
|
|
from artemis.utils.visualization import format_minimal_list_with_elements
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
|
|
class OperatorNoProgressError(RuntimeError):
|
|
"""The model repeatedly returned no usable tool call; stop without device actions."""
|
|
|
|
|
|
DEFERRING_TOOLS = {
|
|
"ask_diagnoser",
|
|
"video_analyzer",
|
|
"read_note",
|
|
"list_notes",
|
|
"search_history",
|
|
"replay_steps",
|
|
"get_step_screenshot",
|
|
"run_adb_command",
|
|
"manage_task",
|
|
"analyze_task_output",
|
|
"ask_explorer",
|
|
}
|
|
|
|
# Bare key names press_key accepts (case-insensitive, optional KEYCODE_ prefix).
|
|
SUPPORTED_PRESS_KEYS = ("ENTER", "BACK", "HOME", "APP_SWITCH")
|
|
|
|
from artemis.agents.operator.prompts import (
|
|
OPERATOR_MAX_TOOL_ITERATIONS,
|
|
PromptBuilder,
|
|
PromptComponent,
|
|
TemplatePromptComponent,
|
|
ObservationPromptComponent,
|
|
PlanRecitationPromptComponent,
|
|
ScreenshotSimilarityPromptComponent,
|
|
HistoricalStateHintPromptComponent,
|
|
FeedbackPromptComponent,
|
|
ExecutionIncidentPromptComponent,
|
|
BackgroundTasksPromptComponent,
|
|
render_plan_ledger_bounce,
|
|
ReasoningReminderPromptComponent,
|
|
unwritten_action_streak,
|
|
ToolLimitWarningPromptComponent,
|
|
InjectedInstructionPromptComponent,
|
|
render_transcript_static_system,
|
|
)
|
|
from artemis.agents.operator.prompts import load_operator_prompts
|
|
|
|
|
|
def _described(value: Any) -> str | None:
|
|
"""Trim target_description, returning None for missing or blank values."""
|
|
if isinstance(value, str) and value.strip():
|
|
return value.strip()
|
|
return None
|
|
|
|
|
|
class OperatorNode:
|
|
def __init__(
|
|
self,
|
|
ctx: ArtemisContext,
|
|
tools: list[BaseTool] | None = None,
|
|
prompt_components: list[PromptComponent] | None = None,
|
|
last_n_detailed: int = 1,
|
|
transcript_config=None,
|
|
):
|
|
self.ctx = ctx
|
|
self.tools = tools or []
|
|
self.prompt_components = prompt_components or []
|
|
self.last_n_detailed = last_n_detailed
|
|
try:
|
|
self.prompts = load_operator_prompts()
|
|
except (OSError, ValueError, json.JSONDecodeError) as e:
|
|
raise RuntimeError(f"Failed to load operator prompts: {e}") from e
|
|
|
|
# M2 transcript flag (agent.memory.transcript.enabled). Off keeps the
|
|
# legacy 2-message prompt path byte-for-byte; an explicit
|
|
# ``transcript_config`` overrides the loaded configuration (tests).
|
|
if transcript_config is not None:
|
|
self._transcript_cfg = transcript_config
|
|
else:
|
|
try:
|
|
from artemis.config import load_agent_config
|
|
|
|
self._transcript_cfg = load_agent_config().memory.transcript
|
|
except Exception:
|
|
from artemis.config.agent import MemoryTranscriptConfig
|
|
|
|
self._transcript_cfg = MemoryTranscriptConfig()
|
|
# Index into the rendered message list where this turn's tail begins;
|
|
# set by the transcript build and consumed after the tool loop.
|
|
self._transcript_turn_base: int | None = None
|
|
self._turn_plan_baseline: str | None = None
|
|
self._turn_ledger_streak: int = 0
|
|
# Whether the previous turn's replies carried no visible reasoning text;
|
|
# the next observation tail then carries a one-line reminder (no extra
|
|
# model call). Legacy-path fallback only: with a transcript ledger the
|
|
# ledger's own ``last_turn_silent`` is authoritative.
|
|
self._previous_turn_silent: bool = False
|
|
|
|
def _previous_turn_was_silent(self) -> bool:
|
|
"""Whether the previous turn carried no visible reasoning text.
|
|
|
|
The transcript ledger judges its committed turns itself; the local flag
|
|
only serves the legacy path, which has no ledger.
|
|
"""
|
|
from artemis.memory import TranscriptLedger
|
|
|
|
ledger = getattr(self.ctx, "transcript_ledger", None)
|
|
if isinstance(ledger, TranscriptLedger):
|
|
return bool(ledger.last_turn_silent)
|
|
return self._previous_turn_silent
|
|
|
|
def _available_device_actions(self) -> frozenset[str]:
|
|
"""Device actions the installed actuator backend provides.
|
|
|
|
Without an explicit ``ctx.actuator`` the full manifest set applies (the
|
|
default AdbActuator implements everything), so behavior is unchanged until a
|
|
partial backend is actually installed.
|
|
"""
|
|
from artemis.mcp.action_manifest import (
|
|
OPTIONAL_ACTIONS,
|
|
REQUIRED_ACTIONS,
|
|
available_device_actions,
|
|
)
|
|
|
|
actuator = getattr(self.ctx, "actuator", None)
|
|
if actuator is not None and callable(getattr(actuator, "capabilities", None)):
|
|
try:
|
|
extension_names = frozenset(
|
|
e.name for e in actuator.extensions() if "operator" in e.targets
|
|
)
|
|
return available_device_actions(actuator) | extension_names
|
|
except Exception as e:
|
|
logger.warning(f"Failed to read actuator capabilities: {e}")
|
|
return REQUIRED_ACTIONS | OPTIONAL_ACTIONS
|
|
|
|
def _get_history_and_plan(self) -> tuple[list[dict], str]:
|
|
steps = []
|
|
if self.ctx.data_engine:
|
|
steps = self.ctx.data_engine.get_agent_friendly_steps()
|
|
|
|
task_plan = "No task plan yet."
|
|
if self.ctx.data_engine:
|
|
task_plan_path = get_note_file_path(self.ctx.data_engine.base_dir, "task_plan")
|
|
|
|
if task_plan_path.exists():
|
|
try:
|
|
task_plan = task_plan_path.read_text(encoding="utf-8")
|
|
except Exception as e:
|
|
task_plan = f"Error reading task plan: {e}"
|
|
|
|
return steps, task_plan
|
|
|
|
async def _build_prompt(
|
|
self,
|
|
state: State,
|
|
latest_screenshot_b64: str,
|
|
fused_xml: list[dict],
|
|
minimal_list: str,
|
|
current_step_num: int,
|
|
steps: list[dict],
|
|
task_plan: str,
|
|
active_background_tasks: list[dict] = None,
|
|
newly_finished_tasks: list[dict] = None,
|
|
) -> list:
|
|
self._transcript_turn_base = None
|
|
if getattr(self._transcript_cfg, "enabled", False):
|
|
try:
|
|
return await self._build_prompt_transcript(
|
|
state,
|
|
latest_screenshot_b64,
|
|
minimal_list,
|
|
current_step_num,
|
|
steps,
|
|
task_plan,
|
|
active_background_tasks=active_background_tasks,
|
|
newly_finished_tasks=newly_finished_tasks,
|
|
)
|
|
except Exception as e:
|
|
logger.error(
|
|
f"Transcript prompt path failed; falling back to the legacy"
|
|
f" 2-message build for this turn: {e}"
|
|
)
|
|
self._transcript_turn_base = None
|
|
|
|
plan_and_history = self._build_plan_and_history(steps, task_plan)
|
|
|
|
builder = PromptBuilder()
|
|
|
|
# One constant component list: the prompt template is never switched by
|
|
# verification results. Check-related components are append-only and
|
|
# render nothing when there is nothing to say.
|
|
components = self.prompt_components
|
|
if not components:
|
|
components = [
|
|
(
|
|
TemplatePromptComponent(),
|
|
{"template_name": "main_template"},
|
|
),
|
|
(ExecutionIncidentPromptComponent(), {}),
|
|
(ObservationPromptComponent(), {}),
|
|
(ScreenshotSimilarityPromptComponent(), {}),
|
|
(HistoricalStateHintPromptComponent(), {}),
|
|
(InjectedInstructionPromptComponent(), {}),
|
|
(BackgroundTasksPromptComponent(), {}),
|
|
(FeedbackPromptComponent(), {}),
|
|
(ReasoningReminderPromptComponent(), {}),
|
|
(ToolLimitWarningPromptComponent(), {}),
|
|
]
|
|
else:
|
|
components = [(c, {}) for c in components] + [
|
|
(ReasoningReminderPromptComponent(), {}),
|
|
(ToolLimitWarningPromptComponent(), {}),
|
|
]
|
|
|
|
# Active and finished tasks are now passed as arguments
|
|
active_tasks = active_background_tasks or []
|
|
newly_finished = newly_finished_tasks or []
|
|
|
|
for component, extra_kwargs in components:
|
|
kwargs_to_pass = {
|
|
"prompts": self.prompts,
|
|
"plan_and_history": plan_and_history,
|
|
"task_plan": task_plan,
|
|
"latest_screenshot_b64": latest_screenshot_b64,
|
|
"minimal_list": minimal_list,
|
|
"current_step_num": current_step_num,
|
|
"active_background_tasks": active_tasks,
|
|
"newly_finished_tasks": newly_finished,
|
|
"previous_turn_silent": self._previous_turn_was_silent(),
|
|
"steps": steps,
|
|
}
|
|
kwargs_to_pass.update(extra_kwargs)
|
|
await component(builder, state, self.ctx, **kwargs_to_pass)
|
|
|
|
return builder.build()
|
|
|
|
# ------------------------------------------------------------------
|
|
# Transcript prompt path (M2, flag agent.memory.transcript.enabled)
|
|
# ------------------------------------------------------------------
|
|
|
|
def _ensure_transcript_ledger(self, state=None):
|
|
"""Session ledger on the composition root, created on first use."""
|
|
from artemis.memory import TranscriptLedger, ensure_step_memory
|
|
|
|
ledger = getattr(self.ctx, "transcript_ledger", None)
|
|
if isinstance(ledger, TranscriptLedger):
|
|
return ledger
|
|
cfg = self._transcript_cfg
|
|
# Anchor the ``T+mm:ss`` clock to the DataEngine session start so the
|
|
# tail offsets agree with the chunk ledger lines, the restored-history
|
|
# block and the video analyzer's action timeline.
|
|
session_start = getattr(self.ctx.data_engine, "session_start_time", None)
|
|
ledger = TranscriptLedger(
|
|
step_memory=ensure_step_memory(self.ctx),
|
|
image_scrub_depth=getattr(cfg, "image_scrub_depth", 3),
|
|
pending_grace_steps=getattr(cfg, "pending_grace_steps", 3),
|
|
xml_scrub_depth=getattr(cfg, "xml_scrub_depth", 1),
|
|
image_scrub_depth_relaxed=getattr(cfg, "image_scrub_depth_relaxed", None),
|
|
context_budget_tokens=getattr(cfg, "context_budget_tokens", None),
|
|
start_ratio=getattr(cfg, "start_ratio", None),
|
|
session_start=session_start if isinstance(session_start, (int, float)) else None,
|
|
)
|
|
# L2/L3 chunk compression (M3) rides the same flag; without a
|
|
# DataEngine there are no step records to chunk, so the ledger simply
|
|
# runs scrub-edge-only.
|
|
if self.ctx.data_engine is not None:
|
|
try:
|
|
from artemis.memory import HistoryChunkManager
|
|
|
|
chunking_cfg = None
|
|
try:
|
|
from artemis.config import load_agent_config
|
|
|
|
chunking_cfg = load_agent_config().memory.chunking
|
|
except Exception as exc:
|
|
logger.debug(
|
|
"Chunking config unavailable; using chunker defaults: %s",
|
|
exc,
|
|
exc_info=True,
|
|
)
|
|
ledger.attach_chunker(
|
|
HistoryChunkManager(
|
|
engine=self.ctx.data_engine,
|
|
ctx=self.ctx,
|
|
chunking_config=chunking_cfg,
|
|
transcript_config=cfg,
|
|
goal=getattr(state, "initial_goal", None) if state else None,
|
|
)
|
|
)
|
|
except Exception as e:
|
|
logger.error(f"History chunk manager unavailable: {e}")
|
|
try:
|
|
self.ctx.transcript_ledger = ledger
|
|
except (AttributeError, TypeError, ValueError):
|
|
pass
|
|
return ledger
|
|
|
|
async def _build_prompt_transcript(
|
|
self,
|
|
state: State,
|
|
latest_screenshot_b64: str,
|
|
minimal_list: str,
|
|
current_step_num: int,
|
|
steps: list[dict],
|
|
task_plan: str,
|
|
active_background_tasks: list[dict] = None,
|
|
newly_finished_tasks: list[dict] = None,
|
|
) -> list:
|
|
"""Build ``S + F + A + tail`` from the session transcript ledger.
|
|
|
|
The static system message renders once per session (byte-stable S
|
|
region); the previous turn is committed into the append-only active
|
|
region together with its step key and validator result; the fresh tail
|
|
carries the current observation plus the task-plan recitation.
|
|
"""
|
|
from langchain_core.messages import HumanMessage
|
|
|
|
from artemis.memory.transcript import mark_ephemeral
|
|
|
|
ledger = self._ensure_transcript_ledger(state)
|
|
|
|
# 1. S region: rendered exactly once per session.
|
|
if not ledger.has_static_prefix:
|
|
static_text = render_transcript_static_system(self.prompts, self.ctx, state)
|
|
ledger.set_static_prefix([SystemMessage(content=static_text)])
|
|
|
|
# 2. F region cold start: an empty ledger over an existing step record
|
|
# trail means the process restarted — freeze the compiled history once.
|
|
# A staged (not yet committed) turn means this is turn 2+ of a live
|
|
# session, not a cold start — step records already exist then, and
|
|
# seeding would be rejected by the empty-ledger invariant.
|
|
if (
|
|
ledger.turn_count == 0
|
|
and not ledger.has_staged_turn
|
|
and not ledger.has_restored_history
|
|
and steps
|
|
):
|
|
restored = build_history_for(
|
|
"operator_cold_start",
|
|
task_plan,
|
|
steps,
|
|
self._get_active_subgoal_hash(),
|
|
)
|
|
ledger.set_restored_history(
|
|
"[Restored history] Rebuilt from the step records of this task"
|
|
" after a process restart; step times below are relative to the"
|
|
" original session start.\n"
|
|
f"{restored}"
|
|
)
|
|
|
|
# 3. Commit the previous turn (its step id and validator result only
|
|
# exist now). A turn without a terminal action never ran the validator,
|
|
# and ``structured_decisions`` is cleared on planner rejection — the
|
|
# gate keeps stale reports out of the ledger.
|
|
validator_result = (
|
|
getattr(state, "last_execution_result", None)
|
|
if getattr(state, "structured_decisions", None)
|
|
else None
|
|
)
|
|
ledger.commit_staged(
|
|
step_key=getattr(state, "current_step_id", None),
|
|
validator_result=validator_result,
|
|
)
|
|
|
|
# 4. Current tail: observation + plan recitation + injected components.
|
|
builder = PromptBuilder()
|
|
builder.add_human_content(f"# CURRENT OBSERVATION [{ledger.elapsed_label()}]")
|
|
components = [
|
|
(PlanRecitationPromptComponent(), {}),
|
|
(ExecutionIncidentPromptComponent(), {}),
|
|
(ObservationPromptComponent(), {}),
|
|
(ScreenshotSimilarityPromptComponent(), {}),
|
|
(HistoricalStateHintPromptComponent(), {}),
|
|
(InjectedInstructionPromptComponent(), {}),
|
|
(BackgroundTasksPromptComponent(), {}),
|
|
(FeedbackPromptComponent(), {}),
|
|
(ReasoningReminderPromptComponent(), {}),
|
|
(ToolLimitWarningPromptComponent(), {}),
|
|
]
|
|
for component, extra_kwargs in components:
|
|
kwargs_to_pass = {
|
|
"prompts": self.prompts,
|
|
"task_plan": task_plan,
|
|
"latest_screenshot_b64": latest_screenshot_b64,
|
|
"minimal_list": minimal_list,
|
|
"current_step_num": current_step_num,
|
|
"active_background_tasks": active_background_tasks or [],
|
|
"newly_finished_tasks": newly_finished_tasks or [],
|
|
"steps": steps,
|
|
"previous_turn_silent": self._previous_turn_was_silent(),
|
|
}
|
|
kwargs_to_pass.update(extra_kwargs)
|
|
await component(builder, state, self.ctx, **kwargs_to_pass)
|
|
|
|
tail_content = []
|
|
for part in builder.human_parts:
|
|
if isinstance(part, str):
|
|
tail_content.append({"type": "text", "text": part})
|
|
else:
|
|
tail_content.append(part)
|
|
tail = HumanMessage(content=tail_content)
|
|
# Per-turn notices (reminders, hints, guidance, incident blocks) are
|
|
# deleted by the scrub edge once the turn leaves the recent window.
|
|
if builder.ephemeral_indices:
|
|
mark_ephemeral(tail, builder.ephemeral_indices)
|
|
|
|
messages = ledger.render([tail])
|
|
self._transcript_turn_base = len(messages) - 1
|
|
return messages
|
|
|
|
def _stage_transcript_turn(self, messages: list) -> None:
|
|
"""Hand this turn's tail + tool-loop products to the ledger."""
|
|
base = self._transcript_turn_base
|
|
self._transcript_turn_base = None
|
|
if base is None:
|
|
return
|
|
try:
|
|
ledger = getattr(self.ctx, "transcript_ledger", None)
|
|
if ledger is not None:
|
|
ledger.stage_turn(messages[base:])
|
|
except Exception as e:
|
|
logger.error(f"Failed to stage transcript turn: {e}")
|
|
|
|
async def _invoke_llm_loop(
|
|
self,
|
|
base_llm,
|
|
current_messages: list,
|
|
traced_tools: list,
|
|
new_subagent_calls: list,
|
|
state: State,
|
|
) -> tuple[list[dict] | None, str | None, str | None, bool]:
|
|
max_iterations = OPERATOR_MAX_TOOL_ITERATIONS
|
|
action_result = None
|
|
raw_thoughts = []
|
|
native_thoughts = []
|
|
tool_limit_exceeded = False
|
|
plan_gate_bounced = False
|
|
no_progress_attempts = 0
|
|
no_progress_limit = settings.ARTEMIS_OPERATOR_MAX_NO_PROGRESS_ATTEMPTS
|
|
|
|
for iteration in range(max_iterations):
|
|
if iteration == max_iterations - 1:
|
|
logger.warning(
|
|
"Operator has reached the maximum number of tool calls. Adding reminder."
|
|
)
|
|
current_messages.append(
|
|
SystemMessage(
|
|
content=(
|
|
"You have reached the maximum number of tool calls"
|
|
" for this turn. You must make a final action, or"
|
|
" save your current research results via notes to"
|
|
" continue later."
|
|
)
|
|
)
|
|
)
|
|
|
|
bound_llm = base_llm.bind_tools(tools=traced_tools)
|
|
response = await invoke_llm_with_timeout_message(acomplete(bound_llm, current_messages))
|
|
|
|
usage = usage_from_message(response)
|
|
# The ledger's compaction thresholds read the operator's own
|
|
# prompt size, never the session-wide meter (which the
|
|
# Planner/Checker/sub-agent calls overwrite). The messages ride
|
|
# along so the ledger can calibrate its chars-per-token ratio.
|
|
transcript_ledger = getattr(self.ctx, "transcript_ledger", None)
|
|
if usage is not None:
|
|
logger.info(
|
|
f"LLM usage: prompt_tokens={usage['prompt_tokens']}"
|
|
f" (cached={usage['cached_tokens']}),"
|
|
f" completion_tokens={usage['completion_tokens']}"
|
|
)
|
|
if hasattr(transcript_ledger, "record_prompt_tokens"):
|
|
transcript_ledger.record_prompt_tokens(
|
|
usage.get("prompt_tokens"), messages=current_messages
|
|
)
|
|
elif hasattr(transcript_ledger, "record_prompt_tokens"):
|
|
# No usage metadata from the provider: record an estimate so
|
|
# the start gate and thresholds still see a context base.
|
|
from artemis.memory.transcript import estimate_prompt_tokens
|
|
|
|
transcript_ledger.record_prompt_tokens(estimate_prompt_tokens(current_messages))
|
|
|
|
if response.content:
|
|
if isinstance(response.content, str):
|
|
raw_thoughts.append(response.content)
|
|
elif isinstance(response.content, list):
|
|
for item in response.content:
|
|
if isinstance(item, dict):
|
|
if item.get("type") == "text":
|
|
raw_thoughts.append(item["text"])
|
|
elif item.get("type") == "thinking":
|
|
native_thoughts.append(item["thinking"])
|
|
|
|
additional_kwargs = getattr(response, "additional_kwargs", None)
|
|
if (
|
|
not response.tool_calls
|
|
and type(additional_kwargs) is dict
|
|
and additional_kwargs.get("function_call")
|
|
):
|
|
fc = additional_kwargs["function_call"]
|
|
try:
|
|
arguments_str = fc.get("arguments", "{}")
|
|
if isinstance(arguments_str, str):
|
|
arguments = json.loads(arguments_str)
|
|
else:
|
|
arguments = arguments_str
|
|
|
|
tool_call = {
|
|
"name": fc["name"],
|
|
"args": arguments,
|
|
"id": f"call_{uuid4().hex[:8]}",
|
|
"type": "tool_call",
|
|
}
|
|
response.tool_calls = [tool_call]
|
|
logger.info(
|
|
"Fallback: Manually parsed function_call from"
|
|
f" additional_kwargs: {tool_call}"
|
|
)
|
|
except Exception as e:
|
|
logger.error(f"Failed to parse function_call fallback: {e}")
|
|
|
|
if not response.tool_calls:
|
|
no_progress_attempts += 1
|
|
if no_progress_attempts >= no_progress_limit:
|
|
raise OperatorNoProgressError(
|
|
f"模型连续 {no_progress_attempts} 次未提供可执行的工具调用，"
|
|
"已停止任务以避免空转；未执行新的设备动作。"
|
|
)
|
|
logger.warning("LLM returned no tool call; requesting a concrete tool call.")
|
|
current_messages.append(response)
|
|
current_messages.append(
|
|
SystemMessage(
|
|
content=(
|
|
"No tool call was returned, so no device action occurred. "
|
|
"Use the available tool interface now, not a prose promise or JSON "
|
|
"inside text. Use an index from the current element list or "
|
|
"normalized coordinates with target_description. If blocked, "
|
|
"use an observation/diagnostic tool; do not invent a click or "
|
|
"claim success. This recovery has a limited retry budget."
|
|
)
|
|
)
|
|
)
|
|
continue
|
|
|
|
current_messages.append(response)
|
|
|
|
# Manifest-driven, so a backend's extension actions are classified as
|
|
# turn-ending device actions without touching this file.
|
|
action_tool_names = sorted(self._available_device_actions())
|
|
|
|
def normalize_name(name: str) -> str:
|
|
return name.split(":")[-1] if ":" in name else name
|
|
|
|
action_calls = [
|
|
tc for tc in response.tool_calls if normalize_name(tc["name"]) in action_tool_names
|
|
]
|
|
other_calls = [
|
|
tc
|
|
for tc in response.tool_calls
|
|
if normalize_name(tc["name"]) not in action_tool_names
|
|
]
|
|
|
|
tool_outputs = []
|
|
validation_errors = False
|
|
|
|
other_tool_failed = False
|
|
other_tool_failure_msg = ""
|
|
|
|
has_terminal_calls = bool(action_calls)
|
|
|
|
if other_calls:
|
|
results = []
|
|
for tc in other_calls:
|
|
tool_name = tc["name"]
|
|
logger.info(f"Operator requested tool: {tool_name}")
|
|
if tool_name in ["ask_diagnoser", "video_analyzer"]:
|
|
new_subagent_calls.append(tool_name)
|
|
|
|
if ":" in tool_name:
|
|
tool_to_run = get_tool_by_name(tool_name, traced_tools)
|
|
else:
|
|
tool_to_run = next(
|
|
(t for t in traced_tools if t.name == tool_name),
|
|
None,
|
|
)
|
|
|
|
if tool_to_run:
|
|
args = dict(tc["args"])
|
|
span = TraceSpan(name=tool_name, trace_type="tool", ctx=self.ctx)
|
|
span.payload = {"args": args}
|
|
|
|
with span:
|
|
try:
|
|
state.operator_raw_thinking = (
|
|
"\n".join(raw_thoughts) if raw_thoughts else None
|
|
)
|
|
state.operator_native_thinking = (
|
|
"\n".join(native_thoughts) if native_thoughts else None
|
|
)
|
|
|
|
result_obj = await invoke_tool_with_injection(
|
|
tool=tool_to_run,
|
|
args=args,
|
|
tool_call_id=tc["id"],
|
|
state=state,
|
|
)
|
|
|
|
content = get_tool_result_content(result_obj)
|
|
# Failure is structural (a ToolMessage status or a
|
|
# ToolFailure text), never sniffed from the words.
|
|
is_err = is_tool_failure(result_obj)
|
|
status = "error" if is_err else "success"
|
|
|
|
if is_err:
|
|
other_tool_failed = True
|
|
other_tool_failure_msg = str(content)
|
|
status = "error"
|
|
|
|
span.status = "failed"
|
|
span.error = str(content)
|
|
else:
|
|
span.result = content
|
|
|
|
results.append((content, status))
|
|
except Exception as e:
|
|
other_tool_failed = True
|
|
other_tool_failure_msg = str(e)
|
|
|
|
span.status = "failed"
|
|
span.error = str(e)
|
|
|
|
results.append(
|
|
(
|
|
f"Error running tool {tool_name}: {e}",
|
|
"error",
|
|
)
|
|
)
|
|
else:
|
|
other_tool_failed = True
|
|
other_tool_failure_msg = f"Tool {tool_name} not supported"
|
|
results.append((f"Error: Tool {tool_name} not supported", "error"))
|
|
|
|
for tc, (content, status) in zip(other_calls, results):
|
|
# A step screenshot (content blocks) travels in the carrier
|
|
# this model's provider accepts; text stays a ToolMessage.
|
|
tool_outputs.extend(
|
|
tool_result_messages(
|
|
tc["id"],
|
|
content,
|
|
name=normalize_name(tc["name"]),
|
|
status=status,
|
|
llm=base_llm,
|
|
)
|
|
)
|
|
|
|
if other_tool_failed:
|
|
if action_calls:
|
|
logger.warning(
|
|
"Scratchpad/helper tool call failed. Deferring"
|
|
f" screen actions: {other_tool_failure_msg}"
|
|
)
|
|
# Every rejected call needs its own ToolMessage, but the
|
|
# full explanation travels once; the rest point at it.
|
|
for idx, tc in enumerate(action_calls):
|
|
tool_outputs.append(
|
|
ToolMessage(
|
|
tool_call_id=tc["id"],
|
|
name=normalize_name(tc["name"]),
|
|
content=(
|
|
"Not executed: this screen action was"
|
|
" rejected because an accompanying tool"
|
|
f" call failed. Error: {other_tool_failure_msg}."
|
|
" No screen action ran this turn. Correct"
|
|
" the error, then re-issue the action."
|
|
if idx == 0
|
|
else "Not executed: see the accompanying"
|
|
" tool-call error above."
|
|
),
|
|
status="error",
|
|
)
|
|
)
|
|
action_calls = []
|
|
|
|
elif has_terminal_calls:
|
|
has_deferring_calls = any(
|
|
normalize_name(tc["name"]) in DEFERRING_TOOLS for tc in other_calls
|
|
)
|
|
if has_deferring_calls:
|
|
logger.info(
|
|
"Operator mixed pre-decision tools"
|
|
" (memory/exploration) and terminal action calls."
|
|
" Deferring terminal actions."
|
|
)
|
|
|
|
for idx, tc in enumerate(action_calls):
|
|
tool_outputs.append(
|
|
ToolMessage(
|
|
tool_call_id=tc["id"],
|
|
name=normalize_name(tc["name"]),
|
|
content=(
|
|
"Not executed: this screen action was"
|
|
" deferred because the same turn also"
|
|
" called a result-dependent pre-decision"
|
|
" tool. No screen action ran this turn."
|
|
" Read the tool results above, then"
|
|
" re-issue the action."
|
|
if idx == 0
|
|
else "Not executed: deferred for the same"
|
|
" reason as the screen action above."
|
|
),
|
|
status="error",
|
|
)
|
|
)
|
|
|
|
action_calls = []
|
|
|
|
if action_calls:
|
|
logger.info(
|
|
f"Operator requested {len(action_calls)} action(s)."
|
|
" Translating and processing all."
|
|
)
|
|
actions_list = []
|
|
burst_cap = self._max_burst_actions()
|
|
bounce = None
|
|
if (
|
|
not plan_gate_bounced
|
|
and iteration < max_iterations - 1
|
|
and len(action_calls) == 1
|
|
and not getattr(state, "open_incident", None)
|
|
and self._plan_changed_this_turn() is False
|
|
):
|
|
bounce = self._plan_ledger_bounce_reason()
|
|
if bounce:
|
|
plan_gate_bounced = True
|
|
validation_errors = True
|
|
logger.info(f"Plan ledger gate: bouncing action. {bounce}")
|
|
for tc in action_calls:
|
|
tool_outputs.append(
|
|
ToolMessage(
|
|
tool_call_id=tc["id"],
|
|
name=normalize_name(tc["name"]),
|
|
content=bounce,
|
|
status="error",
|
|
)
|
|
)
|
|
action_calls_to_translate = []
|
|
elif len(action_calls) > burst_cap:
|
|
# A multi-action turn is a fast-action burst that the Validator
|
|
# fires without the safety net; cap its length before it runs.
|
|
validation_errors = True
|
|
burst_error = (
|
|
f"Error: {len(action_calls)} Turn-Ending Actions in one"
|
|
" turn exceed the fast-action burst limit of"
|
|
f" {burst_cap}. Nothing was executed. Re-issue at most"
|
|
f" {burst_cap} actions (a burst is for sub-second"
|
|
" sequences on transient UI, not for batching normal"
|
|
" steps), or a single vetted action."
|
|
)
|
|
# One ToolMessage per rejected call, full text only on the first.
|
|
for idx, tc in enumerate(action_calls):
|
|
tool_outputs.append(
|
|
ToolMessage(
|
|
tool_call_id=tc["id"],
|
|
name=normalize_name(tc["name"]),
|
|
content=(
|
|
burst_error
|
|
if idx == 0
|
|
else "Not executed: see the burst-limit error above."
|
|
),
|
|
status="error",
|
|
)
|
|
)
|
|
action_calls_to_translate = []
|
|
else:
|
|
if plan_gate_bounced and self._plan_changed_this_turn() is False:
|
|
logger.warning(
|
|
"Plan ledger gate: executing actions after one bounce without"
|
|
" a task_plan change (ledger gap this turn)."
|
|
)
|
|
action_calls_to_translate = action_calls
|
|
for tc in action_calls_to_translate:
|
|
translated_actions, error = self._translate_and_validate_tool(tc, state)
|
|
if error:
|
|
validation_errors = True
|
|
tool_outputs.append(
|
|
ToolMessage(
|
|
tool_call_id=tc["id"],
|
|
name=normalize_name(tc["name"]),
|
|
content=error,
|
|
status="error",
|
|
)
|
|
)
|
|
else:
|
|
actions_list.extend(translated_actions)
|
|
tool_outputs.append(
|
|
ToolMessage(
|
|
tool_call_id=tc["id"],
|
|
name=normalize_name(tc["name"]),
|
|
content="Action Recorded",
|
|
status="success",
|
|
)
|
|
)
|
|
|
|
if not validation_errors:
|
|
action_result = actions_list
|
|
|
|
for tm in tool_outputs:
|
|
current_messages.append(tm)
|
|
|
|
if action_calls and not validation_errors:
|
|
break
|
|
|
|
if validation_errors:
|
|
no_progress_attempts += 1
|
|
if no_progress_attempts >= no_progress_limit:
|
|
raise OperatorNoProgressError(
|
|
f"模型连续 {no_progress_attempts} 次未提供可执行的工具调用，"
|
|
"工具参数校验仍失败；已停止任务，未执行被拒绝的设备动作。"
|
|
)
|
|
logger.warning("Some action calls failed validation. Feeding back to LLM.")
|
|
continue
|
|
no_progress_attempts = 0
|
|
else:
|
|
if no_progress_attempts:
|
|
raise OperatorNoProgressError(
|
|
"模型在本轮工具调用预算内未恢复有效调用，已停止任务以避免空转。"
|
|
)
|
|
tool_limit_exceeded = True
|
|
|
|
raw_thinking = "\n".join(raw_thoughts) if raw_thoughts else None
|
|
native_thinking = "\n".join(native_thoughts) if native_thoughts else None
|
|
return (
|
|
action_result,
|
|
raw_thinking,
|
|
native_thinking,
|
|
tool_limit_exceeded,
|
|
)
|
|
|
|
@wrap_with_callbacks(
|
|
before=lambda: logger.info("Starting Operator Agent..."),
|
|
on_success=lambda _: logger.success("Operator Agent"),
|
|
on_failure=lambda _: logger.error("Operator Agent"),
|
|
)
|
|
@trace(type="agent", name="operator")
|
|
async def __call__(self, state: State):
|
|
state = strict_state(state, "operator")
|
|
# 1. Get perception data from state (populated by Perception node)
|
|
operator_raw_data = getattr(state, "operator_raw_data", {}) or {}
|
|
latest_screenshot_b64 = operator_raw_data.get("screenshot_b64")
|
|
xml_hierarchy = operator_raw_data.get("xml_hierarchy")
|
|
operator_raw_data.get("ocr_results")
|
|
fused_xml = state.latest_ui_hierarchy if hasattr(state, "latest_ui_hierarchy") else []
|
|
state.latest_screenshot if hasattr(state, "latest_screenshot") else None
|
|
|
|
if not latest_screenshot_b64 or xml_hierarchy is None:
|
|
logger.error(
|
|
"Operator requires perception data in state (populated by Perception node)"
|
|
)
|
|
raise ValueError("Operator requires perception data in state")
|
|
|
|
# 3. Get history and plan
|
|
steps, task_plan = self._get_history_and_plan()
|
|
self._turn_plan_baseline = self._plan_ledger_baseline(task_plan)
|
|
self._turn_ledger_streak = unwritten_action_streak(steps)
|
|
|
|
# 4. Format minimal list
|
|
|
|
width = 1080
|
|
height = 2400
|
|
w_raw = operator_raw_data.get("width")
|
|
h_raw = operator_raw_data.get("height")
|
|
if isinstance(w_raw, int) and isinstance(h_raw, int):
|
|
width = w_raw
|
|
height = h_raw
|
|
else:
|
|
try:
|
|
if self.ctx and hasattr(self.ctx, "device") and self.ctx.device:
|
|
w = getattr(self.ctx.device, "device_width", 1080)
|
|
h = getattr(self.ctx.device, "device_height", 2400)
|
|
if isinstance(w, int):
|
|
width = w
|
|
if isinstance(h, int):
|
|
height = h
|
|
except (AttributeError, TypeError):
|
|
pass
|
|
minimal_list, elements, labels = format_minimal_list_with_elements(fused_xml, width, height)
|
|
|
|
# Save transient indexed_points and indexed_elements on the state instance
|
|
state.indexed_points = [el["center"] for el in elements]
|
|
state.indexed_elements = elements
|
|
|
|
current_step_num = len(steps) + 1
|
|
|
|
# 5. Prepare Tools. Device-action shells come from the canonical manifest
|
|
# (artemis/mcp/action_specs.py) and are assembled against the installed
|
|
# actuator backend's capabilities: an action the backend does not implement
|
|
# is simply never declared (and the prompt assembly drops its teaching
|
|
# segments in lockstep).
|
|
available_actions = self._available_device_actions()
|
|
all_tools = [
|
|
operator_shell_tool(name) for name in OPERATOR_SHELL_ORDER if name in available_actions
|
|
] + self.tools
|
|
|
|
# 5. Mount the task-output analyzer and collect background ADB task state.
|
|
# The analyzer is always available: the "output truncated, use
|
|
# analyze_task_output" hint arrives mid-turn, and tools are bound once per
|
|
# turn, so a conditional mount would be one turn late.
|
|
if not any(t.name == "analyze_task_output" for t in all_tools):
|
|
all_tools.append(analyze_task_output_wrapper.tool_fn_getter(self.ctx))
|
|
|
|
traced_tools = [trace_langchain_tool(t, self.ctx) for t in all_tools]
|
|
|
|
adb_registry = get_adb_task_registry(self.ctx)
|
|
active_background_tasks = adb_registry.active_task_summaries()
|
|
newly_finished_tasks = adb_registry.pop_unnotified_finished()
|
|
|
|
# 6. Prepare LLM
|
|
llm = get_llm(ctx=self.ctx, name="operator")
|
|
|
|
# 7. Build Prompt
|
|
messages = await self._build_prompt(
|
|
state,
|
|
latest_screenshot_b64,
|
|
fused_xml,
|
|
minimal_list,
|
|
current_step_num,
|
|
steps,
|
|
task_plan,
|
|
active_background_tasks=active_background_tasks,
|
|
newly_finished_tasks=newly_finished_tasks,
|
|
)
|
|
|
|
# 8. Invoke LLM Loop
|
|
new_subagent_calls = []
|
|
(
|
|
action_result,
|
|
raw_thinking,
|
|
native_thinking,
|
|
tool_limit_exceeded,
|
|
) = await self._invoke_llm_loop(
|
|
base_llm=llm,
|
|
current_messages=messages,
|
|
traced_tools=traced_tools,
|
|
new_subagent_calls=new_subagent_calls,
|
|
state=state,
|
|
)
|
|
self._previous_turn_silent = not (raw_thinking and raw_thinking.strip())
|
|
|
|
# 8b. Transcript path: hold this turn's messages (observation tail +
|
|
# tool-loop products) for commit at the next build, when the turn's
|
|
# step id and validator result exist.
|
|
self._stage_transcript_turn(messages)
|
|
|
|
# 9. Update State
|
|
structured_decisions = json.dumps(action_result) if action_result else None
|
|
|
|
return {
|
|
"structured_decisions": structured_decisions,
|
|
"operator_raw_thinking": raw_thinking,
|
|
"operator_native_thinking": native_thinking,
|
|
"indexed_points": state.indexed_points,
|
|
"indexed_elements": state.indexed_elements,
|
|
"current_step_id": state.current_step_id,
|
|
"subagent_calls": ((state.subagent_calls or []) + new_subagent_calls),
|
|
"operator_tool_limit_exceeded": tool_limit_exceeded,
|
|
}
|
|
|
|
def _plan_ledger_gate_enabled(self) -> bool:
|
|
try:
|
|
from artemis.config import load_agent_config
|
|
|
|
return bool(load_agent_config().pro.execution.plan_ledger_gate)
|
|
except (AttributeError, OSError, TypeError, ValueError):
|
|
return True
|
|
|
|
def _read_task_plan_text(self) -> str | None:
|
|
"""Current ``task_plan`` note text, or ``None`` when it cannot be read."""
|
|
engine = getattr(self.ctx, "data_engine", None)
|
|
if not engine:
|
|
return None
|
|
try:
|
|
path = get_note_file_path(engine.base_dir, "task_plan")
|
|
if not path.exists():
|
|
return None
|
|
return path.read_text(encoding="utf-8")
|
|
except (AttributeError, OSError, TypeError) as e:
|
|
logger.warning(f"Plan ledger gate: failed to read task_plan: {e}")
|
|
return None
|
|
|
|
def _plan_ledger_baseline(self, task_plan: str) -> str | None:
|
|
"""Return the plan text used as the current turn's baseline."""
|
|
if not self._plan_ledger_gate_enabled() or not getattr(self.ctx, "data_engine", None):
|
|
return None
|
|
if not parse_plan(task_plan).has_top_level:
|
|
return None
|
|
return task_plan
|
|
|
|
def _plan_ledger_stale_turns(self) -> int:
|
|
try:
|
|
from artemis.config import load_agent_config
|
|
|
|
return int(load_agent_config().pro.execution.plan_ledger_stale_turns)
|
|
except (AttributeError, OSError, TypeError, ValueError):
|
|
return 4
|
|
|
|
def _plan_ledger_bounce_reason(self) -> str | None:
|
|
"""Bounce message when the (unchanged) ledger is broken or stale, else ``None``."""
|
|
current = self._read_task_plan_text()
|
|
if current is None:
|
|
return None
|
|
return render_plan_ledger_bounce(
|
|
current, self._turn_ledger_streak, self._plan_ledger_stale_turns()
|
|
)
|
|
|
|
def _plan_changed_this_turn(self) -> bool | None:
|
|
"""Whether ``task_plan`` differs from the turn's baseline (``None``: gate inactive)."""
|
|
baseline = self._turn_plan_baseline
|
|
if baseline is None:
|
|
return None
|
|
current = self._read_task_plan_text()
|
|
if current is None:
|
|
return None
|
|
return current != baseline
|
|
|
|
def _get_active_subgoal_hash(self) -> str:
|
|
"""Parses task_plan.md to find the active top-level subgoal hash."""
|
|
|
|
if not self.ctx.data_engine:
|
|
return "default"
|
|
|
|
task_plan_path = get_note_file_path(self.ctx.data_engine.base_dir, "task_plan")
|
|
|
|
if not task_plan_path.exists():
|
|
return "default"
|
|
|
|
try:
|
|
content = task_plan_path.read_text(encoding="utf-8")
|
|
|
|
parent_hash, _ = get_active_subgoal_hashes(content)
|
|
return parent_hash
|
|
except Exception as e:
|
|
logger.error(f"Failed to parse active subgoal: {e}")
|
|
|
|
return "default"
|
|
|
|
def _build_plan_and_history(self, steps: list[dict[str, Any]], task_plan: str) -> str:
|
|
"""Builds a plan checklist and separate execution history."""
|
|
|
|
active_subgoal_hash = self._get_active_subgoal_hash()
|
|
return build_history_for(
|
|
"operator",
|
|
task_plan,
|
|
steps,
|
|
active_subgoal_hash,
|
|
last_n_detailed=self.last_n_detailed,
|
|
)
|
|
|
|
def _max_burst_actions(self) -> int:
|
|
"""Configured ceiling for actions in one fast-action burst (default 4)."""
|
|
try:
|
|
from artemis.config import load_agent_config
|
|
|
|
return int(load_agent_config().pro.execution.max_burst_actions)
|
|
except Exception:
|
|
return 4
|
|
|
|
def _translate_and_validate_tool(self, tc: dict, state: State) -> tuple[list[dict], str | None]:
|
|
actions, err = self._translate_and_validate_tool_inner(tc, state)
|
|
if err:
|
|
return [], err
|
|
return actions, None
|
|
|
|
def _translate_and_validate_tool_inner(
|
|
self, tc: dict, state: State
|
|
) -> tuple[list[dict], str | None]:
|
|
tool_name = tc["name"].split(":")[-1] if ":" in tc["name"] else tc["name"]
|
|
args = tc["args"]
|
|
|
|
# Get device resolution fallbacks
|
|
width = 1080
|
|
height = 2400
|
|
operator_raw_data = getattr(state, "operator_raw_data", {}) or {}
|
|
w_raw = operator_raw_data.get("width")
|
|
h_raw = operator_raw_data.get("height")
|
|
if isinstance(w_raw, int) and isinstance(h_raw, int):
|
|
width = w_raw
|
|
height = h_raw
|
|
else:
|
|
try:
|
|
if self.ctx and hasattr(self.ctx, "device") and self.ctx.device:
|
|
w = getattr(self.ctx.device, "device_width", 1080)
|
|
h = getattr(self.ctx.device, "device_height", 2400)
|
|
if isinstance(w, int):
|
|
width = w
|
|
if isinstance(h, int):
|
|
height = h
|
|
except (AttributeError, TypeError):
|
|
pass
|
|
|
|
def resolve_target_element(target: Any, description: Any) -> tuple[dict | None, str | None]:
|
|
"""Resolves a target to ``{"center": [x, y], **semantics}``.
|
|
|
|
An element index carries observed semantics (text, bounds, id, class
|
|
straight from the indexed list); a coordinate pair carries only the
|
|
model's own ``target_description``. The two never mix.
|
|
"""
|
|
target = normalize_coordinate_target(target)
|
|
if isinstance(target, bool):
|
|
return None, "Error: Boolean values are not valid target indices."
|
|
if isinstance(target, (int, float)):
|
|
try:
|
|
target_int = int(target)
|
|
except (ValueError, TypeError):
|
|
return None, f"Error: Invalid target index {target}."
|
|
indexed_elements = getattr(state, "indexed_elements", None) or []
|
|
if 1 <= target_int <= len(indexed_elements):
|
|
el = indexed_elements[target_int - 1]
|
|
return {
|
|
"center": el.get("center"),
|
|
"target_text": el.get("text"),
|
|
"target_bounds": el.get("bounds"),
|
|
"target_resource_id": el.get("resource_id"),
|
|
"target_class": el.get("class"),
|
|
}, None
|
|
return (
|
|
None,
|
|
(
|
|
f"Error: Invalid target index {target_int}. Active"
|
|
f" index range is 1 to {len(indexed_elements)}."
|
|
),
|
|
)
|
|
elif isinstance(target, list) and len(target) == 2:
|
|
described = _described(description)
|
|
if described is None:
|
|
return (
|
|
None,
|
|
(
|
|
f"Error: Coordinate target {target} requires"
|
|
" 'target_description' (what the element is, in a few"
|
|
" words). Provide it, or address the element by index."
|
|
),
|
|
)
|
|
try:
|
|
nx, ny = map(float, target)
|
|
x = int(max(0, min(width - 1, nx * width / 1000)))
|
|
y = int(max(0, min(height - 1, ny * height / 1000)))
|
|
return {"center": [x, y], "target_description": described}, None
|
|
except (ValueError, TypeError):
|
|
return (
|
|
None,
|
|
(f"Error: Invalid coordinates {target}. Must be list of 2 numbers."),
|
|
)
|
|
return (
|
|
None,
|
|
(
|
|
f"Error: Invalid target parameter {target}. Must be index"
|
|
" (int/float) or coordinates [x, y]."
|
|
),
|
|
)
|
|
|
|
def target_fields(el: dict) -> dict:
|
|
"""The recorded target semantics: observed (index) or described (coords)."""
|
|
return {k: v for k, v in el.items() if k != "center"}
|
|
|
|
if tool_name == "click":
|
|
target = args.get("target")
|
|
times = args.get("times", 1)
|
|
delay_ms = args.get("delay_ms", 100)
|
|
el, err = resolve_target_element(target, args.get("target_description"))
|
|
if err:
|
|
return [], err
|
|
norm_c = [
|
|
int(round(el["center"][0] * 1000.0 / width)),
|
|
int(round(el["center"][1] * 1000.0 / height)),
|
|
]
|
|
return [
|
|
{
|
|
"action": "tap",
|
|
"coordinates": el["center"],
|
|
COORDINATE_SPACE_KEY: COORDINATE_SPACE_PIXEL,
|
|
"normalized_coordinates": norm_c,
|
|
"times": times,
|
|
"delay_ms": delay_ms,
|
|
**target_fields(el),
|
|
}
|
|
], None
|
|
|
|
elif tool_name == "long_press":
|
|
target = args.get("target")
|
|
duration = args.get("duration", 1000)
|
|
el, err = resolve_target_element(target, args.get("target_description"))
|
|
if err:
|
|
return [], err
|
|
norm_c = [
|
|
int(round(el["center"][0] * 1000.0 / width)),
|
|
int(round(el["center"][1] * 1000.0 / height)),
|
|
]
|
|
return [
|
|
{
|
|
"action": "long_press_on",
|
|
"coordinates": el["center"],
|
|
COORDINATE_SPACE_KEY: COORDINATE_SPACE_PIXEL,
|
|
"normalized_coordinates": norm_c,
|
|
"duration": duration,
|
|
**target_fields(el),
|
|
}
|
|
], None
|
|
|
|
elif tool_name == "input_text":
|
|
text = args.get("text")
|
|
target = args.get("target")
|
|
clear_exist = args.get("clear_exist", True)
|
|
|
|
if text is None:
|
|
return [], "Error: Missing required argument 'text'."
|
|
if not isinstance(text, str):
|
|
return (
|
|
[],
|
|
(f"Error: 'text' must be a string, got {type(text).__name__}."),
|
|
)
|
|
|
|
el, err = resolve_target_element(target, args.get("target_description"))
|
|
if err:
|
|
return [], err
|
|
|
|
norm_c = [
|
|
int(round(el["center"][0] * 1000.0 / width)),
|
|
int(round(el["center"][1] * 1000.0 / height)),
|
|
]
|
|
actions = []
|
|
actions.append(
|
|
{
|
|
"action": "focus_and_input_text",
|
|
"coordinates": el["center"],
|
|
COORDINATE_SPACE_KEY: COORDINATE_SPACE_PIXEL,
|
|
"normalized_coordinates": norm_c,
|
|
"text": text,
|
|
"clear_before_input": clear_exist,
|
|
**target_fields(el),
|
|
}
|
|
)
|
|
return actions, None
|
|
|
|
elif tool_name == "swipe":
|
|
kind, target, parsed_duration = parse_swipe_parameters(args, default_duration=None)
|
|
duration = args.get("duration")
|
|
|
|
if duration is not None:
|
|
if not isinstance(duration, (int, float)):
|
|
try:
|
|
duration = int(duration)
|
|
except (ValueError, TypeError):
|
|
return (
|
|
[],
|
|
(f"Error: 'duration' must be a number, got {type(duration).__name__}."),
|
|
)
|
|
else:
|
|
duration = int(duration)
|
|
else:
|
|
duration = parsed_duration
|
|
|
|
# Only a coordinate gesture needs the model to say what it drags; a
|
|
# directional scroll targets the page, not an element.
|
|
described: str | None = None
|
|
if kind == "direction":
|
|
x1, y1, x2, y2, smart_dur = compute_smart_swipe_coordinates(
|
|
direction=target,
|
|
target=args.get("target"),
|
|
indexed_elements=getattr(state, "indexed_elements", None),
|
|
ui_hierarchy=getattr(state, "latest_ui_hierarchy", None),
|
|
width=width,
|
|
height=height,
|
|
duration=duration,
|
|
)
|
|
if duration is None:
|
|
duration = smart_dur
|
|
elif kind == "coords" and isinstance(target, list) and len(target) == 4:
|
|
described = _described(args.get("target_description"))
|
|
if described is None:
|
|
return (
|
|
[],
|
|
(
|
|
f"Error: Coordinate swipe {target} requires"
|
|
" 'target_description' (what is being dragged, in a few"
|
|
" words). Provide it, or use a directional swipe."
|
|
),
|
|
)
|
|
try:
|
|
nx1, ny1, nx2, ny2 = map(float, target)
|
|
x1 = int(max(0, min(width - 1, nx1 * width / 1000)))
|
|
y1 = int(max(0, min(height - 1, ny1 * height / 1000)))
|
|
x2 = int(max(0, min(width - 1, nx2 * width / 1000)))
|
|
y2 = int(max(0, min(height - 1, ny2 * height / 1000)))
|
|
except (ValueError, TypeError):
|
|
return (
|
|
[],
|
|
f"Error: Invalid custom swipe coordinates {target}.",
|
|
)
|
|
else:
|
|
return (
|
|
[],
|
|
(
|
|
f"Error: Invalid swipe gesture parameters {args}. Must be"
|
|
" direction string or custom coordinates."
|
|
),
|
|
)
|
|
|
|
# Calculate duration automatically if not provided (safe range 350-900ms to eliminate Android inertial fling)
|
|
if duration is None:
|
|
dist = ((x2 - x1) ** 2 + (y2 - y1) ** 2) ** 0.5
|
|
if dist <= 300:
|
|
duration = 350
|
|
elif dist <= 600:
|
|
duration = 550
|
|
elif dist <= 1000:
|
|
duration = 800
|
|
else:
|
|
duration = 900
|
|
|
|
nx1, ny1 = int(round(x1 * 1000.0 / width)), int(round(y1 * 1000.0 / height))
|
|
nx2, ny2 = int(round(x2 * 1000.0 / width)), int(round(y2 * 1000.0 / height))
|
|
return [
|
|
{
|
|
"action": "swipe",
|
|
"coordinates": [x1, y1, x2, y2],
|
|
COORDINATE_SPACE_KEY: COORDINATE_SPACE_PIXEL,
|
|
"normalized_coordinates": [nx1, ny1, nx2, ny2],
|
|
"normalized_start_coordinates": [nx1, ny1],
|
|
"normalized_end_coordinates": [nx2, ny2],
|
|
"duration": duration,
|
|
# A directional scroll keeps the direction the agent issued so
|
|
# history and replays render "Swiped up (...)" rather than a
|
|
# bare coordinate path.
|
|
**({"direction": str(target)} if kind == "direction" else {}),
|
|
**({"target_description": described} if described else {}),
|
|
}
|
|
], None
|
|
|
|
elif tool_name == "press_key":
|
|
key = args.get("key") or args.get("keycode")
|
|
if not key:
|
|
return [], "Error: Missing required argument 'key'."
|
|
if not isinstance(key, str):
|
|
return (
|
|
[],
|
|
f"Error: 'key' must be a string, got {type(key).__name__}.",
|
|
)
|
|
|
|
normalized_key = key.strip().upper()
|
|
if normalized_key.startswith("KEYCODE_"):
|
|
normalized_key = normalized_key[len("KEYCODE_") :]
|
|
if normalized_key not in SUPPORTED_PRESS_KEYS:
|
|
return (
|
|
[],
|
|
f"Error: Unsupported key '{key}'. Supported keys:"
|
|
f" {', '.join(SUPPORTED_PRESS_KEYS)}.",
|
|
)
|
|
return [{"action": "press_key", "keycode": f"KEYCODE_{normalized_key}"}], None
|
|
|
|
elif tool_name == "manage_app":
|
|
action = args.get("action")
|
|
app_name = args.get("app_name")
|
|
if action not in ["launch", "stop"]:
|
|
return (
|
|
[],
|
|
f"Error: Unsupported app management action '{action}'.",
|
|
)
|
|
if not app_name or not isinstance(app_name, str):
|
|
return [], "Error: 'app_name' must be a valid non-empty string."
|
|
action_str = "launch_app" if action == "launch" else "stop_app"
|
|
return [{"action": action_str, "app_name": app_name}], None
|
|
|
|
elif tool_name == "wait_for_delay":
|
|
time_in_ms = args.get("time_in_ms")
|
|
if time_in_ms is None:
|
|
return [], "Error: Missing required argument 'time_in_ms'."
|
|
try:
|
|
time_in_ms = int(time_in_ms)
|
|
except (ValueError, TypeError):
|
|
return [], f"Error: Invalid time_in_ms value {time_in_ms}."
|
|
return [{"action": "wait_for_delay", "time_in_ms": time_in_ms}], None
|
|
|
|
return [], f"Error: Unsupported tool name '{tool_name}'."
|